diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c220c3d..a2c3a1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,9 @@ jobs: with: go-version-file: go.mod - name: gofmt - run: test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; } + run: test -z "$(gofmt -l .)" || { gofmt -d .; exit 1; } + - name: Owner admission and post-consumption recovery witnesses + run: go test -race -count=1 -timeout=2m -run '^TestOwner' -v ./pkg/csi - run: go vet ./... - run: go test -race ./... - name: E11 payload accounting and validity regressions diff --git a/cmd/onggi/casync.go b/cmd/onggi/casync.go new file mode 100644 index 0000000..0ce5327 --- /dev/null +++ b/cmd/onggi/casync.go @@ -0,0 +1,184 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/url" + "os" + "time" + + "github.com/folbricht/desync" + + "github.com/ai-computing/onggi/pkg/casync" + "github.com/ai-computing/onggi/pkg/csi" + "github.com/ai-computing/onggi/pkg/digest" + "github.com/ai-computing/onggi/pkg/store" +) + +const casyncUsage = `onggi casync — artifacts in casync format (desync chunk store), with state-aware in-place reuse + + onggi casync publish --store S --index I chunk dir into S; print the manifest digest + onggi casync chunks --store S --index I print the chunk list of every file (JSON) + onggi casync pull --store S --index I [--attest A] + [--base M --base-dir D] take base M over (Keep): write only differing chunks; kept chunks are + read back unless A attests the base unchanged since it was written + [--copy M --copy-dir D] read base M where it is (Copy); it stays intact + [--verify-kept] [--no-sync] [--workers N] [--log] + onggi casync verify --store S --index I hash-check every chunk of dir + onggi casync attest --store S --index I --attest A verify dir, then record it as unchanged +` + +func casyncCmd(args []string, stdout, stderr io.Writer) int { + if len(args) < 1 { + fmt.Fprint(stderr, casyncUsage) + return 2 + } + fs := flag.NewFlagSet("casync "+args[0], flag.ContinueOnError) + fs.SetOutput(stderr) + storeDir := fs.String("store", "", "desync local chunk store") + indexDir := fs.String("index", "", "directory of artifact indexes") + attestDir := fs.String("attest", "", "directory of attestations (none: kept chunks are always read back)") + workers := fs.Int("workers", 12, "parallel chunk reads and writes") + base := fs.String("base", "", "Keep: manifest of the artifact to take over") + baseDir := fs.String("base-dir", "", "Keep: directory holding the base's files") + cp := fs.String("copy", "", "Copy: manifest of an artifact to read where it is") + copyDir := fs.String("copy-dir", "", "Copy: directory holding that artifact's files") + verifyKept := fs.Bool("verify-kept", false, "read back kept chunks even if the base is attested unchanged") + noSync := fs.Bool("no-sync", false, "do not sync written files (desync's default behaviour)") + logPhases := fs.Bool("log", false, "log phase durations to stderr") + remote := fs.String("remote", "", "pull: read chunks from this HTTP chunk server instead of --store") + if err := fs.Parse(args[1:]); err != nil || *storeDir == "" || *indexDir == "" { + fmt.Fprint(stderr, casyncUsage) + return 2 + } + e, err := casync.New(*storeDir, *indexDir, *attestDir, *workers) + if err == nil && *remote != "" { + var u *url.URL + if u, err = url.Parse(*remote); err == nil { + e.Src, err = desync.NewRemoteHTTPStore(u, desync.StoreOptions{N: *workers, Timeout: 5 * time.Minute}) + } + } + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + e.Durable = !*noSync + e.VerifyKept = *verifyKept + if *logPhases { + l := log.New(stderr, "", log.Lmicroseconds) + e.Logf = l.Printf + } + ctx := context.Background() + switch args[0] { + case "publish": + if fs.NArg() != 1 { + fmt.Fprint(stderr, casyncUsage) + return 2 + } + start := time.Now() + d, err := e.Publish(ctx, fs.Arg(0)) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + fmt.Fprintf(stdout, "%s\t%.2fs\n", d, time.Since(start).Seconds()) + return 0 + case "chunks": + // Dump the chunk list of every file of a manifest (start, size, id) as JSON, for capacity + // and transfer accounting outside the engine. + if fs.NArg() != 1 { + fmt.Fprint(stderr, casyncUsage) + return 2 + } + m, err := digest.Parse(fs.Arg(0)) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + a, idxs, err := e.Load(m) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + type fileChunks struct { + Path string `json:"path"` + Size int64 `json:"size"` + Chunks [][3]string `json:"chunks"` + } + out := make([]fileChunks, len(a.Files)) + for i, f := range a.Files { + out[i] = fileChunks{Path: f.Path, Size: f.Size} + for _, c := range idxs[i].Chunks { + out[i].Chunks = append(out[i].Chunks, [3]string{fmt.Sprint(c.Start), fmt.Sprint(c.Size), c.ID.String()}) + } + } + if err := json.NewEncoder(stdout).Encode(out); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + return 0 + case "pull", "verify", "attest": + if fs.NArg() != 2 { + fmt.Fprint(stderr, casyncUsage) + return 2 + } + m, err := digest.Parse(fs.Arg(0)) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + dest := fs.Arg(1) + if args[0] != "pull" { + if err := e.Verify(ctx, m, dest); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + if args[0] == "attest" { + if err := e.Attest(m, dest); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + } + return 0 + } + if err := os.MkdirAll(dest, 0o755); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + var offer *csi.Offer + if *base != "" { + b, err := digest.Parse(*base) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + offer = &csi.Offer{Bases: []store.Base{{Manifest: b, Dir: *baseDir}}, + Claim: func(sb store.Base) (string, bool) { return sb.Dir, true }} + } + if *cp != "" { + c, err := digest.Parse(*cp) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + if offer == nil { + offer = &csi.Offer{} + } + offer.Readable, offer.BaseCopy = []store.Base{{Manifest: c, Dir: *copyDir}}, store.BaseCopyAlways + } + st, err := e.Pull(ctx, m, dest, offer) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + js, _ := json.Marshal(st) + fmt.Fprintln(stdout, string(js)) + return 0 + } + fmt.Fprint(stderr, casyncUsage) + return 2 +} diff --git a/cmd/onggi/csi.go b/cmd/onggi/csi.go index 3c86d4e..a8ce63c 100644 --- a/cmd/onggi/csi.go +++ b/cmd/onggi/csi.go @@ -17,6 +17,7 @@ import ( bfs "github.com/ai-computing/onggi/pkg/backend/fs" "github.com/ai-computing/onggi/pkg/csi" "github.com/ai-computing/onggi/pkg/digest" + "github.com/ai-computing/onggi/pkg/casync" "github.com/ai-computing/onggi/pkg/peer" "github.com/ai-computing/onggi/pkg/store" ) @@ -38,6 +39,13 @@ func csiNode(args []string, stderr io.Writer) int { keepUsed := fs.Duration("keep-used", 7*24*time.Hour, "the node cache keeps what artifacts used within this period need") gcGrace := fs.Duration("gc-grace", time.Hour, "grace period of the node cache's GC") rebaseIdle := fs.Duration("rebase-idle", 5*time.Minute, "a pull may turn an artifact no volume used for this long into the new one, writing only what differs (0: never)") + maxArtifacts := fs.Int("max-artifacts", 0, "artifacts kept per node; a pull takes an idle one over only when the node holds this many, least recently used first (0: no limit)") + verifyKept := fs.Bool("verify-kept", true, "read back and hash-check every chunk a pull leaves in place when it takes an artifact over") + baseCopy := fs.String("base-copy", "remote", "read an idle artifact that is not taken over where it is: remote (only if the store is not local), always, never") + engine := fs.String("engine", "store", "how artifacts are materialized: store (onggi packs) or casync (desync chunk store with state-aware in-place reuse)") + casyncStore := fs.String("casync-store", "", "casync engine: desync local chunk store") + casyncIndex := fs.String("casync-index", "", "casync engine: directory of artifact indexes (onggi casync publish)") + casyncVerify := fs.Bool("casync-verify-always", false, "casync engine: read back kept chunks even when the base is attested unchanged") peerListen := fs.String("peer-listen", "", "serve the node cache to other nodes' pulls on this address, e.g. :7070 (#37)") peers := fs.String("peers", "", "comma-separated peer URLs asked for chunks before the store") peersDNS := fs.String("peers-dns", "", "host:port whose name resolves to the peers, e.g. a headless Service; re-resolved every 30 s") @@ -100,12 +108,31 @@ func csiNode(args []string, stderr io.Writer) int { } } m := &csi.Materializer{Root: *dataDir, Pull: csi.StorePullerPeers(s, openCache, peerList, tmp, *workers), Logf: logger.Printf, MaxPulls: *pulls} + if *engine == "casync" { + e, err := casync.New(*casyncStore, *casyncIndex, filepath.Join(*dataDir, "casync-attest"), *workers) + if err != nil || *casyncStore == "" || *casyncIndex == "" { + logger.Print("casync engine: needs -casync-store and -casync-index: ", err) + return 2 + } + e.VerifyKept, e.Logf = *casyncVerify, logger.Printf + m.Pull = e.Pull + *verifyKept = *casyncVerify // the engine decides from the attestation; the offer forces a read-back only on request + } else if *engine != "store" { + logger.Printf("-engine: %q is not store or casync", *engine) + return 2 + } if err := m.Init(); err != nil { logger.Print("data dir: ", err) return 1 } mounter := csi.NewMounter() m.RebaseIdle, m.Unmount = *rebaseIdle, mounter.Unmount + m.MaxArtifacts, m.VerifyKept = *maxArtifacts, *verifyKept + var ok bool + if m.BaseCopy, ok = parseBaseCopy(*baseCopy); !ok { + logger.Printf("-base-copy: %q is not remote, always or never", *baseCopy) + return 2 + } n := &csi.Node{ Version: version, NodeID: *nodeID, diff --git a/cmd/onggi/main.go b/cmd/onggi/main.go index ae9f0bc..e63bf1e 100644 --- a/cmd/onggi/main.go +++ b/cmd/onggi/main.go @@ -34,6 +34,7 @@ usage: evict from a node cache what recently used artifacts do not need onggi dedup-report [flags] ... measure how much the files would shrink in onggi + onggi casync publish|pull|verify|attest ... casync-format artifacts with state-aware in-place reuse onggi csi --store S [flags] CSI node plugin: serve artifacts as read-only volumes (Linux) onggi peer serve --cache DIR [--listen :7070] serve a node cache to other nodes' pulls (--peers) onggi version @@ -61,6 +62,8 @@ func main() { os.Exit(inspect(os.Args[2:], os.Stdout, os.Stderr)) case "diff": os.Exit(diff(os.Args[2:], os.Stdout, os.Stderr)) + case "casync": + os.Exit(casyncCmd(os.Args[2:], os.Stdout, os.Stderr)) case "csi": os.Exit(csiNode(os.Args[2:], os.Stderr)) case "rm-ref": diff --git a/cmd/onggi/store.go b/cmd/onggi/store.go index 21ba66c..d327dfe 100644 --- a/cmd/onggi/store.go +++ b/cmd/onggi/store.go @@ -26,6 +26,7 @@ import ( type storeFlags struct { url, durability, min, avg, max, sub, subKinds, packSize, frame *string + frameFlat *bool } func addStoreFlags(fs *flag.FlagSet) storeFlags { @@ -38,6 +39,7 @@ func addStoreFlags(fs *flag.FlagSet) storeFlags { sub: fs.String("sub", "4KiB", "average sub-chunk size for two-level chunking, 0 for one level (only when creating a store)"), subKinds: fs.String("sub-kinds", "atom-prefill,atom-decode,atom-other", "file kinds that get sub-chunks, or all (only when creating a store)"), packSize: fs.String("pack-size", "64MiB", "pack object size (only when creating a store)"), + frameFlat: fs.Bool("frame-flat", false, "with --sub 0, compress the new chunks of a one-level store in frames of --frame too (only when creating a store)"), frame: fs.String("frame", "256KiB", "compress new chunks of sub-chunked files together up to this size, 0 for one by one (only when creating a store)"), } } @@ -101,6 +103,13 @@ func (f storeFlags) open(ctx context.Context) (*store.Store, backend.Refs, error } } } + if c.Sub == nil && *f.frameFlat && *f.frame != "0" { + fr, err := size(*f.frame) + if err != nil { + return nil, nil, err + } + c.Frame, c.FrameFlat = int64(fr), true + } s, err := store.OpenWith(ctx, b, c) return s, b, err } @@ -192,6 +201,10 @@ func pull(args []string, stdout, stderr io.Writer) int { peers := fs.String("peers", "", "comma-separated peer URLs (onggi peer serve) asked for chunks before the store") baseRef := fs.String("base", "", "an artifact already restored in --base-dir that the pull may turn into this one, writing only what differs; its files are moved away (#41)") baseDir := fs.String("base-dir", "", "the directory holding --base") + baseIntact := fs.Bool("base-intact", false, "never move --base's files away: read from them where they are") + baseCopy := fs.String("base-copy", "always", "read a base that is not taken over where it is: always, remote (only if the store is not local), never") + baseRange := fs.Int64("base-range", 0, "largest single read from the base's files in bytes (0: 16 MiB; 1: chunk by chunk)") + verifyKept := fs.Bool("verify-kept", false, "read back and hash-check every chunk left in place when --base is taken over") asJSON := fs.Bool("json", false, "print JSON") if err := fs.Parse(args); err != nil || fs.NArg() != 2 { fmt.Fprintln(stderr, "usage: onggi pull --store S [--cache DIR] [--peers URLS] [--base REF --base-dir DIR] ") @@ -220,8 +233,14 @@ func pull(args []string, stdout, stderr io.Writer) int { return 1 } o.Bases = []store.Base{{Manifest: bd, Dir: *baseDir}} - o.Claim = func(b store.Base) (string, bool) { return b.Dir, true } + o.Claim = func(b store.Base) (string, bool) { return b.Dir, !*baseIntact } } + var ok bool + if o.BaseCopy, ok = parseBaseCopy(*baseCopy); !ok { + fmt.Fprintf(stderr, "onggi: --base-copy: %q is not always, remote or never\n", *baseCopy) + return 2 + } + o.VerifyKept, o.BaseRange = *verifyKept, *baseRange if *cacheDir != "" { cb, err := bfs.Open(*cacheDir, backend.DurabilityNone) if err == nil { @@ -430,3 +449,16 @@ func diff(args []string, stdout, stderr io.Writer) int { } return 0 } + +// parseBaseCopy maps a --base-copy value to the rule. +func parseBaseCopy(v string) (store.BaseCopy, bool) { + switch v { + case "always": + return store.BaseCopyAlways, true + case "remote": + return store.BaseCopyRemote, true + case "never": + return store.BaseCopyNever, true + } + return store.BaseCopyAlways, false +} diff --git a/go.mod b/go.mod index 4fe7267..c55c69e 100644 --- a/go.mod +++ b/go.mod @@ -1,32 +1,87 @@ module github.com/ai-computing/onggi -go 1.24 +go 1.25.0 require ( github.com/container-storage-interface/spec v1.12.0 - github.com/klauspost/compress v1.18.0 - github.com/minio/minio-go/v7 v7.0.95 - golang.org/x/sys v0.33.0 - google.golang.org/grpc v1.75.1 + github.com/folbricht/desync v1.1.5 + github.com/klauspost/compress v1.19.2 + github.com/minio/minio-go/v7 v7.3.0 + golang.org/x/sys v0.47.0 + google.golang.org/grpc v1.83.2 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect + cel.dev/expr v0.25.2 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.18.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/storage v1.60.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/dchest/siphash v1.2.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/go-ini/ini v1.67.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/folbricht/tempfile v0.0.1 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.11 // indirect - github.com/minio/crc64nvme v1.0.2 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect + github.com/googleapis/gax-go/v2 v2.17.0 // indirect + github.com/hanwen/go-fuse/v2 v2.11.0 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/philhofer/fwd v1.2.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/sftp v1.13.5 // indirect + github.com/pkg/xattr v0.4.9 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/rs/xid v1.6.0 // indirect - github.com/tinylib/msgp v1.3.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/text v0.26.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect - google.golang.org/protobuf v1.36.6 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/api v0.267.0 // indirect + google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect + oras.land/oras-go/v2 v2.6.2 // indirect ) diff --git a/go.sum b/go.sum index 85baef5..36902fa 100644 --- a/go.sum +++ b/go.sum @@ -1,72 +1,225 @@ +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM= +cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY= +cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8= +cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d h1:zsO4lp+bjv5XvPTF58Vq+qgmZEYZttJK+CWtSZhKenI= +github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d/go.mod h1:f1iKL6ZhUWvbk7PdWVmOaak10o86cqMUYEmn1CZNGEI= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/container-storage-interface/spec v1.12.0 h1:zrFOEqpR5AghNaaDG4qyedwPBqU2fU0dWjLQMP/azK0= github.com/container-storage-interface/spec v1.12.0/go.mod h1:txsm+MA2B2WDa5kW69jNbqPnvTtfvZma7T/zsAZ9qX8= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= +github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/folbricht/desync v1.1.5 h1:FxWghh6PzTglDW6zh6Gx235H33EffX+yxgJu2bBtc+A= +github.com/folbricht/desync v1.1.5/go.mod h1:AvU6wBGs+PrGrInuNo1FX9k+AGNPohdeB6/O4mgke7c= +github.com/folbricht/tempfile v0.0.1 h1:kB3DubP2Fm5e3W7TrWFNZBfzFEHBoKL7Pjn0HvqKxSQ= +github.com/folbricht/tempfile v0.0.1/go.mod h1:/Flpxx/6U+clQJ61jQ3y6Z7L2l6j1/ZSiU4B9EDPgWw= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= +github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8= +github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= +github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= +github.com/hanwen/go-fuse/v2 v2.11.0 h1:CGVkJh9gRz0pTRMADNcqdFl3ec/5QbE/Vx1Gl7ESozM= +github.com/hanwen/go-fuse/v2 v2.11.0/go.mod h1:aU7NkGYZUmuJrZapoI3mEcNve7PZTySUOLBuch/vR6U= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= -github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= -github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= -github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU= +github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go= +github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= +github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= +github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= -github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= -google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= +github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE= +google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +oras.land/oras-go/v2 v2.6.2 h1:N04RXngAp1LJKTG6ifz3xHPipasEkWr+hFmInja5YKo= +oras.land/oras-go/v2 v2.6.2/go.mod h1:PlTtg4JTDJkDe8yVHpM2wz7/YDc00GVas+i4jAW2TZ4= diff --git a/pkg/backend/backend.go b/pkg/backend/backend.go index eb03630..5defad2 100644 --- a/pkg/backend/backend.go +++ b/pkg/backend/backend.go @@ -49,6 +49,8 @@ type Capabilities struct { RangeReads bool AtomicCreate bool // a second Put of the same key can never expose partial or mixed bytes Durability Durability + // Local says the objects are files on this machine, so a range read costs no round trip. + Local bool } // Backend stores immutable objects. diff --git a/pkg/backend/fs/fs.go b/pkg/backend/fs/fs.go index 40d71d1..778767f 100644 --- a/pkg/backend/fs/fs.go +++ b/pkg/backend/fs/fs.go @@ -180,7 +180,7 @@ func (s *Store) Delete(ctx context.Context, k backend.Key) error { // Caps: range reads; atomic create through link or rename. func (s *Store) Caps() backend.Capabilities { - return backend.Capabilities{RangeReads: true, AtomicCreate: true, Durability: s.durability} + return backend.Capabilities{RangeReads: true, AtomicCreate: true, Durability: s.durability, Local: true} } // ---- refs: a lock file per ref makes SetRef a compare-and-swap on a single mount. diff --git a/pkg/casync/casync.go b/pkg/casync/casync.go new file mode 100644 index 0000000..be714e7 --- /dev/null +++ b/pkg/casync/casync.go @@ -0,0 +1,730 @@ +// Package casync materializes artifacts from a casync-format chunk store (desync), and adds what a +// stateless tool cannot do: it knows which sibling on the node is idle, aligned and unchanged since +// it was written, and turns that sibling into the requested artifact by writing only the chunks that +// differ, without reading the ones it keeps. +// +// Layout under Engine.IndexDir: +// +// /artifact.json files of the artifact: path, size, mode, index file +// /.caibx casync index of file i +// +// The manifest digest is the SHA-256 of artifact.json. Chunks live in a desync local store +// (one zstd-compressed file per chunk, named by its SHA512/256). Engine.AttestDir records, for every +// file this engine wrote, the inode, size and modification time it had when the write completed; +// a later Keep trusts the chunks it leaves in place only if the file still has them (state-aware +// in-place), and otherwise reads them back and checks each digest. +package casync + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/folbricht/desync" + + "github.com/ai-computing/onggi/pkg/csi" + "github.com/ai-computing/onggi/pkg/digest" + "github.com/ai-computing/onggi/pkg/store" +) + +// Chunk sizes of desync's defaults (16/64/256 KiB). +const ( + MinChunk = 16 << 10 + AvgChunk = 64 << 10 + MaxChunk = 256 << 10 +) + +// Engine reads and writes artifacts in casync format. +type Engine struct { + Store desync.LocalStore + // Src, if set, is where pulls read chunks from instead of Store (for example a remote HTTP + // chunk server); Store still receives published chunks. + Src desync.Store + IndexDir string + AttestDir string // "" : never trust kept chunks, always read them back + Workers int + MinSame float64 // share of bytes at the same offset needed to take a base (0: store.DefaultMinSame) + // VerifyKept forces reading back kept chunks even when the base is attested unchanged. + VerifyKept bool + // Durable syncs every written file before the pull returns (default true via New). + Durable bool + // Logf, if set, receives one line per phase with its duration. + Logf func(format string, args ...any) +} + +// New opens the chunk store at storeDir and the index and attestation directories. +func New(storeDir, indexDir, attestDir string, workers int) (*Engine, error) { + for _, d := range []string{storeDir, indexDir} { + if err := os.MkdirAll(d, 0o755); err != nil { + return nil, err + } + } + if attestDir != "" { + if err := os.MkdirAll(attestDir, 0o755); err != nil { + return nil, err + } + } + s, err := desync.NewLocalStore(storeDir, desync.StoreOptions{N: max(workers, 1)}) + if err != nil { + return nil, err + } + return &Engine{Store: s, IndexDir: indexDir, AttestDir: attestDir, Workers: max(workers, 1), Durable: true}, nil +} + +// FileEntry is one file of an artifact. +type FileEntry struct { + Path string `json:"path"` + Size int64 `json:"size"` + Mode uint32 `json:"mode"` + Index string `json:"index"` + // IndexSHA256 is the SHA-256 of the index file, so the manifest digest covers the content. + IndexSHA256 string `json:"index_sha256"` +} + +// Artifact is the content of artifact.json. +type Artifact struct { + Files []FileEntry `json:"files"` +} + +func (e *Engine) logf(format string, args ...any) { + if e.Logf != nil { + e.Logf(format, args...) + } +} + +// Publish chunks every regular file under dir into the store and writes the artifact's indexes. +// It returns the manifest digest. +func (e *Engine) Publish(ctx context.Context, dir string) (digest.Digest, error) { + var paths []string + err := filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.Type().IsRegular() { + rel, _ := filepath.Rel(dir, p) + paths = append(paths, filepath.ToSlash(rel)) + } + return nil + }) + if err != nil { + return digest.Digest{}, err + } + sort.Strings(paths) + tmp, err := os.MkdirTemp(e.IndexDir, ".publish-") + if err != nil { + return digest.Digest{}, err + } + defer os.RemoveAll(tmp) + var a Artifact + for i, rel := range paths { + src := filepath.Join(dir, filepath.FromSlash(rel)) + fi, err := os.Stat(src) + if err != nil { + return digest.Digest{}, err + } + idx, _, err := desync.IndexFromFile(ctx, src, e.Workers, MinChunk, AvgChunk, MaxChunk, desync.NullProgressBar{}) + if err != nil { + return digest.Digest{}, fmt.Errorf("index %s: %w", rel, err) + } + if err := desync.ChopFile(ctx, src, idx.Chunks, e.Store, e.Workers, desync.NullProgressBar{}); err != nil { + return digest.Digest{}, fmt.Errorf("chop %s: %w", rel, err) + } + name := fmt.Sprintf("%d.caibx", i) + var buf bytes.Buffer + if _, err := idx.WriteTo(&buf); err != nil { + return digest.Digest{}, err + } + if err := os.WriteFile(filepath.Join(tmp, name), buf.Bytes(), 0o644); err != nil { + return digest.Digest{}, err + } + isum := sha256.Sum256(buf.Bytes()) + a.Files = append(a.Files, FileEntry{Path: rel, Size: fi.Size(), Mode: uint32(fi.Mode().Perm()), Index: name, + IndexSHA256: hex.EncodeToString(isum[:])}) + } + data, _ := json.Marshal(a) + sum := sha256.Sum256(data) + d, err := digest.Parse("sha256:" + hex.EncodeToString(sum[:])) + if err != nil { + return digest.Digest{}, err + } + if err := os.WriteFile(filepath.Join(tmp, "artifact.json"), data, 0o644); err != nil { + return digest.Digest{}, err + } + final := filepath.Join(e.IndexDir, d.Hex()) + if _, err := os.Stat(final); err == nil { + return d, nil // published before: same content + } + if err := os.Rename(tmp, final); err != nil { + return digest.Digest{}, err + } + return d, nil +} + +// Load returns the artifact's file list and the casync index of each file. +func (e *Engine) Load(d digest.Digest) (*Artifact, []desync.Index, error) { + dir := filepath.Join(e.IndexDir, d.Hex()) + data, err := os.ReadFile(filepath.Join(dir, "artifact.json")) + if err != nil { + return nil, nil, err + } + sum := sha256.Sum256(data) + if hex.EncodeToString(sum[:]) != d.Hex() { + return nil, nil, fmt.Errorf("artifact %s: artifact.json does not match its digest", d) + } + var a Artifact + if err := json.Unmarshal(data, &a); err != nil { + return nil, nil, err + } + idxs := make([]desync.Index, len(a.Files)) + for i, f := range a.Files { + raw, err := os.ReadFile(filepath.Join(dir, f.Index)) + if err != nil { + return nil, nil, err + } + if isum := sha256.Sum256(raw); hex.EncodeToString(isum[:]) != f.IndexSHA256 { + return nil, nil, fmt.Errorf("%s: index does not match the manifest", f.Path) + } + idxs[i], err = desync.IndexFromReader(bytes.NewReader(raw)) + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", f.Path, err) + } + } + return &a, idxs, nil +} + +// sameOffset marks the target chunks that the base file holds at the same offset. Both chunk +// lists are sorted by offset, so one pass suffices; no file is read. +func sameOffset(base, target []desync.IndexChunk) ([]bool, int64) { + kept := make([]bool, len(target)) + var same int64 + j := 0 + for i, c := range target { + for j < len(base) && base[j].Start < c.Start { + j++ + } + if j < len(base) && base[j].Start == c.Start && base[j].ID == c.ID && base[j].Size == c.Size { + kept[i] = true + same += int64(c.Size) + } + } + return kept, same +} + +// keepPlan pairs the target's files with a base's files by path. +type keepPlan struct { + base store.Base + files *Artifact + idxs []desync.Index + pair []int // base file per target file, -1: none + kept [][]bool // per target file, chunks the paired base file holds at the same offset + same int64 +} + +func (e *Engine) plan(target *Artifact, tidx []desync.Index, b store.Base) (keepPlan, error) { + ba, bidx, err := e.Load(b.Manifest) + if err != nil { + return keepPlan{}, err + } + byPath := map[string]int{} + for j, f := range ba.Files { + byPath[f.Path] = j + } + p := keepPlan{base: b, files: ba, idxs: bidx, pair: make([]int, len(target.Files)), kept: make([][]bool, len(target.Files))} + for i, f := range target.Files { + j, ok := byPath[f.Path] + if !ok { + p.pair[i] = -1 + continue + } + p.pair[i] = j + var s int64 + p.kept[i], s = sameOffset(bidx[j].Chunks, tidx[i].Chunks) + p.same += s + } + // Variants of different shapes usually name their files differently (decode_S256_B2 vs _B8). + // A target file without a namesake takes the unused base file that holds the most of its + // bytes at the same offset, decided from the two indexes alone. + used := map[int]bool{} + for _, j := range p.pair { + if j >= 0 { + used[j] = true + } + } + for i := range target.Files { + if p.pair[i] >= 0 { + continue + } + best, bestSame, bestShared := -1, int64(0), int64(0) + var bestKept []bool + for j := range ba.Files { + if used[j] { + continue + } + k, s := sameOffset(bidx[j].Chunks, tidx[i].Chunks) + // prefer bytes at the same offset (Keep); otherwise bytes held anywhere (a seed for Copy) + sh := sharedBytes([]desync.Index{bidx[j]}, []desync.Index{tidx[i]}) + if s > bestSame || (s == bestSame && sh > bestShared) { + best, bestSame, bestShared, bestKept = j, s, sh, k + } + } + if bestSame == 0 && bestShared == 0 { + best = -1 + } + if best >= 0 { + p.pair[i], p.kept[i] = best, bestKept + p.same += bestSame + used[best] = true + } + } + return p, nil +} + +// attestation is what a file looked like when this engine finished writing it. +type attestation struct { + Ino uint64 `json:"ino"` + Size int64 `json:"size"` + MtimeNs int64 `json:"mtime_ns"` +} + +func stat(p string) (attestation, error) { + fi, err := os.Stat(p) + if err != nil { + return attestation{}, err + } + a := attestation{Size: fi.Size(), MtimeNs: fi.ModTime().UnixNano()} + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + a.Ino = uint64(st.Ino) + } + return a, nil +} + +func (e *Engine) attestFile(d digest.Digest) string { + return filepath.Join(e.AttestDir, d.Hex()+".json") +} + +func (e *Engine) recordAttestation(d digest.Digest, dir string, a *Artifact) error { + if e.AttestDir == "" { + return nil + } + rec := map[string]attestation{} + for _, f := range a.Files { + s, err := stat(filepath.Join(dir, filepath.FromSlash(f.Path))) + if err != nil { + return err + } + rec[f.Path] = s + } + data, _ := json.Marshal(rec) + tmp := e.attestFile(d) + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, e.attestFile(d)) +} + +// Attest records the files of manifest d under dir as unchanged; call it only after Verify. +func (e *Engine) Attest(d digest.Digest, dir string) error { + a, _, err := e.Load(d) + if err != nil { + return err + } + return e.recordAttestation(d, dir, a) +} + +// unchanged reports whether the base file at p is the one this engine wrote for manifest d. +func (e *Engine) unchanged(d digest.Digest, path, p string) bool { + if e.AttestDir == "" { + return false + } + data, err := os.ReadFile(e.attestFile(d)) + if err != nil { + return false + } + var rec map[string]attestation + if json.Unmarshal(data, &rec) != nil { + return false + } + want, ok := rec[path] + if !ok { + return false + } + got, err := stat(p) + return err == nil && got == want +} + +// Pull writes artifact m into the empty directory dest. With an offer it takes an idle aligned +// base over (Keep), or reads a base that stays intact (Copy); otherwise it assembles from the +// store (Reconstruct). It satisfies csi.Puller. +func (e *Engine) Pull(ctx context.Context, m digest.Digest, dest string, offer *csi.Offer) (st store.PullStats, err error) { + var src desync.Store = e.Store + if e.Src != nil { + src = e.Src + } + cs := &countingStore{Store: src} + defer func() { st.FetchedChunks, st.FetchedBytes = cs.n.Load(), cs.b.Load() }() + return e.pull(ctx, m, dest, offer, cs) +} + +// countingStore counts the chunks a pull takes from its chunk source and their logical size. +type countingStore struct { + desync.Store + n, b atomic.Int64 +} + +func (c *countingStore) GetChunk(id desync.ChunkID) (*desync.Chunk, error) { + ch, err := c.Store.GetChunk(id) + if err == nil { + if d, derr := ch.Data(); derr == nil { + c.n.Add(1) + c.b.Add(int64(len(d))) + } + } + return ch, err +} + +func (e *Engine) pull(ctx context.Context, m digest.Digest, dest string, offer *csi.Offer, src desync.Store) (store.PullStats, error) { + start := time.Now() + a, idxs, err := e.Load(m) + if err != nil { + return store.PullStats{}, err + } + st := store.PullStats{Files: len(a.Files)} + for _, f := range a.Files { + st.Bytes += f.Size + } + finals := make([]string, len(a.Files)) + for i, f := range a.Files { + p := filepath.Join(dest, filepath.FromSlash(f.Path)) + if !filepath.IsLocal(filepath.FromSlash(f.Path)) { + return st, fmt.Errorf("file path %q leaves the artifact", f.Path) + } + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + return st, err + } + finals[i] = p + } + done := make([]bool, len(a.Files)) + minSame := e.MinSame + if minSame <= 0 { + minSame = store.DefaultMinSame + } + + // Keep: the idle base that holds the most bytes at the same offsets + if offer != nil && offer.Claim != nil && len(offer.Bases) > 0 { + var best keepPlan + for _, b := range offer.Bases { + p, err := e.plan(a, idxs, b) + if err == nil && p.same > best.same { + best = p + } + } + st.BaseBest = best.same + if best.same > 0 && float64(best.same) >= minSame*float64(st.Bytes) { + if dir, ok := offer.Claim(best.base); ok { + st.Base = best.base.Manifest.String() + if err := e.keep(ctx, a, idxs, best, dir, finals, done, offer.VerifyKept || e.VerifyKept, &st, src); err != nil { + return st, err + } + } + } + } + // Copy: a base that stays where it is seeds the files Keep did not produce + var release func() + var seedBase *keepPlan + // Reading a sibling instead of the chunk source pays off when that source is remote (Src set, + // BaseCopyRemote) or when asked for explicitly (BaseCopyAlways). + copyOK := offer != nil && (offer.BaseCopy == store.BaseCopyAlways || (offer.BaseCopy == store.BaseCopyRemote && e.Src != nil)) + if copyOK && len(offer.Readable) > 0 { + var most keepPlan + var mostShared int64 + for _, b := range offer.Readable { + if p, err := e.plan(a, idxs, b); err == nil { + // a seed serves chunks at any offset, so count the target bytes the sibling holds anywhere + if sh := sharedBytes(p.idxs, idxs); sh > mostShared { + most, mostShared = p, sh + } + } + } + // the same threshold as Keep (Algorithm 1), on bytes held anywhere in the sibling + if mostShared > 0 && float64(mostShared) >= minSame*float64(st.Bytes) { + ok := true + release = func() {} + if offer.Hold != nil { + release, ok = offer.Hold(most.base) + } + if ok { + seedBase = &most + st.LocalBase = most.base.Manifest.String() + } + } + } + if release != nil { + defer release() + } + // whatever is left: assemble from the store, seeded by the held base if there is one + for i := range a.Files { + if done[i] { + continue + } + partial := finals[i] + ".casync-partial" + os.Remove(partial) + var seeds []desync.Seed + if seedBase != nil && seedBase.pair[i] >= 0 { + j := seedBase.pair[i] + dir, _ := baseDir(offer, seedBase.base) + src := filepath.Join(dir, filepath.FromSlash(seedBase.files.Files[j].Path)) + s, err := desync.NewIndexSeed(partial, src, seedBase.idxs[j]) + if err == nil { + seeds = append(seeds, s) + st.LocalBytes += sharedBytes([]desync.Index{seedBase.idxs[j]}, []desync.Index{idxs[i]}) + } + } + t0 := time.Now() + if _, err := desync.AssembleFile(ctx, partial, idxs[i], src, seeds, desync.AssembleOptions{N: e.Workers, InvalidSeedAction: desync.InvalidSeedActionSkip}); err != nil { + os.Remove(partial) + return st, fmt.Errorf("%s: %w", a.Files[i].Path, err) + } + e.logf("assemble %s (%d seeds): %s", a.Files[i].Path, len(seeds), time.Since(t0).Round(time.Millisecond)) + if err := e.finish(partial, finals[i], a.Files[i]); err != nil { + return st, err + } + done[i] = true + } + if err := e.recordAttestation(m, dest, a); err != nil { + return st, err + } + st.Duration = time.Since(start) + return st, nil +} + +// baseDir is where a readable base's files are: the materializer keeps them under its root, and +// the offer's Base.Dir says where. +func baseDir(_ *csi.Offer, b store.Base) (string, bool) { return b.Dir, b.Dir != "" } + +func countKept(kept []bool, chunks []desync.IndexChunk) int64 { + var n int64 + for i, k := range kept { + if k { + n += int64(chunks[i].Size) + } + } + return n +} + +// keep turns the claimed base's files into the target's: each paired file is moved into place, +// resized, and only its differing chunks are written from the store. Kept chunks are read back and +// hash-checked unless the file is attested unchanged since this engine wrote it. +func (e *Engine) keep(ctx context.Context, a *Artifact, idxs []desync.Index, p keepPlan, dir string, finals []string, done []bool, verify bool, st *store.PullStats, chunkSrc desync.Store) error { + for i, j := range p.pair { + if j < 0 { + continue + } + src := filepath.Join(dir, filepath.FromSlash(p.files.Files[j].Path)) + trusted := !verify && e.unchanged(p.base.Manifest, p.files.Files[j].Path, src) + partial := finals[i] + ".casync-partial" + if err := os.Rename(src, partial); err != nil { + continue // left for the store + } + f, err := os.OpenFile(partial, os.O_RDWR, 0) + if err != nil { + return err + } + if err := f.Chmod(0o644); err != nil { + f.Close() + return err + } + if err := f.Truncate(a.Files[i].Size); err != nil { + f.Close() + return err + } + kept := append([]bool(nil), p.kept[i]...) + t0 := time.Now() + if !trusted { + ok, bad := e.verifyKept(ctx, f, idxs[i].Chunks, kept) + st.Verified += ok + st.Repaired += bad + } + t1 := time.Now() + written, err := e.writeChunks(ctx, f, idxs[i].Chunks, kept, chunkSrc) + if err != nil { + f.Close() + return err + } + t2 := time.Now() + if e.Durable { + if err := f.Sync(); err != nil { + f.Close() + return err + } + } + if err := f.Close(); err != nil { + return err + } + st.Reused += a.Files[i].Size - written + st.ReadBytes += written + e.logf("keep %s: trusted=%v verify %s, write %d B %s, sync %s", a.Files[i].Path, trusted, + t1.Sub(t0).Round(time.Millisecond), written, t2.Sub(t1).Round(time.Millisecond), time.Since(t2).Round(time.Millisecond)) + if err := e.finish(partial, finals[i], a.Files[i]); err != nil { + return err + } + done[i] = true + } + return nil +} + +// verifyKept reads every kept chunk and checks its digest; a chunk that does not match is no +// longer kept, so the write step fetches it. It returns the bytes that matched and that did not. +func (e *Engine) verifyKept(ctx context.Context, f *os.File, chunks []desync.IndexChunk, kept []bool) (ok, bad int64) { + var okN, badN atomic.Int64 + work := make(chan int) + var wg sync.WaitGroup + for w := 0; w < e.Workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + var buf []byte + for i := range work { + c := chunks[i] + if cap(buf) < int(c.Size) { + buf = make([]byte, c.Size) + } + b := buf[:c.Size] + if _, err := f.ReadAt(b, int64(c.Start)); err != nil || desync.ChunkID(desync.Digest.Sum(b)) != c.ID { + kept[i] = false + badN.Add(int64(c.Size)) + continue + } + okN.Add(int64(c.Size)) + } + }() + } + for i := range chunks { + if kept[i] { + select { + case work <- i: + case <-ctx.Done(): + } + } + } + close(work) + wg.Wait() + return okN.Load(), badN.Load() +} + +// writeChunks writes every chunk that is not kept, from the store (each chunk's digest is checked +// when it is read from the store). It returns the bytes written. +func (e *Engine) writeChunks(ctx context.Context, f *os.File, chunks []desync.IndexChunk, kept []bool, src desync.Store) (int64, error) { + var written atomic.Int64 + var firstErr error + var once sync.Once + work := make(chan int) + var wg sync.WaitGroup + for w := 0; w < e.Workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := range work { + c := chunks[i] + ch, err := src.GetChunk(c.ID) + if err == nil { + var b []byte + if b, err = ch.Data(); err == nil { + if uint64(len(b)) != c.Size { + err = fmt.Errorf("chunk %s: %d bytes, index says %d", c.ID, len(b), c.Size) + } else if _, err = f.WriteAt(b, int64(c.Start)); err == nil { + written.Add(int64(len(b))) + } + } + } + if err != nil { + once.Do(func() { firstErr = err }) + } + } + }() + } + for i := range chunks { + if !kept[i] { + select { + case work <- i: + case <-ctx.Done(): + once.Do(func() { firstErr = ctx.Err() }) + } + } + } + close(work) + wg.Wait() + return written.Load(), firstErr +} + +// finish syncs a partial file (when the writer did not), sets its mode, and renames it into place. +func (e *Engine) finish(partial, final string, f FileEntry) error { + if e.Durable { + h, err := os.OpenFile(partial, os.O_RDWR, 0) + if err != nil { + return err + } + if err := h.Sync(); err != nil { + h.Close() + return err + } + h.Close() + } + if err := os.Chmod(partial, os.FileMode(f.Mode)|0o400); err != nil { + return err + } + return os.Rename(partial, final) +} + +// Verify hash-checks every chunk of the artifact's files under dir against its indexes. +func (e *Engine) Verify(ctx context.Context, m digest.Digest, dir string) error { + a, idxs, err := e.Load(m) + if err != nil { + return err + } + for i, f := range a.Files { + h, err := os.Open(filepath.Join(dir, filepath.FromSlash(f.Path))) + if err != nil { + return err + } + kept := make([]bool, len(idxs[i].Chunks)) + for k := range kept { + kept[k] = true + } + _, bad := e.verifyKept(ctx, h, idxs[i].Chunks, kept) + fi, _ := h.Stat() + h.Close() + if bad > 0 || fi.Size() != f.Size { + return fmt.Errorf("%s: %d bytes do not match the index", f.Path, bad) + } + } + return nil +} + +// sharedBytes counts the bytes of the target's chunks that also occur, at any offset, in the base. +func sharedBytes(base, target []desync.Index) int64 { + have := map[desync.ChunkID]bool{} + for _, x := range base { + for _, c := range x.Chunks { + have[c.ID] = true + } + } + var n int64 + for _, x := range target { + for _, c := range x.Chunks { + if have[c.ID] { + n += int64(c.Size) + } + } + } + return n +} diff --git a/pkg/casync/casync_test.go b/pkg/casync/casync_test.go new file mode 100644 index 0000000..cd9330d --- /dev/null +++ b/pkg/casync/casync_test.go @@ -0,0 +1,232 @@ +package casync + +import ( + "bytes" + "context" + "math/rand" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/csi" + "github.com/ai-computing/onggi/pkg/store" +) + +// variants returns a base and an aligned sibling (a middle region rewritten in place) and a shifted +// sibling (a header inserted at the front, so every later byte moves). +func variants() (base, aligned, shifted []byte) { + r := rand.New(rand.NewSource(7)) + base = make([]byte, 6<<20) + r.Read(base) + aligned = append([]byte(nil), base...) + r.Read(aligned[3<<20 : 3<<20+300<<10]) + shifted = append(append(make([]byte, 0, len(base)+4096), bytes.Repeat([]byte{9}, 4096)...), base...) + return +} + +func publish(t *testing.T, e *Engine, root, name string, data []byte) (dir string) { + t.Helper() + dir = filepath.Join(root, "src-"+name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "model.bin"), data, 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func setup(t *testing.T) (*Engine, string) { + t.Helper() + root := t.TempDir() + e, err := New(filepath.Join(root, "store"), filepath.Join(root, "index"), filepath.Join(root, "attest"), 4) + if err != nil { + t.Fatal(err) + } + return e, root +} + +func readFile(t *testing.T, p string) []byte { + t.Helper() + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestPaths(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, shifted := variants() + mA, err := e.Publish(ctx, publish(t, e, root, "a", base)) + if err != nil { + t.Fatal(err) + } + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + mC, _ := e.Publish(ctx, publish(t, e, root, "c", shifted)) + + // Reconstruct: no offer + dA := filepath.Join(root, "art", "A") + os.MkdirAll(dA, 0o755) + st, err := e.Pull(ctx, mA, dA, nil) + if err != nil || !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) || st.Base != "" { + t.Fatalf("reconstruct: %v %+v", err, st) + } + + // Keep, trusted: A is attested unchanged, so kept chunks are not read back + claimed := false + offer := &csi.Offer{Bases: []store.Base{{Manifest: mA, Dir: dA}}, + Claim: func(b store.Base) (string, bool) { claimed = true; return b.Dir, true }} + dB := filepath.Join(root, "art", "B") + os.MkdirAll(dB, 0o755) + st, err = e.Pull(ctx, mB, dB, offer) + if err != nil || !claimed || st.Base != mA.String() || st.Verified != 0 || st.Reused < int64(len(aligned))*8/10 { + t.Fatalf("keep trusted: %v claimed=%v %+v", err, claimed, st) + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) { + t.Fatal("keep trusted: output differs") + } + if _, err := os.Stat(filepath.Join(dA, "model.bin")); !os.IsNotExist(err) { + t.Fatal("keep must consume the base file") + } + + // Keep, tampered: B is changed after it was written; the attestation no longer matches, so kept + // chunks are read back and the corrupted one is fetched again + p := filepath.Join(dB, "model.bin") + os.Chmod(p, 0o644) + f, _ := os.OpenFile(p, os.O_RDWR, 0) + f.WriteAt([]byte("corrupt"), 100) + f.Close() + future := time.Now().Add(time.Second) + os.Chtimes(p, future, future) + dA2 := filepath.Join(root, "art", "A2") + os.MkdirAll(dA2, 0o755) + st, err = e.Pull(ctx, mA, dA2, &csi.Offer{Bases: []store.Base{{Manifest: mB, Dir: dB}}, + Claim: func(b store.Base) (string, bool) { return b.Dir, true }}) + if err != nil || st.Verified == 0 || st.Repaired == 0 { + t.Fatalf("keep tampered: %v %+v", err, st) + } + if !bytes.Equal(readFile(t, filepath.Join(dA2, "model.bin")), base) { + t.Fatal("keep tampered: output differs") + } + + // a shifted sibling keeps nothing at the same offset: no claim, assembled from the store + claimed = false + dC := filepath.Join(root, "art", "C") + os.MkdirAll(dC, 0o755) + st, err = e.Pull(ctx, mC, dC, &csi.Offer{Bases: []store.Base{{Manifest: mA, Dir: dA2}}, + Claim: func(b store.Base) (string, bool) { claimed = true; return b.Dir, true }}) + if err != nil || claimed || st.BaseBest != 0 || !bytes.Equal(readFile(t, filepath.Join(dC, "model.bin")), shifted) { + t.Fatalf("shifted: %v claimed=%v %+v", err, claimed, st) + } + + // Copy: A2 stays intact and seeds B + held := false + dB2 := filepath.Join(root, "art", "B2") + os.MkdirAll(dB2, 0o755) + st, err = e.Pull(ctx, mB, dB2, &csi.Offer{Readable: []store.Base{{Manifest: mA, Dir: dA2}}, BaseCopy: store.BaseCopyAlways, + Hold: func(store.Base) (func(), bool) { held = true; return func() {}, true }}) + if err != nil || !held || st.LocalBase != mA.String() || !bytes.Equal(readFile(t, filepath.Join(dB2, "model.bin")), aligned) { + t.Fatalf("copy: %v held=%v %+v", err, held, st) + } + if !bytes.Equal(readFile(t, filepath.Join(dA2, "model.bin")), base) { + t.Fatal("copy must leave the base intact") + } + if err := e.Verify(ctx, mB, dB2); err != nil { + t.Fatal(err) + } +} + +func TestSameOffset(t *testing.T) { + e, root := setup(t) + base, aligned, _ := variants() + ctx := context.Background() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + _, ia, _ := e.Load(mA) + _, ib, _ := e.Load(mB) + _, same := sameOffset(ia[0].Chunks, ib[0].Chunks) + if same < int64(len(aligned))*8/10 || same == int64(len(aligned)) { + t.Fatalf("same=%d of %d", same, len(aligned)) + } +} + +// TestKeepRenamed: variants of different shapes name their files differently; Keep pairs the +// target file with the base file that holds most of it at the same offset. +func TestKeepRenamed(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + write := func(name, file string, data []byte) string { + d := filepath.Join(root, "src-"+name) + os.MkdirAll(d, 0o755) + if err := os.WriteFile(filepath.Join(d, file), data, 0o644); err != nil { + t.Fatal(err) + } + return d + } + mA, err := e.Publish(ctx, write("a", "decode_B2.bin", base)) + if err != nil { + t.Fatal(err) + } + mB, err := e.Publish(ctx, write("b", "decode_B8.bin", aligned)) + if err != nil { + t.Fatal(err) + } + dA := filepath.Join(root, "art", "A") + os.MkdirAll(dA, 0o755) + if _, err := e.Pull(ctx, mA, dA, nil); err != nil { + t.Fatal(err) + } + offer := &csi.Offer{Bases: []store.Base{{Manifest: mA, Dir: dA}}, + Claim: func(b store.Base) (string, bool) { return b.Dir, true }} + dB := filepath.Join(root, "art", "B") + os.MkdirAll(dB, 0o755) + st, err := e.Pull(ctx, mB, dB, offer) + if err != nil || st.Base != mA.String() || st.Verified != 0 || st.Reused < int64(len(aligned))*8/10 { + t.Fatalf("keep renamed: %v %+v", err, st) + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "decode_B8.bin")), aligned) { + t.Fatal("keep renamed: output differs") + } + if _, err := os.Stat(filepath.Join(dA, "decode_B2.bin")); !os.IsNotExist(err) { + t.Fatal("keep renamed must consume the base file") + } +} + +// TestCopyShiftedFetchesLittle: with a separate chunk source (as a remote store would be), Copy +// seeds from a sibling whose content sits at other offsets and fetches only what it lacks. +func TestCopyShiftedFetchesLittle(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, _, shifted := variants() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mC, _ := e.Publish(ctx, publish(t, e, root, "c", shifted)) + dA := filepath.Join(root, "art", "A") + os.MkdirAll(dA, 0o755) + if _, err := e.Pull(ctx, mA, dA, nil); err != nil { + t.Fatal(err) + } + e.Src = e.Store // a distinct chunk source: Copy is chosen with BaseCopyRemote + dC := filepath.Join(root, "art", "C") + os.MkdirAll(dC, 0o755) + st, err := e.Pull(ctx, mC, dC, &csi.Offer{Readable: []store.Base{{Manifest: mA, Dir: dA}}, BaseCopy: store.BaseCopyRemote}) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFile(t, filepath.Join(dC, "model.bin")), shifted) { + t.Fatal("copy shifted: output differs") + } + if st.FetchedBytes*5 > int64(len(shifted)) || st.LocalBytes == 0 { + t.Fatalf("copy shifted fetched %d of %d B (local %d B)", st.FetchedBytes, len(shifted), st.LocalBytes) + } + // without a sibling the whole target comes from the chunk source + dC2 := filepath.Join(root, "art", "C2") + os.MkdirAll(dC2, 0o755) + st2, err := e.Pull(ctx, mC, dC2, nil) + if err != nil || st2.FetchedBytes < int64(len(shifted)) { + t.Fatalf("reconstruct fetched %d of %d B: %v", st2.FetchedBytes, len(shifted), err) + } +} diff --git a/pkg/casync/guards_test.go b/pkg/casync/guards_test.go new file mode 100644 index 0000000..8e0f31e --- /dev/null +++ b/pkg/casync/guards_test.go @@ -0,0 +1,126 @@ +package casync + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/csi" +) + +// TestKeepRepairsChangedBase: a base that was modified after the node wrote it no longer matches +// its record, so Keep reads the kept chunks back and replaces the ones that fail their digest. +func TestKeepRepairsChangedBase(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond, MaxArtifacts: 1} + if err := m.Init(); err != nil { + t.Fatal(err) + } + dA, err := m.Ensure(ctx, mA) + if err != nil { + t.Fatal(err) + } + // damage a region that A and B share (B differs from A only around 3 MiB) + time.Sleep(20 * time.Millisecond) + f, err := os.OpenFile(filepath.Join(dA, "model.bin"), os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt(bytes.Repeat([]byte{0xEE}, 64<<10), 1<<20); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + dB, err := m.Ensure(ctx, mB) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) { + t.Fatal("Keep carried a changed base into the target") + } + if _, err := os.Stat(dA); !os.IsNotExist(err) { + t.Fatal("the idle victim was not consumed") + } +} + +// TestHardLinkedBaseIsNotOverwritten: a base whose file has a second name is never rebuilt in +// place, so the bytes behind the second name stay those of the base. +func TestHardLinkedBaseIsNotOverwritten(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond, MaxArtifacts: 1} + if err := m.Init(); err != nil { + t.Fatal(err) + } + dA, err := m.Ensure(ctx, mA) + if err != nil { + t.Fatal(err) + } + snap := filepath.Join(m.Root, "snapshot-model.bin") + if err := os.Link(filepath.Join(dA, "model.bin"), snap); err != nil { + t.Skipf("hard links unavailable here: %v", err) + } + time.Sleep(20 * time.Millisecond) + dB, err := m.Ensure(ctx, mB) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) { + t.Fatal("B is wrong") + } + if !bytes.Equal(readFile(t, snap), base) { + t.Fatal("a hard-linked base was rebuilt in place") + } +} + +// TestLeasedBaseSurvivesConcurrentPulls: while a volume holds A, several pulls of B run at once; +// A must stay intact and every pull must return the same correct B. +func TestLeasedBaseSurvivesConcurrentPulls(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond, MaxArtifacts: 2} + if err := m.Init(); err != nil { + t.Fatal(err) + } + dA, err := m.Ensure(ctx, mA) + if err != nil { + t.Fatal(err) + } + if err := m.Acquire("vol-a", mA, "/pods/a"); err != nil { + t.Fatal(err) + } + errs := make(chan error, 4) + dirs := make(chan string, 4) + for i := 0; i < 4; i++ { + go func() { + d, err := m.Ensure(ctx, mB) + dirs <- d + errs <- err + }() + } + for i := 0; i < 4; i++ { + if err := <-errs; err != nil { + t.Fatal(err) + } + if d := <-dirs; !bytes.Equal(readFile(t, filepath.Join(d, "model.bin")), aligned) { + t.Fatal("a concurrent pull returned a wrong B") + } + } + if !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) { + t.Fatal("a leased base changed under concurrent pulls") + } +} diff --git a/pkg/casync/materializer_test.go b/pkg/casync/materializer_test.go new file mode 100644 index 0000000..67c5159 --- /dev/null +++ b/pkg/casync/materializer_test.go @@ -0,0 +1,117 @@ +package casync + +import ( + "bytes" + "context" + "math/rand" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/csi" + "github.com/ai-computing/onggi/pkg/store" +) + +// TestOwnership runs the materializer with the casync engine: a sibling a volume uses is never +// consumed (the new variant is built without destroying it), and once released and idle it is the +// one consumed, writing only what differs. +func TestOwnership(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + third := append([]byte(nil), aligned...) + rand.New(rand.NewSource(9)).Read(third[1<<20 : 1<<20+200<<10]) + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + mC, _ := e.Publish(ctx, publish(t, e, root, "c", third)) + + m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond, + MaxArtifacts: 1, BaseCopy: store.BaseCopyAlways} + if err := m.Init(); err != nil { + t.Fatal(err) + } + dA, err := m.Ensure(ctx, mA) + if err != nil { + t.Fatal(err) + } + if err := m.Acquire("vol-a", mA, "/pods/a"); err != nil { + t.Fatal(err) + } + // A is in use: B must not consume it + dB, err := m.Ensure(ctx, mB) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) { + t.Fatal("a sibling in use was modified") + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) { + t.Fatal("B is wrong") + } + // Release marks A used now, so B is the least recently used idle sibling: C consumes B, not A + if _, _, _, err := m.Release("vol-a"); err != nil { + t.Fatal(err) + } + time.Sleep(20 * time.Millisecond) + dC, err := m.Ensure(ctx, mC) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFile(t, filepath.Join(dC, "model.bin")), third) { + t.Fatal("C is wrong") + } + if _, err := os.Stat(dB); !os.IsNotExist(err) { + t.Fatal("C did not consume B, the least recently used idle sibling") + } + if !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) { + t.Fatal("A changed although it was not the victim") + } + // next, the released A is the least recently used idle sibling and is the one consumed + time.Sleep(20 * time.Millisecond) + dB2, err := m.Ensure(ctx, mB) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(dA); !os.IsNotExist(err) { + t.Fatal("the released sibling A was not consumed when it became the victim") + } + if !bytes.Equal(readFile(t, filepath.Join(dB2, "model.bin")), aligned) { + t.Fatal("B (second time) is wrong") + } +} + +// TestVictimNeedsNoIdleWindow: with a capacity, the eviction victim is consumed as soon as no +// lease holds it, even if it was used a moment ago and the idle window is long. +func TestVictimNeedsNoIdleWindow(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Hour, + MaxArtifacts: 1} + if err := m.Init(); err != nil { + t.Fatal(err) + } + dA, err := m.Ensure(ctx, mA) + if err != nil { + t.Fatal(err) + } + if err := m.Acquire("vol-a", mA, "/pods/a"); err != nil { + t.Fatal(err) + } + if _, _, _, err := m.Release("vol-a"); err != nil { + t.Fatal(err) + } + dB, err := m.Ensure(ctx, mB) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(dA); !os.IsNotExist(err) { + t.Fatal("the just-released victim was not consumed") + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) { + t.Fatal("B is wrong") + } +} diff --git a/pkg/csi/candidates_test.go b/pkg/csi/candidates_test.go new file mode 100644 index 0000000..6cba439 --- /dev/null +++ b/pkg/csi/candidates_test.go @@ -0,0 +1,172 @@ +package csi + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/store" +) + +// An artifact a volume uses is never offered for consumption, but a pull may read it where it is. +func TestLeasedArtifactIsReadNotConsumed(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, c := newMaterializer(t, f) + m.RebaseIdle = time.Hour + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + if err := m.Acquire("vol-a", f.a, "/target-a"); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + before, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")) + if err != nil { + t.Fatal(err) + } + offer, _ := m.offer(f.b) + if offer == nil || len(offer.Bases) != 0 || len(offer.Readable) != 1 || offer.Readable[0].Manifest != f.a { + t.Fatalf("offer %+v: the leased artifact must be readable and not consumable", offer) + } + dir, err := m.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + sameTree(t, f.srcA, m.Dir(f.a)) + last := c.pulls[len(c.pulls)-1] + if last.Base != "" || last.LocalBase != f.a.String() || last.LocalBytes == 0 { + t.Fatalf("the pull did not read the leased artifact where it is: %+v", last) + } + after, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")) + if err != nil || !os.SameFile(before, after) { + t.Fatalf("the leased artifact changed: %v", err) + } + if n, err := m.users(f.a); err != nil || n != 1 { + t.Fatalf("lease lost: %d, %v", n, err) + } +} + +// While a pull reads an artifact, another pull cannot take it over and eviction leaves it alone. +func TestReadHoldBlocksClaimAndEviction(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + m.RebaseIdle = time.Hour + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + offer, claimed := m.offer(f.b) + if offer == nil || len(offer.Bases) != 1 || offer.Hold == nil { + t.Fatalf("offer %+v", offer) + } + release, ok := offer.Hold(offer.Readable[0]) + if !ok { + t.Fatal("could not hold a published artifact") + } + if dir, ok := offer.Claim(offer.Bases[0]); ok || dir != "" || len(*claimed) != 0 { + t.Fatal("an artifact that a pull is reading was taken over") + } + if gone, err := m.Evict(time.Now(), 0, nil); err != nil || len(gone) != 0 { + t.Fatalf("an artifact that a pull is reading was evicted: %v, %v", gone, err) + } + release() + ageOwnerDonor(t, m, f.a) + if dir, ok := offer.Claim(offer.Bases[0]); !ok || dir == "" { + t.Fatal("the artifact could not be taken over after the read ended") + } +} + +// With a limit, the cache policy picks the victim: only the least recently used idle artifact is +// offered for consumption, whatever the others would keep in place. All of them may be read. +func TestOnlyTheEvictionVictimIsOffered(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + m.RebaseIdle = time.Hour + other := t.TempDir() + if err := os.WriteFile(filepath.Join(other, "model.bin"), random(7, 2<<20), 0o644); err != nil { + t.Fatal(err) + } + o, _, err := f.s.Push(ctx, other, store.PushOptions{}) + if err != nil { + t.Fatal(err) + } + for _, d := range []struct { + name string + age time.Duration + }{{"a", 3 * time.Hour}, {"o", 2 * time.Hour}} { + x := f.a + if d.name == "o" { + x = o + } + if _, err := m.Ensure(ctx, x); err != nil { + t.Fatal(err) + } + when := time.Now().Add(-d.age) + if err := os.Chtimes(m.usedFile(x), when, when); err != nil { + t.Fatal(err) + } + } + offer, _ := m.offer(f.b) + if offer == nil || len(offer.Bases) != 2 || len(offer.Readable) != 2 { + t.Fatalf("without a limit both idle artifacts are offered: %+v", offer) + } + m.MaxArtifacts = 2 + offer, _ = m.offer(f.b) + if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a || len(offer.Readable) != 2 { + t.Fatalf("with a limit only the least recently used idle artifact is offered: %+v", offer) + } + // make the other one the least recently used: it becomes the victim, though it shares nothing with B + when := time.Now().Add(-4 * time.Hour) + if err := os.Chtimes(m.usedFile(o), when, when); err != nil { + t.Fatal(err) + } + offer, _ = m.offer(f.b) + if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != o { + t.Fatalf("the victim is the least recently used artifact, not the best aligned one: %+v", offer) + } +} + +// Leases are files: a new materializer on the same root (a restarted plugin) still refuses to +// take over an artifact a volume uses. +func TestLeasesSurviveARestart(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + if err := m.Acquire("vol-a", f.a, "/target-a"); err != nil { + t.Fatal(err) + } + c := &counting{Puller: StorePuller(f.s, f.cache, filepath.Join(m.Root, "tmp"), 4)} + again := &Materializer{Root: m.Root, Pull: c.pull, Logf: t.Logf, RebaseIdle: time.Hour} + if err := again.Init(); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, again, f.a) + if n, err := again.Users(f.a); err != nil || n != 1 { + t.Fatalf("the restarted materializer sees %d leases, %v", n, err) + } + if offer, _ := again.offer(f.b); offer == nil || len(offer.Bases) != 0 { + t.Fatalf("a leased artifact was offered for consumption after a restart: %+v", offer) + } + dir, err := again.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + sameTree(t, f.srcA, again.Dir(f.a)) + if _, _, ok, err := again.Release("vol-a"); err != nil || !ok { + t.Fatalf("the lease could not be released after the restart: %v", err) + } + ageOwnerDonor(t, again, f.a) + if offer, _ := again.offer(f.b); offer == nil || len(offer.Bases) != 1 { + t.Fatalf("the released artifact is not offered: %+v", offer) + } +} diff --git a/pkg/csi/capacity_test.go b/pkg/csi/capacity_test.go new file mode 100644 index 0000000..9839204 --- /dev/null +++ b/pkg/csi/capacity_test.go @@ -0,0 +1,143 @@ +package csi + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/digest" + "github.com/ai-computing/onggi/pkg/store" +) + +// Below MaxArtifacts a pull consumes nothing, however idle the other artifact is: the node has +// room for both. At the limit the idle artifact is taken over, and the node still holds as many +// artifacts as before. +func TestOwnerFreeSlotRefusesConsumption(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, c := newMaterializer(t, f) + m.RebaseIdle = time.Hour + m.MaxArtifacts = 2 + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + dir, err := m.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + sameTree(t, f.srcA, m.Dir(f.a)) + if last := c.pulls[len(c.pulls)-1]; last.Base != "" { + t.Fatalf("a pull consumed an artifact although a slot was free: %+v", last) + } + if ds, err := m.Materialized(); err != nil || len(ds) != 2 { + t.Fatalf("materialized %v, %v; want both", ds, err) + } + + // The node is full now. Drop B and pull it again with the limit at one: A is consumed. + if err := os.RemoveAll(m.Dir(f.b)); err != nil { + t.Fatal(err) + } + m.MaxArtifacts = 1 + ageOwnerDonor(t, m, f.a) + if dir, err = m.Ensure(ctx, f.b); err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + if last := c.pulls[len(c.pulls)-1]; last.Base != f.a.String() { + t.Fatalf("a full node did not take the idle artifact over: %+v", last) + } + if ds, err := m.Materialized(); err != nil || len(ds) != 1 || ds[0] != f.b { + t.Fatalf("materialized %v, %v; want only B", ds, err) + } +} + +// Evict brings a node back under MaxArtifacts by removing the least recently used unused +// artifact, and leaves an artifact a volume uses alone even if that keeps the node over. +func TestEvictAboveMaxArtifacts(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + if _, err := m.Ensure(ctx, f.b); err != nil { + t.Fatal(err) + } + now := time.Now() + older, newer := now.Add(-2*time.Minute), now.Add(-time.Minute) + if err := os.Chtimes(m.usedFile(f.a), older, older); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(m.usedFile(f.b), newer, newer); err != nil { + t.Fatal(err) + } + + // no limit: nothing is idle for an hour, nothing goes + if gone, err := m.Evict(now, time.Hour, nil); err != nil || len(gone) != 0 { + t.Fatalf("evicted %v, %v without a limit", gone, err) + } + // limit one, but the older artifact is in use: the newer, unused one goes instead + m.MaxArtifacts = 1 + if err := m.Acquire("vol-a", f.a, "/target-a"); err != nil { + t.Fatal(err) + } + gone, err := m.Evict(now, time.Hour, nil) + if err != nil || len(gone) != 1 || gone[0] != f.b { + t.Fatalf("evicted %v, %v; want the unused B", gone, err) + } + if _, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")); err != nil { + t.Fatalf("the artifact in use was removed: %v", err) + } + + // both unused: the least recently used goes + if _, _, _, err := m.Release("vol-a"); err != nil { + t.Fatal(err) + } + if _, err := m.Ensure(ctx, f.b); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(m.usedFile(f.a), older, older); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(m.usedFile(f.b), newer, newer); err != nil { + t.Fatal(err) + } + gone, err = m.Evict(now, time.Hour, nil) + if err != nil || len(gone) != 1 || gone[0] != f.a { + t.Fatalf("evicted %v, %v; want the least recently used A", gone, err) + } +} + +// The materializer hands its policies to the pull. +func TestOfferCarriesPolicies(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + m.RebaseIdle = time.Hour + m.VerifyKept, m.BaseCopy = true, store.BaseCopyRemote + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + offer, _ := m.offer(f.b) + if offer == nil || !offer.VerifyKept || offer.BaseCopy != store.BaseCopyRemote { + t.Fatalf("offer %+v does not carry the policies", offer) + } + var seen store.PullStats + inner := m.Pull + m.Pull = func(ctx context.Context, d digest.Digest, dest string, o *Offer) (store.PullStats, error) { + st, err := inner(ctx, d, dest, o) + seen = st + return st, err + } + if _, err := m.Ensure(ctx, f.b); err != nil { + t.Fatal(err) + } + if seen.Base != f.a.String() || seen.Verified <= 0 || seen.Verified != seen.Reused { + t.Fatalf("the pull did not verify what it kept: %+v", seen) + } +} diff --git a/pkg/csi/evict.go b/pkg/csi/evict.go index 446502a..a3ff98e 100644 --- a/pkg/csi/evict.go +++ b/pkg/csi/evict.go @@ -1,6 +1,7 @@ package csi import ( + "sort" "os" "path/filepath" "strings" @@ -61,7 +62,10 @@ func (m *Materializer) materialized() ([]digest.Digest, error) { return out, nil } -// Evict removes the artifacts that no volume uses and that nothing used for at least idle. +// Evict removes the artifacts that no volume uses and that nothing used for at least idle. With +// MaxArtifacts set it then removes the least recently used of the remaining unused artifacts +// until the node holds at most that many; artifacts a volume or a pull uses always stay, so the +// node can be over the limit while they are all in use. // unmount is called on the directory first: on Linux it holds the read-only self-bind that // publishing made. The directory leaves artifacts/ by a rename under the materializer's lock, // so Ensure either finds it complete or pulls it again; deleting the files happens after. @@ -74,38 +78,73 @@ func (m *Materializer) Evict(now time.Time, idle time.Duration, unmount func(dir } var gone []digest.Digest var trash []string - for _, d := range ds { - if _, busy := m.inflight[d]; busy { - continue - } - if n, err := m.users(d); err != nil || n > 0 { - continue - } - if now.Sub(m.lastUsed(d)) < idle { - continue - } + over := map[digest.Digest]bool{} + // remove takes d out of artifacts/. ok false: it stays (logged); err: stop. + remove := func(d digest.Digest) (bool, error) { dir := m.Dir(d) if unmount != nil { if err := unmount(dir); err != nil { m.logf("evict %s: %v", d, err) - continue + return false, nil } } t, err := os.MkdirTemp(filepath.Join(m.Root, "tmp"), "evict-") if err != nil { - m.mu.Unlock() - return gone, err + return false, err } dst := filepath.Join(t, d.Hex()) if err := os.Rename(dir, dst); err != nil { os.Remove(t) m.logf("evict %s: %v", d, err) - continue + return false, nil } os.Remove(m.usedFile(d)) os.Remove(filepath.Join(m.Root, "leases", d.Hex())) trash = append(trash, t) gone = append(gone, d) + return true, nil + } + left := 0 + var spare []digest.Digest // unused, but used too recently for idle + for _, d := range ds { + if _, busy := m.inflight[d]; busy || m.reading[d] > 0 { + left++ + continue + } + if n, err := m.users(d); err != nil || n > 0 { + left++ + continue + } + if now.Sub(m.lastUsed(d)) < idle { + left++ + spare = append(spare, d) + continue + } + ok, err := remove(d) + if err != nil { + m.mu.Unlock() + return gone, err + } + if !ok { + left++ + } + } + if m.MaxArtifacts > 0 && left > m.MaxArtifacts { + sort.Slice(spare, func(i, j int) bool { return m.lastUsed(spare[i]).Before(m.lastUsed(spare[j])) }) + for _, d := range spare { + if left <= m.MaxArtifacts { + break + } + ok, err := remove(d) + if err != nil { + m.mu.Unlock() + return gone, err + } + if ok { + left-- + over[d] = true + } + } } m.mu.Unlock() for _, t := range trash { @@ -114,7 +153,11 @@ func (m *Materializer) Evict(now time.Time, idle time.Duration, unmount func(dir } } for _, d := range gone { - m.logf("evicted %s (no volume, idle %s or more)", d, idle) + if over[d] { + m.logf("evicted %s (no volume, least recently used above the limit of %d)", d, m.MaxArtifacts) + } else { + m.logf("evicted %s (no volume, idle %s or more)", d, idle) + } } return gone, nil } diff --git a/pkg/csi/links_other.go b/pkg/csi/links_other.go new file mode 100644 index 0000000..7698814 --- /dev/null +++ b/pkg/csi/links_other.go @@ -0,0 +1,6 @@ +//go:build !unix + +package csi + +// aliased reports no hard links where the platform does not expose link counts. +func aliased(string) (string, error) { return "", nil } diff --git a/pkg/csi/links_unix.go b/pkg/csi/links_unix.go new file mode 100644 index 0000000..3db22e8 --- /dev/null +++ b/pkg/csi/links_unix.go @@ -0,0 +1,35 @@ +//go:build unix + +package csi + +import ( + "io/fs" + "path/filepath" + "syscall" +) + +// aliased returns a regular file under root that has more than one hard link, or "" if there is +// none. A pull that takes a base over changes its files in place; a second link to one of them +// (a snapshot, a copy made with cp -l) would change with it, outside the artifact and outside +// every lease. +func aliased(root string) (string, error) { + found := "" + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.Type().IsRegular() { + return nil + } + fi, err := d.Info() + if err != nil { + return err + } + if st, ok := fi.Sys().(*syscall.Stat_t); ok && st.Nlink > 1 { + found = p + return fs.SkipAll + } + return nil + }) + return found, err +} diff --git a/pkg/csi/links_unix_test.go b/pkg/csi/links_unix_test.go new file mode 100644 index 0000000..0d83a35 --- /dev/null +++ b/pkg/csi/links_unix_test.go @@ -0,0 +1,67 @@ +//go:build unix + +package csi + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + "time" +) + +// A base whose file has a second hard link is not taken over: changing it in place would change +// the other name too. The pull then builds the target without the base, and both stay intact. +func TestOwnerHardLinkedDonorRefusesConsumption(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, c := newMaterializer(t, f) + m.RebaseIdle = time.Hour + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + inside := filepath.Join(m.Dir(f.a), "model.bin") + want, err := os.ReadFile(inside) + if err != nil { + t.Fatal(err) + } + // The second name lives under the materializer root so that both are on one filesystem. + outside := filepath.Join(m.Root, "snapshot-model.bin") + if err := os.Link(inside, outside); err != nil { + t.Skipf("hard links unavailable here: %v", err) + } + if p, err := aliased(m.Dir(f.a)); err != nil || p != inside { + t.Fatalf("aliased = %q, %v; want %q", p, err, inside) + } + + offer, claimed := m.offer(f.b) + if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a { + t.Fatal("expected the idle A donor to be offered") + } + if dir, ok := offer.Claim(offer.Bases[0]); ok || dir != "" || len(*claimed) != 0 { + t.Fatalf("a hard-linked donor was consumed: dir=%q ok=%v", dir, ok) + } + + dir, err := m.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + sameTree(t, f.srcA, m.Dir(f.a)) + if got, err := os.ReadFile(outside); err != nil || !bytes.Equal(got, want) { + t.Fatalf("the second link changed: err=%v", err) + } + if last := c.pulls[len(c.pulls)-1]; last.Base != "" { + t.Fatalf("the pull took over a hard-linked base: %+v", last) + } + + // Without the second link the same donor is consumed as before. + if err := os.Remove(outside); err != nil { + t.Fatal(err) + } + if p, err := aliased(m.Dir(f.a)); err != nil || p != "" { + t.Fatalf("aliased after unlink = %q, %v", p, err) + } +} diff --git a/pkg/csi/materializer.go b/pkg/csi/materializer.go index 59b86e9..c5729e8 100644 --- a/pkg/csi/materializer.go +++ b/pkg/csi/materializer.go @@ -29,6 +29,13 @@ type Puller func(ctx context.Context, m digest.Digest, dest string, offer *Offer type Offer struct { Bases []store.Base Claim func(store.Base) (dir string, ok bool) + // VerifyKept and BaseCopy are passed to the pull (store.PullOptions). + VerifyKept bool + BaseCopy store.BaseCopy + // Readable are artifacts the pull may read where they are, in use or not; Hold keeps one + // from being taken over or evicted while it is read. + Readable []store.Base + Hold func(store.Base) (release func(), ok bool) } // StorePuller pulls from s through the node cache (nil: no cache), assembling cache packs in tmp. @@ -57,6 +64,8 @@ func StorePullerPeers(s *store.Store, openCache func(context.Context) (*store.St } if offer != nil { o.Bases, o.Claim = offer.Bases, offer.Claim + o.VerifyKept, o.BaseCopy = offer.VerifyKept, offer.BaseCopy + o.ReadBases, o.Hold = offer.Readable, offer.Hold } return s.Pull(ctx, m, dest, o) } @@ -84,9 +93,22 @@ type Materializer struct { // MaxPulls bounds distinct artifacts materialized concurrently (0: one). // Set before the first Ensure; retries for one digest still share one pull. MaxPulls int + // MaxArtifacts is how many artifacts the node keeps (0: no limit). With a limit, a pull takes + // an idle artifact over only when the node already holds that many, least recently used + // first: the artifact it consumes is the one that would be evicted to make room. Below the + // limit nothing is consumed, so a free slot is never filled by destroying a warm artifact. + // Evict removes the least recently used idle artifacts above the limit. + MaxArtifacts int + // VerifyKept makes a pull that takes an artifact over read back and hash-check every chunk it + // leaves in place (store.PullOptions.VerifyKept). + VerifyKept bool + // BaseCopy says when an idle artifact that is not taken over is read where it is + // (store.PullOptions.BaseCopy). + BaseCopy store.BaseCopy mu sync.Mutex inflight map[digest.Digest]*flight + reading map[digest.Digest]int // pulls reading an artifact where it is slots chan struct{} } @@ -370,11 +392,14 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) { } m.mu.Lock() ds, _ := m.materialized() - var bases []store.Base + var bases, readable []store.Base inUse, recent := 0, 0 for _, b := range ds { + if b == d { + continue + } + readable = append(readable, store.Base{Manifest: b, Dir: m.Dir(b)}) switch { - case b == d: case m.idle(b, time.Now()): bases = append(bases, store.Base{Manifest: b, Dir: m.Dir(b)}) case m.busy(b): @@ -384,23 +409,69 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) { } } m.mu.Unlock() + if len(readable) == 0 { + return nil, claimed + } if len(bases) == 0 { - if len(ds) > 0 { - m.logf("pull of %s: no artifact to take over (%d in use, %d used within %s)", d, inUse, recent, m.RebaseIdle) + m.logf("pull of %s: no artifact to take over (%d in use, %d used within %s)", d, inUse, recent, m.RebaseIdle) + } + // To consume: most recently used first. With a limit, only the least recently used idle + // artifact, the one eviction would remove: the cache policy picks the victim, not the pull. + sort.Slice(bases, func(i, j int) bool { + a, b := m.lastUsed(bases[i].Manifest), m.lastUsed(bases[j].Manifest) + if m.MaxArtifacts > 0 { + return a.Before(b) } - return nil, claimed + return a.After(b) + }) + if m.MaxArtifacts > 0 && len(bases) > 1 { + bases = bases[:1] } - sort.Slice(bases, func(i, j int) bool { return m.lastUsed(bases[i].Manifest).After(m.lastUsed(bases[j].Manifest)) }) if len(bases) > 3 { bases = bases[:3] } + // To read: most recently used first, in use or not. + sort.Slice(readable, func(i, j int) bool { + return m.lastUsed(readable[i].Manifest).After(m.lastUsed(readable[j].Manifest)) + }) + if len(readable) > 3 { + readable = readable[:3] + } + hold := func(b store.Base) (func(), bool) { + m.mu.Lock() + defer m.mu.Unlock() + if !isDir(m.Dir(b.Manifest)) { + return nil, false + } + if m.reading == nil { + m.reading = map[digest.Digest]int{} + } + m.reading[b.Manifest]++ + return func() { + m.mu.Lock() + if m.reading[b.Manifest]--; m.reading[b.Manifest] <= 0 { + delete(m.reading, b.Manifest) + } + m.mu.Unlock() + }, true + } claim := func(b store.Base) (string, bool) { m.mu.Lock() defer m.mu.Unlock() if !m.idle(b.Manifest, time.Now()) { return "", false } + if m.MaxArtifacts > 0 { + if ds, err := m.materialized(); err != nil || len(ds) < m.MaxArtifacts { + return "", false // a free slot: consume nothing + } + } dir := m.Dir(b.Manifest) + // A file with a second hard link would change outside the artifact too; leave it alone. + if p, err := aliased(dir); err != nil || p != "" { + m.logf("rebase from %s refused: %q has another hard link (err %v)", b.Manifest, p, err) + return "", false + } if m.Unmount != nil { if err := m.Unmount(dir); err != nil { m.logf("rebase from %s: %v", b.Manifest, err) @@ -422,21 +493,27 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) { *claimed = append(*claimed, t) return dst, true } - return &Offer{Bases: bases, Claim: claim}, claimed + return &Offer{Bases: bases, Claim: claim, VerifyKept: m.VerifyKept, BaseCopy: m.BaseCopy, + Readable: readable, Hold: hold}, claimed } -// idle says whether artifact d may be consumed: materialized, no volume, no pull, unused for -// RebaseIdle. The caller holds m.mu. +// idle says whether artifact d may be consumed: materialized, no volume, no pull, and, on a node +// without a capacity, unused for RebaseIdle. With a capacity the victim is the artifact eviction +// would delete anyway, so waiting would only lose its bytes: no lease is all it needs. +// The caller holds m.mu. func (m *Materializer) idle(d digest.Digest, now time.Time) bool { if m.busy(d) || !isDir(m.Dir(d)) { return false } + if m.MaxArtifacts > 0 { + return true + } return now.Sub(m.lastUsed(d)) >= m.RebaseIdle } // busy says whether a volume or a pull uses artifact d. The caller holds m.mu. func (m *Materializer) busy(d digest.Digest) bool { - if _, ok := m.inflight[d]; ok { + if _, ok := m.inflight[d]; ok || m.reading[d] > 0 { return true } n, err := m.users(d) diff --git a/pkg/csi/owner_protocol_test.go b/pkg/csi/owner_protocol_test.go new file mode 100644 index 0000000..97cc2e5 --- /dev/null +++ b/pkg/csi/owner_protocol_test.go @@ -0,0 +1,261 @@ +package csi + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/digest" + "github.com/ai-computing/onggi/pkg/store" +) + +func ageOwnerDonor(t *testing.T, m *Materializer, d digest.Digest) { + t.Helper() + old := time.Unix(1, 0) + if err := os.Chtimes(m.usedFile(d), old, old); err != nil { + t.Fatal(err) + } +} + +// A stale offer cannot override a lease acquired before its actual claim. +func TestOwnerLeaseAfterOfferRefusesConsumption(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, c := newMaterializer(t, f) + m.RebaseIdle = time.Hour + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + before, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")) + if err != nil { + t.Fatal(err) + } + offer, claimed := m.offer(f.b) + if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a { + t.Fatal("expected the idle A donor to be offered") + } + if err := m.Acquire("late-owner-lease", f.a, "/late-owner-target"); err != nil { + t.Fatal(err) + } + // Keep age eligible so refusal is due to the lease rather than recency. + ageOwnerDonor(t, m, f.a) + if dir, ok := offer.Claim(offer.Bases[0]); ok || dir != "" || len(*claimed) != 0 { + t.Fatalf("stale offer consumed a leased donor: dir=%q ok=%v", dir, ok) + } + if users, err := m.users(f.a); err != nil || users != 1 { + t.Fatalf("lease lost: users=%d err=%v", users, err) + } + dir, err := m.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + sameTree(t, f.srcA, m.Dir(f.a)) + after, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")) + if err != nil || !os.SameFile(before, after) || c.pulls[1].Base != "" { + t.Fatalf("leased donor changed or was consumed: err=%v stats=%+v", err, c.pulls[1]) + } + t.Log("offer preceded lease; saved claim refused; lease/inode/base bytes retained; separate target verified") +} + +// Two distinct target offers contend on the real owner claim, not a stub callback. +func TestOwnerCompetingDonorClaims(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + m.RebaseIdle = time.Hour + m.MaxPulls = 2 + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + otherSource := t.TempDir() + shared, err := os.ReadFile(filepath.Join(f.srcA, "model.bin")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(otherSource, "model.bin"), append(shared, random(3, 100_000)...), 0o644); err != nil { + t.Fatal(err) + } + other, _, err := f.s.Push(ctx, otherSource, store.PushOptions{}) + if err != nil { + t.Fatal(err) + } + first, claimedFirst := m.offer(f.b) + second, claimedSecond := m.offer(other) + for _, offer := range []*Offer{first, second} { + if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a { + t.Fatal("both distinct target offers must name A") + } + } + before, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")) + if err != nil { + t.Fatal(err) + } + type result struct { + dir string + ok bool + } + start := make(chan struct{}) + results := make(chan result, 2) + for _, offer := range []*Offer{first, second} { + go func(o *Offer) { + <-start + dir, ok := o.Claim(o.Bases[0]) + results <- result{dir: dir, ok: ok} + }(offer) + } + close(start) + winner := "" + wins := 0 + for range 2 { + select { + case r := <-results: + if r.ok { + wins++ + winner = r.dir + } else if r.dir != "" { + t.Fatalf("refused claim returned a directory: %q", r.dir) + } + case <-time.After(10 * time.Second): + t.Fatal("competing claims did not return") + } + } + if wins != 1 || len(*claimedFirst)+len(*claimedSecond) != 1 || isDir(m.Dir(f.a)) { + t.Fatalf("donor ownership moved incorrectly: wins=%d first=%v second=%v", wins, *claimedFirst, *claimedSecond) + } + sameTree(t, f.srcA, winner) + after, err := os.Stat(filepath.Join(winner, "model.bin")) + if err != nil || !os.SameFile(before, after) { + t.Fatalf("winner did not receive the original donor inode: %v", err) + } + for _, offer := range []*Offer{first, second} { + if _, ok := offer.Claim(offer.Bases[0]); ok { + t.Fatal("a later claim took the already-transferred donor") + } + } + sameTree(t, f.srcA, winner) + t.Log("two real claims released together; exactly one move; winner bytes/inode intact after repeated refusal") +} + +func ownerPackPaths(t *testing.T, root string) map[string]bool { + t.Helper() + paths := make(map[string]bool) + err := filepath.WalkDir(filepath.Join(root, "packs"), func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.Type().IsRegular() { + paths[path] = true + } + return nil + }) + if err != nil { + t.Fatal(err) + } + return paths +} + +// Actual residual corruption follows a successful real claim; retry has no donor. +func TestOwnerPostConsumptionFailureThenRetry(t *testing.T) { + ctx := context.Background() + storeDir := t.TempDir() + f := &fixture{s: openStore(t, storeDir), srcA: t.TempDir(), srcB: t.TempDir()} + shared := random(71, 3<<20) + if err := os.WriteFile(filepath.Join(f.srcA, "model.bin"), shared, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(f.srcB, "model.bin"), append(shared, random(72, 100_000)...), 0o644); err != nil { + t.Fatal(err) + } + var err error + f.a, _, err = f.s.Push(ctx, f.srcA, store.PushOptions{}) + if err != nil { + t.Fatal(err) + } + basePacks := ownerPackPaths(t, storeDir) + f.b, _, err = f.s.Push(ctx, f.srcB, store.PushOptions{}) + if err != nil { + t.Fatal(err) + } + var residual []string + for path := range ownerPackPaths(t, storeDir) { + if !basePacks[path] { + residual = append(residual, path) + } + } + if len(residual) != 1 { + t.Fatalf("expected one target-only residual pack, got %d", len(residual)) + } + pack := residual[0] + original, err := os.ReadFile(pack) + if err != nil || len(original) == 0 { + t.Fatalf("read residual pack: %v", err) + } + m, _ := newMaterializer(t, f) + m.RebaseIdle = time.Hour + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + // No cache/peer can mask the corrupted target-only store bytes. + realPull := StorePuller(f.s, nil, filepath.Join(m.Root, "tmp"), 2) + claimed := false + var injectionErr, pullErr error + var failedStats store.PullStats + m.Pull = func(ctx context.Context, target digest.Digest, dest string, offer *Offer) (store.PullStats, error) { + if offer != nil { + realClaim := offer.Claim + offer.Claim = func(base store.Base) (string, bool) { + dir, ok := realClaim(base) + if ok { + claimed = true + corrupt := append([]byte(nil), original...) + corrupt[len(corrupt)/2] ^= 0xff + injectionErr = os.Chmod(pack, 0o644) + if injectionErr == nil { + injectionErr = os.WriteFile(pack, corrupt, 0o644) + } + } + return dir, ok + } + } + failedStats, pullErr = realPull(ctx, target, dest, offer) + return failedStats, pullErr + } + _, err = m.Ensure(ctx, f.b) + if !claimed || injectionErr != nil || pullErr == nil || !errors.Is(err, pullErr) { + t.Fatalf("expected actual post-claim construction failure: claimed=%v injection=%v pull=%v ensure=%v", claimed, injectionErr, pullErr, err) + } + if !strings.Contains(pullErr.Error(), "fails its hash check") || !strings.Contains(pullErr.Error(), filepath.Base(pack)) { + t.Fatalf("expected the selected corrupted residual pack's hash failure: pack=%s err=%v", filepath.Base(pack), pullErr) + } + if failedStats.Base != f.a.String() || failedStats.Reused == 0 { + t.Fatalf("failure preceded genuine consumption/preservation: %+v", failedStats) + } + if isDir(m.Dir(f.a)) || isDir(m.Dir(f.b)) { + t.Fatal("consumed donor or incomplete target remained published") + } + if entries, err := os.ReadDir(filepath.Join(m.Root, "tmp")); err != nil || len(entries) != 0 { + t.Fatalf("private/consumed cleanup failed: entries=%v err=%v", entries, err) + } + if err := os.WriteFile(pack, original, 0o644); err != nil { + t.Fatal(err) + } + m.Pull = realPull + dir, err := m.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + if isDir(m.Dir(f.a)) { + t.Fatal("retry manufactured a replacement donor") + } + t.Logf("real consumed-base residual failure=%v reused=%d; no completed target; donor-free retry bytes verified", pullErr, failedStats.Reused) +} diff --git a/pkg/store/pull.go b/pkg/store/pull.go index a90931d..fc05c13 100644 --- a/pkg/store/pull.go +++ b/pkg/store/pull.go @@ -37,8 +37,36 @@ type PullOptions struct { Bases []Base Claim func(Base) (dir string, ok bool) MinSame float64 + // VerifyKept makes a pull that takes a base over read back every chunk it leaves in place and + // check its hash, so nothing reaches the new artifact unchecked. A chunk that fails (the base + // was damaged after it was pulled) is fetched like any other. + VerifyKept bool + // BaseCopy says when a base that is not taken over is read where it is (#58). + BaseCopy BaseCopy + // ReadBases are artifacts on the node the pull may read where they are but never consumes + // (nil: Bases). They may be in use. Hold, if set, is called before one is read and keeps it + // from being taken over or evicted until the returned release is called (ok false: gone). + ReadBases []Base + Hold func(Base) (release func(), ok bool) + // BaseRange is the largest single read from a base's files (0: MaxRange). Chunks that follow + // one another are read together up to this; 1 reads chunk by chunk. + BaseRange int64 } +// BaseCopy is the rule for reading a base that stays intact. +type BaseCopy int + +const ( + // BaseCopyAlways reads the base whenever it holds MinSame of the artifact anywhere. + BaseCopyAlways BaseCopy = iota + // BaseCopyRemote reads the base only if the store is not local. Packs on this machine are + // read in large ranges and decoded faster than a base is read and hashed chunk by chunk: + // on 15-18 GB artifacts a local store took 32-35 s where the base took 44-47 s. + BaseCopyRemote + // BaseCopyNever never reads a base that is not taken over. + BaseCopyNever +) + // PullStats reports a pull. type PullStats struct { Files int `json:"files"` @@ -50,10 +78,14 @@ type PullStats struct { Cached int64 `json:"cached"` // stored bytes added to the node cache Base string `json:"base,omitempty"` // the artifact this pull turned into the new one Reused int64 `json:"reused"` // bytes left in place in the base's files + Verified int64 `json:"verified"` // of those, bytes read back and hash-checked (VerifyKept) + Repaired int64 `json:"repaired"` // bytes of the base that failed that check and were fetched instead BaseBest int64 `json:"base_best"` // bytes the best offered base keeps at the same offsets, taken or not (#49) LocalBase string `json:"local_base,omitempty"` // a base read where it is, not consumed (#58) LocalBytes int64 `json:"local_bytes"` // bytes read from that base's files Ranges int `json:"ranges"` + FetchedChunks int64 `json:"fetched_chunks"` // chunks taken from the chunk source (casync engine) + FetchedBytes int64 `json:"fetched_bytes"` // their logical size Duration time.Duration `json:"duration_ns"` } @@ -104,6 +136,9 @@ func (s *Store) Pull(ctx context.Context, manifest digest.Digest, dest string, o if o.MaxRange <= 0 { o.MaxRange = 16 << 20 } + if o.BaseRange <= 0 { + o.BaseRange = o.MaxRange + } m, err := s.ReadManifest(ctx, manifest) if err != nil { return PullStats{}, err @@ -143,28 +178,50 @@ func (s *Store) Pull(ctx context.Context, manifest digest.Digest, dest string, o return st, err } done := make([]bool, len(nodes)) // in place from a base, or written from peers - if len(o.Bases) > 0 && o.Claim != nil { + readBases := o.ReadBases + if readBases == nil { + readBases = o.Bases + } + if len(o.Bases) > 0 && o.Claim != nil || len(readBases) > 0 { minSame := o.MinSame if minSame <= 0 { minSame = DefaultMinSame } - p, ok, most, okMost := s.chooseBase(ctx, m, extentsOf(nodes, len(m.Files), false), o.Bases, minSame) - st.BaseBest = p.same + newExt := extentsOf(nodes, len(m.Files), false) claimed := false - if ok { - if dir, ok := o.Claim(p.base); ok { - claimed = true - st.Base = p.base.Manifest.String() - if st.Reused, err = rebase(p, dir, m, nodes, handles, done); err != nil { - return st, err + if len(o.Bases) > 0 && o.Claim != nil { + p, ok, _, _ := s.chooseBase(ctx, m, newExt, o.Bases, minSame) + st.BaseBest = p.same + if ok { + if dir, ok := o.Claim(p.base); ok { + claimed = true + st.Base = p.base.Manifest.String() + if st.Reused, err = rebase(p, dir, m, nodes, handles, done); err != nil { + return st, err + } + if o.VerifyKept { + st.Verified, st.Repaired = verifyKept(ctx, nodes, handles, done, o.Workers, o.BaseRange) + st.Reused -= st.Repaired + } } } } - // too little in place, but the base holds the bytes elsewhere: read them from it (#58) - if !claimed && okMost { - st.LocalBase = most.base.Manifest.String() - if st.LocalBytes, err = fromBase(ctx, most, nodes, done, handles, o.Workers); err != nil { - return st, err + // nothing taken over, but an artifact on the node holds the bytes: read them from it (#58) + copyBase := o.BaseCopy == BaseCopyAlways || o.BaseCopy == BaseCopyRemote && !s.B.Caps().Local + if !claimed && copyBase && len(readBases) > 0 { + if _, _, most, okMost := s.chooseBase(ctx, m, newExt, readBases, minSame); okMost { + release, held := func() {}, true + if o.Hold != nil { + release, held = o.Hold(most.base) + } + if held { + st.LocalBase = most.base.Manifest.String() + st.LocalBytes, err = fromBase(ctx, most, nodes, done, handles, o.Workers, o.BaseRange) + release() + if err != nil { + return st, err + } + } } } } diff --git a/pkg/store/push.go b/pkg/store/push.go index 4620e11..c564a2a 100644 --- a/pkg/store/push.go +++ b/pkg/store/push.go @@ -408,7 +408,7 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode var list []ChunkRef var nb int64 var fb *frameBuilder // new chunks of a sub-chunked file go out in compression frames (#52) - if twoLevel && s.Config.Frame > 0 { + if s.Config.Frame > 0 && (twoLevel || s.Config.Sub == nil) { fb = &frameBuilder{limit: int(s.Config.Frame), mode: mode} } for { @@ -430,6 +430,14 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode continue } if !twoLevel { + if fb != nil { // a framed one-level store (Create.FrameFlat) + n, err := fb.add(ctx, ch, d, out) + if err != nil { + return nil, 0, err + } + nb += n + continue + } n, err := send(ctx, ch, d, mode, out) if err != nil { return nil, 0, err diff --git a/pkg/store/rebase.go b/pkg/store/rebase.go index 76ab229..1b945d8 100644 --- a/pkg/store/rebase.go +++ b/pkg/store/rebase.go @@ -110,32 +110,22 @@ func (s *Store) chooseBase(ctx context.Context, m *Manifest, newExt [][]Extent, } // fromBase reads, from the base's files where they are, the stored chunks a pull still needs -// that the base holds anywhere (#58). The base is not consumed. Each chunk is hash-checked; one -// that cannot be read or does not match is left for the other sources. It returns the bytes read. -func fromBase(ctx context.Context, p rebasePlan, nodes []node, done []bool, handles []*os.File, workers int) (int64, error) { - type job struct { - node int32 - file int - off int64 - } - var jobs []job +// that the base holds anywhere (#58). The base is not consumed. Chunks that follow one another in +// a base file are read together. Each chunk is hash-checked; one that cannot be read or does not +// match is left for the other sources. It returns the bytes read. +func fromBase(ctx context.Context, p rebasePlan, nodes []node, done []bool, handles []*os.File, workers int, maxRange int64) (int64, error) { + var jobs []rjob for i := range nodes { if !nodes[i].stored || done[i] { continue } if f, e, ok := p.find(nodes[i].chunk); ok && e.Len == nodes[i].loc.Length { - jobs = append(jobs, job{int32(i), f, e.Off}) + jobs = append(jobs, rjob{int32(i), f, e.Off, int(e.Len)}) } } if len(jobs) == 0 { return 0, nil } - slices.SortFunc(jobs, func(a, b job) int { // read each base file front to back - if a.file != b.file { - return a.file - b.file - } - return cmp.Compare(a.off, b.off) - }) files := make([]*os.File, len(p.files)) defer func() { for _, f := range files { @@ -156,42 +146,53 @@ func fromBase(ctx context.Context, p rebasePlan, nodes []node, done []bool, hand var n atomic.Int64 var mu sync.Mutex var firstErr error - var wg sync.WaitGroup - w := max(workers, 1) - per := (len(jobs) + w - 1) / w - for a := 0; a < len(jobs); a += per { - part := jobs[a:min(a+per, len(jobs))] - wg.Add(1) - go func() { - defer wg.Done() - var buf []byte - for _, j := range part { - if ctx.Err() != nil { - return - } - nd := &nodes[j.node] - if cap(buf) < int(nd.loc.Length) { - buf = make([]byte, nd.loc.Length) - } - b := buf[:nd.loc.Length] - if _, err := files[j.file].ReadAt(b, j.off); err != nil || digest.Of(b) != nd.chunk { - continue - } - if err := emit(nodes, j.node, b, 0, handles); err != nil { - mu.Lock() - firstErr = cmp.Or(firstErr, err) - mu.Unlock() - return - } - done[j.node] = true - n.Add(int64(len(b))) - } - }() - } - wg.Wait() + readSpans(ctx, files, jobs, workers, maxRange, func(j rjob, b []byte) { + if b == nil || digest.Of(b) != nodes[j.node].chunk { + return + } + if err := emit(nodes, j.node, b, 0, handles); err != nil { + mu.Lock() + firstErr = cmp.Or(firstErr, err) + mu.Unlock() + return + } + done[j.node] = true + n.Add(int64(len(b))) + }) return n.Load(), firstErr } +// verifyKept reads back, from the files of the new artifact, every chunk a taken-over base left +// in place, and checks its hash. A chunk that fails in any of its places is no longer done, so +// the pull fetches it and writes it everywhere it goes. It returns the bytes that passed and the +// bytes that did not. +func verifyKept(ctx context.Context, nodes []node, handles []*os.File, done []bool, workers int, maxRange int64) (ok, bad int64) { + var jobs []rjob + for i := range nodes { + if !nodes[i].stored || !done[i] { + continue + } + places(nodes, int32(i), 0, func(t target) { + jobs = append(jobs, rjob{int32(i), int(t.file), t.off, int(nodes[i].loc.Length)}) + }) + } + failed := make([]atomic.Bool, len(nodes)) + readSpans(ctx, handles, jobs, workers, maxRange, func(j rjob, b []byte) { + if b == nil || digest.Of(b) != nodes[j.node].chunk { + failed[j.node].Store(true) + } + }) + for _, j := range jobs { + if failed[j.node].Load() { + done[j.node] = false + bad += int64(j.n) + } else { + ok += int64(j.n) + } + } + return ok, bad +} + // sameAt says whether base holds e's chunk at e's offset. func sameAt(base []Extent, e Extent) bool { k, ok := slices.BinarySearchFunc(base, e.Off, func(x Extent, off int64) int { diff --git a/pkg/store/selector_test.go b/pkg/store/selector_test.go new file mode 100644 index 0000000..c51ea5f --- /dev/null +++ b/pkg/store/selector_test.go @@ -0,0 +1,255 @@ +package store + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/ai-computing/onggi/pkg/backend" + bfs "github.com/ai-computing/onggi/pkg/backend/fs" + "github.com/ai-computing/onggi/pkg/chunker" + "github.com/ai-computing/onggi/pkg/codec" +) + +// A base damaged after it was pulled is not carried into the new artifact: with VerifyKept the +// chunk that no longer matches is fetched, and only the rest is left in place. +func TestRebaseVerifiesWhatItKeeps(t *testing.T) { + ctx := context.Background() + s := openTwoLevel(t, t.TempDir()) + w := random(41, 8<<20) + v := append([]byte(nil), w...) + copy(v[3<<20:], random(42, 200<<10)) + b, nm, src := rebaseFixture(t, s, + map[string][]byte{"decode_B2.rbln": w}, + map[string][]byte{"decode_B8.rbln": v}) + + // flip bytes of the base in a region both artifacts share + f, err := os.OpenFile(filepath.Join(b.Dir, "decode_B2.rbln"), os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt([]byte("damaged after the pull"), 6<<20); err != nil { + t.Fatal(err) + } + f.Close() + + claim := func(x Base) (string, bool) { return x.Dir, true } + out := t.TempDir() + st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: claim, VerifyKept: true}) + if err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + t.Logf("%+v", st) + if st.Base != b.Manifest.String() || st.Repaired <= 0 || st.Repaired > 1<<20 || st.Verified < int64(len(v))*8/10 { + t.Fatalf("damaged chunk not repaired, or too much distrusted: %+v", st) + } + if st.Reused+st.Repaired > int64(len(v)) || st.Reused != st.Verified { + t.Fatalf("kept bytes and verified bytes disagree: %+v", st) + } +} + +// Without VerifyKept the pull reads nothing back, as before. +func TestRebaseDoesNotVerifyByDefault(t *testing.T) { + ctx := context.Background() + s := openTwoLevel(t, t.TempDir()) + w := random(43, 4<<20) + v := append([]byte(nil), w...) + copy(v[1<<20:], random(44, 100<<10)) + b, nm, src := rebaseFixture(t, s, map[string][]byte{"a.rbln": w}, map[string][]byte{"a.rbln": v}) + out := t.TempDir() + st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: func(x Base) (string, bool) { return x.Dir, true }}) + if err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + if st.Verified != 0 || st.Repaired != 0 || st.Reused < int64(len(v))*8/10 { + t.Fatalf("%+v", st) + } +} + +// With a local store, BaseCopyRemote leaves a shifted base alone and reads the packs; the same +// pull under BaseCopyAlways reads the base. Both produce the artifact. +func TestBaseCopyRemoteSkipsALocalStore(t *testing.T) { + ctx := context.Background() + s := openTwoLevel(t, t.TempDir()) + if !s.B.Caps().Local { + t.Fatal("the fs backend must report itself local") + } + w := random(93, 8<<20) + v := append(random(94, 17952), w...) + b, nm, src := rebaseFixture(t, s, map[string][]byte{"rank0.engine": w}, map[string][]byte{"rank0.engine": v}) + never := func(Base) (string, bool) { + t.Fatal("consumed a base that keeps nothing in place") + return "", false + } + for _, c := range []struct { + name string + rule BaseCopy + base bool + }{{"remote", BaseCopyRemote, false}, {"never", BaseCopyNever, false}, {"always", BaseCopyAlways, true}} { + out := t.TempDir() + st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: never, BaseCopy: c.rule}) + if err != nil { + t.Fatal(c.name, err) + } + sameTree(t, src, out) + if got := st.LocalBase != ""; got != c.base { + t.Fatalf("%s: read the base = %v, want %v: %+v", c.name, got, c.base, st) + } + if !c.base && st.ReadBytes == 0 { + t.Fatalf("%s: nothing read from the store: %+v", c.name, st) + } + } + if _, err := os.Stat(filepath.Join(b.Dir, "rank0.engine")); err != nil { + t.Fatalf("the base was touched: %v", err) + } +} + +// Chunks that follow one another in a file are read together, and each still reaches fn alone. +func TestReadSpansJoinsNeighbours(t *testing.T) { + data := random(95, 1<<20) + p := filepath.Join(t.TempDir(), "f") + write(t, p, data) + f, err := os.Open(p) + if err != nil { + t.Fatal(err) + } + defer f.Close() + var jobs []rjob + for off := 0; off < len(data); off += 4096 { // contiguous, then a hole after each 64 KiB + if off/65536%2 == 1 { + continue + } + jobs = append(jobs, rjob{node: int32(off / 4096), off: int64(off), n: 4096}) + } + jobs = append(jobs, rjob{node: -1, off: int64(len(data)) - 100, n: 4096}) // runs past the end + got := make(map[int32]bool) + bad := 0 + done := make(chan struct{}, 1) + done <- struct{}{} + readSpans(context.Background(), []*os.File{f}, jobs, 4, 32<<10, func(j rjob, b []byte) { + <-done + defer func() { done <- struct{}{} }() + if b == nil { + bad++ + return + } + if string(b) != string(data[j.off:j.off+int64(j.n)]) { + t.Errorf("chunk at %d has the wrong bytes", j.off) + } + got[j.node] = true + }) + if bad != 1 || len(got) != len(jobs)-1 { + t.Fatalf("read %d of %d chunks, %d failed", len(got), len(jobs), bad) + } +} + +// Reading a base chunk by chunk and in joined ranges gives the same artifact and the same bytes. +func TestBaseRangeDoesNotChangeTheResult(t *testing.T) { + ctx := context.Background() + s := openTwoLevel(t, t.TempDir()) + w := random(96, 8<<20) + v := append(random(97, 17952), w...) + b, nm, src := rebaseFixture(t, s, map[string][]byte{"rank0.engine": w}, map[string][]byte{"rank0.engine": v}) + never := func(Base) (string, bool) { return "", false } + var local [2]int64 + for i, r := range []int64{1, 0} { + out := t.TempDir() + st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: never, BaseRange: r}) + if err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + local[i] = st.LocalBytes + } + if local[0] != local[1] || local[0] < int64(len(w))*9/10 { + t.Fatalf("bytes read from the base: chunk by chunk %d, joined %d", local[0], local[1]) + } +} + +// A base given only to read is never claimed; it is held while it is read and released after. +func TestReadBasesAreHeldAndNeverClaimed(t *testing.T) { + ctx := context.Background() + s := openTwoLevel(t, t.TempDir()) + w := random(51, 8<<20) + v := append([]byte(nil), w...) + copy(v[3<<20:], random(52, 200<<10)) + b, nm, src := rebaseFixture(t, s, map[string][]byte{"decode_B2.rbln": w}, map[string][]byte{"decode_B8.rbln": v}) + held, released := 0, 0 + out := t.TempDir() + st, err := s.Pull(ctx, nm, out, PullOptions{ReadBases: []Base{b}, + Claim: func(Base) (string, bool) { t.Fatal("claimed a base given only to read"); return "", false }, + Hold: func(x Base) (func(), bool) { + if x.Manifest != b.Manifest { + t.Fatalf("held %s", x.Manifest) + } + held++ + return func() { released++ }, true + }}) + if err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + if held != 1 || released != 1 || st.LocalBase != b.Manifest.String() || st.Base != "" || st.LocalBytes < int64(len(v))*8/10 { + t.Fatalf("held %d, released %d, %+v", held, released, st) + } + if _, err := os.Stat(filepath.Join(b.Dir, "decode_B2.rbln")); err != nil { + t.Fatalf("the base was touched: %v", err) + } + + // a base that cannot be held is not read + out = t.TempDir() + st, err = s.Pull(ctx, nm, out, PullOptions{ReadBases: []Base{b}, Hold: func(Base) (func(), bool) { return nil, false }}) + if err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + if st.LocalBase != "" || st.LocalBytes != 0 { + t.Fatalf("read a base it could not hold: %+v", st) + } +} + +// A one-level store with FrameFlat stores its chunks in frames and gives the same files back; +// compressible data takes less room than without frames. +func TestFlatStoreCanFrameItsChunks(t *testing.T) { + ctx := context.Background() + data := make([]byte, 6<<20) + for i := range data { // compressible, but not chunk by chunk identical + data[i] = byte(i>>9) ^ byte(i%7) + } + src := t.TempDir() + write(t, filepath.Join(src, "decode_B2.rbln"), data) + sizes := map[bool]int64{} + for _, framed := range []bool{false, true} { + b, err := bfs.Open(t.TempDir(), backend.DurabilityNone) + if err != nil { + t.Fatal(err) + } + s, err := OpenWith(ctx, b, Create{Chunker: chunker.Params{Min: 1 << 10, Avg: 4 << 10, Max: 16 << 10}, + PackSize: 1 << 20, Frame: 256 << 10, FrameFlat: framed}) + if err != nil { + t.Fatal(err) + } + if (s.Config.Frame > 0) != framed || s.Config.Sub != nil { + t.Fatalf("framed %v: config %+v", framed, s.Config) + } + m, _, err := s.Push(ctx, src, PushOptions{Compress: codec.ModeAuto}) + if err != nil { + t.Fatal(err) + } + out := t.TempDir() + if _, err := s.Pull(ctx, m, out, PullOptions{}); err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + err = b.List(ctx, backend.Pack, func(_ string, n int64) error { sizes[framed] += n; return nil }) + if err != nil { + t.Fatal(err) + } + } + if sizes[true] <= 0 || sizes[true] >= sizes[false] { + t.Fatalf("packs: %d B framed, %d B unframed", sizes[true], sizes[false]) + } +} diff --git a/pkg/store/spans.go b/pkg/store/spans.go new file mode 100644 index 0000000..d187281 --- /dev/null +++ b/pkg/store/spans.go @@ -0,0 +1,101 @@ +package store + +import ( + "cmp" + "context" + "os" + "slices" + "sync" +) + +// rjob is one chunk to read from a file: n bytes at off. +type rjob struct { + node int32 + file int + off int64 + n int +} + +// readSpans reads the jobs from files, joining chunks that follow one another in a file into +// single reads of at most maxRange bytes, and calls fn with each chunk's bytes. A read of 15 GB +// in 64 KiB chunks is 230 000 system calls one by one and under a thousand joined. fn gets nil +// for a chunk whose read failed. It is called from several goroutines; b is only valid during +// the call. +func readSpans(ctx context.Context, files []*os.File, jobs []rjob, workers int, maxRange int64, fn func(j rjob, b []byte)) { + if len(jobs) == 0 { + return + } + if maxRange <= 0 { + maxRange = 16 << 20 + } + slices.SortFunc(jobs, func(a, b rjob) int { // each file front to back + if a.file != b.file { + return a.file - b.file + } + return cmp.Compare(a.off, b.off) + }) + type span struct{ a, b int } // jobs[a:b] + var spans []span + for k, j := range jobs { + if n := len(spans); n > 0 { + s := &spans[n-1] + first, prev := jobs[s.a], jobs[s.b-1] + if prev.file == j.file && prev.off+int64(prev.n) == j.off && j.off+int64(j.n)-first.off <= maxRange { + s.b = k + 1 + continue + } + } + spans = append(spans, span{k, k + 1}) + } + ch := make(chan span) + var wg sync.WaitGroup + for w := 0; w < max(workers, 1); w++ { + wg.Add(1) + go func() { + defer wg.Done() + var buf []byte + for s := range ch { + if ctx.Err() != nil { + continue + } + first, last := jobs[s.a], jobs[s.b-1] + size := int(last.off + int64(last.n) - first.off) + if cap(buf) < size { + buf = make([]byte, size) + } + b := buf[:size] + _, err := files[first.file].ReadAt(b, first.off) + for _, j := range jobs[s.a:s.b] { + if err != nil { + fn(j, nil) + continue + } + o := int(j.off - first.off) + fn(j, b[o:o+j.n]) + } + } + }() + } + for _, s := range spans { + ch <- s + } + close(ch) + wg.Wait() +} + +// places calls fn for every place chunk id goes: a file and the offset in it. +func places(nodes []node, id int32, base int64, fn func(target)) { + nd := &nodes[id] + if nd.hasT { + fn(target{nd.firstT.file, nd.firstT.off + base}) + for _, t := range nd.moreT { + fn(target{t.file, t.off + base}) + } + } + if nd.hasP { + places(nodes, nd.firstP.node, base+int64(nd.firstP.off), fn) + for _, e := range nd.moreP { + places(nodes, e.node, base+int64(e.off), fn) + } + } +} diff --git a/pkg/store/store.go b/pkg/store/store.go index 124161f..3cc274e 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -52,6 +52,9 @@ type Create struct { SubKinds []layout.Kind // file kinds that get sub-chunks; empty: all PackSize int64 Frame int64 // compression frames for sub-chunked files (#52); used only with Sub + // FrameFlat frames the new chunks of a one-level store too (Sub nil): Frame is then the frame + // size. It exists to compare flat chunking with two levels under the same compression. + FrameFlat bool } // DefaultSubKinds are the kinds whose compile variants rearrange the same bytes (#15). @@ -85,6 +88,8 @@ func OpenWith(ctx context.Context, b backend.Backend, c Create) (*Store, error) cfg = Config{Schema: "onggi.store.v1", Chunker: p, PackSize: packSize, Sub: c.Sub, Created: time.Now().UTC()} if c.Sub != nil { cfg.SubKinds, cfg.Frame = c.SubKinds, c.Frame + } else if c.FrameFlat { + cfg.Frame = c.Frame } data, _ := json.MarshalIndent(cfg, "", " ") if err := b.Put(ctx, configKey, bytes.NewReader(data), int64(len(data))); err != nil { diff --git a/tools/varmorph/README.md b/tools/varmorph/README.md new file mode 100644 index 0000000..108064a --- /dev/null +++ b/tools/varmorph/README.md @@ -0,0 +1,7 @@ +# VarMorph layout coding and device rebuild (prototype) + +- `classify.py`: classifies each 64 KiB window of a compiled artifact (weights by numeric type, code, padding, text) from byte statistics, without parsing the format. +- `layout_coder.py`: codes a target variant against sibling(s) at any byte offset below chunk size (sub-block shift search, motion prediction with merge candidates, run merging, XOR residual, intra codec with bf16 byte planes, fill). It decodes every target and compares it byte for byte. With `VCODEC_DUMP=1` it writes the descriptor (`*.runs.npz`). +- `device_rebuild.py`: rebuilds a target on the device from a resident sibling. `replay` is out of place. `inplace` uses a ring of saved windows, KEEP for unmoved runs, absorb below a measured threshold, folding, and batched copies on NPU-B. +- `npu_d2d_bench.py`: device copy and upload costs per platform. +- `rbln_vmshim.cpp`: C shim over `rbln_memcpy_v2v_multi` / `rbln_memcpy_h2v_multi` of the NPU-B runtime API. diff --git a/tools/varmorph/classify.py b/tools/varmorph/classify.py new file mode 100644 index 0000000..cd6db16 --- /dev/null +++ b/tools/varmorph/classify.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Cohort 2026-11-12 (11): what a compiled binary holds, by content class, without parsing its format. + +Every 64 KiB window of a file is classified from byte statistics alone, so the same rule applies to +any vendor's artifact: + fill one or two distinct byte values (padding, zero-initialized buffers) + text >= 85 % printable ASCII (metadata, names, JSON) + fp16/bf16 16-bit float weights: >= 90 % of 16-bit words have a high byte in 0x04-0x3F or + 0x84-0xBF (|x| < 2 as fp16 or bf16), the two signs are balanced within 10 %, and the + high byte has <= 5.5 bits of entropy. The median magnitude byte tells the two apart + (bf16 weights sit at 0x3A-0x3D, fp16 weights below 0x38). + fp32 the same test on the top byte of 32-bit words (fp32 tensors: norms, tables, biases); + every byte phase is tried, since tensors start at odd offsets in some containers + bf16_in_fp32 fp32 words whose low 16 bits are zero in >= 95 % of words: bf16 values widened + code byte entropy < 7.0 bits (instructions, tables, descriptors; structured, compressible) + dense the rest (high entropy, not weight-like: packed or quantized data, compressed sections) +Per class: bytes, mean byte entropy, mean entropy of the high and low byte of 16-bit words. + +Optional join with a coding result: MODES.npy holds, per 64 KiB window of the target, the number of +64-byte blocks coded as match / fill / residual / intra (from vcodec_server.py). With it, the +output also gives how each class of the target was found in its sibling. +argv: OUT.json FILE [FILE ...] [--modes FILE=MODES.npy ...] +""" +import json +import os +import sys +import time + +import numpy as np + +WIN = 1 << 16 +CLASSES = ["fill", "text", "bf16", "fp32", "code", "dense", "fp16", "bf16_in_fp32"] + + +def entropy_rows(counts): + tot = counts.sum(axis=1, keepdims=True).astype(np.float64) + p = counts / np.maximum(tot, 1) + with np.errstate(divide="ignore", invalid="ignore"): + e = -(np.where(p > 0, p * np.log2(p), 0.0)).sum(axis=1) + return e + + +def hist_rows(a, nw): + """Per-row 256-bin histogram of a (nw, m) uint8 array.""" + off = (np.arange(nw, dtype=np.int64) * 256)[:, None] + return np.bincount((a.astype(np.int64) + off).ravel(), minlength=nw * 256).reshape(nw, 256) + + +def classify(path): + n = os.path.getsize(path) // WIN + mm = np.memmap(path, dtype=np.uint8, mode="r", shape=(n * WIN,)) + cls = np.empty(n, dtype=np.uint8) + feats = np.empty((n, 3), dtype=np.float32) + step = (1 << 28) // WIN + wl_hi = np.zeros(256, dtype=bool) + wl_hi[0x04:0x40] = True + wl_hi[0x84:0xC0] = True + printable = np.zeros(256, dtype=bool) + printable[0x20:0x7F] = True + printable[[9, 10, 13]] = True + for i in range(0, n, step): + j = min(n, i + step) + a = np.asarray(mm[i * WIN:j * WIN]).reshape(j - i, WIN) + nw = j - i + hb = hist_rows(a, nw) + h_odd = hist_rows(a[:, 1::2], nw) # high byte of 16-bit words at phase 0 + h_even = hist_rows(a[:, 0::2], nw) # high byte of 16-bit words at phase 1 + e = entropy_rows(hb) + distinct = (hb > 0).sum(axis=1) + asc = (hb[:, printable].sum(axis=1)) / WIN + + def test16(hh): + w = hh[:, wl_hi].sum(axis=1) / (WIN // 2) + sym = 1 - np.abs(hh[:, :128].sum(axis=1) - hh[:, 128:].sum(axis=1)) / (WIN // 2) + mag = hh[:, :128] + hh[:, 128:] + med = (np.cumsum(mag, axis=1) >= (WIN // 4)).argmax(axis=1) + return (w >= 0.9) & (sym >= 0.9) & (entropy_rows(hh) <= 5.5), med, entropy_rows(hh) + ok0, med0, e0 = test16(h_odd) + ok1, med1, e1 = test16(h_even) + is16 = ok0 | ok1 + med = np.where(ok0, med0, med1) + ehi = np.where(ok1 & ~ok0, e1, e0) + elo = np.where(ok1 & ~ok0, e0, e1) + is32 = np.zeros(nw, dtype=bool) + zlow = np.zeros(nw) + for p in range(4): + h32 = hist_rows(a[:, p::4], nw) + w32 = h32[:, wl_hi].sum(axis=1) / (WIN // 4) + sym32 = 1 - np.abs(h32[:, :128].sum(axis=1) - h32[:, 128:].sum(axis=1)) / (WIN // 4) + ok = (w32 >= 0.9) & (sym32 >= 0.9) + if ok.any(): + lo = (hist_rows(a[:, (p - 3) % 4::4], nw)[:, 0] + hist_rows(a[:, (p - 2) % 4::4], nw)[:, 0]) / (WIN // 2) + zlow = np.where(ok & ~is32, lo, zlow) + is32 |= ok + c = np.full(nw, 5, dtype=np.uint8) + c[e < 7.0] = 4 + c[is32 & ~is16] = 3 + c[is32 & ~is16 & (zlow >= 0.95)] = 7 # fp32 holding bf16 values (low half zero) + c[is16 & (med >= 0x38)] = 2 + c[is16 & (med < 0x38)] = 6 + c[asc >= 0.85] = 1 + c[distinct <= 2] = 0 + cls[i:j] = c + feats[i:j, 0] = e + feats[i:j, 1] = ehi + feats[i:j, 2] = elo + return cls, feats + + +def summary(path, cls, feats, modes=None): + out = {"file": path, "bytes": os.path.getsize(path), "windows": int(len(cls)), "classes": {}} + for k, name in enumerate(CLASSES): + sel = cls == k + if not sel.any(): + continue + row = {"bytes": int(sel.sum()) * WIN, "share": float(sel.mean()), + "entropy": float(feats[sel, 0].mean()), "entropy_hi": float(feats[sel, 1].mean()), + "entropy_lo": float(feats[sel, 2].mean())} + if modes is not None: + m = modes[: len(cls)][sel[: len(modes)]].sum(axis=0).astype(np.int64) * 64 + row["coded_as"] = {"match": int(m[0]), "fill": int(m[1]), "residual": int(m[2]), "intra": int(m[3])} + out["classes"][name] = row + # contiguous extents of one class, to show the layout + brk = np.nonzero(np.diff(cls.astype(np.int16)))[0] + 1 + st = np.concatenate([[0], brk]) + en = np.concatenate([brk, [len(cls)]]) + ext = [{"class": CLASSES[cls[s]], "offset": int(s) * WIN, "bytes": int(e - s) * WIN} for s, e in zip(st, en)] + big = sorted(ext, key=lambda x: -x["bytes"])[:12] + out["extents"] = int(len(ext)) + out["largest_extents"] = sorted(big, key=lambda x: x["offset"]) + return out + + +if __name__ == "__main__": + args = sys.argv[1:] + out = args[0] + files, modes = [], {} + k = 1 + while k < len(args): + if args[k] == "--modes": + f, m = args[k + 1].split("=", 1) + modes[f] = m + k += 2 + else: + files.append(args[k]) + k += 1 + res = [] + for f in files: + t = time.time() + cls, feats = classify(f) + md = np.load(modes[f]) if f in modes else None + res.append(summary(f, cls, feats, md)) + res[-1]["seconds"] = round(time.time() - t, 1) + with open(out, "w") as fh: + json.dump(res, fh, indent=1) + print(time.strftime("%H:%M:%S"), f, {c: round(v["share"], 3) for c, v in res[-1]["classes"].items()}, flush=True) + print("ALL-DONE", flush=True) diff --git a/tools/varmorph/device_rebuild.py b/tools/varmorph/device_rebuild.py new file mode 100644 index 0000000..d2749cb --- /dev/null +++ b/tools/varmorph/device_rebuild.py @@ -0,0 +1,563 @@ +#!/usr/bin/env python3 +"""Rebuild a target variant on the NPU from a resident sibling, with the coder's descriptor. + +The reference (sibling) bytes are uploaded once and left on the device: they stand for weights +that are already resident. The timed part rebuilds the target on the device: + 1. folded copies: match runs of equal length whose target and reference offsets advance by + constant strides are one strided copy (as_strided). The rest are single-run copies. + 2. patch: the 64-byte blocks that match nothing (residual, new content, padding) are uploaded + as one contiguous buffer and put in place with one index_copy_. +The baseline is an upload of the whole target from host memory, in the same 1 GiB pieces. +The result is read back and its SHA-256 compared with the target file. + +Device memory is a list of 1 GiB tensors (int16); a copy never crosses a piece, and copies are +cut at MAXCOPY bytes. Offsets must be even (int16 elements); odd runs are reported and fall +back to the patch. +argv: analyze RUNS.npz OUT.json (folding statistics only, no device) + inplace RUNS.npz OUT.json PLATFORM (in-place rebuild in the sibling's memory) + replay RUNS.npz OUT.json PLATFORM(rngd|rbln) (device replay) +""" +import hashlib +import json +import os +import sys +import time + +import numpy as np + +PIECE = 1 << 30 # bytes per device tensor +MAXCOPY = 256 << 20 # bytes per copy call +BLK = 64 + + +def split_runs(dst, src, ln): + """Cut runs at piece boundaries of both sides and at MAXCOPY.""" + out_d, out_s, out_l = [], [], [] + for d, s, l in zip(dst.tolist(), src.tolist(), ln.tolist()): + while l > 0: + take = min(l, PIECE - d % PIECE, PIECE - s % PIECE, MAXCOPY) + out_d.append(d) + out_s.append(s) + out_l.append(take) + d += take + s += take + l -= take + return np.array(out_d, np.int64), np.array(out_s, np.int64), np.array(out_l, np.int64) + + +GAP = int(os.environ.get("REPLAY_GAP", str(1 << 20))) # bytes of target a merged copy may cover + + +# Measured cost of one copy call and of host upload per platform (notes/npu-d2d-20261112): +# rngd-a3: copy_ per call 120 us, upload 3.7 GB/s; rbln-b3: v2v_multi 1.6 us per copy, upload 21.6 GB/s. +COSTS = {"rngd": dict(call_s=120e-6, h2d_Bps=3.7e9, d2d_Bps=24.4e9), + "rbln": dict(call_s=1.6e-6, h2d_Bps=21.6e9, d2d_Bps=21.7e9), + "rbln-unbatched": dict(call_s=5.0e-6, h2d_Bps=21.6e9, d2d_Bps=21.7e9), + "cpu": dict(call_s=1e-6, h2d_Bps=10e9, d2d_Bps=10e9)} + + +def estimate(platform, n_copies, copy_bytes, patch_bytes, target_bytes): + """Predicted seconds of the rebuild and of a full upload, from the measured cost table.""" + c = COSTS.get(platform, COSTS["cpu"]) + rebuild = n_copies * c["call_s"] + copy_bytes / c["d2d_Bps"] + patch_bytes / c["h2d_Bps"] + upload = target_bytes / c["h2d_Bps"] + return rebuild, upload + + +def absorb_threshold(platform): + """A run shorter than this is cheaper to upload in the patch than to copy with its own call: + the bytes the host link moves in the time of one call.""" + c = COSTS.get(platform, COSTS["cpu"]) + return int(c["call_s"] * c["h2d_Bps"]) // BLK * BLK + + +ABSORB = int(os.environ.get("REPLAY_ABSORB", str(64 << 10))) # bytes: shorter off-motion runs go to the patch +LOOK = int(os.environ.get("REPLAY_LOOK", str(1 << 20))) # bytes of target to look ahead for the motion + + +def absorb(dst, src, ln, limit=ABSORB, look=LOOK): + """Rate-distortion step: a short run whose motion differs from the surrounding segment is not + copied on its own; the segment's copy runs over it and its blocks join the patch. Returns the + kept runs and the absorbed byte ranges (target offset, length).""" + if limit <= 0 or len(dst) == 0: + return dst, src, ln, [] + o = np.argsort(dst, kind="stable") + dst, src, ln = dst[o].tolist(), src[o].tolist(), ln[o].tolist() + mv = [s_ - d for d, s_ in zip(dst, src)] + keep, gone = [], [] + i, n = 0, len(dst) + while i < n: + keep.append(i) + cur = mv[i] + j = i + 1 + while j < n: + if mv[j] == cur: + keep.append(j) + j += 1 + continue + # skip short off-motion runs if the segment's motion resumes within LOOK bytes + k, skipped = j, 0 + while k < n and mv[k] != cur and ln[k] <= limit and dst[k] - dst[j] <= look: + skipped += 1 + k += 1 + if k < n and mv[k] == cur and skipped: + gone.extend(range(j, k)) + j = k + else: + break + i = j + ka = np.array(keep, dtype=np.int64) + d, s_, l = np.array(dst)[ka], np.array(src)[ka], np.array(ln)[ka] + return d, s_, l, [(dst[g], ln[g]) for g in gone] + + +def coalesce(dst, src, ln, gap=GAP): + """Merge runs with the same motion (src - dst) whose target gap is <= gap: one copy covers the + gap, and the patch overwrites the gap blocks afterwards (skip + residual).""" + if gap <= 0 or len(dst) == 0: + return dst, src, ln + o = np.argsort(dst, kind="stable") + dst, src, ln = dst[o], src[o], ln[o] + out = [[dst[0], src[0], ln[0]]] + for d, s_, l in zip(dst[1:].tolist(), src[1:].tolist(), ln[1:].tolist()): + pd, ps, pl = out[-1] + if s_ - d == ps - pd and 0 <= d - (pd + pl) <= gap: + out[-1][2] = d + l - pd + else: + out.append([d, s_, l]) + a = np.array(out, dtype=np.int64) + return a[:, 0], a[:, 1], a[:, 2] + + +def fold(dst, src, ln): + """Greedy: consecutive runs (by target offset) with equal length and constant strides, inside + one piece on each side, form a group (d0, s0, len, count, dd, ds).""" + order = np.argsort(dst, kind="stable") + dst, src, ln = dst[order], src[order], ln[order] + groups = [] + i, n = 0, len(dst) + while i < n: + j = i + 1 + if j < n and ln[j] == ln[i] and dst[j] // PIECE == dst[i] // PIECE and src[j] // PIECE == src[i] // PIECE: + dd, ds = dst[j] - dst[i], src[j] - src[i] + if dd >= ln[i] and ds > 0: + while (j + 1 < n and ln[j + 1] == ln[i] and dst[j + 1] - dst[j] == dd and src[j + 1] - src[j] == ds + and dst[j + 1] // PIECE == dst[i] // PIECE and src[j + 1] // PIECE == src[i] // PIECE + and (j + 1 - i + 1) * ln[i] <= MAXCOPY): + j += 1 + j += 1 + else: + dd = ds = 0 + j = i + 1 + else: + dd = ds = 0 + groups.append((int(dst[i]), int(src[i]), int(ln[i]), j - i, int(dd), int(ds))) + i = j + return groups + + +def load(runs_path): + r = np.load(runs_path) + dst, src, ln = r["dst"].astype(np.int64), r["src"].astype(np.int64), r["len"].astype(np.int64) + odd = (dst % 2 != 0) | (src % 2 != 0) + return r, dst[~odd], src[~odd], ln[~odd], dst[odd], ln[odd] + + +def analyze(runs_path, out): + r, dst, src, ln, odd_d, odd_l = load(runs_path) + dst, src, ln, gone = absorb(dst, src, ln) + dst, src, ln = coalesce(dst, src, ln) + d2, s2, l2 = split_runs(dst, src, ln) + g = fold(d2, s2, l2) + cnt = np.array([x[3] for x in g]) + res = dict(gap=GAP, absorb=ABSORB, absorbed_runs=len(gone), absorbed_bytes=int(sum(x[1] for x in gone)), + coalesced_runs=int(len(dst)), runs=int(len(r["dst"])), odd_runs=int(len(odd_d)), odd_bytes=int(odd_l.sum()), + split_runs=int(len(d2)), groups=len(g), single_run_groups=int((cnt == 1).sum()), + bytes_in_multi_run_groups=int(sum(x[2] * x[3] for x in g if x[3] > 1)), + match_bytes=int(l2.sum()), other_blocks=int(len(r["other"])), v_bytes=int(r["v_bytes"])) + with open(out, "w") as f: + json.dump(res, f, indent=1) + print(json.dumps(res)) + return r, g, odd_d, odd_l, gone + + +def replay(runs_path, out, platform): + r, g, odd_d, odd_l, gone = analyze(runs_path, out + ".fold.json") + import torch + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from npu_d2d_bench import device + dev, sync, pk = device(platform) + ref_path, tgt_path = str(r["refs"][0]), str(r["target"]) + vb = int(r["v_bytes"]) + res = dict(schema="npu-replay.v1", platform=platform, device=dev, ref=ref_path, target=tgt_path, v_bytes=vb, + groups=len(g), load=(os.getloadavg() if hasattr(os, "getloadavg") else None)) + + def pieces_of(nbytes): + return [min(PIECE, nbytes - o) for o in range(0, nbytes, PIECE)] + + def host_bytes(path, nbytes): + a = np.zeros(nbytes + (nbytes % 2), dtype=np.uint8) + with open(path, "rb") as f: + f.readinto(memoryview(a)[:os.path.getsize(path)]) + return a + + ref = host_bytes(ref_path, os.path.getsize(ref_path)) + tgt = host_bytes(tgt_path, vb) + # resident reference (untimed upload) + t = time.perf_counter() + S = [torch.from_numpy(ref[o:o + n].view(np.int16).copy()).to(dev) for o, n in zip(range(0, len(ref), PIECE), pieces_of(len(ref)))] + sync() + res["ref_upload_s"] = time.perf_counter() - t + al = (vb // BLK) * BLK # the device holds the block-aligned part; the tail (< 64 B) stays on the host + D = [torch.empty(n // 2, dtype=torch.int16, device=dev) for n in pieces_of(al)] + # baseline: upload the whole target + host_t = [torch.from_numpy(tgt[o:o + n].view(np.int16).copy()) for o, n in zip(range(0, al, PIECE), pieces_of(al))] + sync() + t = time.perf_counter() + for d, h in zip(D, host_t): + d.copy_(h) + sync() + res["baseline_upload_s"] = time.perf_counter() - t + # patch: every block not covered by an even match run + cover = np.zeros(vb // BLK + 1, dtype=bool) + for d0, s0, l, c, dd, ds in g: + for k in range(c): + a = (d0 + k * dd) // BLK + cover[a:a + l // BLK] = True + need = ~cover[: vb // BLK] + need[r["other"][r["other"] < vb // BLK]] = True + for d0, l in gone: # absorbed runs: the segment's copy wrote over them + need[d0 // BLK:(d0 + l) // BLK] = True + patch_blocks = np.nonzero(need)[0] + pb = tgt[: (vb // BLK) * BLK].reshape(-1, BLK)[patch_blocks].copy() + by_piece = {} + for i, b in enumerate(patch_blocks.tolist()): + by_piece.setdefault((b * BLK) // PIECE, []).append(i) + res["patch_blocks"] = int(len(patch_blocks)) + res["patch_bytes"] = int(len(patch_blocks) * BLK) + tail = tgt[(vb // BLK) * BLK:vb] + # clear the target so the replay cannot pass by accident + for d in D: + d.zero_() + sync() + path = {"strided": 0, "single": 0, "fallback": 0} + t0 = time.perf_counter() + for d0, s0, l, c, dd, ds in g: + dp, do = divmod(d0, PIECE) + sp, so = divmod(s0, PIECE) + le = l // 2 + if c == 1: + D[dp][do // 2:do // 2 + le].copy_(S[sp][so // 2:so // 2 + le]) + path["single"] += 1 + continue + try: + D[dp].as_strided((c, le), (dd // 2, 1), do // 2).copy_(S[sp].as_strided((c, le), (ds // 2, 1), so // 2)) + path["strided"] += 1 + except Exception: + for k in range(c): + a, b = (do + k * dd) // 2, (so + k * ds) // 2 + D[dp][a:a + le].copy_(S[sp][b:b + le]) + path["fallback"] += 1 + sync() + t1 = time.perf_counter() + for p, rows in by_piece.items(): + sel = np.array(rows) + local = (patch_blocks[sel] * BLK - p * PIECE) // BLK + buf = torch.from_numpy(pb[sel].view(np.int16)).to(dev) + idx = torch.from_numpy(local.astype(np.int64)).to(dev) + D[p].view(-1, BLK // 2).index_copy_(0, idx, buf) + sync() + t2 = time.perf_counter() + res.update(copy_s=t1 - t0, patch_s=t2 - t1, rebuild_s=t2 - t0, paths=path) + # verify + h = hashlib.sha256() + got = np.concatenate([d.cpu().numpy().view(np.uint8) for d in D])[: (vb // BLK) * BLK] + h.update(got.tobytes()) + h.update(tail.tobytes()) + want = hashlib.sha256(tgt[:vb].tobytes()).hexdigest() + res.update(sha256=h.hexdigest(), target_sha256=want, correct=h.hexdigest() == want) + res["mismatch_bytes"] = int((got != tgt[: (vb // BLK) * BLK]).sum()) if not res["correct"] else 0 + with open(out, "w") as f: + json.dump(res, f, indent=1) + print(json.dumps({k: v for k, v in res.items() if k != "load"})) + + +WIN = int(os.environ.get("REPLAY_WIN", str(64 << 20))) # bytes of target written per window +RING = int(os.environ.get("REPLAY_RING", str(256 << 20))) # bytes of original content kept behind the window + + +def replay_inplace(runs_path, out, platform): + """Rebuild the target in the memory that holds the sibling, with O(RING) scratch. + + One buffer holds the sibling's bytes and is overwritten with the target window by window, in + target order. Before a window is written, its original bytes are saved in a ring of + RING/WIN + 1 slots. A copy reads a source byte from: + - the live buffer if it lies after the current window (not yet overwritten); + - the ring if it lies in the RING bytes before the window, or in the window itself; + - a far scratch, filled before the first write, for the few sources further back. + Invariant: no original byte is overwritten before it is saved, and every copy that reads the + live buffer reads bytes after the current window. The patch is applied last.""" + import torch + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from npu_d2d_bench import device + r, dst, src, ln, odd_d, odd_l = load(runs_path) + lim = ABSORB if "REPLAY_ABSORB" in os.environ else absorb_threshold(platform) + dst, src, ln, gone = absorb(dst, src, ln, limit=lim) + dst, src, ln = coalesce(dst, src, ln) + dev, sync, pk = device(platform) + ref_path, tgt_path = str(r["refs"][0]), str(r["target"]) + vb = int(r["v_bytes"]) + al = (vb // BLK) * BLK + ref = np.fromfile(ref_path, dtype=np.uint8) + tgt = np.fromfile(tgt_path, dtype=np.uint8) + total = max(len(ref), al) + total += (-total) % WIN + nslots = RING // WIN + 1 + pieces = [] + for d, s_, l in zip(dst.tolist(), src.tolist(), ln.tolist()): + while l > 0: + take = min(l, WIN - d % WIN) + pieces.append((d, s_, take)) + d, s_, l = d + take, s_ + take, l - take + pieces.sort() + kept = [x for x in pieces if x[0] == x[1]] # KEEP: already in place, no copy + pieces_cp = [x for x in pieces if x[0] != x[1]] + segs = [] + far = [] + far_bytes = 0 + need_ring = set() # windows whose original bytes a later copy reads: only these are saved + for d, s_, l in pieces_cp: + w = d // WIN + a = w * WIN + ring_lo = max(a - (nslots - 1) * WIN, 0) + x, dd = s_, d + while x < s_ + l: + if x >= a + WIN: + take = min(s_ + l - x, PIECE - x % PIECE) + segs.append((w, dd, "live", x // PIECE, x % PIECE, take)) + elif x >= ring_lo: + take = min(s_ + l - x, WIN - x % WIN, a + WIN - x) + segs.append((w, dd, "ring", (x // WIN) % nslots, x % WIN, take)) + need_ring.add(x // WIN) + else: + take = min(s_ + l - x, ring_lo - x, PIECE - (far_bytes % PIECE)) + segs.append((w, dd, "far", far_bytes // PIECE, far_bytes % PIECE, take)) + far.append((x, take)) + far_bytes += take + x += take + dd += take + res = dict(schema="npu-replay-inplace.v1", platform=platform, device=dev, ref=ref_path, target=tgt_path, + v_bytes=vb, win=WIN, ring=RING, slots=nslots, copies=len(segs), far_bytes=far_bytes, + absorb_limit=lim, absorbed_bytes=int(sum(x[1] for x in gone)), + keep_bytes=int(sum(x[2] for x in kept)), keep_runs=len(kept), + far_copies=len(far), scratch_bytes=nslots * WIN + far_bytes, buffer_bytes=total, + load=(os.getloadavg() if hasattr(os, "getloadavg") else None)) + host = np.zeros(total, dtype=np.uint8) + host[:len(ref)] = ref + Bf = [torch.from_numpy(host[o:o + PIECE].view(np.int16).copy()).to(dev) for o in range(0, total, PIECE)] + ringt = [torch.empty(WIN // 2, dtype=torch.int16, device=dev) for _ in range(nslots)] + fart = [torch.empty(min(PIECE, far_bytes - o) // 2, dtype=torch.int16, device=dev) for o in range(0, far_bytes, PIECE)] + sync() + del host + # NPU-B: batched copies through the public runtime API (rbln_memcpy_v2v_multi / h2v_multi); + # copies inside one call are unordered, so a call never mixes a window's ring save with its + # copies, and no copy in a call writes where another reads + shim = None + if platform == "rbln" and os.environ.get("RBLN_VMSHIM") and os.environ.get("REPLAY_BATCH", "1") != "0": + import ctypes + shim = ctypes.CDLL(os.environ["RBLN_VMSHIM"]) + for t in ringt + fart: + shim.vmshim_layout_like(ctypes.c_uint64(t.data_ptr()), ctypes.c_uint64(Bf[0].data_ptr())) + if platform == "cpu" and os.environ.get("REPLAY_FAKE_BATCH"): # test the batched addressing on the host + import ctypes + + class _Fake: + @staticmethod + def _run(a_, b_, c_, n): + for i in range(n): + ctypes.memmove(b_[i], a_[i], c_[i]) + return 0 + vmshim_v2v_multi = _run + vmshim_h2v_multi = _run + shim = _Fake() + MAXB = 1024 + batch_calls = [0] + + def flush(ops, h2v=False): + import ctypes + for i in range(0, len(ops), MAXB): + part = ops[i:i + MAXB] + U = ctypes.c_uint64 * len(part) + a_, b_, c_ = U(*[x[0] for x in part]), U(*[x[1] for x in part]), U(*[x[2] for x in part]) + rc = (shim.vmshim_h2v_multi if h2v else shim.vmshim_v2v_multi)(a_, b_, c_, len(part)) + if rc != 0: + raise RuntimeError("batched copy rc=%d" % rc) + batch_calls[0] += 1 + need = np.ones(al // BLK, dtype=bool) + for d, s_, l in pieces: + need[d // BLK:(d + l) // BLK] = False + need[r["other"][r["other"] < al // BLK]] = True + for d0, l in gone: + need[d0 // BLK:(d0 + l) // BLK] = True + # runs at odd offsets cannot be addressed in 16-bit device elements: their blocks are uploaded + for d0, l in zip(odd_d.tolist(), odd_l.tolist()): + need[d0 // BLK:(d0 + l + BLK - 1) // BLK] = True + res["odd_runs"] = int(len(odd_d)) + res["odd_bytes"] = int(odd_l.sum()) + patch_blocks = np.nonzero(need)[0] + pb = tgt[:al].reshape(-1, BLK)[patch_blocks].copy() + res.update(patch_blocks=int(len(patch_blocks)), patch_bytes=int(len(patch_blocks) * BLK)) + nwin = (max(x[0] for x in pieces) // WIN + 1) if pieces else 0 + n_copies = len(segs) + len(far) + len(need_ring) + copy_bytes = sum(sg[5] for sg in segs) + sum(l for _, l in far) + len(need_ring) * WIN + res.update(windows=int(nwin), ring_saves=len(need_ring), relocate_bytes=int(sum(sg[5] for sg in segs))) + est_rebuild, est_upload = estimate(platform, n_copies, copy_bytes, res["patch_bytes"], al) + in_host = os.environ.get("REPLAY_TARGET_IN_HOST") == "1" + # the far scratch is known before the first write; beyond its budget the target is uploaded + far_budget = int(os.environ.get("REPLAY_FAR_BUDGET", str(1 << 30))) + res.update(est_rebuild_s=est_rebuild, est_upload_s=est_upload, target_in_host=in_host, far_budget=far_budget, + choice=("upload" if (in_host and est_upload < est_rebuild) or far_bytes > far_budget else "rebuild")) + + def E(b): + return b // 2 + + def view(origin, key, off, l): + t = {"live": Bf, "ring": ringt, "far": fart}[origin][key] + return t[E(off):E(off) + E(l)] + by_w = {} + for sg in segs: + by_w.setdefault(sg[0], []).append(sg) + # fold, per window: copies from one source tensor with equal length and constant target and + # source strides become one strided copy (as_strided on both sides) + plan = {} + nfold = 0 + for w, lst in by_w.items(): + lst.sort(key=lambda x: x[1]) + out_w, i = [], 0 + while i < len(lst): + _, d0, o0, k0, f0, l0 = lst[i] + j = i + 1 + if j < len(lst) and lst[j][2] == o0 and lst[j][3] == k0 and lst[j][5] == l0: + dd, ds = lst[j][1] - d0, lst[j][4] - f0 + if dd >= l0 and ds > 0 and d0 // PIECE == lst[j][1] // PIECE: + while (j + 1 < len(lst) and lst[j + 1][2] == o0 and lst[j + 1][3] == k0 and lst[j + 1][5] == l0 + and lst[j + 1][1] - lst[j][1] == dd and lst[j + 1][4] - lst[j][4] == ds + and lst[j + 1][1] // PIECE == d0 // PIECE): + j += 1 + j += 1 + else: + j = i + 1 + out_w.append((d0, o0, k0, f0, l0, j - i, (lst[i + 1][1] - d0) if j - i > 1 else 0, + (lst[i + 1][4] - f0) if j - i > 1 else 0)) + nfold += 1 + i = j + plan[w] = out_w + res["calls"] = nfold + path = {"strided": 0, "single": 0, "fallback": 0} + t0 = time.perf_counter() + fo = 0 + if shim is not None: + ops = [] + for x, l in far: + ops.append((Bf[x // PIECE].data_ptr() + x % PIECE, fart[fo // PIECE].data_ptr() + fo % PIECE, l)) + fo += l + flush(ops) + last_w = max(plan) if plan else -1 + for w in range(0, last_w + 1): + a = w * WIN + if w in need_ring: + flush([(Bf[a // PIECE].data_ptr() + a % PIECE, ringt[w % nslots].data_ptr(), WIN)]) + ops = [] + for d, origin, key, off, l, c, dd, ds in plan.get(w, []): + st = {"live": Bf, "ring": ringt, "far": fart}[origin][key] + for k in range(c): + ops.append((st.data_ptr() + off + k * ds, Bf[d // PIECE].data_ptr() + d % PIECE + k * dd, l)) + flush(ops) + path["batched"] = batch_calls[0] + plan = {} + for x, l in far if shim is None else []: + fart[fo // PIECE][E(fo % PIECE):E(fo % PIECE) + E(l)].copy_(Bf[x // PIECE][E(x % PIECE):E(x % PIECE) + E(l)]) + fo += l + last_w = max(plan) if plan else -1 + for w in range(0, last_w + 1): + a = w * WIN + if w in need_ring: + ringt[w % nslots].copy_(Bf[a // PIECE][E(a % PIECE):E(a % PIECE) + E(WIN)]) + for d, origin, key, off, l, c, dd, ds in plan.get(w, []): + dt = Bf[d // PIECE] + st = {"live": Bf, "ring": ringt, "far": fart}[origin][key] + if c == 1: + dt[E(d % PIECE):E(d % PIECE) + E(l)].copy_(st[E(off):E(off) + E(l)]) + path["single"] += 1 + continue + try: + dt.as_strided((c, E(l)), (E(dd), 1), E(d % PIECE)).copy_(st.as_strided((c, E(l)), (E(ds), 1), E(off))) + path["strided"] += 1 + except Exception: + for k in range(c): + a2, b2 = E(d % PIECE + k * dd), E(off + k * ds) + dt[a2:a2 + E(l)].copy_(st[b2:b2 + E(l)]) + path["fallback"] += 1 + sync() + res["paths"] = path + t1 = time.perf_counter() + if shim is not None and len(patch_blocks): + brk = np.nonzero(np.diff(patch_blocks) != 1)[0] + 1 + starts = np.concatenate([[0], brk]) + ends = np.concatenate([brk, [len(patch_blocks)]]) + ops = [] + base = pb.ctypes.data + for a_, b_ in zip(starts.tolist(), ends.tolist()): + off = int(patch_blocks[a_]) * BLK + n_ = (b_ - a_) * BLK + while n_ > 0: # do not cross a device piece + take = min(n_, PIECE - off % PIECE) + ops.append((base + a_ * BLK, Bf[off // PIECE].data_ptr() + off % PIECE, take)) + a_ += take // BLK + off += take + n_ -= take + res["patch_ranges"] = len(ops) + flush(ops, h2v=True) + path["batched"] = batch_calls[0] + else: + by_piece = {} + for i, b in enumerate(patch_blocks.tolist()): + by_piece.setdefault((b * BLK) // PIECE, []).append(i) + for p_, rows in by_piece.items(): + sel = np.array(rows) + local = (patch_blocks[sel] * BLK - p_ * PIECE) // BLK + Bf[p_].view(-1, BLK // 2).index_copy_(0, torch.from_numpy(local.astype(np.int64)).to(dev), + torch.from_numpy(pb[sel].view(np.int16)).to(dev)) + sync() + t2 = time.perf_counter() + got = np.concatenate([b.cpu().numpy().view(np.uint8) for b in Bf])[:al] + want = tgt[:al] + ok = bool(np.array_equal(got, want)) + res.update(copy_s=t1 - t0, patch_s=t2 - t1, rebuild_s=t2 - t0, correct=ok, + mismatch_bytes=0 if ok else int((got != want).sum()), + sha256=hashlib.sha256(got.tobytes() + tgt[al:vb].tobytes()).hexdigest(), + target_sha256=hashlib.sha256(tgt[:vb].tobytes()).hexdigest()) + del got + if res["target_in_host"]: + # the other path: upload the whole target from host memory into the same buffer + hostp = [torch.from_numpy(tgt[o:min(o + PIECE, al)].view(np.int16).copy()) for o in range(0, al, PIECE)] + sync() + t3 = time.perf_counter() + for dst_t, src_t in zip(Bf, hostp): + dst_t[:src_t.numel()].copy_(src_t) + sync() + res["upload_s"] = time.perf_counter() - t3 + res["upload_correct"] = bool(np.array_equal(np.concatenate([b.cpu().numpy().view(np.uint8) for b in Bf])[:al], want)) + res["measured_choice"] = "upload" if res["upload_s"] < res["rebuild_s"] else "rebuild" + with open(out, "w") as f: + json.dump(res, f, indent=1) + print(json.dumps({k: v for k, v in res.items() if k != "load"})) + + +if __name__ == "__main__": + if sys.argv[1] == "analyze": + analyze(sys.argv[2], sys.argv[3]) + elif sys.argv[1] == "inplace": + replay_inplace(sys.argv[2], sys.argv[3], sys.argv[4]) + else: + replay(sys.argv[2], sys.argv[3], sys.argv[4]) diff --git a/tools/varmorph/host_restore.py b/tools/varmorph/host_restore.py new file mode 100644 index 0000000..8b78846 --- /dev/null +++ b/tools/varmorph/host_restore.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Restore a target file on the host from a sibling file and a copy list (runs.npz), timed. + +Modes: + new write the target into a new file: copy every run from the sibling, then the patch + blocks (blocks no run covers), then fsync. Same output contract as `xdelta3 -d`. + inplace turn a private copy of the sibling into the target in place, window by window in + target order, saving each window's original bytes in a ring first (as the device + rebuild does); runs whose move is zero are not touched; then the patch; then fsync. +The patch bytes are taken from the target file at the blocks no run covers (they stand for the +residual and new content that the descriptor carries). The result is compared with the target's +SHA-256. The copy of the sibling for `inplace` is made before the clock and is not timed. +argv: MODE RUNS.npz WORKDIR OUT.json +""" +import hashlib +import json +import os +import shutil +import sys +import time + +import numpy as np + +BLK = 64 +WIN = 64 << 20 +RING = 256 << 20 + + +def sha(path): + h = hashlib.sha256() + with open(path, "rb") as f: + while True: + b = f.read(64 << 20) + if not b: + break + h.update(b) + return h.hexdigest() + + +def main(): + mode, runs_path, work, out = sys.argv[1:5] + r = np.load(runs_path) + dst, src, ln = r["dst"].astype(np.int64), r["src"].astype(np.int64), r["len"].astype(np.int64) + ref, tgt = str(r["refs"][0]), str(r["target"]) + vb = int(r["v_bytes"]) + al = vb // BLK * BLK + T = np.memmap(tgt, dtype=np.uint8, mode="r") + need = np.ones(al // BLK, dtype=bool) + for d, l in zip(dst.tolist(), ln.tolist()): + need[d // BLK:(d + l) // BLK] = False + other = r["other"] + need[other[other < al // BLK]] = True + patch_blocks = np.nonzero(need)[0] + brk = np.nonzero(np.diff(patch_blocks) != 1)[0] + 1 + pr = list(zip(np.concatenate([[0], brk]).tolist(), np.concatenate([brk, [len(patch_blocks)]]).tolist())) + if "patch_data" in r.files: + # bytes carried by the descriptor (vm_apply.py expand): the target file is not read + pd, oth = r["patch_data"], r["other"] + pos = {int(b): i for i, b in enumerate(oth.tolist())} + assert all(int(b) in pos for b in patch_blocks.tolist()), "descriptor does not cover every block" + patch = [] + for a, b in pr: + i0 = pos[int(patch_blocks[a])] + patch.append((int(patch_blocks[a]) * BLK, pd[i0 * BLK:(i0 + (b - a)) * BLK])) + tail = r["tail"] + else: + patch = [(int(patch_blocks[a]) * BLK, np.array(T[int(patch_blocks[a]) * BLK:int(patch_blocks[b - 1]) * BLK + BLK])) + for a, b in pr] + tail = np.array(T[al:vb]) + os.makedirs(work, exist_ok=True) + path = os.path.join(work, os.path.basename(tgt) + "." + mode) + res = dict(mode=mode, runs=runs_path, ref=ref, target=tgt, v_bytes=vb, n_runs=int(len(dst)), + patch_bytes=int(len(patch_blocks) * BLK), patch_ranges=len(pr), + load=os.getloadavg()) + if mode == "inplace": + shutil.copyfile(ref, path) + os.truncate(path, max(vb, os.path.getsize(ref))) + with open(path, "rb") as f: + os.fsync(f.fileno()) + order = np.argsort(dst, kind="stable") + dst, src, ln = dst[order], src[order], ln[order] + t0 = time.perf_counter() + written = 0 + if mode == "new": + R = np.memmap(ref, dtype=np.uint8, mode="r") + O = np.memmap(path, dtype=np.uint8, mode="w+", shape=(vb,)) + for d, s_, l in zip(dst.tolist(), src.tolist(), ln.tolist()): + O[d:d + l] = R[s_:s_ + l] + for off, b in patch: + O[off:off + len(b)] = b + O[al:vb] = tail + written = vb + else: + O = np.memmap(path, dtype=np.uint8, mode="r+") + nslots = RING // WIN + 1 + ring = [None] * nslots + # far sources (before the ring) are saved before the first write + pieces = [] + for d, s_, l in zip(dst.tolist(), src.tolist(), ln.tolist()): + while l > 0: + take = min(l, WIN - d % WIN) + pieces.append((d, s_, take)) + d, s_, l = d + take, s_ + take, l - take + far = {} + for d, s_, l in pieces: + w = d // WIN + lo = max(w * WIN - (nslots - 1) * WIN, 0) + if s_ < lo and d != s_: + far[(s_, l)] = np.array(O[s_:s_ + l]) + by_w = {} + for p in pieces: + by_w.setdefault(p[0] // WIN, []).append(p) + last = (al - 1) // WIN + for w in range(last + 1): + a = w * WIN + lst = [p for p in by_w.get(w, []) if p[0] != p[1]] + ring[w % nslots] = (a, np.array(O[a:a + WIN])) if lst or True else None + for d, s_, l in lst: + if s_ >= a + WIN: + O[d:d + l] = O[s_:s_ + l] + elif (s_, l) in far: + O[d:d + l] = far[(s_, l)] + else: + x, dd, rem = s_, d, l + while rem > 0: + if x >= a + WIN: # not yet overwritten + O[dd:dd + rem] = O[x:x + rem] + break + sw = x // WIN + base, buf = ring[sw % nslots] + take = min(rem, base + WIN - x) + O[dd:dd + take] = buf[x - base:x - base + take] + x, dd, rem = x + take, dd + take, rem - take + written += l + for off, b in patch: + O[off:off + len(b)] = b + written += len(b) + O[al:vb] = tail + O.flush() + del O + with open(path, "rb+") as f: + if mode == "inplace": + f.truncate(vb) + os.fsync(f.fileno()) + t1 = time.perf_counter() + res.update(restore_s=t1 - t0, bytes_written=int(written)) + res["sha256"] = sha(path) + res["target_sha256"] = sha(tgt) + res["correct"] = res["sha256"] == res["target_sha256"] + os.remove(path) + with open(out, "w") as f: + json.dump(res, f, indent=1) + print(json.dumps({k: v for k, v in res.items() if k != "load"})) + + +if __name__ == "__main__": + main() diff --git a/tools/varmorph/layout_coder.py b/tools/varmorph/layout_coder.py new file mode 100644 index 0000000..2a7ce91 --- /dev/null +++ b/tools/varmorph/layout_coder.py @@ -0,0 +1,559 @@ +#!/usr/bin/env python3 +"""Cohort 2026-11-12 (10): a video-coding-style lossless coder for a compiled variant against siblings. + +A target variant v is coded against reference variants (the siblings a node may hold), the way a +P-frame is coded against reference frames. Tools taken from video coding, and why: + +Matching +- block matching: v is split into 64-byte blocks, and each block is searched in the references by + exact hash, verified byte by byte. Exact matching replaces a SAD search, since the result must be + lossless. +- sub-block motion (the "sub-pel" step): content moves by offsets that are not multiples of 64 + bytes (a first, aligned-only run matched 211 KB of a prefill pair that content-defined chunks + share in GB). A rolling hash over every byte position of the references finds a sample of target + blocks. The phases (offset mod 64) of those shifts are then indexed, so the main search stays one + lookup per block. +- MV prediction: the block at the predicted position (the previous match plus the distance since) + is compared first. A hit is a "skip": no lookup and no new vector. +- variable blocks: consecutive blocks whose references are consecutive merge into one run. + +Coding +- MVD + entropy: each run is coded as its length and the difference of its motion vector from the + previous one (zigzag varints), then entropy-coded (zstd). +- inter residual: a block that matches nothing exactly but differs from its predicted block in at + most T bytes is coded as the XOR with that block. +- intra: any other block is raw content. Its bytes are coded with zstd, alone and after splitting + 2-byte words into byte planes (bf16). +- fill (DC): a block of one repeated 8-byte word is coded as that word. +- multi-reference: several references share one index; a vector names its reference through the + global offset. + +Not taken: transform and quantization (lossy) and loop filters. + +Every coded target is decoded and compared with v byte for byte before its size is reported. The +decode is timed: a single-threaded rebuild into host memory from references already in memory. +argv: OUT.json T REF[,REF...] TARGET [REF[,REF...] TARGET ...] +""" +import json +import os +import shutil +import subprocess +import sys +import time + +import numpy as np + +try: + from numba import njit +except ImportError: # slow fallback for small tests + def njit(*a, **k): + if a and callable(a[0]): + return a[0] + return lambda f: f + +B = 64 +W = B // 8 +K = (np.random.default_rng(12).integers(1, 2**63, size=W, dtype=np.uint64) | np.uint64(1)) +OP_MATCH, OP_FILL, OP_RES, OP_INTRA = 0, 1, 2, 3 +BASE = np.uint64(1099511628211) +SAMPLES = 1 << 16 +NMERGE = int(os.environ.get("VCODEC_NMERGE", "4")) # merge candidates (0: off) +NOMV = -(1 << 62) +BITS = 26 + + +def log(*a): + print(time.strftime("%H:%M:%S"), *a, flush=True) + + +def block_hashes(buf): + n = len(buf) // B + h = np.empty(n, dtype=np.uint64) + step = (1 << 28) // B + for i in range(0, n, step): + j = min(n, i + step) + w = np.frombuffer(buf[i * B:j * B].tobytes(), dtype=np.uint64).reshape(-1, W) if buf.ctypes.data % 8 else \ + buf[i * B:j * B].view(np.uint64).reshape(-1, W) + h[i:j] = ((w ^ (w >> np.uint64(29))) * K).sum(axis=1, dtype=np.uint64) + return h + + +@njit(cache=True) +def is_fill(V, o): + for k in range(8, B): + if V[o + k] != V[o + (k & 7)]: + return False + return True + + +@njit(cache=True) +def rk(V, o): + h = np.uint64(0) + for k in range(B): + h = h * BASE + np.uint64(V[o + k]) + return h + + +@njit(cache=True) +def find_shifts(R, V, sample_off, cap): + """Rolling hash over every byte of R; returns (ref_off, tgt_off) of verified sample hits.""" + m = len(sample_off) + sh = np.empty(m, dtype=np.uint64) + for s in range(m): + sh[s] = rk(V, sample_off[s]) + order = np.argsort(sh) + ss = sh[order] + so = sample_off[order] + bitmap = np.zeros(1 << (BITS - 3), dtype=np.uint8) + MIX = np.uint64(0x9E3779B97F4A7C15) + for s in range(m): + b = (ss[s] * MIX) >> np.uint64(64 - BITS) + bitmap[b >> np.uint64(3)] |= np.uint8(1) << np.uint8(b & np.uint64(7)) + pw = np.uint64(1) + for k in range(B): + pw = pw * BASE + out_r = np.empty(cap, dtype=np.int64) + out_t = np.empty(cap, dtype=np.int64) + nout = 0 + n = len(R) + if n < B: + return out_r[:0], out_t[:0] + h = rk(R, 0) + pos = 0 + while True: + b = (h * MIX) >> np.uint64(64 - BITS) + if bitmap[b >> np.uint64(3)] & (np.uint8(1) << np.uint8(b & np.uint64(7))): + lo, hi = 0, m + while lo < hi: + mid = (lo + hi) >> 1 + if ss[mid] < h: + lo = mid + 1 + else: + hi = mid + c = lo + while c < m and ss[c] == h and nout < cap: + t = so[c] + eq = True + for k in range(B): + if R[pos + k] != V[t + k]: + eq = False + break + if eq: + out_r[nout] = pos + out_t[nout] = t + nout += 1 + c += 1 + if pos + B >= n: + break + h = h * BASE + np.uint64(R[pos + B]) - np.uint64(R[pos]) * pw + pos += 1 + return out_r[:nout], out_t[:nout] + + +@njit(cache=True) +def scan(R, V, sr, spos, hv, T): + """Choose a mode per target block. ref holds a byte offset in R (or -1).""" + n = len(hv) + nr = len(sr) + nR = len(R) + mode = np.empty(n, dtype=np.uint8) + ref = np.full(n, -1, dtype=np.int64) + last_i = -1 + last_r = -1 + lookups = 0 + skips = 0 + merges = 0 + mc = np.full(NMERGE, NOMV, dtype=np.int64) + for i in range(n): + o = i * B + if is_fill(V, o): + mode[i] = OP_FILL + continue + p = -1 + if last_r >= 0: + p = last_r + (i - last_i) * B + if p + B > nR: + p = -1 + nz = B + 1 + if p >= 0: + nz = 0 + for k in range(B): + if R[p + k] != V[o + k]: + nz += 1 + if nz == 0: + mode[i] = OP_MATCH + ref[i] = p + last_i = i + last_r = p + skips += 1 + continue + # merge candidates (as in HEVC merge mode): the motions of the last few runs, tried + # before any hash lookup so that a block found elsewhere does not break the motion field + mfound = -1 + for c_ in range(NMERGE): + if mc[c_] == NOMV: + continue + q = o + mc[c_] + if q < 0 or q + B > nR or q == p: + continue + eq = True + for k in range(B): + if R[q + k] != V[o + k]: + eq = False + break + if eq: + mfound = q + break + if mfound >= 0: + mode[i] = OP_MATCH + ref[i] = mfound + last_i = i + last_r = mfound + merges += 1 + continue + lookups += 1 + lo, hi = 0, nr + x = hv[i] + while lo < hi: + mid = (lo + hi) >> 1 + if sr[mid] < x: + lo = mid + 1 + else: + hi = mid + found = -1 + c = lo + while c < nr and c < lo + 8 and sr[c] == x: + q = spos[c] + eq = True + for k in range(B): + if R[q + k] != V[o + k]: + eq = False + break + if eq: + found = q + break + c += 1 + if found >= 0: + mode[i] = OP_MATCH + ref[i] = found + last_i = i + last_r = found + d_ = found - o + if NMERGE > 0 and mc[0] != d_: + for c_ in range(NMERGE - 1, 0, -1): + mc[c_] = mc[c_ - 1] + mc[0] = d_ + continue + if p >= 0 and nz <= T: + mode[i] = OP_RES + ref[i] = p + last_i = i + last_r = p + continue + mode[i] = OP_INTRA + return mode, ref, lookups, skips, merges + + +@njit(cache=True) +def put_varint(buf, pos, x): + while x >= 0x80: + buf[pos] = (x & 0x7F) | 0x80 + x >>= 7 + pos += 1 + buf[pos] = x + return pos + 1 + + +@njit(cache=True) +def build(R, V, mode, ref): + """Merge blocks into runs; write the op stream, the residual stream and the intra stream.""" + n = len(mode) + ops = np.empty(n * 12 + 64, dtype=np.uint8) + res = np.empty(np.sum(mode == OP_RES) * B + 1, dtype=np.uint8) + intra = np.empty(np.sum(mode == OP_INTRA) * B + 1, dtype=np.uint8) + op = 0 + rp = 0 + ip = 0 + prev_d = 0 + runs = np.zeros(4, dtype=np.int64) + i = 0 + while i < n: + m = mode[i] + j = i + 1 + if m == OP_MATCH or m == OP_RES: + while j < n and mode[j] == m and ref[j] == ref[j - 1] + B: + j += 1 + elif m == OP_FILL: + while j < n and mode[j] == OP_FILL: + eq = True + for k in range(8): + if V[j * B + k] != V[i * B + k]: + eq = False + break + if not eq: + break + j += 1 + else: + while j < n and mode[j] == OP_INTRA: + j += 1 + L = j - i + runs[m] += 1 + ops[op] = m + op += 1 + op = put_varint(ops, op, L) + if m == OP_MATCH or m == OP_RES: + d = ref[i] - i * B + dd = d - prev_d + op = put_varint(ops, op, (dd << 1) ^ (dd >> 63)) + prev_d = d + if m == OP_RES: + for b in range(i, j): + for k in range(B): + res[rp] = R[ref[b] + k] ^ V[b * B + k] + rp += 1 + elif m == OP_FILL: + for k in range(8): + ops[op] = V[i * B + k] + op += 1 + else: + intra[ip:ip + L * B] = V[i * B:j * B] + ip += L * B + i = j + return ops[:op], res[:rp], intra[:ip], runs + + +@njit(cache=True) +def decode(R, ops, res, intra, n, tail): + out = np.empty(n * B + len(tail), dtype=np.uint8) + op = 0 + rp = 0 + ip = 0 + prev_d = 0 + i = 0 + while op < len(ops): + m = ops[op] + op += 1 + L = 0 + s = 0 + while True: + b = np.int64(ops[op]) + op += 1 + L |= (b & 0x7F) << s + s += 7 + if b < 0x80: + break + a = i * B + e = a + L * B + if m == OP_MATCH or m == OP_RES: + z = 0 + s = 0 + while True: + b = np.int64(ops[op]) + op += 1 + z |= (b & 0x7F) << s + s += 7 + if b < 0x80: + break + d = prev_d + ((z >> 1) ^ -(z & 1)) + prev_d = d + out[a:e] = R[a + d:e + d] + if m == OP_RES: + for k in range(L * B): + out[a + k] ^= res[rp + k] + rp += L * B + elif m == OP_FILL: + for k in range(a, e): + out[k] = ops[op + ((k - a) & 7)] + op += 8 + else: + out[a:e] = intra[ip:ip + L * B] + ip += L * B + i += L + out[n * B:] = tail + return out + + +def zsize(arr, level=19): + if len(arr) == 0: + return 0 + if shutil.which("zstd"): + lv = ["--ultra", "-22"] if level > 19 else ["-%d" % level] + p = subprocess.run(["zstd", "-q", "-c", "-T16", "--long=27"] + lv, input=arr.tobytes(), capture_output=True, check=True) + return len(p.stdout) + try: + import zstandard + return len(zstandard.ZstdCompressor(level=min(level, 19), threads=16).compress(arr.tobytes())) + except ImportError: + import lzma + return len(lzma.compress(arr.tobytes(), preset=6)) + + +def intra_size(arr): + """Best of zstd on the stream and zstd on its two byte planes (bf16 split).""" + plain = zsize(arr, 9) + if len(arr) < 2: + return plain, "zstd" + a = arr[: len(arr) // 2 * 2] + planes = zsize(np.ascontiguousarray(a[0::2]), 9) + zsize(np.ascontiguousarray(a[1::2]), 9) + (len(arr) - len(a)) + return (planes, "planes") if planes < plain else (plain, "zstd") + + +def load(paths): + sizes = [os.path.getsize(p) for p in paths] + pads = [(-s) % B for s in sizes] + R = np.empty(sum(sizes) + sum(pads), dtype=np.uint8) + o = 0 + for p, s, pd in zip(paths, sizes, pads): + with open(p, "rb") as f: + f.readinto(memoryview(R)[o:o + s]) + R[o + s:o + s + pd] = 0 + o += s + pd + return R + + +def run_stats(mode, ref, nontrivial_bytes): + idx = np.nonzero(mode == OP_MATCH)[0] + if len(idx) == 0: + return {} + r = ref[idx] + brk = np.ones(len(idx), dtype=bool) + brk[1:] = (idx[1:] != idx[:-1] + 1) | (r[1:] != r[:-1] + B) + st = np.nonzero(brk)[0] + run_bytes = np.diff(np.append(st, len(idx))) * B + d = r[st] - idx[st] * B + cov = {str(L): float(run_bytes[run_bytes >= L].sum() / nontrivial_bytes) + for L in [64, 256, 1024, 4096, 16384, 65536, 1 << 20]} + ud, inv = np.unique(d, return_inverse=True) + w = np.bincount(inv, weights=run_bytes) + o = np.argsort(-w) + cw = np.cumsum(w[o]) / run_bytes.sum() + jd = np.diff(d) + out = {"match_runs": int(len(st)), "median_run_bytes": float(np.median(run_bytes)), + "coverage_by_run_ge": cov, "distinct_deltas": int(len(ud)), + "deltas_for_90pct": int(np.searchsorted(cw, 0.9) + 1), + "top_deltas": [{"delta": int(ud[k]), "share": float(w[k] / run_bytes.sum())} for k in o[:10]]} + if len(jd): + uj, invj = np.unique(jd, return_inverse=True) + wj = np.bincount(invj, weights=run_bytes[1:]) + oj = np.argsort(-wj) + cj = np.cumsum(wj[oj]) / max(run_bytes[1:].sum(), 1) + out.update({"distinct_jumps": int(len(uj)), "jumps_for_90pct": int(np.searchsorted(cj, 0.9) + 1), + "top_jumps": [{"jump": int(uj[k]), "share": float(wj[k] / run_bytes[1:].sum())} for k in oj[:10]]}) + return out + + +def code_one(refs, target, T, modes_path=None): + t0 = time.time() + R = load(refs) + V = np.fromfile(target, dtype=np.uint8) + nb = len(V) // B + tail = V[nb * B:].copy() + Vb = V[: nb * B] + # sub-block motion: phases of the shifts found for a sample of target blocks + cand = np.linspace(0, nb - 1, SAMPLES * 2).astype(np.int64) + cand = np.array([c * B for c in cand if not is_fill(Vb, c * B)][:SAMPLES], dtype=np.int64) + hr_, ht_ = find_shifts(R, Vb, cand, 1 << 22) + shift = hr_ - ht_ + ph = np.bincount(((shift % B) + B) % B, minlength=B) + hit_blocks = len(np.unique(ht_)) + order_ph = np.argsort(-ph) + phases = [0] + acc = ph[0] + for p in order_ph: + if os.environ.get("VCODEC_ALIGNED") == "1": # ablation: match only at block-aligned offsets + break + if ph[p] == 0 or len(phases) >= 8 or acc >= 0.99 * max(ph.sum(), 1): + break + if p != 0: + phases.append(int(p)) + acc += ph[p] + us, cs = np.unique(shift, return_counts=True) + oc = np.argsort(-cs)[:10] + t1 = time.time() + log("phases", phases, "sample hits", int(hit_blocks), "of", len(cand)) + hs, ps = [], [] + for p in phases: + n = (len(R) - p) // B + hs.append(block_hashes(R[p:p + n * B])) + ps.append(p + np.arange(n, dtype=np.int64) * B) + hr = np.concatenate(hs) + pr = np.concatenate(ps) + del hs, ps + order = np.argsort(hr, kind="stable") + sr = hr[order] + spos = pr[order] + del hr, pr, order + hv = block_hashes(Vb) + t2 = time.time() + mode, ref, lookups, skips, merges = scan(R, Vb, sr, spos, hv, T) + ops, res, intra, runs = build(R, Vb, mode, ref) + t3 = time.time() + decode(R[:1 << 20], ops[:0], res, intra, 0, tail[:0]) # compile outside the timed call + t4 = time.time() + out = decode(R, ops, res, intra, nb, tail) + t5 = time.time() + ok = bool(np.array_equal(out, V)) + del out + log("decoded", ok, round(t5 - t4, 2), "s") + cnt = np.bincount(mode, minlength=4) * B + shipped = None + if modes_path and os.environ.get("VCODEC_SAVE"): + # the descriptor as one file a node can fetch: the three streams and the tail, zstd -19 + dpath = modes_path.replace(".modes.npy", ".vmd.npz") + np.savez(dpath, ops=ops, res=res, intra=intra, tail=tail, nb=np.int64(nb), v_bytes=np.int64(len(V)), + refs=np.array(refs), target=np.array(target)) + subprocess.run(["zstd", "-q", "-f", "-19", "-T16", "--long=27", dpath, "-o", dpath + ".zst"], check=True) + shipped = os.path.getsize(dpath + ".zst") + if modes_path and os.environ.get("VCODEC_DUMP"): + # the descriptor itself, for a device-side replay: match runs (target byte offset, + # reference byte offset, length) and the other blocks (target block index, mode) + idx = np.nonzero(mode == OP_MATCH)[0] + rr = ref[idx] + brk = np.ones(len(idx), dtype=bool) + brk[1:] = (idx[1:] != idx[:-1] + 1) | (rr[1:] != rr[:-1] + B) + st = np.nonzero(brk)[0] + ln = np.diff(np.append(st, len(idx))) * B + other = np.nonzero(mode != OP_MATCH)[0] + np.savez(modes_path.replace(".modes.npy", ".runs.npz"), dst=idx[st] * B, src=rr[st], len=ln, + other=other, other_mode=mode[other], v_bytes=len(V), refs=np.array(refs), target=np.array(target)) + if modes_path: # per 64 KiB window: blocks coded as match / fill / residual / intra + nw = nb // 1024 + wm = np.bincount((np.arange(nw * 1024) // 1024) * 4 + mode[:nw * 1024], minlength=nw * 4) + np.save(modes_path, wm.reshape(nw, 4).astype(np.uint32)) + s_ops = zsize(ops, 19) + s_res = zsize(res, 19) + s_intra, intra_how = intra_size(intra) + total = s_ops + s_res + s_intra + len(tail) + nontriv = int(len(V) - cnt[1]) + return { + "refs": refs, "target": target, "T": T, "v_bytes": int(len(V)), "lossless": ok, + "shipped_bytes": shipped, "aligned_only": os.environ.get("VCODEC_ALIGNED") == "1", + "sample": {"blocks": int(len(cand)), "hit_blocks": int(hit_blocks), "hits": int(len(shift)), + "phase_hist_top": {int(p): int(ph[p]) for p in order_ph[:8]}, "phases_indexed": phases, + "top_shifts": [{"shift": int(us[k]), "count": int(cs[k])} for k in oc]}, + "bytes_by_mode": {"match": int(cnt[0]), "fill": int(cnt[1]), "residual": int(cnt[2]), "intra": int(cnt[3])}, + "runs_by_mode": {"match": int(runs[0]), "fill": int(runs[1]), "residual": int(runs[2]), "intra": int(runs[3])}, + "lookups": int(lookups), "skips": int(skips), "merges": int(merges), "nmerge": NMERGE, + "match_structure": run_stats(mode, ref, max(nontriv, 1)), + "coded": {"ops_raw": int(len(ops)), "ops": s_ops, "residual_raw": int(len(res)), "residual": s_res, + "intra_raw": int(len(intra)), "intra": s_intra, "intra_codec": intra_how, "total": total}, + "seconds": {"shift_search": round(t1 - t0, 1), "index": round(t2 - t1, 1), "encode": round(t3 - t2, 1), + "decode_host": round(t5 - t4, 2)}, + "decode_GBps": round(len(V) / max(t5 - t4, 1e-9) / 1e9, 2), + } + + +if __name__ == "__main__": + out, T = sys.argv[1], int(sys.argv[2]) + a = sys.argv[3:] + res = [] + for k in range(0, len(a), 2): + refs = a[k].split(",") + log("code", refs, "->", a[k + 1]) + res.append(code_one(refs, a[k + 1], T, "%s.%d.modes.npy" % (out, k // 2))) + with open(out, "w") as f: + json.dump(res, f, indent=1) + r = res[-1] + log("result", json.dumps(r["coded"]), json.dumps(r["bytes_by_mode"]), json.dumps(r["sample"]["phases_indexed"]), + "decode", r["seconds"]["decode_host"]) + log("ALL-DONE") diff --git a/tools/varmorph/npu_d2d_bench.py b/tools/varmorph/npu_d2d_bench.py new file mode 100644 index 0000000..4c19591 --- /dev/null +++ b/tools/varmorph/npu_d2d_bench.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""NPU micro-benchmark (NPU-A or NPU-B): can a target's weights be rebuilt on the device from +resident weights? + +A video-style descriptor turns a variant into copies of runs from a sibling. This measures what +such a decode costs on the card, against uploading the weights from the host. Cases: + h2d_ contiguous host -> device copy (the upload a restart pays) + d2d_ contiguous device -> device copy of one tensor into another + d2d_calls 512 device copies of one 32 KiB run each, one call per run + d2d_strided one call copying every other 32 KiB run of a 64 MiB region (a regular re-tiling + as one descriptor) + d2d_gather one call gathering the 32 KiB runs of a 64 MiB region in a random order + (index_select) + use_256 a device reduction over a 256 MiB tensor already on the device + d2duse_256 a device copy into that tensor, then the same reduction (timed together): if the + copy left the data on the host side, the reduction pays the upload + h2duse_256 a host upload into that tensor, then the reduction + +Every case runs in its own process, so a device error in one case does not leave the card in a bad +state for the next. Each case syncs the card around the timed region and repeats REPS times after +a warm-up. Values are finite bf16 (from a normal distribution), and results are compared as int16 +bit patterns: a fast path that changes the bytes is reported as wrong. +argv: OUT.json PLATFORM(rngd|rbln) [REPS, default 5] (driver) + --case NAME PLATFORM REPS OUT_CASE.json (one case, internal) +""" +import json +import os +import subprocess +import sys +import time +import traceback + +RUN = 16384 # bf16 elements per 32 KiB run +CASES = ["h2d_64", "h2d_1024", "d2d_1", "d2d_64", "d2d_256", "d2d_1024", + "d2d_calls", "d2d_strided", "d2d_gather", + "use_256", "d2duse_256", "h2duse_256"] +if os.environ.get("RBLN_VMSHIM"): # NPU-B batched copies through the public runtime API + CASES += ["v2vmulti_1", "v2vmulti_64", "v2vmulti_1024", "v2vmultis_1024", "h2vmulti_1024"] + + +def device(platform): + import torch + if platform == "cpu": # local test of the drivers + return "cpu", (lambda: None), ("torch",) + if platform == "rngd": + from rngd_pd import devices + from rngd_pd.engines.fsdk import dev_str, open_device, sync + chip = devices.resolve_chip(-1) + dev = dev_str(chip) + open_device(8, 2) + return dev, (lambda: sync(chip)), ("torch", "furiosa-torch") + import rebel # noqa: F401 + import torch_rbln # noqa: F401 registers the rbln device + dev = "rbln:%d" % int(os.environ.get("RBLN_BENCH_DEVICE", "0")) + return dev, (lambda: torch.rbln.synchronize(dev)), ("torch", "rebel-compiler") + + +def run_case(name, platform, reps, out): + import torch + from importlib.metadata import version + torch.set_num_threads(4) + row = dict(case=name, s=[]) + try: + dev, sync, pk = device(platform) + row.update(device=dev, packages={k: version(k) for k in pk}) + g = torch.Generator().manual_seed(7) + bits = lambda t: t.view(torch.int16) + + def host(nelem): + return torch.randn(nelem, generator=g, dtype=torch.float32).to(torch.bfloat16) + + kind, _, arg = name.partition("_") + if kind in ("v2vmulti", "v2vmultis", "h2vmulti"): + import ctypes + lib = ctypes.CDLL(os.environ["RBLN_VMSHIM"]) + k = int(arg) + rows = 2048 # 64 MiB region of 32 KiB runs + n = rows * RUN + h = host(n) + src = h.to(dev) + dst = torch.empty(n, dtype=torch.bfloat16, device=dev) + dst.copy_(src) # give dst a device view + sync() + if kind != "h2vmulti": + row["layout_like_rc"] = lib.vmshim_layout_like(ctypes.c_uint64(dst.data_ptr()), ctypes.c_uint64(src.data_ptr())) + perm = torch.randperm(rows, generator=g)[:k].tolist() + U64 = ctypes.c_uint64 * k + sz = U64(*([RUN * 2] * k)) + dsta = U64(*[dst.data_ptr() + i * RUN * 2 * (2 if kind == "v2vmultis" else 1) for i in range(k)]) + if kind == "h2vmulti": + hb = h.view(rows, RUN) + srca = U64(*[hb[r].data_ptr() for r in perm]) + fn = lambda: lib.vmshim_h2v_multi(srca, dsta, sz, k) + else: + srca = U64(*[src.data_ptr() + r * RUN * 2 for r in perm]) + fn = lambda: lib.vmshim_v2v_multi(srca, dsta, sz, k) + nbytes = k * RUN * 2 + row["calls"] = 1 + step = 2 if kind == "v2vmultis" else 1 + check = lambda: all(torch.equal(bits(dst[(i * step) * RUN:(i * step + 1) * RUN].cpu()), bits(h[perm[i] * RUN:(perm[i] + 1) * RUN])) + for i in sorted({0, k // 2, k - 1})) + rc = fn() + row["rc"] = rc + elif kind in ("use", "d2duse", "h2duse"): + n = int(arg) * (1 << 20) // 2 + h = host(n) + src = h.to(dev) + dst = torch.empty(n, dtype=torch.bfloat16, device=dev) + dst.copy_(src) + sync() + red = lambda: dst.sum() + if kind == "use": + fn = lambda: red() + elif kind == "d2duse": + fn = lambda: (dst.copy_(src), red()) + else: + fn = lambda: (dst.copy_(h), red()) + nbytes = n * 2 + check = lambda: torch.equal(bits(dst[-RUN:].cpu()), bits(h[-RUN:])) + elif kind == "h2d": + n = int(arg) * (1 << 20) // 2 + h = host(n) + dst = torch.empty(n, dtype=torch.bfloat16, device=dev) + fn = lambda: dst.copy_(h) + nbytes = n * 2 + check = lambda: torch.equal(bits(dst[-RUN:].cpu()), bits(h[-RUN:])) + else: + n = (int(arg) if arg.isdigit() else 64) * (1 << 20) // 2 + n = max(n // RUN, 2) * RUN + rows = n // RUN + h = host(n) + src = h.to(dev) + dst = torch.empty(n, dtype=torch.bfloat16, device=dev) + sync() + if arg.isdigit(): + fn = lambda: dst.copy_(src) + nbytes = n * 2 + check = lambda: torch.equal(bits(dst[-RUN:].cpu()), bits(h[-RUN:])) + elif arg == "calls": + k = min(512, rows) + perm = torch.randperm(rows, generator=g)[:k].tolist() + + def fn(): + for i, r in enumerate(perm): + dst[i * RUN:(i + 1) * RUN].copy_(src[r * RUN:(r + 1) * RUN]) + nbytes = k * RUN * 2 + row["calls"] = k + check = lambda: torch.equal(bits(dst[5 * RUN:6 * RUN].cpu()), bits(h[perm[5] * RUN:(perm[5] + 1) * RUN])) + elif arg == "strided": + s2, d2 = src.view(rows // 2, 2, RUN), dst.view(rows // 2, 2, RUN) + fn = lambda: d2[:, 0, :].copy_(s2[:, 1, :]) + nbytes = (rows // 2) * RUN * 2 + check = lambda: torch.equal(bits(d2[:, 0, :].cpu()), bits(h.view(rows // 2, 2, RUN)[:, 1, :])) + else: # gather + pidx = torch.randperm(rows, generator=g) + pdev = pidx.to(dev) + sv, dv = src.view(rows, RUN), dst.view(rows, RUN) + fn = lambda: dv.copy_(torch.index_select(sv, 0, pdev)) + nbytes = n * 2 + check = lambda: torch.equal(bits(dv[7].cpu()), bits(h.view(rows, RUN)[pidx[7]])) + row["bytes"] = nbytes + fn() + sync() + for _ in range(reps): + sync() + t = time.perf_counter() + fn() + sync() + row["s"].append(time.perf_counter() - t) + row["median_s"] = sorted(row["s"])[len(row["s"]) // 2] + row["GBps"] = nbytes / row["median_s"] / 1e9 + if "calls" in row: + row["us_per_call"] = row["median_s"] / row["calls"] * 1e6 + row["correct"] = bool(check()) + except Exception as e: + row["error"] = repr(e)[:400] + row["trace"] = traceback.format_exc()[-1500:] + with open(out, "w") as f: + json.dump(row, f, indent=1) + + +def driver(out, platform, reps): + res = dict(schema="npu-d2d.v2", platform=platform, reps=reps, load=os.getloadavg(), cases={}) + for name in CASES: + part = out + "." + name + ".json" + t = time.time() + try: + p = subprocess.run([sys.executable, __file__, "--case", name, platform, str(reps), part], + capture_output=True, text=True, timeout=600) + row = json.load(open(part)) if os.path.exists(part) else dict(error="no output", rc=p.returncode) + if p.returncode: + row["rc"] = p.returncode + row["stderr"] = p.stderr[-800:] + except subprocess.TimeoutExpired: + row = dict(error="timeout 600 s") + row["wall_s"] = round(time.time() - t, 1) + res["cases"][name] = row + with open(out, "w") as f: + json.dump(res, f, indent=1) + print(name, {k: v for k, v in row.items() if k not in ("s", "trace", "stderr")}, flush=True) + res["finished"] = time.time() + with open(out, "w") as f: + json.dump(res, f, indent=1) + + +if __name__ == "__main__": + if sys.argv[1] == "--case": + run_case(sys.argv[2], sys.argv[3], int(sys.argv[4]), sys.argv[5]) + else: + driver(sys.argv[1], sys.argv[2], int(sys.argv[3]) if len(sys.argv) > 3 else 5) diff --git a/tools/varmorph/rbln_vmshim.cpp b/tools/varmorph/rbln_vmshim.cpp new file mode 100644 index 0000000..547599c --- /dev/null +++ b/tools/varmorph/rbln_vmshim.cpp @@ -0,0 +1,38 @@ +// C shim over the public NPU-B runtime API (rebel/runtime/api/rbln_runtime_api.h, SDK 0.12.0) for +// batched copies, which torch-rbln does not bind to Python. Loaded with ctypes inside a process +// that has imported torch_rbln; tensor data pointers are vmem virtual addresses. +// g++ -O2 -shared -fPIC -I$SP/rebel/include rbln_vmshim.cpp -o librbln_vmshim.so \ +// -L$SP/tvm -l:librbln.so -Wl,-rpath,$SP/tvm +#include + +#include +#include +#include + +extern "C" { + +// n copies (src_vaddr[i] -> dst_vaddr[i], size[i]); on the device for <= 1024 per destination. +int vmshim_v2v_multi(const uint64_t* src, const uint64_t* dst, const uint64_t* size, int n) { + std::vector> v; + v.reserve(n); + for (int i = 0; i < n; ++i) v.emplace_back(src[i], dst[i], size[i]); + return static_cast(rbln::rbln_memcpy_v2v_multi(v)); +} + +// n patches (host pointer -> dst_vaddr, size). +int vmshim_h2v_multi(const uint64_t* host, const uint64_t* dst, const uint64_t* size, int n) { + std::vector> v; + v.reserve(n); + for (int i = 0; i < n; ++i) v.emplace_back(static_cast(host[i]), dst[i], size[i]); + return static_cast(rbln::rbln_memcpy_h2v_multi(v)); +} + +int vmshim_layout_like(uint64_t target, uint64_t ref) { + return static_cast(rbln::rbln_set_device_alloc_layout_like(target, ref)); +} + +int vmshim_sync(uint32_t torch_device_id) { + return static_cast(rbln::rbln_device_synchronize(torch_device_id)); +} + +} // extern "C" diff --git a/tools/varmorph/reach.py b/tools/varmorph/reach.py new file mode 100644 index 0000000..f577b73 --- /dev/null +++ b/tools/varmorph/reach.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Backward reach of the moves of a copy list and the scratch an in-place rebuild needs. +argv: OUT.json RUNS.npz [RUNS.npz ...] (window 64 MiB, ring 256 MiB as in npu_replay.py)""" +import json, sys +import numpy as np +WIN, RING = 64 << 20, 256 << 20 +res = [] +for p in sys.argv[2:]: + r = np.load(p) + d, s, l = r["dst"].astype(np.int64), r["src"].astype(np.int64), r["len"].astype(np.int64) + mv = s - d + moved = mv != 0 + a = (d // WIN) * WIN # start of the window being written + back = np.where(moved, a - s, 0) # > 0: the source lies before the window (already overwritten) + ring_lo = np.maximum(a - RING, 0) + far = moved & (s < ring_lo) + far_bytes = int(np.minimum(l[far], ring_lo[far] - s[far]).sum()) + res.append(dict(runs=p, target=str(r["target"]), n_runs=int(len(d)), keep_bytes=int(l[~moved].sum()), + moved_bytes=int(l[moved].sum()), max_back_bytes=int(back.max()) if len(back) else 0, + back_p99_bytes=int(np.percentile(back[moved], 99)) if moved.any() else 0, + far_runs=int(far.sum()), far_bytes=far_bytes, scratch_bytes=RING + WIN + far_bytes, + v_bytes=int(r["v_bytes"]), ref_bytes=None)) +json.dump(res, open(sys.argv[1], "w"), indent=1) +for x in res: print(json.dumps(x)) diff --git a/tools/varmorph/vcdiff_to_runs.py b/tools/varmorph/vcdiff_to_runs.py new file mode 100644 index 0000000..f90481f --- /dev/null +++ b/tools/varmorph/vcdiff_to_runs.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Turn an xdelta3 (VCDIFF) patch into the copy list that the device rebuild reads (runs.npz). + +Reads `xdelta3 printdelta PATCH` on stdin. printdelta prints the target offset and S@a +(source offset) as absolute values, and T@a relative to the start of the current VCDIFF window +("window at offset"); a T@ copy repeats bytes already decoded in that window. A copy from the target (T@) is resolved to the source bytes it repeats, +piece by piece, so the result is generous to the delta: every byte a copy chain carries from the +source becomes a run. ADD and RUN bytes, and copies that resolve to them, have no source and go +to the patch. Runs are trimmed to whole 64-byte target blocks; blocks not fully covered form +`other`. The instruction counts of the patch as written are stored too. +argv: OUT.npz REF TARGET < printdelta.txt +""" +import bisect +import json +import os +import sys + +import numpy as np + +BLK = 64 + + +class Window: + def __init__(self, t0, src0): + self.t0, self.src0 = t0, src0 + self.st, self.en, self.kind, self.src = [], [], [], [] + + def emit(self, t, l, kind, s): + if l <= 0: + return + if self.st and self.en[-1] == t and self.kind[-1] == kind and \ + (kind == 0 or self.src[-1] + (self.en[-1] - self.st[-1]) == s): + self.en[-1] = t + l + return + self.st.append(t) + self.en.append(t + l) + self.kind.append(kind) + self.src.append(s) + + +def main(): + out, ref, target = sys.argv[1], sys.argv[2], sys.argv[3] + vb = os.path.getsize(target) + st = dict(windows=0, cpy_s=0, cpy_t=0, add=0, run=0, cpy_s_bytes=0, cpy_t_bytes=0, add_bytes=0, run_bytes=0) + rd, rs, rl = [], [], [] + w = None + next_t0, cw_off, tlen, win_at = 0, 0, 0, 0 + + def close(win): + if win is None: + return + for a, e, k, s in zip(win.st, win.en, win.kind, win.src): + if k == 1: + rd.append(win.t0 + a) + rs.append(win.src0 + s) + rl.append(e - a) + + for line in sys.stdin: + if line.startswith("VCDIFF window number"): + close(w) + w = None + st["windows"] += 1 + continue + if line.startswith("VCDIFF window at offset"): + win_at = int(line.split(":")[1]) + continue + if line.startswith("VCDIFF copy window offset"): + cw_off = int(line.split(":")[1]) + continue + if line.startswith("VCDIFF target window length"): + tlen = int(line.split(":")[1]) + w = Window(0, 0) # printdelta prints absolute target offsets and addresses + next_t0 += tlen + cw_off = 0 + continue + f = line.split() + if w is None or len(f) < 4 or not f[0].isdigit(): + continue + t = int(f[0]) + rest = f[2:] + while rest: + kind = rest[0] + if kind in ("ADD", "RUN"): + size, addr, rest = int(rest[1]), None, rest[2:] + st["add" if kind == "ADD" else "run"] += 1 + st[("add" if kind == "ADD" else "run") + "_bytes"] += size + w.emit(t, size, 0, 0) + else: + size, addr, rest = int(rest[1]), rest[2], rest[3:] + if addr.startswith("S@"): + st["cpy_s"] += 1 + st["cpy_s_bytes"] += size + w.emit(t, size, 1, int(addr[2:])) + else: + st["cpy_t"] += 1 + st["cpy_t_bytes"] += size + p, done = win_at + int(addr[2:]), 0 # T@ is relative to the window + while done < size: + i = bisect.bisect_right(w.st, p) - 1 + avail = min(w.en[i] - p, size - done, (t + done) - p) + if w.kind[i] == 1: + w.emit(t + done, avail, 1, w.src[i] + (p - w.st[i])) + else: + w.emit(t + done, avail, 0, 0) + p += avail + done += avail + t += size + close(w) + d = np.array(rd, np.int64) + s = np.array(rs, np.int64) + l = np.array(rl, np.int64) + st["source_runs_raw"] = int(len(d)) + st["source_bytes"] = int(l.sum()) + # merge across windows: consecutive runs with the same move + o = np.argsort(d, kind="stable") + d, s, l = d[o], s[o], l[o] + md, ms, ml = [], [], [] + for a, b, c in zip(d.tolist(), s.tolist(), l.tolist()): + if md and md[-1] + ml[-1] == a and ms[-1] + ml[-1] == b: + ml[-1] += c + else: + md.append(a), ms.append(b), ml.append(c) + d, s, l = np.array(md, np.int64), np.array(ms, np.int64), np.array(ml, np.int64) + st["source_runs_merged"] = int(len(d)) + # trim to whole target blocks + a = (d + BLK - 1) // BLK * BLK + e = (d + l) // BLK * BLK + keep = e > a + s = s + (a - d) + d, s, l = a[keep], s[keep], (e - a)[keep] + al = vb // BLK + covered = np.zeros(al, dtype=bool) + for a_, l_ in zip(d.tolist(), l.tolist()): + covered[a_ // BLK:(a_ + l_) // BLK] = True + other = np.nonzero(~covered)[0].astype(np.int64) + R = np.memmap(ref, dtype=np.uint8, mode="r") + T = np.memmap(target, dtype=np.uint8, mode="r") + bad = sum(1 for a_, b_, l_ in zip(d.tolist(), s.tolist(), l.tolist()) + if not np.array_equal(R[b_:b_ + l_], T[a_:a_ + l_])) + st["runs_mismatched"] = int(bad) + st["runs"] = int(len(d)) + st["run_bytes_trimmed"] = int(l.sum()) + st["other_blocks"] = int(len(other)) + st["v_bytes"] = vb + np.savez(out, dst=d, src=s, len=l, other=other, other_mode=np.zeros(len(other), np.uint8), + v_bytes=np.int64(vb), refs=np.array([ref]), target=np.array(target)) + with open(out + ".json", "w") as fh: + json.dump(st, fh, indent=1) + print(json.dumps(st)) + + +if __name__ == "__main__": + main() diff --git a/tools/varmorph/vm_apply.py b/tools/varmorph/vm_apply.py new file mode 100644 index 0000000..55c6294 --- /dev/null +++ b/tools/varmorph/vm_apply.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Apply a VarMorph descriptor file on a node: expand it, then restore the target file. + + expand DESC.vmd.npz[.zst] REF OUT.runs.npz + Read the descriptor (ops, residual, intra, tail) and the sibling REF and write the copy list + the restorers read: match runs (dst, src, len) and, for every other block, its bytes + (fill, residual XOR the sibling block it predicts from, new content). Only the sibling + blocks that residuals predict from are read. Timed: decompress + expand. + new|inplace DESC REF WORKDIR OUT.json [COLD] + expand, then restore the target into a new file / over a private copy of the sibling + (host_restore.py), sync, and compare with the SHA-256 recorded in the descriptor's name + file (the target itself, read after the clock). COLD=1 evicts the sibling from the page + cache first (posix_fadvise DONTNEED). +The copy of the sibling for `inplace` is made before the clock. +""" +import io +import json +import os +import subprocess +import sys +import time + +import numpy as np +from numba import njit + +B = 64 +OP_MATCH, OP_FILL, OP_RES, OP_INTRA = 0, 1, 2, 3 + + +@njit(cache=True) +def count(ops): + """Number of match runs, and of non-match blocks.""" + op, nm, nb_other = 0, 0, 0 + while op < len(ops): + m = ops[op] + op += 1 + L, s = 0, 0 + while True: + b = np.int64(ops[op]) + op += 1 + L |= (b & 0x7F) << s + s += 7 + if b < 0x80: + break + if m == OP_MATCH or m == OP_RES: + while True: + b = np.int64(ops[op]) + op += 1 + if b < 0x80: + break + if m == OP_MATCH: + nm += 1 + else: + nb_other += L + elif m == OP_FILL: + op += 8 + nb_other += L + else: + nb_other += L + return nm, nb_other + + +@njit(cache=True) +def expand(R, ops, res, intra, nm, nb_other): + dst = np.empty(nm, np.int64) + src = np.empty(nm, np.int64) + ln = np.empty(nm, np.int64) + other = np.empty(nb_other, np.int64) + data = np.empty(nb_other * B, np.uint8) + op, rp, ip, prev_d, i, km, ko = 0, 0, 0, 0, 0, 0, 0 + while op < len(ops): + m = ops[op] + op += 1 + L, s = 0, 0 + while True: + b = np.int64(ops[op]) + op += 1 + L |= (b & 0x7F) << s + s += 7 + if b < 0x80: + break + a = i * B + if m == OP_MATCH or m == OP_RES: + z, s = 0, 0 + while True: + b = np.int64(ops[op]) + op += 1 + z |= (b & 0x7F) << s + s += 7 + if b < 0x80: + break + d = prev_d + ((z >> 1) ^ -(z & 1)) + prev_d = d + if m == OP_MATCH: + dst[km] = a + src[km] = a + d + ln[km] = L * B + km += 1 + else: + for k in range(L * B): + data[ko * B + k] = R[a + d + k] ^ res[rp + k] + rp += L * B + for k in range(L): + other[ko + k] = i + k + ko += L + elif m == OP_FILL: + for k in range(L * B): + data[ko * B + k] = ops[op + (k & 7)] + op += 8 + for k in range(L): + other[ko + k] = i + k + ko += L + else: + data[ko * B:(ko + L) * B] = intra[ip:ip + L * B] + ip += L * B + for k in range(L): + other[ko + k] = i + k + ko += L + i += L + return dst, src, ln, other, data + + +def read_desc(path): + if path.endswith(".zst"): + raw = subprocess.run(["zstd", "-q", "-d", "-c", "--long=27", path], capture_output=True, check=True).stdout + return np.load(io.BytesIO(raw)) + return np.load(path) + + +def do_expand(desc, ref, out): + t0 = time.perf_counter() + d = read_desc(desc) + ops, res, intra, tail = d["ops"], d["res"], d["intra"], d["tail"] + t1 = time.perf_counter() + R = np.memmap(ref, dtype=np.uint8, mode="r") + nm, no = count(ops) + dst, src, ln, other, data = expand(R, ops, res, intra, nm, no) + t2 = time.perf_counter() + np.savez(out, dst=dst, src=src, len=ln, other=other, other_mode=np.zeros(len(other), np.uint8), + patch_data=data, tail=tail, v_bytes=d["v_bytes"], refs=np.array([ref]), target=d["target"]) + return dict(desc_bytes=os.path.getsize(desc), decompress_s=t1 - t0, expand_s=t2 - t1, runs=int(nm), + other_blocks=int(no), patch_bytes=int(no * B)) + + +if __name__ == "__main__": + mode = sys.argv[1] + here = os.path.dirname(os.path.abspath(__file__)) + if mode == "expand": + expand(np.zeros(64, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), 0, 0) + print(json.dumps(do_expand(sys.argv[2], sys.argv[3], sys.argv[4]))) + else: + desc, ref, work, out = sys.argv[2:6] + cold = len(sys.argv) > 6 and sys.argv[6] == "1" + os.makedirs(work, exist_ok=True) + expand(np.zeros(64, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), 0, 0) + count(np.zeros(0, np.uint8)) + runs = os.path.join(work, "apply.runs.npz") + if cold: + fd = os.open(ref, os.O_RDONLY) + os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) + os.close(fd) + t0 = time.perf_counter() + e = do_expand(desc, ref, runs) + t1 = time.perf_counter() + p = subprocess.run([sys.executable, os.path.join(here, "host_restore.py"), mode, runs, work, out + ".restore"], + capture_output=True, text=True, check=True) + t2 = time.perf_counter() + r = json.load(open(out + ".restore")) + res = dict(mode=mode, cold=cold, expand=e, expand_total_s=t1 - t0, restore_s=r["restore_s"], + process_s=t2 - t1, apply_s=(t1 - t0) + r["restore_s"], correct=r["correct"], + bytes_written=r["bytes_written"], load=os.getloadavg()) + json.dump(res, open(out, "w"), indent=1) + print(json.dumps(res))