Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
103f8c6
test(csi): exercise owner admission races and consumed-base recovery
gspark-etri Nov 2, 2026
4b32c96
Require the selected residual pack hash failure in recovery witness
gspark-etri Nov 2, 2026
2d0ed88
Take a base over only in place of an eviction, and check what is kept
gspark-etri Nov 11, 2026
00456d4
cli: expose base policies on onggi pull; add a base read size
gspark-etri Nov 11, 2026
6d78193
Separate what a pull may consume from what it may read
gspark-etri Nov 11, 2026
57bb2b8
casync engine: desync chunk store with state-aware in-place reuse
gspark-etri Nov 11, 2026
b329e9f
casync: manifest covers index content; add desync dependency
gspark-etri Nov 11, 2026
4626388
casync engine in the CSI plugin; ownership integration test
gspark-etri Nov 11, 2026
ef2f798
casync: honour BaseCopyRemote with a local store; test the victim choice
gspark-etri Nov 11, 2026
50867a7
casync: pair renamed files by same-offset bytes so Keep works across …
gspark-etri Nov 11, 2026
a4f2730
csi: with a capacity the eviction victim needs no idle window; casync…
gspark-etri Nov 11, 2026
53d842d
casync: chunks subcommand dumps the chunk list of a manifest for capa…
gspark-etri Nov 11, 2026
8aaa0c7
casync: chunk source may be a remote HTTP chunk server (--remote); pu…
gspark-etri Nov 12, 2026
b87c6e4
casync: keep's chunk source parameter no longer shadowed by the base …
gspark-etri Nov 12, 2026
892ab54
tools/varmorph: layout coder, artifact classifier, device rebuild (in…
gspark-etri Nov 12, 2026
ef1c2ea
casync backend: tests for repair of a changed base, a hard-linked bas…
gspark-etri Nov 12, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ jobs:
with:
go-version-file: go.mod
- name: gofmt
run: test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; }
run: test -z "$(gofmt -l .)" || { gofmt -d .; exit 1; }
- name: Owner admission and post-consumption recovery witnesses
run: go test -race -count=1 -timeout=2m -run '^TestOwner' -v ./pkg/csi
- run: go vet ./...
- run: go test -race ./...
- name: E11 payload accounting and validity regressions
Expand Down
184 changes: 184 additions & 0 deletions cmd/onggi/casync.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
package main

import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/url"
"os"
"time"

"github.com/folbricht/desync"

"github.com/ai-computing/onggi/pkg/casync"
"github.com/ai-computing/onggi/pkg/csi"
"github.com/ai-computing/onggi/pkg/digest"
"github.com/ai-computing/onggi/pkg/store"
)

const casyncUsage = `onggi casync — artifacts in casync format (desync chunk store), with state-aware in-place reuse

onggi casync publish --store S --index I <dir> chunk dir into S; print the manifest digest
onggi casync chunks --store S --index I <manifest> print the chunk list of every file (JSON)
onggi casync pull --store S --index I [--attest A] <manifest> <dir>
[--base M --base-dir D] take base M over (Keep): write only differing chunks; kept chunks are
read back unless A attests the base unchanged since it was written
[--copy M --copy-dir D] read base M where it is (Copy); it stays intact
[--verify-kept] [--no-sync] [--workers N] [--log]
onggi casync verify --store S --index I <manifest> <dir> hash-check every chunk of dir
onggi casync attest --store S --index I --attest A <manifest> <dir> verify dir, then record it as unchanged
`

func casyncCmd(args []string, stdout, stderr io.Writer) int {
if len(args) < 1 {
fmt.Fprint(stderr, casyncUsage)
return 2
}
fs := flag.NewFlagSet("casync "+args[0], flag.ContinueOnError)
fs.SetOutput(stderr)
storeDir := fs.String("store", "", "desync local chunk store")
indexDir := fs.String("index", "", "directory of artifact indexes")
attestDir := fs.String("attest", "", "directory of attestations (none: kept chunks are always read back)")
workers := fs.Int("workers", 12, "parallel chunk reads and writes")
base := fs.String("base", "", "Keep: manifest of the artifact to take over")
baseDir := fs.String("base-dir", "", "Keep: directory holding the base's files")
cp := fs.String("copy", "", "Copy: manifest of an artifact to read where it is")
copyDir := fs.String("copy-dir", "", "Copy: directory holding that artifact's files")
verifyKept := fs.Bool("verify-kept", false, "read back kept chunks even if the base is attested unchanged")
noSync := fs.Bool("no-sync", false, "do not sync written files (desync's default behaviour)")
logPhases := fs.Bool("log", false, "log phase durations to stderr")
remote := fs.String("remote", "", "pull: read chunks from this HTTP chunk server instead of --store")
if err := fs.Parse(args[1:]); err != nil || *storeDir == "" || *indexDir == "" {
fmt.Fprint(stderr, casyncUsage)
return 2
}
e, err := casync.New(*storeDir, *indexDir, *attestDir, *workers)
if err == nil && *remote != "" {
var u *url.URL
if u, err = url.Parse(*remote); err == nil {
e.Src, err = desync.NewRemoteHTTPStore(u, desync.StoreOptions{N: *workers, Timeout: 5 * time.Minute})
}
}
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
e.Durable = !*noSync
e.VerifyKept = *verifyKept
if *logPhases {
l := log.New(stderr, "", log.Lmicroseconds)
e.Logf = l.Printf
}
ctx := context.Background()
switch args[0] {
case "publish":
if fs.NArg() != 1 {
fmt.Fprint(stderr, casyncUsage)
return 2
}
start := time.Now()
d, err := e.Publish(ctx, fs.Arg(0))
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
fmt.Fprintf(stdout, "%s\t%.2fs\n", d, time.Since(start).Seconds())
return 0
case "chunks":
// Dump the chunk list of every file of a manifest (start, size, id) as JSON, for capacity
// and transfer accounting outside the engine.
if fs.NArg() != 1 {
fmt.Fprint(stderr, casyncUsage)
return 2
}
m, err := digest.Parse(fs.Arg(0))
if err != nil {
fmt.Fprintln(stderr, err)
return 2
}
a, idxs, err := e.Load(m)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
type fileChunks struct {
Path string `json:"path"`
Size int64 `json:"size"`
Chunks [][3]string `json:"chunks"`
}
out := make([]fileChunks, len(a.Files))
for i, f := range a.Files {
out[i] = fileChunks{Path: f.Path, Size: f.Size}
for _, c := range idxs[i].Chunks {
out[i].Chunks = append(out[i].Chunks, [3]string{fmt.Sprint(c.Start), fmt.Sprint(c.Size), c.ID.String()})
}
}
if err := json.NewEncoder(stdout).Encode(out); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
return 0
case "pull", "verify", "attest":
if fs.NArg() != 2 {
fmt.Fprint(stderr, casyncUsage)
return 2
}
m, err := digest.Parse(fs.Arg(0))
if err != nil {
fmt.Fprintln(stderr, err)
return 2
}
dest := fs.Arg(1)
if args[0] != "pull" {
if err := e.Verify(ctx, m, dest); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
if args[0] == "attest" {
if err := e.Attest(m, dest); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
}
return 0
}
if err := os.MkdirAll(dest, 0o755); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
var offer *csi.Offer
if *base != "" {
b, err := digest.Parse(*base)
if err != nil {
fmt.Fprintln(stderr, err)
return 2
}
offer = &csi.Offer{Bases: []store.Base{{Manifest: b, Dir: *baseDir}},
Claim: func(sb store.Base) (string, bool) { return sb.Dir, true }}
}
if *cp != "" {
c, err := digest.Parse(*cp)
if err != nil {
fmt.Fprintln(stderr, err)
return 2
}
if offer == nil {
offer = &csi.Offer{}
}
offer.Readable, offer.BaseCopy = []store.Base{{Manifest: c, Dir: *copyDir}}, store.BaseCopyAlways
}
st, err := e.Pull(ctx, m, dest, offer)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
js, _ := json.Marshal(st)
fmt.Fprintln(stdout, string(js))
return 0
}
fmt.Fprint(stderr, casyncUsage)
return 2
}
27 changes: 27 additions & 0 deletions cmd/onggi/csi.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
bfs "github.com/ai-computing/onggi/pkg/backend/fs"
"github.com/ai-computing/onggi/pkg/csi"
"github.com/ai-computing/onggi/pkg/digest"
"github.com/ai-computing/onggi/pkg/casync"
"github.com/ai-computing/onggi/pkg/peer"
"github.com/ai-computing/onggi/pkg/store"
)
Expand All @@ -38,6 +39,13 @@ func csiNode(args []string, stderr io.Writer) int {
keepUsed := fs.Duration("keep-used", 7*24*time.Hour, "the node cache keeps what artifacts used within this period need")
gcGrace := fs.Duration("gc-grace", time.Hour, "grace period of the node cache's GC")
rebaseIdle := fs.Duration("rebase-idle", 5*time.Minute, "a pull may turn an artifact no volume used for this long into the new one, writing only what differs (0: never)")
maxArtifacts := fs.Int("max-artifacts", 0, "artifacts kept per node; a pull takes an idle one over only when the node holds this many, least recently used first (0: no limit)")
verifyKept := fs.Bool("verify-kept", true, "read back and hash-check every chunk a pull leaves in place when it takes an artifact over")
baseCopy := fs.String("base-copy", "remote", "read an idle artifact that is not taken over where it is: remote (only if the store is not local), always, never")
engine := fs.String("engine", "store", "how artifacts are materialized: store (onggi packs) or casync (desync chunk store with state-aware in-place reuse)")
casyncStore := fs.String("casync-store", "", "casync engine: desync local chunk store")
casyncIndex := fs.String("casync-index", "", "casync engine: directory of artifact indexes (onggi casync publish)")
casyncVerify := fs.Bool("casync-verify-always", false, "casync engine: read back kept chunks even when the base is attested unchanged")
peerListen := fs.String("peer-listen", "", "serve the node cache to other nodes' pulls on this address, e.g. :7070 (#37)")
peers := fs.String("peers", "", "comma-separated peer URLs asked for chunks before the store")
peersDNS := fs.String("peers-dns", "", "host:port whose name resolves to the peers, e.g. a headless Service; re-resolved every 30 s")
Expand Down Expand Up @@ -100,12 +108,31 @@ func csiNode(args []string, stderr io.Writer) int {
}
}
m := &csi.Materializer{Root: *dataDir, Pull: csi.StorePullerPeers(s, openCache, peerList, tmp, *workers), Logf: logger.Printf, MaxPulls: *pulls}
if *engine == "casync" {
e, err := casync.New(*casyncStore, *casyncIndex, filepath.Join(*dataDir, "casync-attest"), *workers)
if err != nil || *casyncStore == "" || *casyncIndex == "" {
logger.Print("casync engine: needs -casync-store and -casync-index: ", err)
return 2
}
e.VerifyKept, e.Logf = *casyncVerify, logger.Printf
m.Pull = e.Pull
*verifyKept = *casyncVerify // the engine decides from the attestation; the offer forces a read-back only on request
} else if *engine != "store" {
logger.Printf("-engine: %q is not store or casync", *engine)
return 2
}
if err := m.Init(); err != nil {
logger.Print("data dir: ", err)
return 1
}
mounter := csi.NewMounter()
m.RebaseIdle, m.Unmount = *rebaseIdle, mounter.Unmount
m.MaxArtifacts, m.VerifyKept = *maxArtifacts, *verifyKept
var ok bool
if m.BaseCopy, ok = parseBaseCopy(*baseCopy); !ok {
logger.Printf("-base-copy: %q is not remote, always or never", *baseCopy)
return 2
}
n := &csi.Node{
Version: version,
NodeID: *nodeID,
Expand Down
3 changes: 3 additions & 0 deletions cmd/onggi/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ usage:
evict from a node cache what recently used
artifacts do not need
onggi dedup-report [flags] <path>... measure how much the files would shrink in onggi
onggi casync publish|pull|verify|attest ... casync-format artifacts with state-aware in-place reuse
onggi csi --store S [flags] CSI node plugin: serve artifacts as read-only volumes (Linux)
onggi peer serve --cache DIR [--listen :7070] serve a node cache to other nodes' pulls (--peers)
onggi version
Expand Down Expand Up @@ -61,6 +62,8 @@ func main() {
os.Exit(inspect(os.Args[2:], os.Stdout, os.Stderr))
case "diff":
os.Exit(diff(os.Args[2:], os.Stdout, os.Stderr))
case "casync":
os.Exit(casyncCmd(os.Args[2:], os.Stdout, os.Stderr))
case "csi":
os.Exit(csiNode(os.Args[2:], os.Stderr))
case "rm-ref":
Expand Down
34 changes: 33 additions & 1 deletion cmd/onggi/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (

type storeFlags struct {
url, durability, min, avg, max, sub, subKinds, packSize, frame *string
frameFlat *bool
}

func addStoreFlags(fs *flag.FlagSet) storeFlags {
Expand All @@ -38,6 +39,7 @@ func addStoreFlags(fs *flag.FlagSet) storeFlags {
sub: fs.String("sub", "4KiB", "average sub-chunk size for two-level chunking, 0 for one level (only when creating a store)"),
subKinds: fs.String("sub-kinds", "atom-prefill,atom-decode,atom-other", "file kinds that get sub-chunks, or all (only when creating a store)"),
packSize: fs.String("pack-size", "64MiB", "pack object size (only when creating a store)"),
frameFlat: fs.Bool("frame-flat", false, "with --sub 0, compress the new chunks of a one-level store in frames of --frame too (only when creating a store)"),
frame: fs.String("frame", "256KiB", "compress new chunks of sub-chunked files together up to this size, 0 for one by one (only when creating a store)"),
}
}
Expand Down Expand Up @@ -101,6 +103,13 @@ func (f storeFlags) open(ctx context.Context) (*store.Store, backend.Refs, error
}
}
}
if c.Sub == nil && *f.frameFlat && *f.frame != "0" {
fr, err := size(*f.frame)
if err != nil {
return nil, nil, err
}
c.Frame, c.FrameFlat = int64(fr), true
}
s, err := store.OpenWith(ctx, b, c)
return s, b, err
}
Expand Down Expand Up @@ -192,6 +201,10 @@ func pull(args []string, stdout, stderr io.Writer) int {
peers := fs.String("peers", "", "comma-separated peer URLs (onggi peer serve) asked for chunks before the store")
baseRef := fs.String("base", "", "an artifact already restored in --base-dir that the pull may turn into this one, writing only what differs; its files are moved away (#41)")
baseDir := fs.String("base-dir", "", "the directory holding --base")
baseIntact := fs.Bool("base-intact", false, "never move --base's files away: read from them where they are")
baseCopy := fs.String("base-copy", "always", "read a base that is not taken over where it is: always, remote (only if the store is not local), never")
baseRange := fs.Int64("base-range", 0, "largest single read from the base's files in bytes (0: 16 MiB; 1: chunk by chunk)")
verifyKept := fs.Bool("verify-kept", false, "read back and hash-check every chunk left in place when --base is taken over")
asJSON := fs.Bool("json", false, "print JSON")
if err := fs.Parse(args); err != nil || fs.NArg() != 2 {
fmt.Fprintln(stderr, "usage: onggi pull --store S [--cache DIR] [--peers URLS] [--base REF --base-dir DIR] <ref|sha256:...> <dir>")
Expand Down Expand Up @@ -220,8 +233,14 @@ func pull(args []string, stdout, stderr io.Writer) int {
return 1
}
o.Bases = []store.Base{{Manifest: bd, Dir: *baseDir}}
o.Claim = func(b store.Base) (string, bool) { return b.Dir, true }
o.Claim = func(b store.Base) (string, bool) { return b.Dir, !*baseIntact }
}
var ok bool
if o.BaseCopy, ok = parseBaseCopy(*baseCopy); !ok {
fmt.Fprintf(stderr, "onggi: --base-copy: %q is not always, remote or never\n", *baseCopy)
return 2
}
o.VerifyKept, o.BaseRange = *verifyKept, *baseRange
if *cacheDir != "" {
cb, err := bfs.Open(*cacheDir, backend.DurabilityNone)
if err == nil {
Expand Down Expand Up @@ -430,3 +449,16 @@ func diff(args []string, stdout, stderr io.Writer) int {
}
return 0
}

// parseBaseCopy maps a --base-copy value to the rule.
func parseBaseCopy(v string) (store.BaseCopy, bool) {
switch v {
case "always":
return store.BaseCopyAlways, true
case "remote":
return store.BaseCopyRemote, true
case "never":
return store.BaseCopyNever, true
}
return store.BaseCopyAlways, false
}
Loading
Loading