diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1bb3856..682e7d9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ name: test # smoke - correctness: dotnet test + corpus smoke on every platform / ISA tier (+ Metal) # bench - performance: corpus benchmark (.github/scripts/bench.ps1), several replicas per platform # report - merges every result into the run summary (tests/Sdcb.SimdGcdrNet.TestReport) -# Job budget: 35 jobs (40 concurrent); macOS: 2 smoke + 3 bench (5 concurrent). +# Job budget: 36 jobs (40 concurrent); macOS: 2 smoke + 3 bench (5 concurrent). on: push: @@ -33,6 +33,9 @@ concurrency: env: DOTNET_NOLOGO: 1 + # jobs build against the pinned Sdcb.SimdOnnx package (what ships); the engine-head smoke builds + # sdcb/SimdOnnx main from source next to this checkout (MSBuild reads env vars as properties) + SimdOnnxProject: ${{ github.workspace }}/_simdonnx/src/Sdcb.SimdOnnx/Sdcb.SimdOnnx.csproj DOTNET_CLI_TELEMETRY_OPTOUT: 1 DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 @@ -57,11 +60,15 @@ jobs: name: smoke ${{ matrix.name }} runs-on: ${{ matrix.os }} timeout-minutes: 30 + env: + UseLocalSimdOnnx: ${{ matrix.enginehead && 'true' || 'false' }} strategy: fail-fast: false matrix: include: - { name: win-x64, os: windows-2025, count: 5 } + # the same tests against sdcb/SimdOnnx main from source: engine changes that break this library + - { name: linux-x64-engine-head, os: ubuntu-24.04, count: 2, enginehead: true } - { name: win-arm64, os: windows-11-arm, count: 5 } - { name: linux-x64, os: ubuntu-24.04, count: 5, pack: true } - { name: linux-arm64, os: ubuntu-24.04-arm, count: 5 } @@ -77,6 +84,9 @@ jobs: - { name: linux-arm64-scalar, os: ubuntu-24.04-arm, count: 2, maxmp: 1, isa: DOTNET_EnableHWIntrinsic=0 } steps: - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + if: matrix.enginehead + with: { repository: sdcb/SimdOnnx, path: _simdonnx } - uses: actions/download-artifact@v8 with: { name: realdae, path: tests/testdata/realdae } - name: Build diff --git a/Directory.Build.props b/Directory.Build.props index 4d17f13..914a541 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -4,8 +4,15 @@ never force a models release). Edit here and keep build/00-common.linq in sync. --> - 1.0.0 + 2.0.0 1.0.0 + + 0.1.0-preview.4 + $(MSBuildThisFileDirectory)..\Sdcb.SimdOnnx\src\Sdcb.SimdOnnx\Sdcb.SimdOnnx.csproj + + $(NoWarn);NU5104 @@ -39,7 +46,7 @@ Sdcb.SimdGcdrNet $(SdcbSimdGcdrNetVersion) true - Pure C# GCDRNet inference: document image shadow removal / appearance enhancement. Self-contained ONNX graph engine with CPU SIMD (AVX-512/AVX2/NEON/Vector) and GPU (Vulkan, Metal) backends; no onnxruntime dependency. Accepts interleaved BGR24 / RGB24 / BGRA32 / RGBA32. Models ship separately in Sdcb.SimdGcdrNet.Models. + Pure C# GCDRNet inference: document image shadow removal / appearance enhancement. Runs on the pure C# Sdcb.SimdOnnx engine (CPU SIMD AVX-512/AVX2/NEON/Vector, GPU Vulkan/Metal) with GCDRNet-specific fused kernels; no onnxruntime dependency. Accepts interleaved BGR24 / RGB24 / BGRA32 / RGBA32. Models ship separately in Sdcb.SimdGcdrNet.Models. Sdcb;GCDRNet;document;document-enhancement;shadow-removal;image-enhancement;onnx;inference;deep-learning;SIMD;AVX512;NEON;wasm;AOT;Vulkan;Metal;linqpad-samples diff --git a/README.md b/README.md index 48d63a2..883ba42 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ **Sdcb.SimdGcdrNet** is a pure C# implementation of [GCDRNet](https://github.com/ZZZHANG-jx/GCDRNet) (*Appearance Enhancement for Camera-captured Document Images in the Wild*, IEEE TAI 2023) — document-image shadow removal and appearance enhancement **without onnxruntime or any other -third-party inference engine**. It ships its own compiled-model inference engine, inspired by -`Sdcb.SimdPaddleOCR`. +third-party inference engine**. Inference runs on [Sdcb.SimdOnnx](https://github.com/sdcb/SimdOnnx), +a pure C# engine shared with `Sdcb.SimdPaddleOCR` (it grew out of this repository's own engine). The model is a two-stage pipeline: @@ -110,19 +110,20 @@ dotnet run --project examples/PaddleOCR.Console -- [image] [output.jpg] ### Layout -- `src/Sdcb.SimdGcdrNet/Onnx` — minimal protobuf reader + ONNX model/graph objects -- `src/Sdcb.SimdGcdrNet/Engine` — `GraphCompiler` (specialize → fold → fuse → arena-plan), - `InferenceSession` -- `src/Sdcb.SimdGcdrNet/Kernels` — Scalar / `Vector` / AVX2 / AVX-512 kernels (`Ops`, `Avx2Kernels`, `Nhwc`, - `PixelOps`, `SimdMath`) -- `src/Sdcb.SimdGcdrNet/Backends/Vulkan` — `Vk` P/Invoke layer, `VkDevice`, `VkShaders` (embedded - SPIR-V), `VkGraph` dispatch lowering, `VkSession`, `VkPipeline2` (fused pipeline), `VkBackend` -- `src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders` — GLSL sources; `tools/build-shaders.bat` - recompiles `.comp` → `.spv` (committed; dev-time only, not part of `dotnet build`) -- `src/Sdcb.SimdGcdrNet/Backends/Metal` — `ObjC`/`Metal` P/Invoke layer, `MtlGraph` dispatch - lowering, `MtlSession`, `MtlPipeline` (fused pipeline), `MetalBackend` -- `src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders` — MSL sources (`.metal`, embedded resources, - compiled to PSOs at run time — no pre-build step) +The ONNX engine (protobuf reader, graph compiler, CPU SIMD kernels, Vulkan / Metal backends) is +[Sdcb.SimdOnnx](https://github.com/sdcb/SimdOnnx); this library holds the GCDRNet-specific parts: + +- `src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs` — public API: backend selection, per-size session pools, + gc → dr stitching, arena stacking +- `src/Sdcb.SimdGcdrNet/Extensions` — engine extensions (`GcdrExtensions`): fused operators + (`UpHeadOp`, `ChannelShiftOp`) with their CPU kernels (Vector / AVX-512 / NEON / WASM SIMD) and + graph passes +- `src/Sdcb.SimdGcdrNet/Kernels` — `PixelOps` (interleaved pixels ↔ tensors, padding) +- `src/Sdcb.SimdGcdrNet/Backends/Vulkan` — `VkPipeline2` (fused gc + dr pipeline on SimdOnnx's + Vulkan backend); `Shaders` holds the GCDRNet GLSL kernels, `tools/build-shaders.bat` recompiles + `.comp` → `.spv` (committed; dev-time only, not part of `dotnet build`) +- `src/Sdcb.SimdGcdrNet/Backends/Metal` — `MtlPipeline` (fused pipeline on SimdOnnx's Metal + backend); `Shaders` holds MSL sources (embedded, compiled to PSOs at run time) - `bench/Sdcb.SimdGcdrNet.Benchmark` — `--check`, `--bench` (CPU), `--benchvk` (Vulkan e2e), `--benchmtl` / `--checkgpu` / `--mtlprof` / `--mtrace` (Metal e2e, diff check, per-shader profile, CPU-vs-Metal per-node trace), `--vkprof` (per-shader/per-dispatch GPU timing), `--vkcheck` / `--vknodes` @@ -173,8 +174,10 @@ runs two waves in parallel, then merges them into one report (the run's summary - Metal path requires macOS on Apple Silicon with Apple7+ (simdgroup-MMA, i.e. M1+; an x64 process under Rosetta also qualifies); `Auto` checks `SupportsApple7` before choosing it, `GcdrBackend.Metal` selects it unconditionally. Set `GCDRNET_MTL_VERBOSE=1` to print the chosen device. -- Thread-safety: `GcdrEnhancer` is thread-safe (CPU: per-size session pools; Vulkan: size-agnostic worker pool); individual - `InferenceSession`/`VkInferenceSession` objects are not. +- Thread-safety: `GcdrEnhancer` is thread-safe (CPU: per-size session pools; Vulkan: size-agnostic worker pool). +- 2.0 moved the engine out: `InferenceSession`, `CompiledModel`, `OnnxModel`, `VkInferenceSession` and + `MtlInferenceSession` are no longer in `Sdcb.SimdGcdrNet.*`; use the same types from + `Sdcb.SimdOnnx.*` (a dependency of this package). `GcdrEnhancer`'s API is unchanged. - Not covered: batched inference (N>1), fp16/int8. - Native AOT works (`IsAotCompatible`, no trim/AOT warnings; CPU and Vulkan output verified identical to JIT). On **x64** the executable project must set the instruction set, otherwise ILC compiles for the SSE baseline, folds @@ -202,8 +205,8 @@ are from the RealDAE dataset of the same paper. ## 中文 **Sdcb.SimdGcdrNet** 是 [GCDRNet](https://github.com/ZZZHANG-jx/GCDRNet)(相机拍摄文档去阴影/外观增强, -IEEE TAI 2023)的**纯 C# 推理库**——不依赖 onnxruntime 等第三方推理引擎,像 `Sdcb.SimdPaddleOCR` -一样自带 CompiledModel 推理引擎。 +IEEE TAI 2023)的**纯 C# 推理库**——不依赖 onnxruntime 等第三方推理引擎,与 `Sdcb.SimdPaddleOCR` +共用纯 C# 推理引擎 [Sdcb.SimdOnnx](https://github.com/sdcb/SimdOnnx)(2.0 起引擎类型移到 `Sdcb.SimdOnnx.*`)。 模型为两级串联:**GC-Net** 估计逐像素光照/阴影图 `I_sm` → `I_GC = clamp(I_s/I_sm)` 除法校正 → **DR-Net** 消费 6 通道 `[I_s; I_GC]` 输出增强图。 diff --git a/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs b/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs index ad03c02..6aad87e 100644 --- a/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs +++ b/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs @@ -195,7 +195,7 @@ private static JsonObject HostInfo() string isa = RuntimeInformation.ProcessArchitecture switch { Architecture.X64 or Architecture.X86 => - Kernels.Nhwc.UseAvx512 ? "avx512" : Avx2.IsSupported ? "avx2" : Avx.IsSupported ? "avx" : + Sdcb.SimdOnnx.Kernels.Nhwc.UseAvx512 ? "avx512" : Avx2.IsSupported ? "avx2" : Avx.IsSupported ? "avx" : System.Numerics.Vector.IsHardwareAccelerated ? "sse" : "scalar", Architecture.Arm64 => AdvSimd.IsSupported ? "neon" : "scalar", _ => System.Numerics.Vector.IsHardwareAccelerated ? "vector" : "scalar", diff --git a/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs b/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs index 100c298..1f3caa5 100644 --- a/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs +++ b/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs @@ -1,7 +1,8 @@ using System.Diagnostics; using System.Runtime.InteropServices; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Onnx; namespace Sdcb.SimdGcdrNet.Benchmark; @@ -44,7 +45,7 @@ public static int Main(string[] args) var m = OnnxModel.Load(mpath); int c = which == "gc" ? 3 : 6; // run the SAME (gpu-layout) plan on CPU and Metal so node indices align - var cpuPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdGcdrNet.Backends.Metal.MetalBackend.PlanOptions); + var cpuPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdOnnx.Backends.Metal.MetalBackend.PlanOptions with { Extensions = GcdrExtensions.Instance }); using var cpu = new InferenceSession(cpuPlan); var rng = new Random(0); if (args.Length > 4) @@ -56,9 +57,9 @@ public static int Main(string[] args) } else for (int i = 0; i < cpu.Input.Length; i++) cpu.Input[i] = rng.NextSingle(); - var mbe = Sdcb.SimdGcdrNet.Backends.Metal.MetalBackend.Default; - var mPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdGcdrNet.Backends.Metal.MetalBackend.PlanOptions); - using var mtl = mbe.NewSession(m, [1, c, h, w]); + var mbe = Sdcb.SimdOnnx.Backends.Metal.MetalBackend.Default; + var mPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdOnnx.Backends.Metal.MetalBackend.PlanOptions with { Extensions = GcdrExtensions.Instance }); + using var mtl = mbe.NewSession(m, [1, c, h, w], GcdrExtensions.Instance); cpu.Input.CopyTo(mtl.Input); mtl.DebugUploadInput(); if (Environment.GetEnvironmentVariable("MTRACE_NODES") == "1") @@ -106,9 +107,9 @@ public static int Main(string[] args) var exp = new float[wsrc.Length]; for (int o2 = 0; o2 < oc_; o2++) for (int c2 = 0; c2 < ic_; c2++) for (int t = 0; t < taps; t++) exp[(t * ic_ + c2) * oc_ + o2] = wsrc[(o2 * ic_ + c2) * taps + t]; - var mw = mtl.DebugConst(nd_.In[1], 2, wsrc.Length); + var mw = mtl.DebugConst(nd_.In[1], ConstKind.ConvW, wsrc.Length); int wb = -1; for (int k = 0; k < exp.Length; k++) if (Math.Abs(mw[k] - exp[k]) > 1e-3) { wb = k; break; } - Console.WriteLine($" w-check: pack len {exp.Length} off {mtl.DebugConstOff(nd_.In[1], 2)} first-diff {wb} head {string.Join(' ', mw.Take(8).Select(x => $"{x:F3}"))} vs exp {string.Join(' ', exp.Take(8).Select(x => $"{x:F3}"))}"); + Console.WriteLine($" w-check: pack len {exp.Length} off {mtl.DebugConstOff(nd_.In[1], ConstKind.ConvW)} first-diff {wb} head {string.Join(' ', mw.Take(8).Select(x => $"{x:F3}"))} vs exp {string.Join(' ', exp.Take(8).Select(x => $"{x:F3}"))}"); if (wb >= 0) Console.WriteLine($" @{wb}: got {mw[wb]:F4} exp {exp[wb]:F4}; around got {string.Join(' ', mw.Skip(wb).Take(6).Select(x => $"{x:F3}"))} exp {string.Join(' ', exp.Skip(wb).Take(6).Select(x => $"{x:F3}"))}"); } if (bad < 0 && (rmse > 1e-2 || nan > 0 || nanw > 0)) @@ -177,7 +178,7 @@ public static int Main(string[] args) // --node gc|dr h w idx reps : time one node in isolation (after a full warmup run) bool isDr = args[1] == "dr"; int nh = int.Parse(args[2]), nw = int.Parse(args[3]), idx = int.Parse(args[4]), nreps = int.Parse(args[5]); - var cm = CompiledModel.Compile(OnnxModel.Load(isDr ? drPath : gcPath), [1, isDr ? 6 : 3, nh, nw]); + var cm = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(isDr ? drPath : gcPath), [1, isDr ? 6 : 3, nh, nw], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); var ss = cm.CreateSession(); new Random(0).NextBytes(MemoryMarshal.AsBytes(ss.Input)); if (args.Length > 6 && args[6] == "u") { var rr = new Random(0); var inp = ss.Input; for (int i = 0; i < inp.Length; i++) inp[i] = rr.NextSingle(); } @@ -202,8 +203,8 @@ public static int Main(string[] args) foreach ((string tag, string mp, int cin) in new[] { ("gc", gcPath, 3), ("dr", drPath, 6) }) { var m = OnnxModel.Load(mp); - var cpu = CompiledModel.Compile(m, [1, cin, ch, cw]).CreateSession(); - using var vk = Backends.Vulkan.VkBackend.Default.NewSession(m, [1, cin, ch, cw]); + var cpu = new CompiledModel(GraphCompiler.Compile(m, [1, cin, ch, cw], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })).CreateSession(); + using var vk = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.NewSession(m, [1, cin, ch, cw], GcdrExtensions.Instance); var rng = new Random(7); for (int i = 0; i < cpu.Input.Length; i++) cpu.Input[i] = vk.Input[i] = rng.NextSingle(); cpu.Run(); vk.Run(); @@ -227,14 +228,14 @@ public static int Main(string[] args) int nh = args.Length > 2 ? int.Parse(args[2]) : 128, nw = args.Length > 3 ? int.Parse(args[3]) : 192; var m = OnnxModel.Load(isDr ? drPath : gcPath); int[] shape = [1, isDr ? 6 : 3, nh, nw]; - var plan = GraphCompiler.Compile(m, shape, Backends.Vulkan.VkBackend.PlanOptions); + var plan = GraphCompiler.Compile(m, shape, Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.PlanOptions with { Extensions = GcdrExtensions.Instance }); using var cpu = new InferenceSession(plan); - using var gpu = Backends.Vulkan.VkBackend.Default.NewSession(m, shape); + using var gpu = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); var rng = new Random(7); for (int i = 0; i < cpu.Input.Length; i++) cpu.Input[i] = gpu.Input[i] = rng.NextSingle(); gpu.DebugUploadInput(); - var ranges = Backends.Vulkan.VkBackend.Default.GraphFor(m, shape).NodeRanges; - var gd = Backends.Vulkan.VkBackend.Default.GraphFor(m, shape).Dispatches; + var ranges = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.GraphFor(m, shape, ext: GcdrExtensions.Instance).NodeRanges; + var gd = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.GraphFor(m, shape, ext: GcdrExtensions.Instance).Dispatches; for (int i = 0; i < plan.Nodes.Length; i++) { var nd = plan.Nodes[i]; @@ -304,11 +305,11 @@ public static int Main(string[] args) { int h = args.Length > 1 ? int.Parse(args[1]) : 1152; int w = args.Length > 2 ? int.Parse(args[2]) : 1600; - var be = Backends.Metal.MetalBackend.Default; + var be = Sdcb.SimdOnnx.Backends.Metal.MetalBackend.Default; foreach ((string tag, string mp, int cin) in new[] { ("gc", gcPath, 3), ("dr", drPath, 6) }) { var m = OnnxModel.Load(mp); - using var s = be.NewSession(m, [1, cin, h, w]); + using var s = be.NewSession(m, [1, cin, h, w], GcdrExtensions.Instance); s.Run(); // warm var agg = s.DebugProfile(); Console.WriteLine($"--- {tag} metal per-shader (total {agg.Values.Sum():F1} ms) ---"); @@ -387,7 +388,7 @@ public static int Main(string[] args) // destroy, recreate — any hash change = stale-read/nondeterminism at session level. private static int VkDup(string gcPath, string drPath, int h, int w, bool useDr) { - var be = Backends.Vulkan.VkBackend.Default; + var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default; string path = useDr ? drPath : gcPath; int c = useDr ? 6 : 3; var m = OnnxModel.Load(path); @@ -399,7 +400,7 @@ static ulong Fhash(ReadOnlySpan d) } for (int round = 0; round < 4; round++) { - using var s = be.NewSession(m, [1, c, h, w]); + using var s = be.NewSession(m, [1, c, h, w], GcdrExtensions.Instance); var rng = new Random(42); for (int i = 0; i < s.Input.Length; i++) s.Input[i] = rng.NextSingle(); s.Run(); @@ -409,7 +410,7 @@ static ulong Fhash(ReadOnlySpan d) // interleave: run smaller sessions to churn memory foreach (int cs in new[] { 1024, 1312, 1152 }) { - using var s2 = be.NewSession(m, [1, c, cs, cs]); + using var s2 = be.NewSession(m, [1, c, cs, cs], GcdrExtensions.Instance); new Random(0).NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(s2.Input)); s2.Run(); } @@ -423,7 +424,7 @@ private static void Analyze(string gcPath, string drPath, int h, int w) { foreach (var (path, c) in new[] { (gcPath, 3), (drPath, 6) }) { - var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w]); + var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }); var written = new List<(long s, long e)>(); bool Covered(long s, long e) => written.Any(r => r.s <= s && r.e >= e); @@ -463,7 +464,7 @@ bool Covered(long s, long e) => // stage first diverges. private static int Probe(string gcPath, string drPath, string imgPath, int[] churnPx) { - var be = Backends.Vulkan.VkBackend.Default; + var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default; var gc = OnnxModel.Load(gcPath); var dr = OnnxModel.Load(drPath); int iw, ih; byte[] bgr; (iw, ih) = ImageSize(imgPath); @@ -488,10 +489,10 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag) { p.Enhance(bgr, iw * 3, dst, iw * 3); p.DebugIntermediates(gcI, gcO, drI); - Console.WriteLine($"{tag}: dst={Fb(dst):x16} gcIn={Ff(gcI):x16} gcOut={Ff(gcO):x16} drIn={Ff(drI):x16} live={Backends.Vulkan.VkDevice.LiveDeviceBytes >> 20}MB"); + Console.WriteLine($"{tag}: dst={Fb(dst):x16} gcIn={Ff(gcI):x16} gcOut={Ff(gcO):x16} drIn={Ff(drI):x16} live={Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.LiveDeviceBytes >> 20}MB"); } - var p1 = be.NewPipe(gc, dr, ih, iw); + var p1 = NewVkPipe(be, gc, dr, ih, iw); Run(p1, "p1-run1"); Run(p1, "p1-run2"); var churn = new List(); @@ -499,7 +500,7 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag) { int cw = Math.Max(64, (int)Math.Sqrt(px * 16.0 / 9) / 32 * 32); int chh = Math.Max(64, px / Math.Max(cw, 1) / 32 * 32); - var cp = be.NewPipe(gc, dr, chh, cw); + var cp = NewVkPipe(be, gc, dr, chh, cw); byte[] cb = new byte[chh * cw * 3]; new Random(px).NextBytes(cb); byte[] cd = new byte[chh * cw * 3]; cp.Enhance(cb, cw * 3, cd, cw * 3); @@ -507,7 +508,7 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag) if (churn.Count > 1) { churn[0].Dispose(); churn.RemoveAt(0); } } foreach (var c in churn) c.Dispose(); - var p2 = be.NewPipe(gc, dr, ih, iw); + var p2 = NewVkPipe(be, gc, dr, ih, iw); Run(p2, "p2-after-churn"); p2.Dispose(); p1.Dispose(); return 0; @@ -516,14 +517,14 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag) // bisect which plan value first diverges between two pipeline instances. private static int Bisect(string gcPath, string drPath, string imgPath, int[] churnPx) { - var be = Backends.Vulkan.VkBackend.Default; + var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default; var gc = OnnxModel.Load(gcPath); var dr = OnnxModel.Load(drPath); int iw, ih; (iw, ih) = ImageSize(imgPath); byte[] bgr = DecodeBgr(imgPath); byte[] dst = new byte[bgr.Length]; - var p1 = be.NewPipe(gc, dr, ih, iw); + var p1 = NewVkPipe(be, gc, dr, ih, iw); p1.Enhance(bgr, iw * 3, dst, iw * 3); // replicate corpus allocation history: run every corpus image sorted by px // (up to `churnPx[0]` predecessors, or all when empty) with real (h,w) sizes @@ -537,7 +538,7 @@ private static int Bisect(string gcPath, string drPath, string imgPath, int[] ch var alive = new List(); foreach (var e in seq.Take(take)) { - var cp = be.NewPipe(gc, dr, e.h, e.w); + var cp = NewVkPipe(be, gc, dr, e.h, e.w); byte[] cb = new byte[e.h * e.w * 3]; new Random(e.w * e.h).NextBytes(cb); byte[] cd = new byte[e.h * e.w * 3]; cp.Enhance(cb, e.w * 3, cd, e.w * 3); @@ -545,7 +546,7 @@ private static int Bisect(string gcPath, string drPath, string imgPath, int[] ch if (alive.Count > 1) { alive[0].Dispose(); alive.RemoveAt(0); } } foreach (var c in alive) c.Dispose(); - var p2 = be.NewPipe(gc, dr, ih, iw); + var p2 = NewVkPipe(be, gc, dr, ih, iw); p2.Enhance(bgr, iw * 3, dst, iw * 3); // dump BOTH whole gc arenas in one shot each, diff per plan value on host @@ -749,7 +750,7 @@ static ulong Ff(Span d) }; } - Backends.Vulkan.VkDevice.ResetPeakDeviceBytes(); + Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.ResetPeakDeviceBytes(); long gc0 = GC.GetTotalMemory(true); double wsLast = wsLoaded; var hashes = new Dictionary(); @@ -824,8 +825,8 @@ static ulong Ff(Span d) Console.WriteLine($" ws: init {wsInit0:F0} -> loaded {wsLoaded:F0} MB ({wsLoaded - wsInit0:+0;-0}), " + $"loaded -> last {wsLast:F0} MB (delta {wsLast - wsLoaded:+0;-0} MB), peak {wsPeak:F0} MB"); Console.WriteLine($" mem: managed {gcD / 1048576.0:+0;-0} MB (after forced GC), " + - $"vkDevice live {Backends.Vulkan.VkDevice.LiveDeviceBytes / 1048576.0:F0} MB " + - $"(peak {Backends.Vulkan.VkDevice.PeakDeviceBytes / 1048576.0:F0} MB)"); + $"vkDevice live {Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.LiveDeviceBytes / 1048576.0:F0} MB " + + $"(peak {Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.PeakDeviceBytes / 1048576.0:F0} MB)"); return 0; } @@ -948,7 +949,7 @@ private static int Trace(string gcPath, string drPath, int h, int w) { foreach (var (path, c) in new[] { (gcPath, 3), (drPath, 6) }) { - var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w]); + var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }); long maxT = plan.Values.Max(v => v.Shape.Aggregate(1L, (a, x) => a * x)); Console.WriteLine($"max tensor {maxT * 4 / 1048576.0:F0} MB, arena {plan.ArenaFloats * 4L / 1048576.0:F0} MB"); using var s = new InferenceSession(plan); @@ -972,8 +973,8 @@ private static int Trace(string gcPath, string drPath, int h, int w) private static int Bench(string gcPath, string drPath, int h, int w, int reps) { - var gc = CompiledModel.Compile(OnnxModel.Load(gcPath), [1, 3, h, w]); - var dr = CompiledModel.Compile(OnnxModel.Load(drPath), [1, 6, h, w]); + var gc = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(gcPath), [1, 3, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); + var dr = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(drPath), [1, 6, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); var s1 = gc.CreateSession(); var s2 = dr.CreateSession(); new Random(0).NextBytes(MemoryMarshal.AsBytes(s1.Input)); new Random(1).NextBytes(MemoryMarshal.AsBytes(s2.Input)); @@ -1033,11 +1034,11 @@ private static int BenchE2e(string gcPath, string drPath, int h, int w, int reps private static int VkProf(string gcPath, string drPath, int h, int w) { - var be = Backends.Vulkan.VkBackend.Default; + var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default; foreach ((string tag, string mp, int cin) in new[] { ("gc", gcPath, 3), ("dr", drPath, 6) }) { var m = OnnxModel.Load(mp); - using var s = be.NewSession(m, [1, cin, h, w]); + using var s = be.NewSession(m, [1, cin, h, w], GcdrExtensions.Instance); s.Run(); // warm var agg = s.DebugProfile(); double tot = agg.Values.Sum(); @@ -1078,4 +1079,11 @@ private static string FindTestdata() } throw new DirectoryNotFoundException("testdata not found"); } -} + + private static Backends.Vulkan.VkGcdrPipeline NewVkPipe(Sdcb.SimdOnnx.Backends.Vulkan.VkBackend be, OnnxModel gc, OnnxModel dr, int ih, int iw) + { + var p = new Backends.Vulkan.VkGcdrPipeline(be, gc, dr); + try { p.Bind(ih, iw); return p; } + catch { p.Dispose(); throw; } + } +} \ No newline at end of file diff --git a/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs b/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs index 0362e48..af12d38 100644 --- a/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs +++ b/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs @@ -5,9 +5,11 @@ using System.Runtime.Intrinsics.Wasm; using System.Text.Json; using System.Text.Json.Nodes; -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Kernels; using Sdcb.SimdGcdrNet.Kernels; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Onnx; using Sdcb.SimdGcdrNet.Testing; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; @@ -274,8 +276,8 @@ private static async Task Corpus(string backend, int count, JsonObject doc) private static async Task Bench(int h, int w, int reps, JsonObject doc) { var (gcBytes, drBytes) = await Models(); - var gc = CompiledModel.Compile(OnnxModel.Load(gcBytes), [1, 3, h, w]); - var dr = CompiledModel.Compile(OnnxModel.Load(drBytes), [1, 6, h, w]); + var gc = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(gcBytes), [1, 3, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); + var dr = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(drBytes), [1, 6, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); using var s1 = gc.CreateSession(); using var s2 = dr.CreateSession(); new Random(0).NextBytes(MemoryMarshal.AsBytes(s1.Input)); @@ -381,7 +383,7 @@ private static async Task TiersCore(int h, int w, JsonObject doc) bool ok = true; foreach (var (tag, model, c) in new[] { ("gc", gcBytes, 3), ("dr", drBytes, 6) }) { - var cm = CompiledModel.Compile(OnnxModel.Load(model), [1, c, h, w]); + var cm = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(model), [1, c, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); var outs = new float[2][]; for (int pass = 0; pass < 2; pass++) { diff --git a/build/00-common.linq b/build/00-common.linq index 49e400d..d6593b2 100644 --- a/build/00-common.linq +++ b/build/00-common.linq @@ -25,7 +25,7 @@ static void Run(string exe, string args, Encoding encoding) => Util.Cmd(exe, arg // dependencies) only when the models change. Keep the Directory.Build.props defaults in sync. static ProjectVersion[] Projects = { - new("Sdcb.SimdGcdrNet", "1.0.0"), + new("Sdcb.SimdGcdrNet", "2.0.0"), new("Sdcb.SimdGcdrNet.Models", "1.0.0"), }; diff --git a/docs/rockchip-rk3399.md b/docs/rockchip-rk3399.md index ecf8a9a..1b05efb 100644 --- a/docs/rockchip-rk3399.md +++ b/docs/rockchip-rk3399.md @@ -84,7 +84,7 @@ CPU 档位(`--bench 512 512`,gc+dr 推理,锁 1008 MHz): NEON 档位(`Nhwc.UseNeon = AdvSimd.Arm64.IsSupported`)沿用 x86 各档的算子结构,以下是只在 NEON 上走的部分: -1. **6×16 micro-kernel `TileNeonK`**(`Kernels/Nhwc.Neon.cs`) +1. **6×16 micro-kernel `TileNeonK`**(Sdcb.SimdOnnx `src/Sdcb.SimdOnnx/Kernels/Nhwc.Neon.cs`) - 每像素一次 128 位读入 4 个输入通道,`fmla` by element 累加到 24 个寄存器累加器(共 32 个向量寄存器)。 - 下一组权重向量提前一组读入,照顾按序执行的 A53。`ic % 4` 的余数用 `ld1r` 广播。 - 单核微基准(`taps × ic` 次累加,L1 内):A72 6.3–7.1 GFLOP/s,约为峰值 7.5 的 85–95%;A53 3.5–3.8 GFLOP/s,约为峰值的 55%。 @@ -97,7 +97,7 @@ NEON 档位(`Nhwc.UseNeon = AdvSimd.Arm64.IsSupported`)沿用 x86 各档的 3. **Winograd** - GEMM 走 `TileNeonK`,输出 epilogue 改为向量化。 - V 块预算降到 16 KB(ic ≥ 32 时 T = 6)。4 个 A53 共享 512 KB L2,块大了会互相挤出。 - - U(36·oc·ic)超过 2 MB 的深层改走三阶段分组调度 `WinogradGrouped`(`Kernels/NhwcWinograd.Neon.cs`):输入变换 → 每个 (xi, oc 块) 一个 GEMM 任务,每块 U 每组只读一次 → 输出变换。原来是每 6 个 tile 就把整块 U 从 DRAM 流一遍。 + - U(36·oc·ic)超过 2 MB 的深层改走三阶段分组调度 `WinogradGrouped`(Sdcb.SimdOnnx `src/Sdcb.SimdOnnx/Kernels/NhwcWinograd.Neon.cs`):输入变换 → 每个 (xi, oc 块) 一个 GEMM 任务,每块 U 每组只读一次 → 输出变换。原来是每 6 个 tile 就把整块 U 从 DRAM 流一遍。 - 分组调度与原调度逐位相同:变换、k 序都一样。512→256 层(1600×1152,锁频)从 105 ms 降到 83 ms,256→128 从 120 ms 降到 93 ms。 4. **向量化激活**:`SimdMath` 增加 Vector128 版 Exp / Sigmoid / Erf / GELU,公式与 AVX2 版一致,用 FMA。 - 深度卷积 + GELU 原来在非 AVX 档逐元素走标量 `ErfScalar`,现在向量化:gc 128 通道那一层从 21.6 ms 降到 8.5 ms(512²,锁频)。 diff --git a/docs/wasm.md b/docs/wasm.md index e64fa47..606c68e 100644 --- a/docs/wasm.md +++ b/docs/wasm.md @@ -89,7 +89,7 @@ Transpose 现在是 gc / dr 里仅次于 Conv 的算子(未优化,两档相 各平台的 CPU 档位:x64 上 AVX-512 / AVX2,arm64 上 NEON,browser-wasm 上 **PackedSimd**;没有专门内核的算子(Transpose、Resize、MaxPool 等)仍走 `Vector`,再往下是标量。PackedSimd 档(`Nhwc.UsePacked = PackedSimd.IsSupported`)参照 SimdPaddleOCR 的 WASM 优化(`Nhwc.Conv.PackedSimd`),结构上对应本仓库 NEON 档已有的特化点: -1. **4 像素 × 8 输出通道 micro-kernel `TilePkK`**(`Kernels/Nhwc.PackedSimd.cs`) +1. **4 像素 × 8 输出通道 micro-kernel `TilePkK`**(Sdcb.SimdOnnx `src/Sdcb.SimdOnnx/Kernels/Nhwc.PackedSimd.cs`) - V8 在 x64 上最多给 v128 分 15 个 XMM 寄存器,6 行 tile 会在内层循环溢出。改为 8 个累加器 + 2 个权重 + 1 个 splat,每个 16 通道打包块跑两遍。 - 输入直接 `v128.load32_splat` 从内存广播,乘、加分开(WASM SIMD 无 FMA),按 k 升序累加:与 `Vector` 档逐 lane 运算顺序相同。 - 用于 1×1 / Gemm(`GemmPk`)、直接卷积(`DenseRowPk`)和 Winograd 的 36 个小 GEMM(tile 行数随之改为 4)。 diff --git a/nuget.config b/nuget.config new file mode 100644 index 0000000..9085df2 --- /dev/null +++ b/nuget.config @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Metal.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/Metal.cs deleted file mode 100644 index 0263c36..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Metal.cs +++ /dev/null @@ -1,337 +0,0 @@ -using System.Runtime.InteropServices; - -namespace Sdcb.SimdGcdrNet.Backends.Metal; - -// Thin Metal wrappers over ObjC msgSend. +1-owned objects are released on -// Dispose; autoreleased objects (commandBuffer, computeCommandEncoder, …) are -// only valid inside the current NSAutoreleasePool — all public entry points -// that create them take their own pool. - -// MTLResourceOptions: StorageModeShared = 0 << 4 = 0 (the zero value), which -// is what the whole backend uses — unified memory means the same allocation -// is CPU- and GPU-visible without didModifyRange/coherence calls. -internal static class MtlStorageMode { public const nuint Shared = 0; } - -// MTLDataType values (MTLDataType.h) — only what NewPipeline's intConstants -// path uses. -internal static class MtlDataType -{ - public const nuint Int = 29; -} - -internal sealed class MtlDevice : IDisposable -{ - public readonly IntPtr H; - private readonly IntPtr _queue; - // shader library + PSO cache, shared by every MetalGraphModel on this - // device (each MtlGcdrPipeline carries gc+dr models — per-model - // libraries would recompile ~2.5k lines of MSL three times and leak the - // whole set on every release). Disposed with the device. - private MtlLibrary? _lib; - private readonly Dictionary _pipes = new(); - - private MtlDevice(IntPtr h) - { - H = h; - // newCommandQueue is +1 — owned, released in Dispose. - _queue = ObjC.Send0(h, ObjC.S.NewCommandQueue); - if (_queue == 0) throw new InvalidOperationException("Metal: newCommandQueue returned nil"); - } - - public IntPtr Queue => _queue; - - public string Name => ObjC.NsToString(ObjC.Send0(H, ObjC.S.Name)) ?? "?"; - - // MTLGPUFamily enum values (MTLGPUFamily.h) - public bool SupportsFamily(int family) => ObjC.SendBool1L(H, ObjC.S.SupportsFamily, family) != 0; - public bool SupportsApple1 => SupportsFamily(1001); - public bool SupportsApple2 => SupportsFamily(1002); - public bool SupportsApple3 => SupportsFamily(1003); - public bool SupportsApple4 => SupportsFamily(1004); - public bool SupportsApple5 => SupportsFamily(1005); - public bool SupportsApple6 => SupportsFamily(1006); - public bool SupportsApple7 => SupportsFamily(1007); - public bool SupportsApple8 => SupportsFamily(1008); - public bool SupportsApple9 => SupportsFamily(1009); - public bool SupportsMetal3 => SupportsFamily(5001); - public bool SupportsMetal4 => SupportsFamily(5002); - - public nuint MaxThreadgroupMemoryLength => ObjC.SendNuint(H, ObjC.S.MaxThreadgroupMemoryLength); - - /// Defensive check for optional selectors — an unrecognized - /// selector on this paravirt driver raises an ObjC exception that kills - /// the process, so never call a maybe-absent method unconditionally. - public bool RespondsTo(IntPtr sel) => ObjC.SendBool1P(H, ObjC.S.RespondsToSelector, sel) != 0; - public bool HasUnifiedMemory => ObjC.SendBool(H, ObjC.S.HasUnifiedMemory) != 0; - public ulong RecommendedMaxWorkingSetSize => (ulong)ObjC.SendNuint(H, ObjC.S.RecommendedMaxWorkingSetSize); - public nuint CurrentAllocatedSize => ObjC.SendNuint(H, ObjC.S.CurrentAllocatedSize); - public ulong RegistryID => (ulong)ObjC.SendNuint(H, ObjC.S.RegistryID); - - public static MtlDevice? Probe() - { - if (!OperatingSystem.IsMacOS()) return null; - using var pool = AutoReleasePool.Create(); - // MTLCreateSystemDefaultDevice follows the Create rule: already +1. - IntPtr h = ObjC.CreateSystemDefaultDevice(); - return h == 0 ? null : new MtlDevice(h); - } - - /// Pipeline for a kernel in the embedded shader library, - /// compiled once and cached per device. - internal MtlPipeline Pipe(string name) - { - lock (_pipes) - { - if (!_pipes.TryGetValue(name, out MtlPipeline? p)) - { - p = NewPipeline(ShaderLibrary(), name); - // every kernel is dispatched at 256 threads/threadgroup - if (p.MaxTotalThreadsPerThreadgroup < 256) - { - p.Dispose(); - throw new NotSupportedException( - $"Metal: {name} pso supports {p.MaxTotalThreadsPerThreadgroup} threads/group"); - } - _pipes[name] = p; - } - return p; - } - } - - private MtlLibrary ShaderLibrary() - { - if (_lib is not null) return _lib; - var asm = typeof(MtlDevice).Assembly; - var names = asm.GetManifestResourceNames() - .Where(n => n.StartsWith("Sdcb.SimdGcdrNet.Backends.Metal.Shaders.") - && n.EndsWith(".metal")) - .OrderBy(n => n).ToArray(); - if (names.Length == 0) - throw new FileNotFoundException("Metal: no embedded .metal shaders"); - var sb = new System.Text.StringBuilder(); - foreach (var n in names) - { - using Stream s = asm.GetManifestResourceStream(n)!; - using var r = new StreamReader(s); - sb.AppendLine(r.ReadToEnd()); - } - return _lib = NewLibrary(sb.ToString()); - } - - public MtlBuffer NewBuffer(nuint bytes) - { - // MTLResourceStorageModeShared = 0 options. - IntPtr b = ObjC.Send1N1N(H, ObjC.S.NewBufferWithLengthOptions, bytes, 0); - if (b == 0) throw new InvalidOperationException($"Metal: newBufferWithLength({bytes}) failed"); - return new MtlBuffer(b, bytes); - } - - public unsafe MtlBuffer NewBuffer(void* src, nuint bytes) - { - IntPtr b = ObjC.Send1P2N(H, ObjC.S.NewBufferWithBytesLengthOptions, (IntPtr)src, bytes, 0); - if (b == 0) throw new InvalidOperationException($"Metal: newBufferWithBytes({bytes}) failed"); - return new MtlBuffer(b, bytes); - } - - /// Compile MSL source into a library. Throws with the NSError - /// localizedDescription on failure — MSL compile errors must never be - /// silently swallowed. - public unsafe MtlLibrary NewLibrary(string source, bool fastMath = false) - { - using var pool = AutoReleasePool.Create(); - IntPtr src = ObjC.NsStr(source); - IntPtr opts = MakeCompileOptions(fastMath); - IntPtr err = 0; - IntPtr lib; - lib = ObjC.Send3P(H, ObjC.S.NewLibraryWithSourceOptionsError, src, opts, (IntPtr)(&err)); - ObjC.Release(src); - ObjC.Release(opts); - if (lib == 0) throw new InvalidOperationException("MSL compile failed: " + DescribeError(err)); - return new MtlLibrary(lib); - } - - private static IntPtr MakeCompileOptions(bool fastMath) - { - // MTLCompileOptions: mathMode is a private-ish key; the public knob is - // fastMathEnabled (deprecated in MSL 3.x but still honored) plus - // languageVersion. MSL 3.1 = 0x30001 << 16? No — MTLLanguageVersion is - // (major<<16)+minor: 2.4=0x20004, 3.1=0x30001. We request 3.1 when - // supported for simdgroup_matrix; harmless if clamped. - IntPtr o = ObjC.Send0(ObjC.Send0(ObjC.ClassMTLCompileOptions, ObjC.S.Alloc), ObjC.S.Init); - ObjC.SendV1N(o, ObjC.S.SetLanguageVersion, (nuint)0x00030001); // MSL 3.1 - ObjC.SendV1B(o, ObjC.S.SetFastMathEnabled, (byte)(fastMath ? 1 : 0)); - return o; - } - - public unsafe MtlPipeline NewPipeline(MtlLibrary lib, string fnName, int[]? intConstants = null) - { - using var pool = AutoReleasePool.Create(); - IntPtr name = ObjC.NsStr(fnName); - IntPtr fn; - if (intConstants is { Length: > 0 }) - { - IntPtr fc = ObjC.Send0(ObjC.Send0(ObjC.ClassMTLFunctionConstantValues, ObjC.S.Alloc), ObjC.S.Init); - for (int i = 0; i < intConstants.Length; ++i) - { - int v = intConstants[i]; - // setConstantValue:type:atIndex: — (const void*, MTLDataType, NSUInteger). - ObjC.Send3P(fc, ObjC.S.SetConstantValueTypeAtIndex, (IntPtr)(&v), (IntPtr)(nint)(long)MtlDataType.Int, (IntPtr)(nint)i); - } - IntPtr err = 0; - fn = ObjC.Send3P(lib.H, ObjC.S.NewFunctionWithNameConstantValuesError, name, fc, (IntPtr)(&err)); - ObjC.Release(fc); - if (fn == 0) { ObjC.Release(name); throw new InvalidOperationException($"Metal: function '{fnName}' failed: {DescribeError(err)}"); } - } - else - { - fn = ObjC.Send1P(lib.H, ObjC.S.NewFunctionWithName, name); - } - ObjC.Release(name); - if (fn == 0) throw new InvalidOperationException($"Metal: no function '{fnName}' in library"); - - IntPtr err2 = 0; - IntPtr pso = ObjC.Send2P(H, ObjC.S.NewComputePipelineStateWithFunctionError, fn, (IntPtr)(&err2)); - ObjC.Release(fn); - if (pso == 0) throw new InvalidOperationException($"Metal: PSO '{fnName}' failed: {DescribeError(err2)}"); - return new MtlPipeline(pso, fnName); - } - - internal static unsafe string DescribeError(IntPtr err) - { - if (err == 0) return "(no NSError)"; - IntPtr desc = ObjC.Send0(err, ObjC.S.LocalizedDescription); - return ObjC.NsToString(desc) ?? "(unreadable NSError)"; - } - - /// Begin a command buffer + compute encoder pair. Both are - /// autoreleased — caller must hold an AutoReleasePool open. - public MtlEncoder BeginCommands() - { - IntPtr cmd = ObjC.Send0(_queue, ObjC.S.CommandBuffer); - IntPtr enc = ObjC.Send0(cmd, ObjC.S.ComputeCommandEncoder); - return new MtlEncoder(cmd, enc); - } - - /// Begin encoder with a specific dispatch type - /// (MTLDispatchType: 0=Serial, 1=Concurrent — concurrent allows the GPU to - /// overlap dispatches that don't depend on each other; serial inserts an - /// implicit barrier between dispatches which is what the schedule assumes). - public MtlEncoder BeginCommands(bool concurrent) - { - IntPtr cmd = ObjC.Send0(_queue, ObjC.S.CommandBuffer); - IntPtr enc = concurrent - ? ObjC.Send1N(cmd, ObjC.S.ComputeCommandEncoderWithDispatchType, 1) - : ObjC.Send0(cmd, ObjC.S.ComputeCommandEncoder); - return new MtlEncoder(cmd, enc); - } - - public void Dispose() - { - lock (_pipes) - { - foreach (MtlPipeline p in _pipes.Values) p.Dispose(); - _pipes.Clear(); - } - _lib?.Dispose(); - ObjC.Release(_queue); - ObjC.Release(H); - } -} - -internal sealed class MtlBuffer : IDisposable -{ - public IntPtr H; - public readonly nuint Length; - public MtlBuffer(IntPtr h, nuint length) { H = h; Length = length; } - public IntPtr Contents => ObjC.Send0(H, ObjC.S.Contents); - public void Dispose() { ObjC.Release(H); H = 0; } -} - -internal sealed class MtlLibrary : IDisposable -{ - public readonly IntPtr H; - public MtlLibrary(IntPtr h) => H = h; - public void Dispose() => ObjC.Release(H); -} - -internal sealed class MtlPipeline : IDisposable -{ - public readonly IntPtr Pso; - public readonly string Name; - public readonly nuint MaxTotalThreadsPerThreadgroup; - public readonly nuint ThreadExecutionWidth; - public MtlPipeline(IntPtr pso, string name) - { - Pso = pso; - Name = name; - MaxTotalThreadsPerThreadgroup = ObjC.SendNuint(pso, ObjC.S.MaxTotalThreadsPerThreadgroup); - ThreadExecutionWidth = ObjC.SendNuint(pso, ObjC.S.ThreadExecutionWidth); - } - public void Dispose() => ObjC.Release(Pso); -} - -/// One-shot command-buffer + compute-encoder pair. The objects are -/// autoreleased; keep the enclosing AutoReleasePool alive until Commit has -/// been issued (and waitUntilCompleted returned if you read results). -internal readonly struct MtlEncoder -{ - public readonly IntPtr Cmd; - public readonly IntPtr Enc; - - public MtlEncoder(IntPtr cmd, IntPtr enc) { Cmd = cmd; Enc = enc; } - - public void SetPipeline(MtlPipeline p) => ObjC.Send1P(Enc, ObjC.S.SetComputePipelineState, p.Pso); - - public void SetBuffer(MtlBuffer buf, ulong byteOffset, int index) => - ObjC.SendV1P2N(Enc, ObjC.S.SetBufferOffsetAtIndex, buf.H, (nuint)byteOffset, (nuint)index); - - public unsafe void SetPc(in T value, int index) where T : unmanaged - { - fixed (T* p = &value) - ObjC.SendV1P2N(Enc, ObjC.S.SetBytesLengthAtIndex, (IntPtr)p, (nuint)sizeof(T), (nuint)index); - } - - public unsafe void SetPc(void* data, nuint bytes, int index) => - ObjC.SendV1P2N(Enc, ObjC.S.SetBytesLengthAtIndex, (IntPtr)data, bytes, (nuint)index); - - public void SetThreadgroupMemory(nuint bytes, int index) => - ObjC.SendV1N1N(Enc, ObjC.S.SetThreadgroupMemoryLengthAtIndex, bytes, (nuint)index); - - public void Dispatch(nuint gx, nuint gy, nuint gz, nuint tx, nuint ty, nuint tz) => - ObjC.SendV2Size(Enc, ObjC.S.DispatchThreadgroupsThreadsPerThreadgroup, - new ObjC.MTLSize(gx, gy, gz), new ObjC.MTLSize(tx, ty, tz)); - - /// MTLBarrierScope buffers(=1): visibility barrier between - /// dispatches — needed when a later dispatch consumes a buffer written by - /// an earlier one and we used a Concurrent encoder. Serial encoders imply - /// this already. - public void BarrierBuffers() => ObjC.SendV1N(Enc, ObjC.S.MemoryBarrierWithScope, 1); - - public void End() => ObjC.Send0(Enc, ObjC.S.EndEncoding); - - public void Commit() => ObjC.Send0(Cmd, ObjC.S.Commit); - - /// Wait for completion and check status; throws with NSError on - /// any non-completed terminal state (incl. GPU watchdog kills). - public void CommitAndWait() - { - Commit(); - ObjC.Send0(Cmd, ObjC.S.WaitUntilCompleted); - long status = (long)ObjC.SendLong(Cmd, ObjC.S.Status); - // MTLCommandBufferStatus: 4=Completed; 5=Error; others shouldn't occur post-wait. - if (status != 4) - { - IntPtr err = ObjC.Send0(Cmd, ObjC.S.Error); - throw new InvalidOperationException($"Metal command buffer failed (status={status}): {MtlDevice.DescribeError(err)}"); - } - } - - /// GPU-side wall time of the buffer (seconds). 0 when timestamps - /// are unavailable (paravirt drivers often return 0). - public double GpuDurationSeconds() - { - double s = ObjC.SendDouble(Cmd, ObjC.S.GpuStartTime); - double e = ObjC.SendDouble(Cmd, ObjC.S.GpuEndTime); - return e > s ? e - s : 0; - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MetalBackend.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MetalBackend.cs deleted file mode 100644 index 735e478..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MetalBackend.cs +++ /dev/null @@ -1,86 +0,0 @@ -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; - -namespace Sdcb.SimdGcdrNet.Backends.Metal; - -/// -/// Metal backend root: the device, the per-(model, shape) graph cache, and -/// factories for inference sessions and fused enhancer pipelines. Thread-safe; -/// dispose with the process or when the backend is torn down. -/// -internal sealed class MetalBackend : IDisposable -{ - internal readonly MtlDevice Dev; - private readonly Dictionary<(OnnxModel m, string shape, bool physIo), MtlGraphModel> _graphs = new(); - private readonly object _gate = new(); - - private static readonly Lazy s_default = new(() => new MetalBackend()); - internal static MetalBackend Default => s_default.Value; - - internal static CompileOptions PlanOptions => new(Nhwc: true, PhysicalIo: false, Gpu: true); - internal static CompileOptions PipeOptions => new(Nhwc: true, PhysicalIo: true, Gpu: true); - - internal MetalBackend() - { - Dev = MtlDevice.Probe() ?? throw new PlatformNotSupportedException("Metal: no device"); - if (Environment.GetEnvironmentVariable("GCDRNET_MTL_VERBOSE") == "1") - Console.Error.WriteLine($"[metal] device: {Dev.Name}"); - } - - /// New inference session for a model and input shape (non-physical I/O plan). - public MtlInferenceSession NewSession(OnnxModel m, int[] shape) - { - MtlGraphModel g = AcquireGraph(m, shape, physIo: false); // caller ref: later evictions can't free it under us - try { return new MtlInferenceSession(Dev, g.Plan, g); } - finally { g.Release(); } - } - - /// The cached graph for (model, shape); acquires a reference for the caller. - internal MtlGraphModel GraphFor(OnnxModel m, int[] shape, bool physIo = false) - { - var key = (m, string.Join('x', shape), physIo); - lock (_gate) - { - if (_graphs.TryGetValue(key, out var gg)) - { - gg.Acquire(); - return gg; - } - } - ExecPlan plan = GraphCompiler.Compile(m, shape, physIo ? PipeOptions : PlanOptions); - var g = new MtlGraphModel(Dev, plan, constKey: m); - lock (_gate) - { - if (_graphs.TryGetValue(key, out var won)) { g.Release(); won.Acquire(); return won; } - _graphs[key] = g; - // bound the cache: graphs share the model's const store, so eviction only - // frees the small per-size plan+dispatch list - while (_graphs.Count > MaxGraphs) - { - var victim = _graphs.Keys.First(k => !k.Equals(key)); - _graphs.Remove(victim, out var old); - old?.Release(); - } - g.Acquire(); - return g; - } - } - private const int MaxGraphs = 32; - - /// Alias: acquires the graph for the caller (caller must Release). - internal MtlGraphModel AcquireGraph(OnnxModel m, int[] shape, bool physIo) - => GraphFor(m, shape, physIo); - - /// New fused enhancer worker (size-agnostic; binds per frame). - internal MtlGcdrPipeline NewWorker(OnnxModel gc, OnnxModel dr) => new(this, gc, dr); - - /// New fused enhancer worker pre-bound to a size. - internal MtlGcdrPipeline NewPipe(OnnxModel gc, OnnxModel dr, int ih, int iw) - { - var p = new MtlGcdrPipeline(this, gc, dr); - p.Bind(ih, iw); - return p; - } - - public void Dispose() => Dev.Dispose(); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlGraph.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlGraph.cs deleted file mode 100644 index e06e77c..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlGraph.cs +++ /dev/null @@ -1,798 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using Sdcb.SimdGcdrNet.Engine; - -namespace Sdcb.SimdGcdrNet.Backends.Metal; - -/// Binding slots resolved per-session: which underlying buffer. -internal enum MtlSlot : int { Arena = 0, Consts = 1, Dummy = 2, Direct = 3 } - -internal readonly struct MtlBind -{ - public readonly MtlSlot Slot; - public readonly MtlBuffer? Buf; // when Slot == Direct - public readonly ulong Offset, Size; - public MtlBind(MtlSlot s, ulong off, ulong size) { Slot = s; Offset = off; Size = size; } - public MtlBind(MtlBuffer b, ulong off, ulong size) { Slot = MtlSlot.Direct; Buf = b; Offset = off; Size = size; } -} - -/// One recorded compute dispatch. -internal sealed class MtlDispatch -{ - public required MtlPipeline Pipe; - public required MtlBind[] Binds; - public required byte[] Pc; - public uint Gx, Gy = 1, Gz = 1; - public string Name = ""; -} - -/// -/// Device-side const tensors (backend-packed weights) plus a 64B zero buffer, -/// shared by every of one model: per-size plans copy -/// the same initializers, so instead of re-packing and re-uploading the weights -/// per input size, a new graph whose const list is element-for-element identical -/// (same consumer packing, same contents) reuses the existing buffer. A graph -/// whose consts differ gets a private, unshared store. Ref-counted; the -/// buffers are freed when the last graph releases. -/// -internal sealed class MtlConstStore -{ - private static readonly Dictionary<(MtlDevice dev, object key), MtlConstStore> s_shared = new(); - - private readonly (MtlDevice, object)? _key; - private readonly float[][] _src; - private readonly int[] _kinds; - private int _refs = 1; - internal readonly MtlBuffer Consts, Dummy; - internal readonly ulong[] Offsets; - internal readonly ulong Bytes; - - private MtlConstStore(MtlDevice dev, (MtlDevice, object)? key, float[][] src, int[] kinds, - ulong[] offsets, byte[] host) - { - _key = key; - _src = src; - _kinds = kinds; - Offsets = offsets; - Bytes = (ulong)host.LongLength; - unsafe - { - fixed (byte* hp = host) Consts = dev.NewBuffer(hp, (nuint)host.Length); - byte* z = stackalloc byte[64]; - new Span(z, 64).Clear(); - Dummy = dev.NewBuffer(z, 64); - } - } - - /// - /// Returns a store holding (kinds = consumer packing ids), - /// shared under when an identical list was uploaded before. - /// (offsets, host bytes) only runs when a new store is built. - /// - internal static MtlConstStore Acquire(MtlDevice dev, object? key, float[][] src, int[] kinds, - Func<(ulong[] offsets, byte[] host)> pack) - { - if (key is null) - { - var (o, h) = pack(); - return new MtlConstStore(dev, null, src, kinds, o, h); - } - var k = (dev, key); - lock (s_shared) - { - if (s_shared.TryGetValue(k, out MtlConstStore? s) && s.Matches(src, kinds)) - { - s._refs++; - return s; - } - var (offs, host) = pack(); - bool share = s is null; // keep the first registered store on a mismatch - var ns = new MtlConstStore(dev, share ? k : null, src, kinds, offs, host); - if (share) s_shared[k] = ns; - return ns; - } - } - - private bool Matches(float[][] src, int[] kinds) - { - if (src.Length != _src.Length || !kinds.AsSpan().SequenceEqual(_kinds)) return false; - for (int i = 0; i < src.Length; i++) - if (!ReferenceEquals(src[i], _src[i]) && !src[i].AsSpan().SequenceEqual(_src[i])) return false; - return true; - } - - internal void Release() - { - lock (s_shared) - { - if (--_refs > 0) return; - if (_key is { } k && s_shared.TryGetValue(k, out MtlConstStore? cur) && cur == this) - s_shared.Remove(k); - } - Consts.Dispose(); - Dummy.Dispose(); - } -} - -/// -/// Per-(device, ExecPlan) half of the Metal backend: backend-private constant -/// packing/upload and the node list compiled to -/// records. Sessions own the activation arena and resolve slot ids to live -/// buffers. Activations and packed weights are stored as fp16; all shader math -/// is fp32; conv bias and LayerNorm params stay fp32. -/// -internal sealed class MtlGraphModel : IDisposable -{ - private readonly MtlDevice _dev; - private readonly ExecPlan _plan; - private int _refs = 1; // the graph cache holds the first reference - private readonly MtlConstStore _store; - internal readonly MtlBuffer Consts; // all const tensors, backend-packed (shared store) - private readonly Dictionary<(int v, CK k), ulong> _constOff = new(); - internal readonly MtlBuffer Dummy; // 64B zero buffer for unused bindings - internal readonly MtlDispatch[] Dispatches; - internal readonly ulong ConstBytes; - /// Activations + weights stored as float16 (fp32 math in every shader). - internal const bool F16 = true; - /// Bytes per stored activation element. - internal int Es => 2; - - /// How a const is laid out on the device (decided by its consumer). - private enum CK { Raw, F32, ConvW, DwW, ConvWT, GemmNK } - - /// Metal device the graph is lowered for. - /// Compiled execution plan to lower. - /// Sharing key for the const store (the source model): graphs of the - /// same model at other sizes reuse its uploaded weights. Null = private store. - internal MtlGraphModel(MtlDevice dev, ExecPlan plan, object? constKey = null) - { - _dev = dev; - _plan = plan; - if (!plan.Nhwc) throw new NotSupportedException("Metal backend requires a layout-planned (NHWC) plan."); - - // ---- consts: per-consumer packing ---- - var need = new List<(int v, CK k)>(); - void Need(int v, CK k) - { - if (v < 0 || plan.Values[v].Floats is null) return; - if (!need.Contains((v, k))) need.Add((v, k)); - } - foreach (ExecNode n in plan.Nodes) - { - switch (n.P) - { - case ConvP cp: - Need(n.In[0], CK.Raw); - Need(n.In[1], cp.Group > 1 ? CK.DwW : UseMmConv(plan, n, cp) ? CK.ConvWT : CK.ConvW); - Need(cp.BiasValue, CK.F32); - Need(cp.AddValue, CK.Raw); - break; - case GemmP gp: - Need(n.In[0], CK.Raw); Need(n.In[1], UseMmGemm(plan, n, gp) ? CK.GemmNK : CK.Raw); - Need(gp.BiasValue, CK.F32); - break; - case LnP: - Need(n.In[0], CK.Raw); Need(n.In[1], CK.F32); Need(n.In[2], CK.F32); - break; - case UpHeadP up: - Need(n.In[2], CK.Raw); - if (n.In.Length > 3) Need(n.In[3], CK.F32); - if (up.SkipConv is not null) { Need(up.SkipW, CK.ConvW); Need(up.SkipB, CK.F32); } - break; - default: - foreach (int iv in n.In) Need(iv, CK.Raw); - break; - } - } - foreach (var (v, k) in need) - { - CValue cv = plan.Values[v]; - if (k == CK.Raw && cv.Layout is { } l && !Engine.Layout.IsIdentity(l)) - throw new NotSupportedException($"Metal: const '{cv.Name}' with non-identity layout"); - } - float[][] srcs = need.Select(x => plan.Values[x.v].Floats!).ToArray(); - int[] kinds = need.Select(x => (int)x.k).ToArray(); - _store = MtlConstStore.Acquire(dev, constKey, srcs, kinds, () => - { - ulong total = 0; - var offs = new ulong[need.Count]; - var items = new List<(ulong off, float[] data, bool half)>(); - for (int i = 0; i < need.Count; i++) - { - var (v, k) = need[i]; - CValue cv = plan.Values[v]; - float[] data = k switch - { - CK.ConvW => PackConv(cv), - CK.ConvWT => PackConvT(cv), - CK.GemmNK => PackGemmNK(cv, plan.Nodes.First(m => m.P is GemmP && m.In[1] == v)), - CK.DwW => PackDw(cv), - _ => cv.Floats!, - }; - bool half = k != CK.F32; - total = (total + 63) & ~63UL; - offs[i] = total; - items.Add((total, data, half)); - total += (ulong)data.Length * (half ? 2UL : 4UL); - } - byte[] host = new byte[Math.Max((total + 63) & ~63UL, 64)]; - foreach (var (off, data, half) in items) - { - if (half) - { - Span dst = MemoryMarshal.Cast(host.AsSpan(checked((int)off), data.Length * 2)); - for (int i = 0; i < data.Length; i++) dst[i] = (Half)data[i]; - } - else - MemoryMarshal.AsBytes(data.AsSpan()).CopyTo(host.AsSpan(checked((int)off))); - } - return (offs, host); - }); - for (int i = 0; i < need.Count; i++) _constOff[need[i]] = _store.Offsets[i]; - Consts = _store.Consts; - Dummy = _store.Dummy; - ConstBytes = _store.Bytes; - - var disps = new List(); - NodeRanges = new (int start, int count)[plan.Nodes.Length]; - for (int i = 0; i < plan.Nodes.Length; i++) - { - int start = disps.Count; - disps.AddRange(Emit(plan.Nodes[i])); - NodeRanges[i] = (start, disps.Count - start); - } - Dispatches = disps.ToArray(); - } - - // ONNX [OC][IC][KH][KW] -> [K][OC], k = (ky*KW + kx)*IC + ic - private static float[] PackConv(CValue w) - { - int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3]; - float[] src = w.Floats!, dst = new float[src.Length]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < taps; t++) - dst[(t * ic + c) * oc + o] = src[(o * ic + c) * taps + t]; - return dst; - } - - // ONNX [OC][IC][KH][KW] -> [OC][Kpad], k = (ky*KW + kx)*IC + ic, Kpad = K rounded up to 8 - private static float[] PackConvT(CValue w) - { - int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3], k = (taps * ic + 7) & ~7; - float[] src = w.Floats!, dst = new float[oc * k]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < taps; t++) - dst[o * k + t * ic + c] = src[(o * ic + c) * taps + t]; - return dst; - } - - /// simdgroup-MMA conv eligibility (conv_mm): dense, output channels in - /// 16 B quads, an epilogue it implements, M within the grid limit. cin%4!=0 uses - /// the scalar-gather variant; constant pad must be zero. - private static bool UseMmConv(ExecPlan plan, ExecNode n, ConvP cp) - { - if (cp.Group != 1 || cp.Pool is not null || cp.Dilations.Any(d => d != 1)) return false; - if (cp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false; - if (cp.PadMode == PadMode.Constant && cp.PadValue != 0f) return false; - int[] so = plan.Values[n.Out[0]].PhysShape; // [1,h,w,c] - long m = (long)so[1] * so[2]; - return so[3] % 8 == 0 && m / 64 <= int.MaxValue; - } - - // Gemm B -> [N][K] (k fastest), the conv_mm weight layout - private static float[] PackGemmNK(CValue b, ExecNode n) - { - var gp = (GemmP)n.P!; - if (gp.TransB) return b.Floats!; // already [N][K] - int k = b.Shape[0], nn = b.Shape[1]; - float[] src = b.Floats!, dst = new float[src.Length]; - for (int i = 0; i < k; i++) - for (int j = 0; j < nn; j++) dst[j * k + i] = src[i * nn + j]; - return dst; - } - - /// MMA Gemm (a 1x1 conv over M "pixels" through conv_mm): const B, - /// K/N in 16 B quads, relu/gelu/none epilogue. - private static bool UseMmGemm(ExecPlan plan, ExecNode n, GemmP gp) - { - if (plan.Values[n.In[1]].Floats is null || plan.Values[n.In[1]].Shape.Length != 2) return false; - // B packed per value: only when no other node reads it (its transB could differ) - if (plan.Nodes.Count(m => m.In.Contains(n.In[1])) != 1) return false; - if (gp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false; - CValue a = plan.Values[n.In[0]]; - int k = a.Shape[^1], nn = plan.Values[n.Out[0]].Shape[^1]; - if (a.PhysShape[^1] != k) return false; - long m = a.ElementCount / k; - return k % 8 == 0 && nn % 8 == 0 && m / 64 <= int.MaxValue; - } - - // ONNX [C][1][KH][KW] -> [tap][C] - private static float[] PackDw(CValue w) - { - int c = w.Shape[0], taps = w.Shape[2] * w.Shape[3]; - float[] src = w.Floats!, dst = new float[src.Length]; - for (int ch = 0; ch < c; ch++) - for (int t = 0; t < taps; t++) dst[t * c + ch] = src[ch * taps + t]; - return dst; - } - - /// Dispatch index range per source ExecNode (for per-node debug runs). - internal readonly (int start, int count)[] NodeRanges; - - // ---- value → bind resolution (byte offsets/sizes: all long math) ---- - private MtlBind Bind(int valueId, CK kind = CK.Raw) - { - CValue v = _plan.Values[valueId]; - if (v.Buf >= 0) - return new MtlBind(MtlSlot.Arena, (ulong)_plan.BufOffset[v.Buf] * (ulong)Es, - (ulong)Math.Max(v.ElementCount, 1) * (ulong)Es); - if (_constOff.TryGetValue((valueId, kind), out ulong off)) - { - ulong es = kind != CK.F32 ? 2UL : 4UL; - // ConvWT rows are K-padded to 8: bind the packed length, not the logical one - long elems = kind == CK.ConvWT - ? (long)v.Shape[0] * ((v.Shape[1] * v.Shape[2] * v.Shape[3] + 7) & ~7) - : Math.Max(v.ElementCount, 1); - return new MtlBind(MtlSlot.Consts, off, (ulong)elems * es); - } - if (v.Floats != null) - throw new InvalidOperationException($"const value '{v.Name}' not packed as {kind}"); - throw new InvalidOperationException($"value '{v.Name}' has no GPU storage (ints must be folded)"); - } - - private static byte[] PcOf(params uint[] ints) - { - byte[] b = new byte[ints.Length * 4]; - Buffer.BlockCopy(ints, 0, b, 0, b.Length); - return b; - } - - private static byte[] PcOfF(uint[] ints, params float[] floats) - { - byte[] b = new byte[(ints.Length + floats.Length) * 4]; - Buffer.BlockCopy(ints, 0, b, 0, ints.Length * 4); - Buffer.BlockCopy(floats, 0, b, ints.Length * 4, floats.Length * 4); - return b; - } - - private static uint[] U(params int[] v) { uint[] u = new uint[v.Length]; for (int i = 0; i < v.Length; i++) u[i] = (uint)v[i]; return u; } - - private static uint G(uint n) => (n + 255) / 256; - - /// - /// Broadcast strides (padded rank-4, storage order of the output) of operand - /// onto output . Output physical dim i - /// holds logical dim o.Layout[i]; the operand's logical dim is that minus the - /// numpy right-alignment offset. - /// - private static int[] PhysBroadcastStrides(CValue a, CValue o) - { - int rank = o.Shape.Length, off = rank - a.Shape.Length; - if (rank > 4) throw new NotSupportedException("Metal elementwise: rank > 4"); - int[] lo = o.Layout ?? Engine.Layout.Identity(rank); - int[] la = a.Layout ?? Engine.Layout.Identity(a.Shape.Length); - int[] ap = a.PhysShape; - int[] aStr = new int[ap.Length]; - for (int i = ap.Length - 1, acc = 1; i >= 0; i--) { aStr[i] = acc; acc *= ap[i]; } - int[] r4 = new int[4]; - for (int i = 0; i < rank; i++) - { - int d = lo[i] - off; - r4[4 - rank + i] = d < 0 || a.Shape[d] == 1 ? 0 : aStr[Array.IndexOf(la, d)]; - } - return r4; - } - - private IEnumerable Emit(ExecNode n) - { - CValue[] V = _plan.Values; - // all addressing on storage-order shapes - int[] shapeIn(int k) => V[n.In[k]].PhysShape; - int[] shapeOut() => V[n.Out[0]].PhysShape; - int nOut() => checked((int)V[n.Out[0]].ElementCount); - - switch (n.Op) - { - case ExecOp.Copy: - yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)nOut(), 4), G((uint)nOut())); - break; - - case ExecOp.Relu or ExecOp.Sigmoid or ExecOp.Erf or ExecOp.Gelu: - { - uint mode = n.Op switch - { - ExecOp.Relu => 0, ExecOp.Sigmoid => 1, ExecOp.Erf => 2, _ => 3, - }; - yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)nOut(), mode), G((uint)nOut())); - break; - } - - case ExecOp.Elementwise: - { - var p = (EwP)n.P!; - CValue a = V[n.In[0]], bv = V[n.In[1]], o = V[n.Out[0]]; - int[] os = Pad4(shapeOut()); - int[] ostr = Strides(os); - int[] astr = PhysBroadcastStrides(a, o); - int[] bstr = PhysBroadcastStrides(bv, o); - uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, _ => 0u }; - // same-storage fast path: vec4 (n % 4 guard — no OOB tail writes) - bool dense = astr.SequenceEqual(ostr) && bstr.SequenceEqual(ostr); - if (nOut() % 4 == 0 && dense) - { - yield return ND("ew2v", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])], - PcOf((uint)(nOut() / 4), (uint)p.Op, act), G((uint)(nOut() / 4))); - break; - } - yield return ND("ew2", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - astr[0], astr[1], astr[2], astr[3], - bstr[0], bstr[1], bstr[2], bstr[3], (int)p.Op, (int)act)), - G((uint)nOut())); - break; - } - - case ExecOp.Conv: - { - var p = (ConvP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - if (p.Pool is { } pool) - { - // small-ic stem conv + 2x2/2 max pool (GraphCompiler.GpuSmallConv) - int[] ci = shapeIn(0), co = shapeOut(), cw = V[n.In[1]].Shape; - if (!GraphCompiler.GpuSmallConv(p, cw, ci[3]) || p.AddValue >= 0 || ci[1] != co[1] * 2 || ci[2] != co[2] * 2) - throw new NotSupportedException("Metal convpool: unsupported conv"); - uint pf = (p.BiasValue >= 0 ? 1u : 0u) | (p.PadMode == PadMode.Reflect ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) | (pool.FuseRelu ? 8u : 0u); - bool k3 = p.Kernel[0] == 3 && p.Kernel[1] == 3; - yield return ND(k3 ? "convpool_k3" : "convpool", - [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOfF(U(ci[3], ci[1], ci[2], cw[0], co[1], co[2], p.Kernel[0], p.Kernel[1], p.Pads[0], p.Pads[1], (int)pf), - p.PadMode == PadMode.Constant ? p.PadValue : 0f), - G((uint)co[1] * (uint)co[2] * (uint)(cw[0] / 8)), name: "convpool"); - break; - } - if (p.AddValue >= 0) RequireNhwc(p.AddValue); - int[] si = shapeIn(0), sw_ = V[n.In[1]].Shape, so = shapeOut(); // [1,h,w,c] - uint ih = (uint)si[1], iw = (uint)si[2], ic = (uint)si[3]; - uint oc = (uint)sw_[0], oh = (uint)so[1], ow = (uint)so[2]; - if (p.PadMode == PadMode.Constant && p.PadValue != 0f) - throw new NotSupportedException("Metal conv: nonzero constant pad"); - if (p.Dilations.Any(d => d != 1)) - throw new NotSupportedException("Metal conv: dilation"); - if (p.Act == ActivationKind.Sigmoid) - throw new NotSupportedException("Metal conv: sigmoid epilogue"); - uint flags = (p.BiasValue >= 0 ? 1u : 0u) - | (p.PadMode == PadMode.Reflect ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) - | (p.AddValue >= 0 ? 8u : 0u) - | (p.Act == ActivationKind.Gelu ? 16u : 0u); - if (p.Group > 1) - { - if (p.Group != ic || oc != ic) throw new NotSupportedException("grouped non-depthwise conv"); - if (p.AddValue >= 0) throw new NotSupportedException("depthwise conv + residual"); - bool v4 = ic % 4 == 0; - yield return ND(v4 ? "depthwise4" : "depthwise", - [Bind(n.In[0]), Bind(n.In[1], CK.DwW), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOfF(U((int)ic, (int)ih, (int)iw, (int)oh, (int)ow, p.Kernel[0], p.Kernel[1], - p.Strides[0], p.Strides[1], p.Pads[0], p.Pads[1], (int)(flags & ~8u)), p.PadValue), - G(ic * oh * ow / (v4 ? 4u : 1u)), name: "depthwise"); - break; - } - if (UseMmConv(_plan, n, p)) - { - // BM = 64 rows (pixels); BN by oc divisibility; SCK when cin%4!=0 - uint sck = ic % 4 != 0 ? 1u : 0u; - (string cm, uint bn) = oc % 128 == 0 ? ("conv_mm_128", 128u) - : oc % 64 == 0 ? ("conv_mm_64", 64u) : ("conv_mm_32", 32u); - cm += "_" + sck; - uint m = oh * ow, kr = (uint)(p.Kernel[0] * p.Kernel[1]) * ic, kd = (kr + 7u) & ~7u; - uint cflags = (p.BiasValue >= 0 ? 1u : 0u) | (p.AddValue >= 0 ? 2u : 0u) - | (p.PadMode == PadMode.Reflect ? 4u : 0u) - | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4); - yield return ND(cm, - [Bind(n.In[0]), Bind(n.In[1], CK.ConvWT), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])], - PcOf(m, oc, kd, cflags, ow, iw, ih, ic, (uint)p.Kernel[1], - (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], kr), - (m + 63) / 64, (oc + bn - 1) / bn, name: "conv_mm"); - break; - } - yield return ND("conv_tile", - [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])], - PcOf(ic, ih, iw, oc, oh, ow, (uint)p.Kernel[0], (uint)p.Kernel[1], - (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], - flags, 0), - (ow + 15) / 16, (oh + 7) / 8, (oc + 63) / 64, name: "conv"); - break; - } - - case ExecOp.MaxPool: - { - var p = (PoolP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0), so = shapeOut(); - bool v4 = si[3] % 4 == 0; - yield return ND(v4 ? "maxpool4" : "maxpool", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], - (uint)p.Kernel[0], (uint)p.Kernel[1], (uint)p.Strides[0], (uint)p.Strides[1], - (uint)p.Pads[0], (uint)p.Pads[1], p.FuseRelu ? 1u : 0u), - G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "maxpool"); - break; - } - - case ExecOp.LayerNorm: - { - var p = (LnP)n.P!; - uint len = (uint)V[n.In[0]].Shape[^1]; - if ((uint)shapeIn(0)[^1] != len) throw new NotSupportedException("Metal LayerNorm: last dim not innermost"); - uint rows = (uint)(V[n.In[0]].ElementCount / len); - yield return ND("layernorm", - [Bind(n.In[0]), Bind(n.In[1], CK.F32), Bind(n.In[2], CK.F32), Bind(n.Out[0])], - PcOfF([len, rows], p.Eps), (rows + 7) / 8, name: "layernorm"); - break; - } - - case ExecOp.Gemm: - { - var p = (GemmP)n.P!; - uint k = (uint)V[n.In[0]].Shape[^1]; - if ((uint)shapeIn(0)[^1] != k) throw new NotSupportedException("Metal Gemm: K not innermost"); - uint m = (uint)(V[n.In[0]].ElementCount / k); - uint nn = (uint)V[n.Out[0]].Shape[^1]; - if (UseMmGemm(_plan, n, p)) - { - (string cm, uint bn) = nn % 128 == 0 ? ("conv_mm_128", 128u) - : nn % 64 == 0 ? ("conv_mm_64", 64u) : ("conv_mm_32", 32u); - cm += "_0"; - uint cflags = (p.BiasValue >= 0 ? 1u : 0u) - | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4); - // [M,K] x [N,K]^T as a 1x1 conv over a 1 x M image - yield return ND(cm, - [Bind(n.In[0]), Bind(n.In[1], CK.GemmNK), B(p.BiasValue, CK.F32), B(-1), Bind(n.Out[0])], - PcOf(m, nn, k, cflags, m, m, 1u, k, 1u, 1u, 1u, 0u, 0u, k), - (m + 63) / 64, (nn + bn - 1) / bn, name: "gemm_mm"); - break; - } - uint gflags = (p.TransB ? 1u : 0u) | (p.BiasValue >= 0 ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) - | (p.Act == ActivationKind.Gelu ? 8u : 0u); - yield return ND("gemm", - [Bind(n.In[0]), Bind(n.In[1]), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOf(m, nn, k, gflags), - (nn + 63) / 64, (m + 63) / 64, name: "gemm"); - break; - } - - case ExecOp.ResizeBilinear: - { - var p = (ResizeP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - bool hasAdd = n.In.Length > 1; - if (hasAdd) RequireNhwc(n.In[1]); - int[] si = shapeIn(0), so = shapeOut(); - uint flags = (p.FuseRelu ? 1u : 0u) | (hasAdd ? 2u : 0u) | (p.AddRelu ? 4u : 0u); - bool v4 = si[3] % 4 == 0; - yield return ND(v4 ? "bilinear4" : "bilinear", [Bind(n.In[0]), hasAdd ? Bind(n.In[1]) : B(-1), Bind(n.Out[0])], - PcOfF([(uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], flags], - p.Scales[2], p.Scales[3]), - G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "bilinear"); - break; - } - - case ExecOp.UpHead: - { - var p = (UpHeadP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.In[1]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0), so = shapeOut(); - uint c = (uint)si[3], oc = (uint)so[3]; - if (oc > 8 || c > 128 || c % 4 != 0) throw new NotSupportedException($"Metal UpHead: c={c} oc={oc}"); - bool hasBias = n.In.Length > 3; - uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, ActivationKind.None => 0u, - _ => throw new NotSupportedException($"Metal UpHead: {p.Act}") }; - uint flags = (p.ResizeRelu ? 1u : 0u) | (p.SkipRelu ? 2u : 0u) | (hasBias ? 4u : 0u) | (act << 3); - if (p.SkipConv is { } sc) - { - // skip recomputed from the stem conv's input (GraphCompiler.GpuSmallConv, c <= 32) - int[] xi = shapeIn(1), cw = V[p.SkipW].Shape; - if (!GraphCompiler.GpuSmallConv(sc, cw, xi[3]) || cw[0] != c || c > 32 || xi[1] != so[1] || xi[2] != so[2]) - throw new NotSupportedException("Metal UpHead: unsupported skip conv"); - uint cf = (sc.BiasValue >= 0 ? 1u : 0u) | (sc.PadMode == PadMode.Reflect ? 2u : 0u) - | (sc.Act == ActivationKind.Relu ? 4u : 0u); - yield return ND("uphead_sc", - [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0]), - Bind(p.SkipW, CK.ConvW), B(p.SkipB, CK.F32)], - PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], - p.Scales[2], p.Scales[3]).Concat(PcOfF(U(xi[3], sc.Kernel[0], sc.Kernel[1], sc.Pads[0], sc.Pads[1], (int)cf), - sc.PadMode == PadMode.Constant ? sc.PadValue : 0f)).ToArray(), - G((uint)so[1] * (uint)so[2]), name: "uphead_sc"); - break; - } - yield return ND("uphead", - [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0])], - PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], - p.Scales[2], p.Scales[3]), - G((uint)so[1] * (uint)so[2])); - break; - } - - case ExecOp.Pad: - { - // generic rank-4 pad in storage order (pads permuted to physical dims) - var p = (PadP)n.P!; - CValue x = V[n.In[0]]; - if (x.Shape.Length != 4) throw new NotSupportedException("Metal pad: rank != 4"); - int[] l = x.Layout ?? Engine.Layout.Identity(4); - int[] id4 = shapeIn(0), od4 = shapeOut(); - int[] pb4 = new int[4]; - for (int d = 0; d < 4; d++) pb4[d] = p.Pads[l[d]]; - yield return ND("pad", [Bind(n.In[0]), Bind(n.Out[0])], - PcOfF(U(id4[0], id4[1], id4[2], id4[3], od4[0], od4[1], od4[2], od4[3], - pb4[0], pb4[1], pb4[2], pb4[3], p.Mode == PadMode.Reflect ? 1 : 0), - p.Value), - G((uint)V[n.Out[0]].ElementCount)); - break; - } - - case ExecOp.Transpose: - { - // only planner relayouts reach here; the perm is physical - var p = (TransposeP)n.P!; - int[] is4 = Pad4(shapeIn(0)); - int[] os4 = Pad4(shapeOut()); - int[] ostr = Strides(os4), istr = Strides(is4); - int[] perm4 = PermuteTo4(p.Perm, shapeIn(0).Length); - yield return ND("transpose", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - istr[perm4[0]], istr[perm4[1]], istr[perm4[2]], istr[perm4[3]])), - G((uint)nOut())); - break; - } - - case ExecOp.Slice: - { - // SliceP.Axes are physical after the layout pass - var p = (SliceP)n.P!; - int[] is4 = Pad4(shapeIn(0)), os4 = Pad4(shapeOut()); - int[] ostr = Strides(os4), istr = Strides(is4); - uint[] st = [0, 0, 0, 0], sp = [1, 1, 1, 1]; - int shift = 4 - shapeIn(0).Length; - for (int i = 0; i < p.Axes.Length; i++) - { - st[p.Axes[i] + shift] = (uint)p.Starts[i]; - sp[p.Axes[i] + shift] = (uint)p.Steps[i]; - } - yield return ND("slice", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - (int)st[0], (int)st[1], (int)st[2], (int)st[3], - (int)sp[0], (int)sp[1], (int)sp[2], (int)sp[3], - istr[0], istr[1], istr[2], istr[3])), - G((uint)nOut())); - break; - } - - case ExecOp.Concat: - { - // ConcatP.Axis is physical after the layout pass - var p = (ConcatP)n.P!; - int[] os = shapeOut(); - int axis = p.Axis; - int inner = 1, dstAxis = os[axis]; - for (int d = axis + 1; d < os.Length; d++) inner *= os[d]; - int axisOff = 0; - foreach (int iv in n.In) - { - int[] ishape = V[iv].PhysShape; - int cnt = checked((int)V[iv].ElementCount); - if (cnt > 0) - { - yield return ND("concat_ax", [Bind(iv), Bind(n.Out[0])], - PcOf((uint)cnt, (uint)inner, (uint)ishape[axis], (uint)dstAxis, (uint)axisOff), - G((uint)cnt)); - } - axisOff += ishape[axis]; - } - break; - } - - case ExecOp.ChannelShift: - { - var p = (ShiftP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0); // [1,h,w,c] - var ge = p.GroupEnds; var of = p.Offsets; - if (ge.Length > 5) throw new NotSupportedException("Metal shift: > 5 groups"); - var pc = new List { (uint)si[3], (uint)si[1], (uint)si[2], (uint)p.Axis }; - for (int i = 0; i < 5; i++) pc.Add((uint)(i < ge.Length ? ge[i] : si[3])); - for (int i = 0; i < 5; i++) pc.Add((uint)(i < of.Length ? of[i] : 0)); - bool v4 = si[3] % 4 == 0 && ge.All(e => e % 4 == 0); - yield return ND(v4 ? "shift4" : "shift", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(pc.ToArray()), G((uint)nOut() / (v4 ? 4u : 1u)), name: "shift"); - break; - } - - default: - throw new NotSupportedException($"Metal: ExecOp {n.Op} not implemented"); - } - - MtlDispatch ND(string shader, MtlBind[] binds, byte[] pc, uint gx, uint gy = 1, uint gz = 1, string? name = null) - { - return new MtlDispatch - { - Pipe = _dev.Pipe(shader), - Pc = pc, Binds = binds, Gx = gx, Gy = gy, Gz = gz, - Name = $"{shader}({n.Name})", - }; - } - - MtlBind B(int valueId, CK kind = CK.Raw) => valueId >= 0 ? Bind(valueId, kind) : new MtlBind(MtlSlot.Dummy, 0, 64); - } - - private void RequireNhwc(int v) - { - CValue cv = _plan.Values[v]; - if (cv.Shape.Length != 4 || !Engine.Layout.Equivalent(cv.Shape, cv.Layout ?? Engine.Layout.Identity(4), Engine.Layout.Nhwc)) - throw new NotSupportedException($"Metal: '{cv.Name}' expected NHWC storage, got {Engine.Layout.Str(cv.Layout)}"); - } - - // pad a shape to rank 4 with leading 1s - private static int[] Pad4(int[] s) - { - if (s.Length == 4) return s; - int[] r = [1, 1, 1, 1]; - s.CopyTo(r, 4 - s.Length); - return r; - } - - private static int[] Strides(int[] shape4) - { - int[] s = new int[4]; s[3] = 1; - for (int i = 2; i >= 0; i--) s[i] = s[i + 1] * shape4[i + 1]; - return s; - } - - // lift an original-rank perm into padded rank-4 (leading dims map to themselves) - private static int[] PermuteTo4(int[] perm, int rank) - { - int shift = 4 - rank; - int[] p4 = [0, 1, 2, 3]; - for (int i = 0; i < rank; i++) p4[i + shift] = perm[i] + shift; - return p4; - } - - /// Sessions hold a reference so cache eviction can't free live const/dummy buffers. - public void Acquire() - { - if (Interlocked.Increment(ref _refs) <= 1) - throw new ObjectDisposedException(nameof(MtlGraphModel), "const buffers already freed"); - } - /// Debug/test: const buffer + byte offset of (value, packing-kind) for diagnostics. - internal (MtlBuffer buf, ulong off, bool half) DebugConst(int valueId, int kind) - { - return _constOff.TryGetValue((valueId, (CK)kind), out ulong o) - ? (Consts, o, (CK)kind != CK.F32) - : (Dummy, ulong.MaxValue, false); - } - - /// Acquire unless already freed (no resurrection; for racing cache readers). - public bool TryAcquire() - { - while (true) - { - int r = Volatile.Read(ref _refs); - if (r <= 0) return false; - if (Interlocked.CompareExchange(ref _refs, r + 1, r) == r) return true; - } - } - public void Release() - { - if (Interlocked.Decrement(ref _refs) == 0) _store.Release(); - } - - /// The plan this graph was compiled from (arena offsets, I/O values). - internal ExecPlan Plan => _plan; - public void Dispose() => Release(); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs index b55f67c..6e49615 100644 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs +++ b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs @@ -1,5 +1,7 @@ -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Backends.Metal; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Onnx; namespace Sdcb.SimdGcdrNet.Backends.Metal; @@ -56,9 +58,9 @@ internal void Bind(int ih, int iw) if (_gcG is null || h != _h || w != _w) { // a new size compiles two plans + graphs: build them side by side - var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], physIo: true)); + var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], physIo: true, GcdrExtensions.Instance)); MtlGraphModel gd; - try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], physIo: true); } + try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], physIo: true, GcdrExtensions.Instance); } catch { try { tg.GetAwaiter().GetResult().Release(); } catch { } throw; } MtlGraphModel gg; try { gg = tg.GetAwaiter().GetResult(); } @@ -109,7 +111,7 @@ private void RecordAll(MtlEncoder enc, int srcStride) uint h = (uint)_h, w = (uint)_w; // bgr2nhwc -> gc arena input - enc.SetPipeline(_dev.Pipe("bgr2nhwc")); + enc.SetPipeline(_dev.Pipe("bgr2nhwc", GcdrExtensions.Owner)); enc.SetBuffer(_bgrIn!, 0, 0); enc.SetBuffer(_arena!, _gcBase + InOff(gcPlan), 1); var pc = stackalloc uint[7] { h, w, (uint)_ih, (uint)_iw, (uint)_pt, (uint)_pl, (uint)srcStride }; @@ -121,7 +123,7 @@ private void RecordAll(MtlEncoder enc, int srcStride) // div6: gc input + gc output -> dr arena input uint hw = h * w; - enc.SetPipeline(_dev.Pipe("div6")); + enc.SetPipeline(_dev.Pipe("div6", GcdrExtensions.Owner)); enc.SetBuffer(_arena!, _gcBase + InOff(gcPlan), 0); enc.SetBuffer(_arena!, _gcBase + OutOff(gcPlan), 1); enc.SetBuffer(_arena!, InOff(drPlan), 2); @@ -134,7 +136,7 @@ private void RecordAll(MtlEncoder enc, int srcStride) // nhwc2bgr -> bgrOut uint nwords = ((uint)_ih * (uint)_iw * 3u + 3u) / 4u; - enc.SetPipeline(_dev.Pipe("nhwc2bgr")); + enc.SetPipeline(_dev.Pipe("nhwc2bgr", GcdrExtensions.Owner)); enc.SetBuffer(_arena!, OutOff(drPlan), 0); enc.SetBuffer(_bgrOut!, 0, 1); var pc3 = stackalloc uint[7] { h, w, (uint)_ih, (uint)_iw, (uint)_pt, (uint)_pl, nwords }; diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlSession.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlSession.cs deleted file mode 100644 index ceaab1e..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlSession.cs +++ /dev/null @@ -1,250 +0,0 @@ -using Sdcb.SimdGcdrNet.Engine; - -namespace Sdcb.SimdGcdrNet.Backends.Metal; - -/// -/// GPU counterpart of : one activation arena in -/// shared (CPU+GPU-visible) memory; Run() writes the input window straight -/// into the arena, records the graph's dispatches into one command buffer and -/// waits once — no staging copies (unified memory). Not thread-safe. -/// -public sealed unsafe class MtlInferenceSession : IDisposable -{ - private readonly MtlDevice _dev; - private readonly MtlGraphModel _graph; - private readonly ExecPlan _plan; - - internal readonly MtlBuffer Arena; // shared-memory activation arena - private readonly float[] _inHost, _outHost; - private readonly ulong _inOff, _outOff; - private readonly int _inFloats, _outFloats; - internal int Es => _graph.Es; // bytes per stored element (2 = fp16) - - internal MtlInferenceSession(MtlDevice dev, ExecPlan plan, MtlGraphModel graph) - { - _dev = dev; - _plan = plan; - _graph = graph; - _graph.Acquire(); - - Arena = dev.NewBuffer((nuint)((ulong)plan.ArenaFloats * (ulong)Es + 64)); - _inFloats = checked((int)plan.Values[plan.InputValue].ElementCount); - _outFloats = checked((int)plan.Values[plan.OutputValue].ElementCount); - _inOff = (ulong)plan.BufOffset[plan.Values[plan.InputValue].Buf] * (ulong)Es; - _outOff = (ulong)plan.BufOffset[plan.Values[plan.OutputValue].Buf] * (ulong)Es; - _inHost = new float[_inFloats]; - _outHost = new float[_outFloats]; - } - - internal MtlBuffer ResolveBuf(MtlBind b) => Resolve(b).buf; - internal ulong ResolveOff(MtlBind b) => b.Offset; - internal (MtlBuffer buf, ulong off, ulong size) Resolve(MtlBind b) => b.Slot switch - { - MtlSlot.Arena => (Arena, b.Offset, b.Size), - MtlSlot.Consts => (_graph.Consts, b.Offset, b.Size), - MtlSlot.Direct => (b.Buf!, b.Offset, b.Size), - _ => (_graph.Dummy, b.Offset, b.Size), - }; - - /// Input tensor shape (e.g. [1,3,H,W]). - public int[] InputShape => _plan.Values[_plan.InputValue].Shape; - /// Output tensor shape (e.g. [1,3,H,W]). - public int[] OutputShape => _plan.Values[_plan.OutputValue].Shape; - - /// The writable input window (staged into the arena on ). - public Span Input => _inHost; - /// The output tensor after . - public ReadOnlySpan Output => _outHost; - - internal ulong InputOffset => _inOff; - internal ulong OutputOffset => _outOff; - internal ulong InputBytes => (ulong)_inFloats * (ulong)Es; - internal ulong OutputBytes => (ulong)_outFloats * (ulong)Es; - - /// Host float -> stored elements (fp16 narrowing). - internal static void ToStorage(ReadOnlySpan src, void* dst) - { - var d = new Span(dst, src.Length); - for (int i = 0; i < src.Length; i++) d[i] = (Half)src[i]; - } - - /// Stored elements -> host float (fp16 widening). - internal static void FromStorage(void* src, Span dst) - { - var sp = new ReadOnlySpan(src, dst.Length); - for (int i = 0; i < dst.Length; i++) dst[i] = (float)sp[i]; - } - - /// Downloads dst.Length stored elements at byte offset . - internal void DownloadElems(MtlBuffer buf, ulong off, Span dst) - => FromStorage((byte*)buf.Contents + off, dst); - - /// Record one dispatch on an open encoder. - internal static void RecordOne(MtlEncoder enc, MtlGraphModel g, - MtlDispatch d, MtlBuffer arena, ulong arenaBase = 0) - { - enc.SetPipeline(d.Pipe); - for (int b = 0; b < d.Binds.Length; b++) - { - MtlBind mb = d.Binds[b]; - (MtlBuffer buf, ulong off, _) = mb.Slot switch - { - MtlSlot.Arena => (arena, arenaBase + mb.Offset, mb.Size), - MtlSlot.Consts => (g.Consts, mb.Offset, mb.Size), - MtlSlot.Direct => (mb.Buf!, mb.Offset, mb.Size), - _ => (g.Dummy, mb.Offset, mb.Size), - }; - enc.SetBuffer(buf, off, b); - } - if (d.Pc.Length > 0) - fixed (byte* pc = d.Pc) - enc.SetPc(pc, (nuint)d.Pc.Length, d.Binds.Length); - enc.Dispatch(d.Gx, d.Gy, d.Gz, 256, 1, 1); - } - - /// Record all dispatches on an open encoder (serial dispatch order). - internal void RecordCompute(MtlEncoder enc) - { - foreach (var d in _graph.Dispatches) - RecordOne(enc, _graph, d, Arena); - } - - /// Debug/test: run the dispatches of ExecNode i (sequential state persists). - internal void DebugRunNode(int i) - { - var (start, count) = _graph.NodeRanges[i]; - if (count > 0) DebugRunRange(start, count); - } - - /// Debug/test: a value's arena window as host floats. - internal float[] DebugValue(int valueId) - { - var dst = new float[checked((int)_plan.Values[valueId].ElementCount)]; - DebugDownload(valueId, dst); - return dst; - } - - /// Debug/test: run a single dispatch-list slice on its own command buffer. - internal void DebugRunRange(int dStart, int dCount) - { - using var pool = AutoReleasePool.Create(); - var enc = _dev.BeginCommands(); - for (int i = dStart; i < dStart + dCount; i++) - RecordOne(enc, _graph, _graph.Dispatches[i], Arena); - enc.End(); - enc.CommitAndWait(); - } - - /// - /// Debug/test: time each dispatch by running it alone (commit + wait) and - /// wall-timing; includes ~45us launch overhead per dispatch — rank only. - /// Returns name→best-of-2 ms. - /// - internal Dictionary DebugProfile() - { - int n = _graph.Dispatches.Length; - var best = new double[n]; - Array.Fill(best, double.MaxValue); - // warm pass to keep state identical to production (input already in arena) - using (var pool = AutoReleasePool.Create()) - { - var enc = _dev.BeginCommands(); - foreach (var d in _graph.Dispatches) RecordOne(enc, _graph, d, Arena); - enc.End(); - enc.CommitAndWait(); - } - var sw = new System.Diagnostics.Stopwatch(); - for (int rep = 0; rep < 2; rep++) - for (int i = 0; i < n; i++) - { - var d = _graph.Dispatches[i]; - using var pool = AutoReleasePool.Create(); - var enc = _dev.BeginCommands(); - RecordOne(enc, _graph, d, Arena); - enc.End(); - sw.Restart(); - enc.CommitAndWait(); - double ms = sw.Elapsed.TotalMilliseconds; - if (ms < best[i]) best[i] = ms; - } - var agg = new Dictionary(); - PerDispatch = new (string, double)[n]; - for (int i = 0; i < n; i++) - { - string name = _graph.Dispatches[i].Name ?? $"d{i}"; - PerDispatch[i] = (name, best[i]); - name = name[..(name.IndexOf('(') is int cut && cut > 0 ? cut : name.Length)]; - agg[name] = agg.TryGetValue(name, out double v) ? v + best[i] : best[i]; - } - return agg; - } - - /// Per-dispatch timings from the last call. - internal (string Name, double Ms)[]? PerDispatch; - - /// Debug: the compiled plan backing this session. - internal ExecPlan DebugPlan => _plan; - - /// Debug/test: const offset (ulong.MaxValue when not packed). - internal ulong DebugConstOff(int valueId, int kind) => _graph.DebugConst(valueId, kind).off; - - /// Debug/test: const storage of a value (kind 0 raw,1 f32,2 convw,3 dww,4 convwt,5 gemmnk). - internal float[] DebugConst(int valueId, int kind, int len) - { - var (buf, off, half) = _graph.DebugConst(valueId, kind); - var dst = new float[len]; - if (half) - { - var h = new ReadOnlySpan((byte*)buf.Contents + off, len); - for (int i = 0; i < len; i++) dst[i] = (float)h[i]; - } - else - new ReadOnlySpan((byte*)buf.Contents + off, len).CopyTo(dst); - return dst; - } - - /// Debug/test: dispatch names for ExecNode i. - internal string[] DebugNodeDispatches(int i) - { - var (st, cnt) = _graph.NodeRanges[i]; - return _graph.Dispatches.Skip(st).Take(cnt).Select(d => d.Name ?? "").ToArray(); - } - - /// Debug: read the whole arena (ArenaFloats floats) into dst. - internal void DebugArena(Span dst) - => FromStorage(Arena.Contents.ToPointer(), dst.Slice(0, _plan.ArenaFloats)); - - /// Debug: graph consts buffer + the session's arena buffer. - internal (MtlBuffer consts, MtlBuffer arena) DebugBuffers => (_graph.Consts, Arena); - - /// Debug/test: download an activation value's region into dst. - internal void DebugDownload(int valueId, float[] dst) - { - CValue v = _plan.Values[valueId]; - FromStorage((byte*)Arena.Contents + (ulong)_plan.BufOffset[v.Buf] * (ulong)Es, - dst.AsSpan(0, checked((int)v.ElementCount))); - } - - /// Debug/test: copy host input into the arena's input region. - internal void DebugUploadInput() - => ToStorage(_inHost, (byte*)Arena.Contents + _inOff); - - /// Write input into the arena and run all dispatches. - public void Run() - { - ToStorage(_inHost, (byte*)Arena.Contents + _inOff); - using var pool = AutoReleasePool.Create(); - var enc = _dev.BeginCommands(); - RecordCompute(enc); - enc.End(); - enc.CommitAndWait(); - FromStorage((byte*)Arena.Contents + _outOff, _outHost); - } - - /// Releases the session's device arena and its reference to the shared graph. - public void Dispose() - { - Arena.Dispose(); - _graph.Release(); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/ObjC.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/ObjC.cs deleted file mode 100644 index d0852a9..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/ObjC.cs +++ /dev/null @@ -1,229 +0,0 @@ -using System.Runtime.InteropServices; - -namespace Sdcb.SimdGcdrNet.Backends.Metal; - -// Minimal libobjc P/Invoke layer — pure net10.0 C#, no macOS workload. -// Ownership follows Cocoa rules: alloc/init/newXxx/copy return +1 (caller -// releases); everything else is autoreleased within the enclosing -// NSAutoreleasePool. Selectors and class handles are cached in static -// readonly fields — a dispatch already costs enough objc_msgSend calls that -// re-registering selector strings per call is wasted marshaling. -internal static unsafe partial class ObjC -{ - private const string LibObjC = "libobjc.dylib"; - private const string LibMetal = "/System/Library/Frameworks/Metal.framework/Metal"; - - [LibraryImport(LibObjC, EntryPoint = "objc_getClass")] - private static partial IntPtr GetClassRaw(IntPtr name); - - [LibraryImport(LibObjC, EntryPoint = "sel_registerName")] - private static partial IntPtr RegSelRaw(IntPtr name); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send0(IntPtr recv, IntPtr sel); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send1P(IntPtr recv, IntPtr sel, IntPtr a); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send2P(IntPtr recv, IntPtr sel, IntPtr a, IntPtr b); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send3P(IntPtr recv, IntPtr sel, IntPtr a, IntPtr b, IntPtr c); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send1N(IntPtr recv, IntPtr sel, nuint a); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send1P1N(IntPtr recv, IntPtr sel, IntPtr a, nuint b); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send2P1N(IntPtr recv, IntPtr sel, IntPtr a, IntPtr b, nuint c); - - // newBufferWithBytes:length:options: / newBufferWithLength:options: - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send1P2N(IntPtr recv, IntPtr sel, IntPtr a, nuint b, nuint c); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send1N1N(IntPtr recv, IntPtr sel, nuint a, nuint b); - - // setBuffer:offset:atIndex: / setBytes:length:atIndex: - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial void SendV1P2N(IntPtr recv, IntPtr sel, IntPtr a, nuint b, nuint c); - - // dispatchThreadgroups:threadsPerThreadgroup: — MTLSize by value (24B) - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial void SendV2Size(IntPtr recv, IntPtr sel, MTLSize a, MTLSize b); - - // setThreadgroupMemoryLength:atIndex: - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial void SendV1N1N(IntPtr recv, IntPtr sel, nuint a, nuint b); - - // single-NSUInteger void calls (setLanguageVersion:, memoryBarrierWithScope:, …) - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial void SendV1N(IntPtr recv, IntPtr sel, nuint a); - - // single-BOOL void calls (setFastMathEnabled:…) — BOOL is 1 byte on arm64. - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial void SendV1B(IntPtr recv, IntPtr sel, byte a); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial byte SendBool(IntPtr recv, IntPtr sel); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial byte SendBool1P(IntPtr recv, IntPtr sel, IntPtr a); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial byte SendBool1N(IntPtr recv, IntPtr sel, nuint a); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial nuint SendNuint(IntPtr recv, IntPtr sel); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial double SendDouble(IntPtr recv, IntPtr sel); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial long SendLong(IntPtr recv, IntPtr sel); - - // supportsFamily: — (id,SEL,NSInteger) - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial byte SendBool1L(IntPtr recv, IntPtr sel, nint a); - - [LibraryImport(LibMetal, EntryPoint = "MTLCreateSystemDefaultDevice")] - public static partial IntPtr CreateSystemDefaultDevice(); - - [StructLayout(LayoutKind.Sequential)] - public readonly struct MTLSize - { - public readonly nuint X, Y, Z; - public MTLSize(nuint x, nuint y, nuint z) { X = x; Y = y; Z = z; } - } - - [StructLayout(LayoutKind.Sequential)] - public readonly struct NSRange - { - public readonly nuint Location, Length; - public NSRange(nuint location, nuint length) { Location = location; Length = length; } - } - - // ---- cached class/selector tables ------------------------------------ - // sel_registerName returns the same SEL for the same string, so register - // once. Classes likewise are process-wide stable. - - private static readonly IntPtr s_classNsString = Cls("NSString"); - private static readonly IntPtr s_classNsAutoreleasePool = Cls("NSAutoreleasePool"); - private static readonly IntPtr s_classMtlCompileOptions = Cls("MTLCompileOptions"); - private static readonly IntPtr s_classMtlFunctionConstantValues = Cls("MTLFunctionConstantValues"); - - public static IntPtr ClassNSString => s_classNsString; - public static IntPtr ClassNSAutoreleasePool => s_classNsAutoreleasePool; - public static IntPtr ClassMTLCompileOptions => s_classMtlCompileOptions; - public static IntPtr ClassMTLFunctionConstantValues => s_classMtlFunctionConstantValues; - - public static IntPtr Cls(string name) - { - IntPtr n = Marshal.StringToHGlobalAnsi(name); - try { return GetClassRaw(n); } finally { Marshal.FreeHGlobal(n); } - } - - public static IntPtr Sel(string name) - { - IntPtr n = Marshal.StringToHGlobalAnsi(name); - try { return RegSelRaw(n); } finally { Marshal.FreeHGlobal(n); } - } - - public static class S - { - public static readonly IntPtr Alloc = Sel("alloc"); - public static readonly IntPtr Init = Sel("init"); - public static readonly IntPtr Release = Sel("release"); - public static readonly IntPtr Retain = Sel("retain"); - public static readonly IntPtr Drain = Sel("drain"); - public static readonly IntPtr InitWithUTF8String = Sel("initWithUTF8String:"); - public static readonly IntPtr Utf8String = Sel("UTF8String"); - public static readonly IntPtr LocalizedDescription = Sel("localizedDescription"); - public static readonly IntPtr Length = Sel("length"); - - public static readonly IntPtr Name = Sel("name"); - public static readonly IntPtr RespondsToSelector = Sel("respondsToSelector:"); - public static readonly IntPtr SupportsFamily = Sel("supportsFamily:"); - public static readonly IntPtr MaxThreadgroupMemoryLength = Sel("maxThreadgroupMemoryLength"); - public static readonly IntPtr MaxThreadsPerThreadgroup = Sel("maxThreadsPerThreadgroup"); - public static readonly IntPtr MaxTotalThreadsPerThreadgroup = Sel("maxTotalThreadsPerThreadgroup"); - public static readonly IntPtr ThreadExecutionWidth = Sel("threadExecutionWidth"); - public static readonly IntPtr HasUnifiedMemory = Sel("hasUnifiedMemory"); - public static readonly IntPtr RecommendedMaxWorkingSetSize = Sel("recommendedMaxWorkingSetSize"); - public static readonly IntPtr CurrentAllocatedSize = Sel("currentAllocatedSize"); - public static readonly IntPtr RegistryID = Sel("registryID"); - - public static readonly IntPtr NewCommandQueue = Sel("newCommandQueue"); - public static readonly IntPtr NewBufferWithBytesLengthOptions = Sel("newBufferWithBytes:length:options:"); - public static readonly IntPtr NewBufferWithLengthOptions = Sel("newBufferWithLength:options:"); - public static readonly IntPtr Contents = Sel("contents"); - public static readonly IntPtr NewLibraryWithSourceOptionsError = Sel("newLibraryWithSource:options:error:"); - public static readonly IntPtr NewFunctionWithName = Sel("newFunctionWithName:"); - public static readonly IntPtr NewFunctionWithNameConstantValuesError = Sel("newFunctionWithName:constantValues:error:"); - public static readonly IntPtr NewComputePipelineStateWithFunctionError = Sel("newComputePipelineStateWithFunction:error:"); - public static readonly IntPtr FunctionName = Sel("functionName"); - public static readonly IntPtr FunctionConstantsDictionary = Sel("functionConstantsDictionary"); - public static readonly IntPtr SetConstantValueTypeAtIndex = Sel("setConstantValue:type:atIndex:"); - - public static readonly IntPtr SetFastMathEnabled = Sel("setFastMathEnabled:"); - public static readonly IntPtr SetMathMode = Sel("setMathMode:"); - public static readonly IntPtr SetLanguageVersion = Sel("setLanguageVersion:"); - - public static readonly IntPtr CommandBuffer = Sel("commandBuffer"); - public static readonly IntPtr ComputeCommandEncoder = Sel("computeCommandEncoder"); - public static readonly IntPtr ComputeCommandEncoderWithDispatchType = Sel("computeCommandEncoderWithDispatchType:"); - public static readonly IntPtr SetComputePipelineState = Sel("setComputePipelineState:"); - public static readonly IntPtr SetBufferOffsetAtIndex = Sel("setBuffer:offset:atIndex:"); - public static readonly IntPtr SetBytesLengthAtIndex = Sel("setBytes:length:atIndex:"); - public static readonly IntPtr DispatchThreadgroupsThreadsPerThreadgroup = Sel("dispatchThreadgroups:threadsPerThreadgroup:"); - public static readonly IntPtr DispatchThreadsThreadsPerThreadgroup = Sel("dispatchThreads:threadsPerThreadgroup:"); - public static readonly IntPtr SetThreadgroupMemoryLengthAtIndex = Sel("setThreadgroupMemoryLength:atIndex:"); - public static readonly IntPtr MemoryBarrierWithScope = Sel("memoryBarrierWithScope:"); - public static readonly IntPtr EndEncoding = Sel("endEncoding"); - public static readonly IntPtr Commit = Sel("commit"); - public static readonly IntPtr WaitUntilCompleted = Sel("waitUntilCompleted"); - public static readonly IntPtr Status = Sel("status"); - public static readonly IntPtr Error = Sel("error"); - public static readonly IntPtr GpuStartTime = Sel("GPUStartTime"); - public static readonly IntPtr GpuEndTime = Sel("GPUEndTime"); - public static readonly IntPtr Label = Sel("label"); - public static readonly IntPtr SetLabel = Sel("setLabel:"); - } - - // NSString* with UTF-8 — caller owns (+1 via alloc/init); pass owns:false - // to consume within the current autorelease pool. - public static IntPtr NsStr(string s) - { - IntPtr obj = Send0(s_classNsString, S.Alloc); - IntPtr cstr = Marshal.StringToHGlobalAnsi(s); - try { return Send1P(obj, S.InitWithUTF8String, cstr); } - finally { Marshal.FreeHGlobal(cstr); } - } - - public static string? NsToString(IntPtr nsString) - { - if (nsString == 0) return null; - IntPtr utf8 = Send0(nsString, S.Utf8String); - return utf8 == 0 ? null : Marshal.PtrToStringUTF8(utf8); - } - - public static void Release(IntPtr obj) { if (obj != 0) Send0(obj, S.Release); } - public static IntPtr Retain(IntPtr obj) => Send0(obj, S.Retain); -} - -/// Scoped NSAutoreleasePool — .NET worker threads have none, so every -/// public entry that allocates autoreleased ObjC objects wraps itself in one. -/// Drain at Dispose releases the pool itself. -internal readonly struct AutoReleasePool : IDisposable -{ - private readonly IntPtr _pool; - private AutoReleasePool(IntPtr pool) => _pool = pool; - - public static AutoReleasePool Create() => - new(ObjC.Send0(ObjC.Send0(ObjC.ClassNSAutoreleasePool, ObjC.S.Alloc), ObjC.S.Init)); - - public void Dispose() => ObjC.Send0(_pool, ObjC.S.Drain); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/common.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/common.metal deleted file mode 100644 index b8b9f95..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/common.metal +++ /dev/null @@ -1,43 +0,0 @@ -// Common helpers for the GCDRNet Metal kernels (fp16 storage, fp32 math — -// same contract as the Vulkan *_h shaders: half is storage-only, every load -// widens to float and all arithmetic runs in fp32). -#include -using namespace metal; - -static inline float ferf1(float v) -{ - float s = v < 0.0f ? -1.0f : 1.0f; - float ax = fabs(v); - float t = 1.0f / (1.0f + 0.3275911f * ax); - float poly = ((((1.061405429f * t - 1.453152027f) * t + 1.421413741f) * t - - 0.284496736f) * t + 0.254829592f) * t; - return (1.0f - poly * exp(-ax * ax)) * s; -} - -static inline float fexp1(float v) -{ - v = clamp(v, -87.3f, 88.7f); - float fx = v * 1.4426950408889634f + 0.5f; - float nf = floor(fx); - float r = v - nf * 0.693359375f - nf * -0.00021219444005469058f; - float poly = (((((1.9875691500e-4f * r + 1.3981999507e-3f) * r + 8.3334519073e-3f) * r - + 4.1665795894e-2f) * r + 1.6666665459e-1f) * r + 5.0000001201e-1f) * r * r + r + 1.0f; - return poly * as_type((uint(int(nf)) + 127u) << 23); -} - -// Conv epilogue activation: bits 4-6 (relu=1, gelu=2, sigmoid=3). -static inline float cact(float v, uint flags) -{ - uint a = (flags >> 4) & 7u; - if (a == 1u) return max(v, 0.0f); - if (a == 2u) return 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f)); - if (a == 3u) return 1.0f / (1.0f + fexp1(-v)); - return v; -} - -static inline int mirror1(int i, int d) -{ - if (d <= 1) return 0; - while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; - return i; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv.metal deleted file mode 100644 index 9cb02a7..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv.metal +++ /dev/null @@ -1,333 +0,0 @@ -// Conv kernels: conv_tile (implicit-GEMM tiled fallback, conv_gen/k33/k11 -// equivalent), depthwise(4), convpool(_k3) fused stem+pool. fp16 storage, fp32 math. - -struct PcConv { - uint ic, ih, iw; - uint oc, oh, ow; - uint kh, kw, sh, sw; - uint pt, pl; - uint flags; // bit0 bias, bit1 reflectPad, bit2 relu, bit3 hasAdd, bit4 gelu - uint pad0; -}; - -// conv_tile: O[px,oc] = sum_k X[px,k]*W[k,oc], k = (ky*kw+kx)*ic+ci (coalesced). -// W backend-packed [K][OC] oc-fastest. 3D dispatch: gx=x-tile(16), gy=y-tile(8), -// gz=oc-tile(64); 256 threads, each 4oc x 8px. K-chunk 32 staged. -kernel void conv_tile(device const half* x [[buffer(0)]], - device const half* w [[buffer(1)]], - device const float* b [[buffer(2)]], - device const half* r [[buffer(3)]], - device half* o [[buffer(4)]], - constant PcConv& p [[buffer(5)]], - uint3 wg [[threadgroup_position_in_grid]], - uint3 tp [[thread_position_in_threadgroup]]) -{ - const uint tid = tp.x; - const uint BM = 64u, BH = 8u, BW = 16u, BN = BH * BW, BK = 32u, BKP = 36u; - threadgroup float4 Asv[BK * BM / 4]; - threadgroup float4 Bsv[BN * BKP / 4]; - uint oc0 = wg.z * BM; - uint oy0 = wg.y * BH; - uint ox0 = wg.x * BW; - uint K = p.kh * p.kw * p.ic; - uint tm = tid & 15u, tn = tid >> 4; - float4 acc[8] = { float4(0.0f), float4(0.0f), float4(0.0f), float4(0.0f), - float4(0.0f), float4(0.0f), float4(0.0f), float4(0.0f) }; - bool reflect = (p.flags & 2u) != 0u; - uint bk = tid & 31u; - - for (uint k0 = 0u; k0 < K; k0 += BK) - { - for (uint e = tid; e < BK * BM; e += 256u) - { - uint k = e >> 6, m = e & 63u; - uint gm = oc0 + m, gk = k0 + k; - ((threadgroup float*)Asv)[e] = (gm < p.oc && gk < K) ? float(w[gk * p.oc + gm]) : 0.0f; - } - uint gk = k0 + bk; - bool kok = gk < K; - uint tap = kok ? gk / p.ic : 0u; - uint ch = gk - tap * p.ic; - uint ky = tap / p.kw, kx = tap - ky * p.kw; - for (uint n = tid >> 5; n < BN; n += 8u) - { - float v = 0.0f; - uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u); - if (kok && oy < p.oh && ox < p.ow) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - int sx = int(ox * p.sw + kx) - int(p.pl); - if (reflect) - { - if (uint(sy) >= p.ih) sy = mirror1(sy, int(p.ih)); - if (uint(sx) >= p.iw) sx = mirror1(sx, int(p.iw)); - v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]); - } - else if (uint(sy) < p.ih && uint(sx) < p.iw) - v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]); - } - uint si = n * BKP + bk; - ((threadgroup float*)Bsv)[si] = v; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint k = 0u; k < BK; k += 4u) - { - float4 a0 = Asv[k * 16u + tm]; - float4 a1 = Asv[(k + 1u) * 16u + tm]; - float4 a2 = Asv[(k + 2u) * 16u + tm]; - float4 a3 = Asv[(k + 3u) * 16u + tm]; - for (int j = 0; j < 8; j++) - { - float4 bv = Bsv[(tn * 8u + uint(j)) * (BKP / 4u) + (k >> 2)]; - acc[j] += a0 * bv.x + a1 * bv.y + a2 * bv.z + a3 * bv.w; - } - } - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - bool hasBias = (p.flags & 1u) != 0u; - uint gm0 = oc0 + tm * 4u; - float4 bias = float4( - hasBias && gm0 < p.oc ? b[gm0] : 0.0f, - hasBias && gm0 + 1u < p.oc ? b[gm0 + 1u] : 0.0f, - hasBias && gm0 + 2u < p.oc ? b[gm0 + 2u] : 0.0f, - hasBias && gm0 + 3u < p.oc ? b[gm0 + 3u] : 0.0f); - for (int j = 0; j < 8; j++) - { - uint n = tn * 8u + uint(j); - uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u); - if (oy >= p.oh || ox >= p.ow || gm0 >= p.oc) continue; - uint base = (oy * p.ow + ox) * p.oc + gm0; - uint full = gm0 + 3u < p.oc ? 4u : p.oc - gm0; - for (uint l = 0u; l < 4u && l < full; l++) - { - float v = acc[j][l] + bias[l]; - if ((p.flags & 8u) != 0u) v += float(r[base + l]); - if ((p.flags & 4u) != 0u) v = max(v, 0.0f); - if ((p.flags & 16u) != 0u) v = 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f)); - o[base + l] = half(v); - } - } -} - -// ---- depthwise / depthwise4: out[oy,ox,c] = bias[c] + sum w[tap,c]*x[sy,sx,c] ---- -struct PcDw { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl; - uint flags; // bit0 hasBias, bit1 reflect, bit2 relu, bit4 gelu - float padVal; -}; -kernel void depthwise(device const half* x [[buffer(0)]], - device const half* w [[buffer(1)]], - device const float* b [[buffer(2)]], - device half* o [[buffer(3)]], - constant PcDw& p [[buffer(4)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.c * p.oh * p.ow) return; - uint ch = gid % p.c, px = gid / p.c; - uint ox = px % p.ow, oy = px / p.ow; - bool refl = (p.flags & 2u) != 0u; - float acc = ((p.flags & 1u) != 0u) ? b[ch] : 0.0f; - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - float wv = float(w[(ky * p.kw + kx) * p.c + ch]); - int yy = sy, xx = sx; - if (uint(sy) >= p.ih || uint(sx) >= p.iw) - { - if (!refl) { acc += wv * p.padVal; continue; } - if (uint(sy) >= p.ih) yy = mirror1(sy, int(p.ih)); - if (uint(sx) >= p.iw) xx = mirror1(sx, int(p.iw)); - } - acc += wv * float(x[(uint(yy) * p.iw + uint(xx)) * p.c + ch]); - } - } - if ((p.flags & 4u) != 0u) acc = max(acc, 0.0f); - if ((p.flags & 16u) != 0u) acc = 0.5f * acc * (1.0f + ferf1(acc * 0.7071067811865475f)); - o[gid] = half(acc); -} -kernel void depthwise4(device const half4* x [[buffer(0)]], - device const half4* w [[buffer(1)]], - device const float4* b [[buffer(2)]], - device half4* o [[buffer(3)]], - constant PcDw& p [[buffer(4)]], - uint gid [[thread_position_in_grid]]) -{ - uint c4 = p.c >> 2; - if (gid >= c4 * p.oh * p.ow) return; - uint ch = gid % c4, px = gid / c4; - uint ox = px % p.ow, oy = px / p.ow; - bool refl = (p.flags & 2u) != 0u; - float4 acc = ((p.flags & 1u) != 0u) ? b[ch] : float4(0.0f); - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - float4 wv = float4(w[(ky * p.kw + kx) * c4 + ch]); - int yy = sy, xx = sx; - if (uint(sy) >= p.ih || uint(sx) >= p.iw) - { - if (!refl) { acc += wv * p.padVal; continue; } - if (uint(sy) >= p.ih) yy = mirror1(sy, int(p.ih)); - if (uint(sx) >= p.iw) xx = mirror1(sx, int(p.iw)); - } - acc += wv * float4(x[(uint(yy) * p.iw + uint(xx)) * c4 + ch]); - } - } - if ((p.flags & 4u) != 0u) acc = max(acc, float4(0.0f)); - if ((p.flags & 16u) != 0u) - for (int l = 0; l < 4; l++) acc[l] = 0.5f * acc[l] * (1.0f + ferf1(acc[l] * 0.7071067811865475f)); - o[gid] = half4(acc); -} - -// ---- convpool / convpool_k3: small-ic stem conv + fused 2x2/2 maxpool ---- -struct PcConvPool { - uint ic, ih, iw, oc, ph, pw, kh, kw, pt, pl, flags; - float padv; -}; -kernel void convpool(device const half* x [[buffer(0)]], - device const half* w [[buffer(1)]], - device const float* b [[buffer(2)]], - device half* o [[buffer(3)]], - constant PcConvPool& p [[buffer(4)]], - uint gid [[thread_position_in_grid]], - uint tid [[thread_position_in_threadgroup]]) -{ - threadgroup float4 ws[1024]; - uint K = p.kh * p.kw * p.ic, oc4 = p.oc >> 2; - for (uint e = tid; e < K * p.oc; e += 256u) ((threadgroup float*)ws)[e] = float(w[e]); - threadgroup_barrier(mem_flags::mem_threadgroup); - uint groups = p.oc >> 3; - if (gid >= p.ph * p.pw * groups) return; - uint g = gid % groups, pp = gid / groups; - uint pxx = pp % p.pw, pyy = pp / p.pw; - bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u; - uint n0 = g * 8u; - float4 b0 = float4(0.0f), b1 = float4(0.0f); - if ((p.flags & 1u) != 0u) - { - b0 = float4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]); - b1 = float4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]); - } - float4 m0 = float4(-3.4e38f), m1 = float4(-3.4e38f); - for (uint d = 0u; d < 4u; d++) - { - uint oy = pyy * 2u + (d >> 1), ox = pxx * 2u + (d & 1u); - float4 a0 = b0, a1 = b1; - for (uint ky = 0u; ky < p.kh; ky++) - { - int iy = int(oy + ky) - int(p.pt); - bool vy = iy >= 0 && iy < int(p.ih); - if (!vy && reflect) { iy = mirror1(iy, int(p.ih)); vy = true; } - for (uint kx = 0u; kx < p.kw; kx++) - { - int ix = int(ox + kx) - int(p.pl); - bool vx = ix >= 0 && ix < int(p.iw); - if (!vx && reflect) { ix = mirror1(ix, int(p.iw)); vx = true; } - bool valid = vy && vx; - uint base = valid ? (uint(iy) * p.iw + uint(ix)) * p.ic : 0u; - uint kb = (ky * p.kw + kx) * p.ic; - for (uint ci = 0u; ci < p.ic; ci++) - { - float xv = valid ? float(x[base + ci]) : p.padv; - uint wr = (kb + ci) * oc4 + g * 2u; - a0 += xv * ws[wr]; - a1 += xv * ws[wr + 1u]; - } - } - } - if (crelu) { a0 = max(a0, float4(0.0f)); a1 = max(a1, float4(0.0f)); } - m0 = max(m0, a0); m1 = max(m1, a1); - } - if ((p.flags & 8u) != 0u) { m0 = max(m0, float4(0.0f)); m1 = max(m1, float4(0.0f)); } - uint ob = pp * p.oc + n0; - for (uint l = 0u; l < 4u; l++) { o[ob + l] = half(m0[l]); o[ob + 4u + l] = half(m1[l]); } -} - -kernel void convpool_k3(device const half* x [[buffer(0)]], - device const half* w [[buffer(1)]], - device const float* b [[buffer(2)]], - device half* o [[buffer(3)]], - constant PcConvPool& p [[buffer(4)]], - uint gid [[thread_position_in_grid]], - uint tid [[thread_position_in_threadgroup]]) -{ - threadgroup float4 ws[1024]; - uint K = 9u * p.ic, oc4 = p.oc >> 2; - for (uint e = tid; e < K * p.oc; e += 256u) ((threadgroup float*)ws)[e] = float(w[e]); - threadgroup_barrier(mem_flags::mem_threadgroup); - uint groups = p.oc >> 3; - if (gid >= p.ph * p.pw * groups) return; - uint g = gid % groups, pp = gid / groups; - uint pxx = pp % p.pw, pyy = pp / p.pw; - bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u; - int ro[4], co[4]; - bool rv[4], cv[4]; - for (uint r = 0u; r < 4u; r++) - { - int iy = int(pyy * 2u + r) - int(p.pt); - bool vy = iy >= 0 && iy < int(p.ih); - if (!vy && reflect) { iy = mirror1(iy, int(p.ih)); vy = true; } - rv[r] = vy; ro[r] = vy ? iy * int(p.iw) : 0; - int ix = int(pxx * 2u + r) - int(p.pl); - bool vx = ix >= 0 && ix < int(p.iw); - if (!vx && reflect) { ix = mirror1(ix, int(p.iw)); vx = true; } - cv[r] = vx; co[r] = vx ? ix : 0; - } - uint n0 = g * 8u; - float4 b0 = float4(0.0f), b1 = float4(0.0f); - if ((p.flags & 1u) != 0u) - { - b0 = float4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]); - b1 = float4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]); - } - float4 a0[4], a1[4]; - for (uint d = 0u; d < 4u; d++) { a0[d] = b0; a1[d] = b1; } - for (uint r = 0u; r < 4u; r++) - { - float xr[4][8]; - for (uint c = 0u; c < 4u; c++) - { - bool v = rv[r] && cv[c]; - uint base = uint(ro[r] + co[c]) * p.ic; - for (uint ci = 0u; ci < 8u; ci++) - xr[c][ci] = ci < p.ic ? (v ? float(x[base + ci]) : p.padv) : 0.0f; - } - for (uint dy = 0u; dy < 2u; dy++) - { - if (r < dy || r - dy > 2u) continue; - uint ky = r - dy; - for (uint kx = 0u; kx < 3u; kx++) - { - uint kb = (ky * 3u + kx) * p.ic; - for (uint ci = 0u; ci < 8u; ci++) - { - if (ci >= p.ic) break; - uint wr = (kb + ci) * oc4 + g * 2u; - float4 w0 = ws[wr], w1 = ws[wr + 1u]; - for (uint dx = 0u; dx < 2u; dx++) - { - float xv = xr[kx + dx][ci]; - a0[dy * 2u + dx] += xv * w0; - a1[dy * 2u + dx] += xv * w1; - } - } - } - } - } - float4 m0 = float4(-3.4e38f), m1 = float4(-3.4e38f); - for (uint d = 0u; d < 4u; d++) - { - if (crelu) { a0[d] = max(a0[d], float4(0.0f)); a1[d] = max(a1[d], float4(0.0f)); } - m0 = max(m0, a0[d]); m1 = max(m1, a1[d]); - } - if ((p.flags & 8u) != 0u) { m0 = max(m0, float4(0.0f)); m1 = max(m1, float4(0.0f)); } - uint ob = pp * p.oc + n0; - for (uint l = 0u; l < 4u; l++) { o[ob + l] = half(m0[l]); o[ob + 4u + l] = half(m1[l]); } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv_mm.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv_mm.metal deleted file mode 100644 index f784851..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv_mm.metal +++ /dev/null @@ -1,229 +0,0 @@ -// conv_mm: simdgroup-MMA implicit-GEMM NHWC conv (Metal port of conv_cm.comp). -// A gathers input pixels through tap-major im2col addressing — no im2col -// materialization; W is backend-packed [oc][Kd] tap-major, Kd = K rounded up -// to 8 (zero-filled rows make pad k-slices contribute 0 for any gathered -// pixel). BM = 64 output pixels x BN output channels, 8 simdgroups -// (256 threads), fp16 storage + simdgroup fp16xfp32 MMA. -// cflags: bit0 bias, bit1 residual, bit2 reflect pad, bits4-6 act (1 relu, 2 gelu). -struct PcConvMm { - uint mDim, nDim, kDim, flags; - uint outW, inW, inH, cin, kW; - uint sH, sW, pT, pL, kReal; -}; - -template -static inline void conv_mm_body(device const half* x, - device const half* w, - device const float* bias, - device const half* res, - device half* o, - constant PcConvMm& p, - uint2 wg, uint tid, - threadgroup half* tA, threadgroup half* tB, - threadgroup float* outst) -{ - const uint JN = BN / 16u; // acc columns per simdgroup row-half - uint sgid = tid >> 5; - uint sm = (sgid & 3u) * 16u; // 4 rows of 16 -> 64 M - uint sn = (sgid >> 2) * (BN / 2u); // 2 cols of BN/2 - uint m0 = wg.x * 64u + sm; // m tiles on gx (grid allows 2^31 in x) - uint n0 = wg.y * BN + sn; - - simdgroup_float8x8 acc[2][8]; - for (uint i = 0u; i < 2u; i++) for (uint j = 0u; j < 8u; j++) - acc[i][j] = simdgroup_float8x8(0.0f); - - bool reflect = (p.flags & 4u) != 0u; - - for (uint kb = 0u; kb < p.kDim; kb += 16u) - { - if (SCK == 0) - { - // vec4 gather: 64 rows x 16 k as one half4 per thread - uint am = tid >> 2, ak = (tid & 3u) * 4u; - uint gm = wg.x * 64u + am; - uint gk = kb + ak; - half4 av = half4(0.0h); - if (gm < p.mDim && gk < p.kReal) - { - uint tap = gk / p.cin, ci = gk - tap * p.cin; - uint dy = tap / p.kW, dx = tap - dy * p.kW; - uint oy = gm / p.outW, ox = gm - oy * p.outW; - int iy = int(oy * p.sH + dy) - int(p.pT); - int ix = int(ox * p.sW + dx) - int(p.pL); - if (reflect) - { - if (uint(iy) >= p.inH) iy = mirror1(iy, int(p.inH)); - if (uint(ix) >= p.inW) ix = mirror1(ix, int(p.inW)); - av = *(device const half4*)(x + (((uint(iy) * p.inW + uint(ix)) * (p.cin >> 2)) + (ci >> 2)) * 4u); - } - else if (iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW)) - av = *(device const half4*)(x + (((uint(iy) * p.inW + uint(ix)) * (p.cin >> 2)) + (ci >> 2)) * 4u); - } - *(threadgroup half4*)(tA + am * 20 + ak) = av; - } - else - { - // scalar gather for cin % 4 != 0 (RGB stems): 64x16 one half per thread x4 - for (uint e = tid; e < 64u * 16u; e += 256u) - { - uint am = e >> 4, ak = e & 15u; - uint gm = wg.x * 64u + am; - uint gk = kb + ak; - half av = 0.0h; - if (gm < p.mDim && gk < p.kReal) - { - uint tap = gk / p.cin, ci = gk - tap * p.cin; - uint dy = tap / p.kW, dx = tap - dy * p.kW; - uint oy = gm / p.outW, ox = gm - oy * p.outW; - int iy = int(oy * p.sH + dy) - int(p.pT); - int ix = int(ox * p.sW + dx) - int(p.pL); - if (reflect) - { - if (uint(iy) >= p.inH) iy = mirror1(iy, int(p.inH)); - if (uint(ix) >= p.inW) ix = mirror1(ix, int(p.inW)); - av = x[(uint(iy) * p.inW + uint(ix)) * p.cin + ci]; - } - else if (iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW)) - av = x[(uint(iy) * p.inW + uint(ix)) * p.cin + ci]; - } - tA[am * 20 + ak] = av; - } - } - // B stage: [N][Kd] rows, one half4 per thread covers 64x16; BN>64 loops - for (uint rr = tid >> 2; rr < uint(BN); rr += 64u) - { - uint bk = (tid & 3u) * 4u; - uint gn = wg.y * BN + rr; - uint gkB = kb + bk; - half4 wv = half4(0.0h); - if (gn < p.nDim && gkB < p.kDim) - wv = *(device const half4*)(w + (ulong)gn * p.kDim + gkB); - *(threadgroup half4*)(tB + rr * 20 + bk) = wv; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint kk = 0u; kk < 16u; kk += 8u) - { - simdgroup_half8x8 a0, a1; - simdgroup_load(a0, tA + sm * 20 + kk, 20); - simdgroup_load(a1, tA + (sm + 8u) * 20 + kk, 20); - simdgroup_half8x8 bfr[8]; - for (uint j = 0u; j < JN; j++) - simdgroup_load(bfr[j], tB + (sn + j * 8u) * 20 + kk, 20, ulong2(0, 0), true); - for (uint j = 0u; j < JN; j++) - { - simdgroup_multiply_accumulate(acc[0][j], a0, bfr[j], acc[0][j]); - simdgroup_multiply_accumulate(acc[1][j], a1, bfr[j], acc[1][j]); - } - } - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - uint lane = tid & 31u; - for (uint i = 0u; i < 2u; i++) - for (uint j = 0u; j < JN; j++) - { - threadgroup_barrier(mem_flags::mem_threadgroup); - simdgroup_store(acc[i][j], outst + sgid * 64, 8); - threadgroup_barrier(mem_flags::mem_threadgroup); - uint r = lane >> 2, c2 = (lane & 3u) * 2u; - for (uint e = 0u; e < 2u; e++) - { - uint mm = m0 + i * 8u + r; - uint nn = n0 + j * 8u + c2 + e; - if (mm >= p.mDim || nn >= p.nDim) continue; - float v = outst[sgid * 64 + r * 8 + c2 + e]; - if ((p.flags & 1u) != 0u) v += bias[nn]; - if ((p.flags & 2u) != 0u) v += float(res[mm * p.nDim + nn]); - o[mm * p.nDim + nn] = half(cact(v, p.flags)); - } - } -} - -#define CONV_MM_INST(BN, SCK) \ -kernel void conv_mm_##BN##_##SCK(device const half* x [[buffer(0)]], \ - device const half* w [[buffer(1)]], device const float* bias [[buffer(2)]], \ - device const half* res [[buffer(3)]], device half* o [[buffer(4)]], \ - constant PcConvMm& p [[buffer(5)]], uint2 wg [[threadgroup_position_in_grid]], \ - uint2 tp [[thread_position_in_threadgroup]]) \ -{ \ - threadgroup half tA[64 * 20]; \ - threadgroup half tB[BN * 20]; \ - threadgroup float outst[8 * 64]; \ - conv_mm_body(x, w, bias, res, o, p, wg, tp.x, tA, tB, outst); \ -} - -CONV_MM_INST(128, 0) -CONV_MM_INST(64, 0) -CONV_MM_INST(32, 0) -CONV_MM_INST(128, 1) -CONV_MM_INST(64, 1) -CONV_MM_INST(32, 1) - -// ---- gemm: C[M,N] = A[M,K]*B + bias[N]; B [K,N] or [N,K] (transB) ---- -// tiled 64x64x16, 256 threads -> 4x4 each (fallback; MMA path uses conv_mm). -struct PcGemm { uint m, n, k, flags; }; // bit0 transB bit1 bias bit2 relu bit3 gelu -kernel void gemm(device const half* a [[buffer(0)]], - device const half* b [[buffer(1)]], - device const float* bs [[buffer(2)]], - device half* o [[buffer(3)]], - constant PcGemm& p [[buffer(4)]], - uint2 wg [[threadgroup_position_in_grid]], - uint2 tp [[thread_position_in_threadgroup]]) -{ - const uint tid = tp.x; - const uint BM = 64u, BN = 64u, BK = 16u; - threadgroup float As[BK * BM]; - threadgroup float Bs[BK * BN]; - uint m0 = wg.y * BM; - uint n0 = wg.x * BN; - uint ty = tid >> 4, tx = tid & 15u; - bool transB = (p.flags & 1u) != 0u; - float acc[4][4]; - for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) acc[i][j] = 0.0f; - for (uint k0 = 0u; k0 < p.k; k0 += BK) - { - for (uint e = tid; e < BK * BM; e += 256u) - { - uint kk = e >> 6, mm = e & 63u; - uint gm = m0 + mm, gk = k0 + kk; - As[e] = (gm < p.m && gk < p.k) ? float(a[gm * p.k + gk]) : 0.0f; - } - for (uint e = tid; e < BK * BN; e += 256u) - { - uint kk = e >> 6, nn = e & 63u; - uint gn = n0 + nn, gk = k0 + kk; - float v = 0.0f; - if (gn < p.n && gk < p.k) - v = float(transB ? b[gn * p.k + gk] : b[gk * p.n + gn]); - Bs[e] = v; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint kk = 0u; kk < BK; kk++) - { - float av[4], bv[4]; - for (int i = 0; i < 4; i++) av[i] = As[kk * BM + ty * 4u + uint(i)]; - for (int j = 0; j < 4; j++) bv[j] = Bs[kk * BN + tx * 4u + uint(j)]; - for (int i = 0; i < 4; i++) - for (int j = 0; j < 4; j++) - acc[i][j] += av[i] * bv[j]; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - } - bool hasBias = (p.flags & 2u) != 0u; - for (int i = 0; i < 4; i++) - { - uint gm = m0 + ty * 4u + uint(i); - if (gm >= p.m) continue; - for (int j = 0; j < 4; j++) - { - uint gn = n0 + tx * 4u + uint(j); - if (gn >= p.n) continue; - float v = acc[i][j]; - if (hasBias) v += bs[gn]; - if ((p.flags & 4u) != 0u) v = max(v, 0.0f); - if ((p.flags & 8u) != 0u) v = 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f)); - o[gm * p.n + gn] = half(v); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal index 02142af..e0bd563 100644 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal +++ b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal @@ -1,4 +1,4 @@ -// Pipeline glue + elementwise/shape kernels. All activation storage is fp16 +// GCDR pipeline glue kernels. All activation storage is fp16 // (half/half4); math fp32 — see common.metal. // ---- bgr2nhwc: BGR u8 HWC -> NHWC [h][w][3] /255, top/left replicate pad ---- @@ -67,333 +67,3 @@ kernel void nhwc2bgr(device const half* x [[buffer(0)]], o[gid] = packed; } -// ---- unary: mode 0 relu, 1 sigmoid, 2 erf, 3 gelu, 4 copy ---- -struct PcUnary { uint n, mode; }; -kernel void unary(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcUnary& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.n) return; - float v = float(x[gid]); - float r; - if (p.mode == 0u) r = max(v, 0.0f); - else if (p.mode == 1u) r = 1.0f / (1.0f + fexp1(-v)); - else if (p.mode == 2u) r = ferf1(v); - else if (p.mode == 3u) r = 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f)); - else r = v; - o[gid] = half(r); -} - -// ---- ew2: elementwise binary + act, rank<=4 broadcast ---- -struct PcEw2 { - uint n, s0, s1, s2; - uint as0, as1, as2, as3; - uint bs0, bs1, bs2, bs3; - uint op, act; -}; -kernel void ew2(device const half* a [[buffer(0)]], - device const half* b [[buffer(1)]], - device half* o [[buffer(2)]], - constant PcEw2& p [[buffer(3)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.n) return; - uint d0 = gid / p.s0, r0 = gid % p.s0; - uint d1 = r0 / p.s1, r1 = r0 % p.s1; - uint d2 = r1 / p.s2, d3 = r1 % p.s2; - float va = float(a[d0 * p.as0 + d1 * p.as1 + d2 * p.as2 + d3 * p.as3]); - float vb = float(b[d0 * p.bs0 + d1 * p.bs1 + d2 * p.bs2 + d3 * p.bs3]); - float v = p.op == 0u ? va + vb : p.op == 1u ? va - vb : p.op == 2u ? va * vb : va / vb; - if (p.act == 1u) v = max(v, 0.0f); - else if (p.act == 2u) v = 1.0f / (1.0f + exp(-v)); - o[gid] = half(v); -} - -// ---- ew2v: same-shape vec4 fast path ---- -struct PcEw2v { uint nv4, op, act; }; -kernel void ew2v(device const half4* a [[buffer(0)]], - device const half4* b [[buffer(1)]], - device half4* o [[buffer(2)]], - constant PcEw2v& p [[buffer(3)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.nv4) return; - float4 va = float4(a[gid]), vb = float4(b[gid]), v; - if (p.op == 0u) v = va + vb; - else if (p.op == 1u) v = va - vb; - else if (p.op == 2u) v = va * vb; - else v = va / vb; - if (p.act == 1u) v = max(v, float4(0.0f)); - else if (p.act == 2u) v = 1.0f / (1.0f + exp(-v)); - o[gid] = half4(v); -} - -// ---- transpose: rank<=4 permute ---- -struct PcTranspose { uint n, os0, os1, os2, ps0, ps1, ps2, ps3; }; -kernel void transpose(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcTranspose& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.n) return; - uint d0 = gid / p.os0, r0 = gid % p.os0; - uint d1 = r0 / p.os1, r1 = r0 % p.os1; - uint d2 = r1 / p.os2, d3 = r1 % p.os2; - o[gid] = x[d0 * p.ps0 + d1 * p.ps1 + d2 * p.ps2 + d3 * p.ps3]; -} - -// ---- slice: rank<=4 strided slice ---- -struct PcSlice { - uint n, os0, os1, os2; - uint st0, st1, st2, st3; - uint sp0, sp1, sp2, sp3; - uint is0, is1, is2, is3; -}; -kernel void slice(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcSlice& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.n) return; - uint d0 = gid / p.os0, r0 = gid % p.os0; - uint d1 = r0 / p.os1, r1 = r0 % p.os1; - uint d2 = r1 / p.os2, d3 = r1 % p.os2; - o[gid] = x[(p.st0 + d0 * p.sp0) * p.is0 + (p.st1 + d1 * p.sp1) * p.is1 - + (p.st2 + d2 * p.sp2) * p.is2 + (p.st3 + d3 * p.sp3) * p.is3]; -} - -// ---- concat_ax: copy one input into dst at axis offset ---- -struct PcConcat { uint n, inner, srcAxis, dstAxis, axisOff; }; -kernel void concat_ax(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcConcat& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.n) return; - uint ii = gid % p.inner; - uint a = (gid / p.inner) % p.srcAxis; - uint oi = gid / (p.inner * p.srcAxis); - o[(oi * p.dstAxis + a + p.axisOff) * p.inner + ii] = x[gid]; -} - -// ---- pad: rank-4 all-dims pad, constant or reflect ---- -struct PcPad { - uint id0, id1, id2, id3; - uint od0, od1, od2, od3; - uint pb0, pb1, pb2, pb3; - uint mode; // 0 const 1 reflect - float value; -}; -static inline int padcoord(uint oidx, uint pb, uint idim, uint mode, thread bool& miss) -{ - int s = int(oidx) - int(pb); - if (uint(s) >= idim) - { - if (mode == 0u) { miss = true; return 0; } - s = mirror1(s, int(idim)); - } - miss = false; return s; -} -kernel void pad(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcPad& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - uint n = p.od0 * p.od1 * p.od2 * p.od3; - if (gid >= n) return; - uint s2 = p.od3, s1 = p.od2 * s2, s0 = p.od1 * s1; - uint d0 = gid / s0, r0 = gid % s0; - uint d1 = r0 / s1, r1 = r0 % s1; - uint d2 = r1 / s2, d3 = r1 % s2; - bool m0, m1, m2, m3; - int c0 = padcoord(d0, p.pb0, p.id0, p.mode, m0); - int c1 = padcoord(d1, p.pb1, p.id1, p.mode, m1); - int c2 = padcoord(d2, p.pb2, p.id2, p.mode, m2); - int c3 = padcoord(d3, p.pb3, p.id3, p.mode, m3); - if (m0 || m1 || m2 || m3) { o[gid] = half(p.value); return; } - uint si2 = p.id3, si1 = p.id2 * si2, si0 = p.id1 * si1; - o[gid] = x[uint(c0) * si0 + uint(c1) * si1 + uint(c2) * si2 + uint(c3)]; -} - -// ---- shift: NHWC channel-shift (fused shift-MLP) ---- -struct PcShift { - uint c, h, w, axis; - uint ge0, ge1, ge2, ge3, ge4; - int of0, of1, of2, of3, of4; -}; -static inline int shiftoff(uint ch, constant PcShift& p) -{ - return ch < p.ge0 ? p.of0 : ch < p.ge1 ? p.of1 : ch < p.ge2 ? p.of2 : ch < p.ge3 ? p.of3 : ch < p.ge4 ? p.of4 : 0; -} -kernel void shift(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcShift& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.c * p.h * p.w) return; - uint ch = gid % p.c, px = gid / p.c; - uint ox = px % p.w, oy = px / p.w; - int off = shiftoff(ch, p); - int sy = int(oy), sx = int(ox); - if (p.axis == 2u) sy += off; else sx += off; - o[gid] = half((uint(sy) < p.h && uint(sx) < p.w) ? float(x[(uint(sy) * p.w + uint(sx)) * p.c + ch]) : 0.0f); -} -kernel void shift4(device const half4* x [[buffer(0)]], - device half4* o [[buffer(1)]], - constant PcShift& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - uint c4 = p.c >> 2; - if (gid >= c4 * p.h * p.w) return; - uint q = gid % c4, px = gid / c4, ch = q * 4u; - uint ox = px % p.w, oy = px / p.w; - int off = shiftoff(ch, p); - int sy = int(oy), sx = int(ox); - if (p.axis == 2u) sy += off; else sx += off; - o[gid] = (uint(sy) < p.h && uint(sx) < p.w) ? x[(uint(sy) * p.w + uint(sx)) * c4 + q] : half4(0.0h); -} - -// ---- bilinear / bilinear4: NHWC half-pixel bilinear + fused add ---- -struct PcBilinear { uint c, ih, iw, oh, ow, flags; float shy, shx; }; -kernel void bilinear(device const half* x [[buffer(0)]], - device const half* a [[buffer(1)]], - device half* o [[buffer(2)]], - constant PcBilinear& p [[buffer(3)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.c * p.oh * p.ow) return; - uint ch = gid % p.c, px = gid / p.c; - uint ox = px % p.ow, oy = px / p.ow; - float fy = (float(oy) + 0.5f) / p.shy - 0.5f; - float fx = (float(ox) + 0.5f) / p.shx - 0.5f; - int fy0 = int(floor(fy)), fx0 = int(floor(fx)); - float dy = fy - float(fy0), dx = fx - float(fx0); - uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1)); - uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1)); - float v00 = float(x[(y0 * p.iw + x0) * p.c + ch]); - float v01 = float(x[(y0 * p.iw + x1) * p.c + ch]); - float v10 = float(x[(y1 * p.iw + x0) * p.c + ch]); - float v11 = float(x[(y1 * p.iw + x1) * p.c + ch]); - float v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy); - if ((p.flags & 1u) != 0u) v = max(v, 0.0f); - if ((p.flags & 2u) != 0u) - { - float s = float(a[gid]); - v += (p.flags & 4u) != 0u ? max(s, 0.0f) : s; - } - o[gid] = half(v); -} -kernel void bilinear4(device const half4* x [[buffer(0)]], - device const half4* a [[buffer(1)]], - device half4* o [[buffer(2)]], - constant PcBilinear& p [[buffer(3)]], - uint gid [[thread_position_in_grid]]) -{ - uint c4 = p.c >> 2; - if (gid >= c4 * p.oh * p.ow) return; - uint ch = gid % c4, px = gid / c4; - uint ox = px % p.ow, oy = px / p.ow; - float fy = (float(oy) + 0.5f) / p.shy - 0.5f; - float fx = (float(ox) + 0.5f) / p.shx - 0.5f; - int fy0 = int(floor(fy)), fx0 = int(floor(fx)); - float dy = fy - float(fy0), dx = fx - float(fx0); - uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1)); - uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1)); - float4 v00 = float4(x[(y0 * p.iw + x0) * c4 + ch]); - float4 v01 = float4(x[(y0 * p.iw + x1) * c4 + ch]); - float4 v10 = float4(x[(y1 * p.iw + x0) * c4 + ch]); - float4 v11 = float4(x[(y1 * p.iw + x1) * c4 + ch]); - float4 v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy); - if ((p.flags & 1u) != 0u) v = max(v, float4(0.0f)); - if ((p.flags & 2u) != 0u) - { - float4 s = float4(a[gid]); - v += (p.flags & 4u) != 0u ? max(s, float4(0.0f)) : s; - } - o[gid] = half4(v); -} - -// ---- maxpool / maxpool4: NHWC max pool + optional relu ---- -struct PcPool { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl, relu; -}; -kernel void maxpool(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcPool& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.c * p.oh * p.ow) return; - uint ch = gid % p.c, px = gid / p.c; - uint ox = px % p.ow, oy = px / p.ow; - float m = -3.4e38f; - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - if (uint(sy) >= p.ih) continue; - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - if (uint(sx) >= p.iw) continue; - m = max(m, float(x[(uint(sy) * p.iw + uint(sx)) * p.c + ch])); - } - } - if (m == -3.4e38f) m = 0.0f; - if (p.relu != 0u && m < 0.0f) m = 0.0f; - o[gid] = half(m); -} -kernel void maxpool4(device const half4* x [[buffer(0)]], - device half4* o [[buffer(1)]], - constant PcPool& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - uint c4 = p.c >> 2; - if (gid >= c4 * p.oh * p.ow) return; - uint ch = gid % c4, px = gid / c4; - uint ox = px % p.ow, oy = px / p.ow; - float4 m = float4(-3.4e38f); - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - if (uint(sy) >= p.ih) continue; - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - if (uint(sx) >= p.iw) continue; - m = max(m, float4(x[(uint(sy) * p.iw + uint(sx)) * c4 + ch])); - } - } - if (m.x == -3.4e38f) m = float4(0.0f); - if (p.relu != 0u) m = select(m, float4(0.0f), m < float4(0.0f)); - o[gid] = half4(m); -} - -// ---- layernorm: one workgroup (256 threads) per row ---- -struct PcLn { uint len; uint rows; float eps; }; -kernel void layernorm(device const half* x [[buffer(0)]], - device const float* w [[buffer(1)]], - device const float* b [[buffer(2)]], - device half* o [[buffer(3)]], - constant PcLn& p [[buffer(4)]], - uint wg [[threadgroup_position_in_grid]], - uint tid [[thread_position_in_threadgroup]]) -{ - // one simdgroup (32 lanes) per row, 8 rows per threadgroup: lane-strided - // accumulate + simd_sum only, no threadgroup barrier at all. - uint lane = tid & 31u; - uint row = wg * 8u + (tid >> 5); - if (row >= p.rows) return; - uint base = row * p.len; - float s = 0.0f, s2 = 0.0f; - for (uint i = lane; i < p.len; i += 32u) - { - float v = float(x[base + i]); - s += v; s2 += v * v; - } - s = simd_sum(s); s2 = simd_sum(s2); - float mean = s / float(p.len); - float inv = rsqrt(s2 / float(p.len) - mean * mean + p.eps); - for (uint i = lane; i < p.len; i += 32u) - o[base + i] = half((float(x[base + i]) - mean) * inv * w[i] + b[i]); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/shift.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/shift.metal new file mode 100644 index 0000000..a6509c3 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/shift.metal @@ -0,0 +1,38 @@ +// ---- shift: NHWC channel-shift (fused shift-MLP) ---- +struct PcShift { + uint c, h, w, axis; + uint ge0, ge1, ge2, ge3, ge4; + int of0, of1, of2, of3, of4; +}; +static inline int shiftoff(uint ch, constant PcShift& p) +{ + return ch < p.ge0 ? p.of0 : ch < p.ge1 ? p.of1 : ch < p.ge2 ? p.of2 : ch < p.ge3 ? p.of3 : ch < p.ge4 ? p.of4 : 0; +} +kernel void shift(device const half* x [[buffer(0)]], + device half* o [[buffer(1)]], + constant PcShift& p [[buffer(2)]], + uint gid [[thread_position_in_grid]]) +{ + if (gid >= p.c * p.h * p.w) return; + uint ch = gid % p.c, px = gid / p.c; + uint ox = px % p.w, oy = px / p.w; + int off = shiftoff(ch, p); + int sy = int(oy), sx = int(ox); + if (p.axis == 2u) sy += off; else sx += off; + o[gid] = half((uint(sy) < p.h && uint(sx) < p.w) ? float(x[(uint(sy) * p.w + uint(sx)) * p.c + ch]) : 0.0f); +} +kernel void shift4(device const half4* x [[buffer(0)]], + device half4* o [[buffer(1)]], + constant PcShift& p [[buffer(2)]], + uint gid [[thread_position_in_grid]]) +{ + uint c4 = p.c >> 2; + if (gid >= c4 * p.h * p.w) return; + uint q = gid % c4, px = gid / c4, ch = q * 4u; + uint ox = px % p.w, oy = px / p.w; + int off = shiftoff(ch, p); + int sy = int(oy), sx = int(ox); + if (p.axis == 2u) sy += off; else sx += off; + o[gid] = (uint(sy) < p.h && uint(sx) < p.w) ? x[(uint(sy) * p.w + uint(sx)) * c4 + q] : half4(0.0h); +} + diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.comp deleted file mode 100644 index 28ef98b..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.comp +++ /dev/null @@ -1,44 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// NHWC bilinear resize, half-pixel centers: src_coord = (dst + 0.5) / scale - 0.5. -// flags: bit0 relu on the resized value, bit1 add same-shape A, bit2 relu on A. -// out = [relu](resize(x)) + [relu](a) -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) readonly buffer A { ET a[]; }; // addend (dummy if none) -layout(set = 0, binding = 2) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint c, ih, iw, oh, ow, flags; - float shy, shx; // scales -} p; - -void main() -{ - uint id = gl_GlobalInvocationID.x; - if (id >= p.c * p.oh * p.ow) return; - uint ch = id % p.c, px = id / p.c; - uint ox = px % p.ow, oy = px / p.ow; - - float fy = (float(oy) + 0.5) / p.shy - 0.5; - float fx = (float(ox) + 0.5) / p.shx - 0.5; - int fy0 = int(floor(fy)), fx0 = int(floor(fx)); - float dy = fy - float(fy0), dx = fx - float(fx0); - uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1)); - uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1)); - - float v00 = float(x[(y0 * p.iw + x0) * p.c + ch]); - float v01 = float(x[(y0 * p.iw + x1) * p.c + ch]); - float v10 = float(x[(y1 * p.iw + x0) * p.c + ch]); - float v11 = float(x[(y1 * p.iw + x1) * p.c + ch]); - float v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy); - if ((p.flags & 1u) != 0u) v = max(v, 0.0); - if ((p.flags & 2u) != 0u) - { - float s = float(a[id]); - v += (p.flags & 4u) != 0u ? max(s, 0.0) : s; - } - o[id] = ET(v); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.spv deleted file mode 100644 index a252fef..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.comp deleted file mode 100644 index 0a84eb4..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.comp +++ /dev/null @@ -1,44 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// bilinear.comp with 4 channels per invocation (c % 4 == 0), same per-element math. -// flags: bit0 relu on the resized value, bit1 add same-shape A, bit2 relu on A. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET4 x[]; }; -layout(set = 0, binding = 1) readonly buffer A { ET4 a[]; }; // addend (dummy if none) -layout(set = 0, binding = 2) writeonly buffer O { ET4 o[]; }; - -layout(push_constant) uniform PC { - uint c, ih, iw, oh, ow, flags; - float shy, shx; // scales -} p; - -void main() -{ - uint id = gl_GlobalInvocationID.x; - uint c4 = p.c >> 2; - if (id >= c4 * p.oh * p.ow) return; - uint ch = id % c4, px = id / c4; - uint ox = px % p.ow, oy = px / p.ow; - - float fy = (float(oy) + 0.5) / p.shy - 0.5; - float fx = (float(ox) + 0.5) / p.shx - 0.5; - int fy0 = int(floor(fy)), fx0 = int(floor(fx)); - float dy = fy - float(fy0), dx = fx - float(fx0); - uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1)); - uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1)); - - vec4 v00 = vec4(x[(y0 * p.iw + x0) * c4 + ch]); - vec4 v01 = vec4(x[(y0 * p.iw + x1) * c4 + ch]); - vec4 v10 = vec4(x[(y1 * p.iw + x0) * c4 + ch]); - vec4 v11 = vec4(x[(y1 * p.iw + x1) * c4 + ch]); - vec4 v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy); - if ((p.flags & 1u) != 0u) v = max(v, vec4(0.0)); - if ((p.flags & 2u) != 0u) - { - vec4 s = vec4(a[id]); - v += (p.flags & 4u) != 0u ? max(s, vec4(0.0)) : s; - } - o[id] = ET4(v); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.spv deleted file mode 100644 index 274db15..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4_h.spv deleted file mode 100644 index bbbaff9..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear_h.spv deleted file mode 100644 index c7258e9..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.comp deleted file mode 100644 index 8b1f173..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.comp +++ /dev/null @@ -1,27 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Concat along an axis: one dispatch per input. Copies this input's slice into -// dst at axisOffset. dst[o, a, i] = src[o, a - axisOff, i]. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint n; // src element count - uint inner; // product of dims after axis - uint srcAxis; // src extent on axis - uint dstAxis; // dst extent on axis - uint axisOff; // dst offset of this input on axis -} p; - -void main() -{ - uint i = gl_GlobalInvocationID.x; - if (i >= p.n) return; - uint ii = i % p.inner; - uint a = (i / p.inner) % p.srcAxis; - uint oi = i / (p.inner * p.srcAxis); - o[(oi * p.dstAxis + a + p.axisOff) * p.inner + ii] = x[i]; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.spv deleted file mode 100644 index efde93b..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax_h.spv deleted file mode 100644 index 3fdf16a..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm.comp deleted file mode 100644 index c843790..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm.comp +++ /dev/null @@ -1,358 +0,0 @@ -#version 450 - -#extension GL_KHR_cooperative_matrix : require -#extension GL_KHR_memory_scope_semantics : require -#extension GL_KHR_shader_subgroup_basic : require -#extension GL_EXT_shader_16bit_storage : require -#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require -#extension GL_EXT_control_flow_attributes : require - -// Tensor-core implicit-GEMM dense KxK conv on NHWC fp16 activations -// (after SimdPaddleOCR's convk_cm_sg32): -// OUT[m = oy*outW + ox, n] = act(sum_k A[m, k] * W[n, k] + bias[n] + res[m, n]) -// k = (dy*kW + dx)*cin + ci (tap-major), A gathered from the input (no im2col), -// zero padding or reflect padding (flag), fp16 in memory, fp32 accumulation. -// SGX*SGY 32-lane subgroups, each owning a TSM*16 x TSN*16 strip of 16x16 -// accumulators; A/B k-slices of 32 halves are staged through double-buffered -// shared memory with coalesced 16 B loads. Requires cin % 8 == 0 (a 16 B quad -// never straddles a tap), nDim % 8 == 0 and 16 B aligned bindings. W is the -// [N][kDim] tap-major repack (kDim = kH*kW*cin rounded up to 8, zero-filled). -// SCALARK: any cin (the RGB/6-channel stems) — each k element of a quad is -// gathered individually, k >= kReal reads as zero. -// LITE (wave64-capable parts / 32 KB shared memory, after SimdPaddleOCR's sg32l): single- -// buffered staging, and the epilogue stage aliases the staging memory (one uvec4 array; the -// coopmat load/store element type may differ from the matrix type), so a 512-thread / 16- -// subgroup tile fits in 20 KB; scalar epilogue; workgroups rastered in groups of RASTER -// N-columns so the tiles in flight share A strips in L2. Same k order and epilogue math as -// the default build. -// SG16 (with LITE; Intel Xe2 XMX, after SimdPaddleOCR's conv1x1_cm): 16-lane subgroups -// (requiredSubgroupSize = 16) and the 8x16x16 shape — A and accumulator fragments are 8 -// rows, so a TSM*8 x TSN*16 strip per subgroup; staging, k order and epilogue as LITE. -// flags: bit0 bias, bit1 residual, bit2 reflect pad, bits[4:6] act (0 none, 1 relu, 2 gelu) - -#ifndef TSM -#define TSM 4u -#endif -#ifndef TSN -#define TSN 2u -#endif -#ifndef SGX -#define SGX 4u -#endif -#ifndef SGY -#define SGY 2u -#endif -#ifndef NTHR -#define NTHR 256 -#endif -#ifdef SG16 -#ifndef LITE -#error SG16 requires LITE -#endif -#define SGSZ 16u -#define AM 8 // A / accumulator fragment rows -#define AMU 8u -#else -#define SGSZ 32u -#define AM 16 -#define AMU 16u -#endif -#define RASTER 4u - -layout(local_size_x = NTHR) in; - -const uint STRM = TSM * AMU; -const uint STRN = TSN * 16u; -const uint BM = SGY * STRM; -const uint BN = SGX * STRN; -const uint BKV = 8u; // k-slice in vec4 units (32 halves) -const uint LDS = BKV + 2u; // padded smem row stride (vec4 units) -const uint BKQ = BKV / 2u; // 16 B quads per slice row -const uint RSTEP = NTHR / BKQ; -const uint LA = BM / RSTEP; -const uint LB = BN / RSTEP; - -layout(set = 0, binding = 0) readonly buffer X8 { uvec4 x8[]; }; -layout(set = 0, binding = 0) readonly buffer X1 { float16_t x1[]; }; -layout(set = 0, binding = 1) readonly buffer W8 { uvec4 w8[]; }; -layout(set = 0, binding = 2) readonly buffer Bias { float bias[]; }; -layout(set = 0, binding = 3) readonly buffer Res8 { uvec4 res8[]; }; -layout(set = 0, binding = 4) writeonly buffer O8 { uvec4 o8[]; }; -#ifdef LITE -layout(set = 0, binding = 3) readonly buffer Res1 { float16_t res1[]; }; -layout(set = 0, binding = 4) writeonly buffer O1 { float16_t o1[]; }; -#endif - -layout(push_constant) uniform PC { - uint mDim; uint nDim; uint kDim; uint flags; - uint outW; uint inW; uint inH; uint cin; uint kW; - uint sH; uint sW; uint pT; uint pL; uint kReal; -} p; - -#ifdef LITE -const uint LDS4 = LDS / 2u; // row stride in 16 B units -const uint SM_AB = (BM + BN) * LDS4; // A rows, then B rows (single buffer) -const uint SM_ST = NTHR / SGSZ * (AMU * 4u); // one AMx16 fp32 tile per subgroup (epilogue) -shared uvec4 smem[SM_AB > SM_ST ? SM_AB : SM_ST]; -#else -shared f16vec4 sA[2u * BM * LDS]; -shared f16vec4 sB[2u * BN * LDS]; -shared float stage[NTHR / SGSZ * 16 * 16]; -#endif - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -float actf(float v, uint act) -{ - if (act == 1u) return max(v, 0.0); - if (act == 2u) return 0.5 * v * (1.0 + ferf(v * 0.7071067811865475)); - return v; -} - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -int rowY[LA], rowX[LA]; -bool rowOk[LA]; - -void loadSlice(out uvec4 ra[LA], out uvec4 rb[LB], out uint va, out uint vb, - uint kt, uint n0, uint r0, uint kq, bool reflect) -{ - uint q = kt * BKQ + kq; // global 16 B quad index along k - uint kv2 = p.kDim >> 3; - bool kok = q < kv2; - uint qc = min(q, kv2 - 1u); - va = 0u; vb = 0u; - uint k8 = qc * 8u; -#ifdef SCALARK -#ifdef LITE - // the k -> (tap, ci) split does not depend on the row: once per element, not per row - uint edy[8], edx[8], eci[8]; - [[unroll]] for (uint e = 0u; e < 8u; e++) { - uint k = k8 + e; - uint tap = k / p.cin; - eci[e] = k - tap * p.cin; - edy[e] = tap / p.kW; - edx[e] = tap - edy[e] * p.kW; - } -#endif - [[unroll]] for (uint i = 0u; i < LA; i++) { - float16_t h[8]; - [[unroll]] for (uint e = 0u; e < 8u; e++) { - uint k = k8 + e; -#ifdef LITE - uint ci = eci[e], dy = edy[e], dx = edx[e]; -#else - uint tap = k / p.cin, ci = k - tap * p.cin; - uint dy = tap / p.kW, dx = tap - dy * p.kW; -#endif - int iy = rowY[i] + int(dy), ix = rowX[i] + int(dx); - bool inb = iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW); - if (reflect && rowOk[i] && !inb) { iy = mirror(iy, int(p.inH)); ix = mirror(ix, int(p.inW)); inb = true; } - bool ok = kok && rowOk[i] && inb && k < p.kReal; - h[e] = x1[ok ? (uint(iy) * p.inW + uint(ix)) * p.cin + ci : 0u]; - h[e] = ok ? h[e] : float16_t(0.0); - } - ra[i] = uvec4(packFloat2x16(f16vec2(h[0], h[1])), packFloat2x16(f16vec2(h[2], h[3])), - packFloat2x16(f16vec2(h[4], h[5])), packFloat2x16(f16vec2(h[6], h[7]))); - va |= kok && rowOk[i] ? 1u << i : 0u; - } -#else - uint tap = k8 / p.cin, ci = k8 - tap * p.cin; - uint dy = tap / p.kW, dx = tap - dy * p.kW; - uint cq = p.cin >> 3; - [[unroll]] for (uint i = 0u; i < LA; i++) { - int iy = rowY[i] + int(dy), ix = rowX[i] + int(dx); - bool inb = iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW); - if (reflect && rowOk[i] && !inb) { iy = mirror(iy, int(p.inH)); ix = mirror(ix, int(p.inW)); inb = true; } - bool ok = kok && rowOk[i] && inb; - uint px = ok ? uint(iy) * p.inW + uint(ix) : 0u; - ra[i] = x8[px * cq + (ci >> 3)]; - va |= ok ? 1u << i : 0u; - } -#endif - [[unroll]] for (uint i = 0u; i < LB; i++) { - uint gn = n0 + r0 + i * RSTEP; - bool rok = gn < p.nDim; - gn = min(gn, p.nDim - 1u); - rb[i] = w8[gn * kv2 + qc]; - vb |= rok && kok ? 1u << i : 0u; - } -} - -void storeSlice(uvec4 ra[LA], uvec4 rb[LB], uint va, uint vb, uint buf, uint r0, uint kq) -{ -#ifdef LITE - [[unroll]] for (uint i = 0u; i < LA; i++) - smem[(r0 + i * RSTEP) * LDS4 + kq] = (va & (1u << i)) != 0u ? ra[i] : uvec4(0u); - [[unroll]] for (uint i = 0u; i < LB; i++) - smem[(BM + r0 + i * RSTEP) * LDS4 + kq] = (vb & (1u << i)) != 0u ? rb[i] : uvec4(0u); -#else - [[unroll]] for (uint i = 0u; i < LA; i++) { - uint o = buf * BM * LDS + (r0 + i * RSTEP) * LDS + kq * 2u; - bool ok = (va & (1u << i)) != 0u; - sA[o] = ok ? f16vec4(unpackFloat2x16(ra[i].x), unpackFloat2x16(ra[i].y)) : f16vec4(0.0hf); - sA[o + 1u] = ok ? f16vec4(unpackFloat2x16(ra[i].z), unpackFloat2x16(ra[i].w)) : f16vec4(0.0hf); - } - [[unroll]] for (uint i = 0u; i < LB; i++) { - uint o = buf * BN * LDS + (r0 + i * RSTEP) * LDS + kq * 2u; - bool ok = (vb & (1u << i)) != 0u; - sB[o] = ok ? f16vec4(unpackFloat2x16(rb[i].x), unpackFloat2x16(rb[i].y)) : f16vec4(0.0hf); - sB[o + 1u] = ok ? f16vec4(unpackFloat2x16(rb[i].z), unpackFloat2x16(rb[i].w)) : f16vec4(0.0hf); - } -#endif -} - -void main() -{ - uint tid = gl_LocalInvocationID.x; - uint sg = gl_SubgroupID; - uint lane = gl_SubgroupInvocationID; - uint sgx = sg & (SGX - 1u), sgy = sg / SGX; -#ifdef LITE - // walk RASTER N-columns at a time, M fastest within the group - uint gmt = gl_NumWorkGroups.x; - uint lin = gl_WorkGroupID.y * gmt + gl_WorkGroupID.x; - uint grp = lin / (RASTER * gmt); - uint gw = min(gl_NumWorkGroups.y - grp * RASTER, RASTER); - uint gr = lin - grp * RASTER * gmt; - uint m0 = gr / gw * BM; - uint n0 = (grp * RASTER + gr % gw) * BN; -#else - uint m0 = gl_WorkGroupID.x * BM; - uint n0 = gl_WorkGroupID.y * BN; -#endif - - uint act = (p.flags >> 4) & 7u; - bool hasBias = (p.flags & 1u) != 0u; - bool hasRes = (p.flags & 2u) != 0u; - bool reflect = (p.flags & 4u) != 0u; - - coopmat acc[TSM][TSN]; - [[unroll]] for (uint i = 0u; i < TSM; i++) - [[unroll]] for (uint j = 0u; j < TSN; j++) - acc[i][j] = coopmat(0.0f); - - uint kq = tid % BKQ; - uint r0 = tid / BKQ; - [[unroll]] for (uint i = 0u; i < LA; i++) { - uint gm = m0 + r0 + i * RSTEP; - uint oy = gm / p.outW, ox = gm - oy * p.outW; - rowOk[i] = gm < p.mDim; - rowY[i] = int(oy * p.sH) - int(p.pT); - rowX[i] = int(ox * p.sW) - int(p.pL); - } - uint nk = (p.kDim + 31u) / 32u; - - uvec4 ra[LA], rb[LB]; - uint va, vb; - loadSlice(ra, rb, va, vb, 0u, n0, r0, kq, reflect); - storeSlice(ra, rb, va, vb, 0u, r0, kq); - barrier(); - - coopmat ma[TSM]; - coopmat mb[TSN]; -#ifdef LITE - uint ab = sgy * STRM * LDS4, bb = (BM + sgx * STRN) * LDS4; - for (uint kt = 0u; kt < nk; kt++) { - bool more = kt + 1u < nk; - if (more) loadSlice(ra, rb, va, vb, kt + 1u, n0, r0, kq, reflect); - [[unroll]] for (uint kk = 0u; kk < BKV / 2u; kk += 2u) { - [[unroll]] for (uint i = 0u; i < TSM; i++) - coopMatLoad(ma[i], smem, ab + i * AMU * LDS4 + kk, LDS4, gl_CooperativeMatrixLayoutRowMajor); - [[unroll]] for (uint j = 0u; j < TSN; j++) - coopMatLoad(mb[j], smem, bb + j * 16u * LDS4 + kk, LDS4, gl_CooperativeMatrixLayoutColumnMajor); - [[unroll]] for (uint i = 0u; i < TSM; i++) - [[unroll]] for (uint j = 0u; j < TSN; j++) - acc[i][j] = coopMatMulAdd(ma[i], mb[j], acc[i][j]); - } - barrier(); // one buffer: every read done before the refill - if (more) storeSlice(ra, rb, va, vb, 0u, r0, kq); - barrier(); - } -#else - uint aw = sgy * STRM * LDS, bw = sgx * STRN * LDS; - for (uint kt = 0u; kt < nk; kt++) { - uint cur = kt & 1u; - bool more = kt + 1u < nk; - if (more) loadSlice(ra, rb, va, vb, kt + 1u, n0, r0, kq, reflect); - uint ab = cur * BM * LDS + aw, bb = cur * BN * LDS + bw; - [[unroll]] for (uint kk = 0u; kk < BKV; kk += 4u) { - [[unroll]] for (uint i = 0u; i < TSM; i++) - coopMatLoad(ma[i], sA, ab + i * 16u * LDS + kk, LDS, gl_CooperativeMatrixLayoutRowMajor); - [[unroll]] for (uint j = 0u; j < TSN; j++) - coopMatLoad(mb[j], sB, bb + j * 16u * LDS + kk, LDS, gl_CooperativeMatrixLayoutColumnMajor); - [[unroll]] for (uint i = 0u; i < TSM; i++) - [[unroll]] for (uint j = 0u; j < TSN; j++) - acc[i][j] = coopMatMulAdd(ma[i], mb[j], acc[i][j]); - } - if (more) storeSlice(ra, rb, va, vb, cur ^ 1u, r0, kq); - barrier(); - } -#endif - -#ifdef LITE - // lane owns elements lane + SGSZ*l of each AMx16 tile: scalar fp16 stores, coalesced rows - [[unroll]] for (uint i = 0u; i < TSM; i++) { - [[unroll]] for (uint j = 0u; j < TSN; j++) { - coopMatStore(acc[i][j], smem, sg * (AMU * 4u), 4u, gl_CooperativeMatrixLayoutRowMajor); - subgroupMemoryBarrierShared(); - subgroupBarrier(); - uint fr = m0 + sgy * STRM + i * AMU, cn = n0 + sgx * STRN + j * 16u; - for (uint l = 0u; l < AMU * 16u / SGSZ; l++) { - uint e = l * SGSZ + lane; - uint gmr = fr + (e >> 4), gn = cn + (e & 15u); - if (gmr < p.mDim && gn < p.nDim) { - float v = uintBitsToFloat(smem[sg * (AMU * 4u) + (e >> 2)][e & 3u]); - if (hasBias) v += bias[gn]; - uint oe = gmr * p.nDim + gn; - if (hasRes) v += float(res1[oe]); - o1[oe] = float16_t(actf(v, act)); - } - } - subgroupMemoryBarrierShared(); - subgroupBarrier(); - } - } -#else - // epilogue through a per-subgroup row-major stage: each lane owns 8 - // consecutive columns of one row, so bias / residual / outputs move as 16 B - uint mm = lane >> 1, nb = (lane & 1u) * 8u; - [[unroll]] for (uint i = 0u; i < TSM; i++) { - [[unroll]] for (uint j = 0u; j < TSN; j++) { - coopMatStore(acc[i][j], stage, sg * 256u, 16u, gl_CooperativeMatrixLayoutRowMajor); - subgroupMemoryBarrierShared(); - subgroupBarrier(); - uint gmr = m0 + sgy * STRM + i * 16u + mm; - uint gnc = n0 + sgx * STRN + j * 16u + nb; - uint sb = sg * 256u + mm * 16u + nb; - if (gmr < p.mDim && gnc < p.nDim) { - float v[8]; - [[unroll]] for (uint l = 0u; l < 8u; l++) v[l] = stage[sb + l]; - if (hasBias) - [[unroll]] for (uint l = 0u; l < 8u; l++) v[l] += bias[gnc + l]; - uint oi = (gmr * p.nDim + gnc) >> 3; - if (hasRes) { - uvec4 rq = res8[oi]; - [[unroll]] for (uint l = 0u; l < 8u; l += 2u) { - vec2 r2 = vec2(unpackFloat2x16(rq[l >> 1])); - v[l] += r2.x; v[l + 1u] += r2.y; - } - } - uvec4 oq; - [[unroll]] for (uint l = 0u; l < 8u; l += 2u) - oq[l >> 1] = packFloat2x16(f16vec2(actf(v[l], act), actf(v[l + 1u], act))); - o8[oi] = oq; - } - subgroupMemoryBarrierShared(); - subgroupBarrier(); - } - } -#endif -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_h.spv deleted file mode 100644 index 8b3f3b8..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_h.spv deleted file mode 100644 index 9cc7c44..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32_h.spv deleted file mode 100644 index 53a037b..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32s_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32s_h.spv deleted file mode 100644 index 756df1d..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32s_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n64_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n64_h.spv deleted file mode 100644 index 424f6fb..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n64_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32_h.spv deleted file mode 100644 index 12a63e4..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32s_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32s_h.spv deleted file mode 100644 index a025194..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32s_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n64_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n64_h.spv deleted file mode 100644 index ca2e6f7..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n64_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_h.spv deleted file mode 100644 index 41ebb12..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32_h.spv deleted file mode 100644 index c99e99f..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32s_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32s_h.spv deleted file mode 100644 index 33d4f36..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32s_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n64_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n64_h.spv deleted file mode 100644 index dae8855..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n64_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot.comp deleted file mode 100644 index f6ae97b..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot.comp +++ /dev/null @@ -1,146 +0,0 @@ -#version 450 -#extension GL_EXT_shader_16bit_storage : require -#extension GL_EXT_control_flow_attributes : require -// Direct NHWC 3x3 conv for GPUs without cooperative matrices whose wide waves -// (Adreno: 64/128 lanes, 32 KB shared memory) starve on conv_tile's shared-memory -// staging: no shared memory, every operand is an f16vec4 load that the L1/texture -// cache serves. Each invocation computes TM consecutive output pixels of one row x -// TN*4 output channels; the TM*SW+2 input pixels of a kernel row are loaded once -// per 4 input channels and reused by the three kx taps. Invocations of one wave -// share pixels and step through oc (n4 fastest), so input loads broadcast and -// weight loads coalesce. Requires fp16 storage, ic % 4 == 0 and oc % (4*TN) == 0. -// Accumulation is fp32. Same push constants / weight layout ([K][OC], k = tap*IC + ic) -// as conv_tile.comp. Compile-time: -DSW= (1 or 2), -DTM=, -DTN=. -layout(local_size_x = 64) in; - -layout(set = 0, binding = 0) readonly buffer X { f16vec4 x[]; }; -layout(set = 0, binding = 1) readonly buffer W { f16vec4 w[]; }; -layout(set = 0, binding = 2) readonly buffer B { float b[]; }; -layout(set = 0, binding = 3) readonly buffer R { f16vec4 r[]; }; -layout(set = 0, binding = 4) writeonly buffer O { f16vec4 o[]; }; - -layout(push_constant) uniform PC { - uint ic, ih, iw; - uint oc, oh, ow; - uint kh, kw, sh, sw; - uint pt, pl; - uint flags; // bit0 hasBias, bit1 reflectPad, bit2 relu, bit3 hasAdd, bit4 gelu - uint pad0; -} p; - -#ifndef SW -#define SW 1u -#endif -#ifndef TM -#define TM 4 -#endif -#ifndef TN -#define TN 2 -#endif -const int NX = (TM - 1) * int(SW) + 3; // input pixels of one kernel row - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -vec4 gelu4(vec4 v) -{ - return vec4(0.5 * v.x * (1.0 + ferf(v.x * 0.7071067811865475)), - 0.5 * v.y * (1.0 + ferf(v.y * 0.7071067811865475)), - 0.5 * v.z * (1.0 + ferf(v.z * 0.7071067811865475)), - 0.5 * v.w * (1.0 + ferf(v.w * 0.7071067811865475))); -} - -void main() -{ - uint ng = p.oc / uint(4 * TN); // oc groups - uint xg = (p.ow + uint(TM) - 1u) / uint(TM); // pixel groups per row - uint gid = gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * gl_WorkGroupSize.x; - if (gid >= ng * xg * p.oh) return; - uint g = gid % ng; - uint t = gid / ng; - uint xb = t % xg, oy = t / xg; - uint ox0 = xb * uint(TM); - uint c4 = p.ic >> 2, o4 = p.oc >> 2; - uint n40 = g * uint(TN); - bool reflect = (p.flags & 2u) != 0u; - - vec4 acc[TM * TN]; - [[unroll]] for (int i = 0; i < TM * TN; i++) acc[i] = vec4(0.0); - - // input column per window slot (clamped / mirrored once; zero-pad slots masked) - int sx0 = int(ox0 * SW) - int(p.pl); - uint col[NX]; - bool cok[NX]; - [[unroll]] for (int i = 0; i < NX; i++) - { - int sx = sx0 + i; - cok[i] = reflect || uint(sx) < p.iw; - if (reflect && uint(sx) >= p.iw) sx = mirror(sx, int(p.iw)); - col[i] = uint(clamp(sx, 0, int(p.iw) - 1)); - } - - // no `continue`: Adreno 750 mis-executed a `continue` in this ky loop (one accumulator - // of the 4x2 tile was lost) and in depthwise's tap loop; zero-pad rows are masked loads - for (uint ky = 0u; ky < 3u; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - bool rok = reflect || uint(sy) < p.ih; - sy = reflect ? mirror(sy, int(p.ih)) : clamp(sy, 0, int(p.ih) - 1); - uint rowBase = uint(sy) * p.iw; - uint wk = ky * 3u * p.ic; // k of (ky, kx = 0, ci = 0) - for (uint ci = 0u; ci < c4; ci++) - { - vec4 xs[NX]; - [[unroll]] for (int i = 0; i < NX; i++) - xs[i] = cok[i] && rok ? vec4(x[(rowBase + col[i]) * c4 + ci]) : vec4(0.0); - [[unroll]] for (uint kx = 0u; kx < 3u; kx++) - { - uint wrow = (wk + kx * p.ic + ci * 4u) * o4 + n40; - [[unroll]] for (int j = 0; j < TN; j++) - { - vec4 w0 = vec4(w[wrow + uint(j)]); - vec4 w1 = vec4(w[wrow + o4 + uint(j)]); - vec4 w2 = vec4(w[wrow + 2u * o4 + uint(j)]); - vec4 w3 = vec4(w[wrow + 3u * o4 + uint(j)]); - [[unroll]] for (int m = 0; m < TM; m++) - { - vec4 a = xs[m * int(SW) + int(kx)]; - acc[m * TN + j] += a.x * w0 + a.y * w1 + a.z * w2 + a.w * w3; - } - } - } - } - } - - bool hasBias = (p.flags & 1u) != 0u; - bool relu = (p.flags & 4u) != 0u; - bool hasAdd = (p.flags & 8u) != 0u; - bool gelu = (p.flags & 16u) != 0u; - [[unroll]] for (int j = 0; j < TN; j++) - { - uint n4 = n40 + uint(j); - uint oc0 = n4 * 4u; - vec4 bias = hasBias ? vec4(b[oc0], b[oc0 + 1u], b[oc0 + 2u], b[oc0 + 3u]) : vec4(0.0); - [[unroll]] for (int m = 0; m < TM; m++) - { - uint ox = ox0 + uint(m); - if (ox < p.ow) - { - uint oi = (oy * p.ow + ox) * o4 + n4; - vec4 v = acc[m * TN + j] + bias; - if (hasAdd) v += vec4(r[oi]); - if (relu) v = max(v, vec4(0.0)); - if (gelu) v = gelu4(v); - o[oi] = f16vec4(v); - } - } - } -} \ No newline at end of file diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x1_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x1_h.spv deleted file mode 100644 index 38307d1..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x1_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x2_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x2_h.spv deleted file mode 100644 index 3a5a359..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x2_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x1_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x1_h.spv deleted file mode 100644 index 8923448..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x1_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x2_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x2_h.spv deleted file mode 100644 index bfcc4bc..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x2_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen.spv deleted file mode 100644 index 3e630a8..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen_h.spv deleted file mode 100644 index 1255943..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1.spv deleted file mode 100644 index 95df567..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1_h.spv deleted file mode 100644 index d94d45a..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1.spv deleted file mode 100644 index 8d3b620..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1_h.spv deleted file mode 100644 index ec1fc64..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2.spv deleted file mode 100644 index e47861f..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2_h.spv deleted file mode 100644 index 5740460..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_tile.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_tile.comp deleted file mode 100644 index 52ca6ab..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_tile.comp +++ /dev/null @@ -1,166 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -#extension GL_EXT_control_flow_attributes : require -// NHWC implicit-GEMM conv: O[px, oc] = sum_k X[px, k] * W[k, oc] with -// k = (ky*KW + kx)*IC + ic, so consecutive k are consecutive channels of one -// input pixel (coalesced loads). W is backend-packed [K][OC] (oc fastest). -// 3D dispatch: gx = x-tile(16), gy = y-tile(8), gz = oc-tile(64); 256 threads, -// each 4 oc x 8 pixels (vec4 accumulators over oc). K-chunk 32 staged in -// shared: As[k][m] (vec4 over m), Bs[n][k] (row stride 36 floats, vec4 over k). -// Accumulation is always fp32. Compile-time kernel/stride: -DKH= -DKW= -DSH= -DSW=. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) readonly buffer W { ET w[]; }; -layout(set = 0, binding = 2) readonly buffer B { float b[]; }; // per-OC bias (dummy if none) -layout(set = 0, binding = 3) readonly buffer R { ET r[]; }; // same-shape residual (dummy if none) -layout(set = 0, binding = 4) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint ic, ih, iw; - uint oc, oh, ow; - uint kh, kw, sh, sw; - uint pt, pl; - uint flags; // bit0 hasBias, bit1 reflectPad, bit2 relu, bit3 hasAdd, bit4 gelu - uint pad0; -} p; - -#ifndef KH -#define KH (p.kh) -#endif -#ifndef KW -#define KW (p.kw) -#endif -#ifndef SH -#define SH (p.sh) -#endif -#ifndef SW -#define SW (p.sw) -#endif - -const uint BM = 64u; // oc tile -const uint BH = 8u, BW = 16u; // spatial tile -const uint BN = BH * BW; // 128 pixels per wg -const uint BK = 32u; -const uint BKP = 36u; // Bs row stride in floats (vec4 aligned) -shared vec4 Asv[BK * BM / 4]; -shared vec4 Bsv[BN * BKP / 4]; - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -void main() -{ - uint oc0 = gl_WorkGroupID.z * BM; - uint oy0 = gl_WorkGroupID.y * BH; - uint ox0 = gl_WorkGroupID.x * BW; - uint K = KH * KW * p.ic; - uint tid = gl_LocalInvocationID.x; - uint tm = tid & 15u, tn = tid >> 4; // tm: 4 oc each, tn: 8 pixels each - - vec4 acc[8] = vec4[8](vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0)); - - bool reflect = (p.flags & 2u) != 0u; - uint bk = tid & 31u; // fixed k lane for Bs loads (256 % 32 == 0) - - for (uint k0 = 0u; k0 < K; k0 += BK) - { - // As[k][m] = W[k0+k][oc0+m] - for (uint e = tid; e < BK * BM; e += 256u) - { - uint k = e >> 6, m = e & 63u; - uint gm = oc0 + m, gk = k0 + k; - Asv[e >> 2][e & 3u] = (gm < p.oc && gk < K) ? float(w[gk * p.oc + gm]) : 0.0; - } - // Bs[n][k]: a warp reads 32 consecutive k = channels of one pixel tap - uint gk = k0 + bk; - bool kok = gk < K; - uint tap = kok ? gk / p.ic : 0u; - uint ch = gk - tap * p.ic; - uint ky = tap / KW, kx = tap - ky * KW; - for (uint n = tid >> 5; n < BN; n += 8u) - { - float v = 0.0; - uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u); - if (kok && oy < p.oh && ox < p.ow) - { - int sy = int(oy * SH + ky) - int(p.pt); - int sx = int(ox * SW + kx) - int(p.pl); - if (reflect) - { - if (uint(sy) >= p.ih) sy = mirror(sy, int(p.ih)); - if (uint(sx) >= p.iw) sx = mirror(sx, int(p.iw)); - v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]); - } - else if (uint(sy) < p.ih && uint(sx) < p.iw) - v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]); - } - uint si = n * BKP + bk; - Bsv[si >> 2][si & 3u] = v; - } - barrier(); - - [[unroll]] for (uint k = 0u; k < BK; k += 4u) - { - vec4 a0 = Asv[k * 16u + tm]; - vec4 a1 = Asv[(k + 1u) * 16u + tm]; - vec4 a2 = Asv[(k + 2u) * 16u + tm]; - vec4 a3 = Asv[(k + 3u) * 16u + tm]; - [[unroll]] for (int j = 0; j < 8; j++) - { - vec4 bv = Bsv[(tn * 8u + uint(j)) * (BKP / 4u) + (k >> 2)]; - acc[j] += a0 * bv.x + a1 * bv.y + a2 * bv.z + a3 * bv.w; - } - } - barrier(); - } - - bool hasBias = (p.flags & 1u) != 0u; - bool relu = (p.flags & 4u) != 0u; - bool hasAdd = (p.flags & 8u) != 0u; - bool gelu = (p.flags & 16u) != 0u; - uint gm0 = oc0 + tm * 4u; - // constant component indices only: keeps acc[] in registers (no local-memory spill) - vec4 bias = vec4( - hasBias && gm0 < p.oc ? b[gm0] : 0.0, - hasBias && gm0 + 1u < p.oc ? b[gm0 + 1u] : 0.0, - hasBias && gm0 + 2u < p.oc ? b[gm0 + 2u] : 0.0, - hasBias && gm0 + 3u < p.oc ? b[gm0 + 3u] : 0.0); - bool full = gm0 + 3u < p.oc; - [[unroll]] for (int j = 0; j < 8; j++) - { - uint n = tn * 8u + uint(j); - uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u); - if (oy >= p.oh || ox >= p.ow || gm0 >= p.oc) continue; - uint base = (oy * p.ow + ox) * p.oc + gm0; - vec4 v = acc[j] + bias; - if (hasAdd) - { - v.x += float(r[base]); - if (full) { v.y += float(r[base + 1u]); v.z += float(r[base + 2u]); v.w += float(r[base + 3u]); } - else - { - if (gm0 + 1u < p.oc) v.y += float(r[base + 1u]); - if (gm0 + 2u < p.oc) v.z += float(r[base + 2u]); - } - } - if (relu) v = max(v, vec4(0.0)); - if (gelu) v = vec4(0.5 * v.x * (1.0 + ferf(v.x * 0.7071067811865475)), - 0.5 * v.y * (1.0 + ferf(v.y * 0.7071067811865475)), - 0.5 * v.z * (1.0 + ferf(v.z * 0.7071067811865475)), - 0.5 * v.w * (1.0 + ferf(v.w * 0.7071067811865475))); - o[base] = ET(v.x); - if (gm0 + 1u < p.oc) o[base + 1u] = ET(v.y); - if (gm0 + 2u < p.oc) o[base + 2u] = ET(v.z); - if (full) o[base + 3u] = ET(v.w); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.comp deleted file mode 100644 index 85421b3..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.comp +++ /dev/null @@ -1,135 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -#extension GL_EXT_control_flow_attributes : require -// Small-ic conv (stem: ic < 8, stride 1) with a fused 2x2/2 max pool, NHWC: -// o[py][px][n] = [relu] max_{dy,dx<2} [relu](b[n] + sum_k x[2py+dy, 2px+dx; k] w[k][n]) -// One thread per (pooled pixel, 8 output channels); weights staged in shared -// memory as vec4 over n ([K][oc/4], K * oc <= 4096). The full-resolution conv -// output is never stored. fp32 math. -// flags: bit0 bias, bit1 reflect pad, bit2 conv relu, bit3 pool relu -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; // [ih][iw][ic] -layout(set = 0, binding = 1) readonly buffer W { ET w[]; }; // [K][oc], k = (ky*kw + kx)*ic + ci -layout(set = 0, binding = 2) readonly buffer B { float b[]; }; // [oc] (dummy if none) -layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; // [ph][pw][oc] - -layout(push_constant) uniform PC { - uint ic, ih, iw, oc, ph, pw, kh, kw, pt, pl, flags; - float padv; -} p; - -shared vec4 ws[1024]; - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -void main() -{ - uint tid = gl_LocalInvocationID.x; - uint K = p.kh * p.kw * p.ic, oc4 = p.oc >> 2; - for (uint e = tid; e < K * p.oc; e += 256u) ws[e >> 2][e & 3u] = float(w[e]); - barrier(); - - uint groups = p.oc >> 3; - uint idx = gl_GlobalInvocationID.x; - if (idx >= p.ph * p.pw * groups) return; - uint g = idx % groups, pp = idx / groups; - uint pxx = pp % p.pw, pyy = pp / p.pw; - bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u; - uint n0 = g * 8u; - vec4 b0 = vec4(0.0), b1 = vec4(0.0); - if ((p.flags & 1u) != 0u) - { - b0 = vec4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]); - b1 = vec4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]); - } - vec4 m0 = vec4(-3.4e38), m1 = vec4(-3.4e38); - uint oh = p.ph * 2u, ow = p.pw * 2u; // conv output = input size (stride 1, 'same') -#ifdef PIXELS_INNER - // the four pool pixels share each weight load (wide-wave GPUs: shared/L1 loads, not - // FMAs, bound the per-pixel loop); same per-pixel summation order - vec4 a[8]; - [[unroll]] for (uint d = 0u; d < 4u; d++) { a[2u * d] = b0; a[2u * d + 1u] = b1; } - for (uint ky = 0u; ky < p.kh; ky++) - { - uint rb[2]; - bool rv[2]; - [[unroll]] for (uint dy = 0u; dy < 2u; dy++) - { - int iy = int(pyy * 2u + dy + ky) - int(p.pt); - rv[dy] = iy >= 0 && iy < int(p.ih); - if (!rv[dy] && reflect) { iy = mirror(iy, int(p.ih)); rv[dy] = true; } - rb[dy] = uint(clamp(iy, 0, int(p.ih) - 1)) * p.iw; - } - for (uint kx = 0u; kx < p.kw; kx++) - { - uint base[4]; - bool valid[4]; - [[unroll]] for (uint dx = 0u; dx < 2u; dx++) - { - int ix = int(pxx * 2u + dx + kx) - int(p.pl); - bool vx = ix >= 0 && ix < int(p.iw); - if (!vx && reflect) { ix = mirror(ix, int(p.iw)); vx = true; } - uint cx = uint(clamp(ix, 0, int(p.iw) - 1)); - [[unroll]] for (uint dy = 0u; dy < 2u; dy++) - { - valid[dy * 2u + dx] = rv[dy] && vx; - base[dy * 2u + dx] = (rb[dy] + cx) * p.ic; - } - } - uint kb = (ky * p.kw + kx) * p.ic; - for (uint ci = 0u; ci < p.ic; ci++) - { - uint wr = (kb + ci) * oc4 + g * 2u; - vec4 w0 = ws[wr], w1 = ws[wr + 1u]; - [[unroll]] for (uint d = 0u; d < 4u; d++) - { - float xv = valid[d] ? float(x[base[d] + ci]) : p.padv; - a[2u * d] += xv * w0; - a[2u * d + 1u] += xv * w1; - } - } - } - } - [[unroll]] for (uint d = 0u; d < 4u; d++) - { - vec4 a0 = a[2u * d], a1 = a[2u * d + 1u]; - if (crelu) { a0 = max(a0, vec4(0.0)); a1 = max(a1, vec4(0.0)); } - m0 = max(m0, a0); m1 = max(m1, a1); - } -#else - [[unroll]] for (uint d = 0u; d < 4u; d++) - { - uint oy = pyy * 2u + (d >> 1), ox = pxx * 2u + (d & 1u); - vec4 a0 = b0, a1 = b1; - for (uint ky = 0u; ky < p.kh; ky++) - { - int iy = int(oy + ky) - int(p.pt); - bool vy = iy >= 0 && iy < int(p.ih); - if (!vy && reflect) { iy = mirror(iy, int(p.ih)); vy = true; } - for (uint kx = 0u; kx < p.kw; kx++) - { - int ix = int(ox + kx) - int(p.pl); - bool vx = ix >= 0 && ix < int(p.iw); - if (!vx && reflect) { ix = mirror(ix, int(p.iw)); vx = true; } - bool valid = vy && vx; - uint base = valid ? (uint(iy) * p.iw + uint(ix)) * p.ic : 0u; - uint kb = (ky * p.kw + kx) * p.ic; - for (uint ci = 0u; ci < p.ic; ci++) - { - float xv = valid ? float(x[base + ci]) : p.padv; - uint wr = (kb + ci) * oc4 + g * 2u; - a0 += xv * ws[wr]; - a1 += xv * ws[wr + 1u]; - } - } - } - if (crelu) { a0 = max(a0, vec4(0.0)); a1 = max(a1, vec4(0.0)); } - m0 = max(m0, a0); m1 = max(m1, a1); - } -#endif - if ((p.flags & 8u) != 0u) { m0 = max(m0, vec4(0.0)); m1 = max(m1, vec4(0.0)); } - uint ob = pp * p.oc + n0; - [[unroll]] for (uint l = 0u; l < 4u; l++) { o[ob + l] = ET(m0[l]); o[ob + 4u + l] = ET(m1[l]); } -} \ No newline at end of file diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.spv deleted file mode 100644 index c90b3a6..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_h.spv deleted file mode 100644 index 7435cd6..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.comp deleted file mode 100644 index d904a8b..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.comp +++ /dev/null @@ -1,104 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -#extension GL_EXT_control_flow_attributes : require -// convpool.comp specialised for 3x3 kernels (ic <= 7): the 4x4 input window is read one row -// at a time into registers and each staged weight pair is applied to both pool columns, so -// every input element is loaded once (16 vs 36 taps) and shared-memory weight reads halve. -// Same thread mapping (pooled pixel, 8 output channels) and the same per-output accumulation -// order (bias, then k = (ky*3 + kx)*ic + ci ascending) as convpool.comp: -// o[py][px][n] = [relu] max_{dy,dx<2} [relu](b[n] + sum_k x[2py+dy, 2px+dx; k] w[k][n]) -// flags: bit0 bias, bit1 reflect pad, bit2 conv relu, bit3 pool relu -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; // [ih][iw][ic] -layout(set = 0, binding = 1) readonly buffer W { ET w[]; }; // [K][oc], k = (ky*3 + kx)*ic + ci -layout(set = 0, binding = 2) readonly buffer B { float b[]; }; // [oc] (dummy if none) -layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; // [ph][pw][oc] - -layout(push_constant) uniform PC { - uint ic, ih, iw, oc, ph, pw, kh, kw, pt, pl, flags; - float padv; -} p; - -shared vec4 ws[1024]; - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -void main() -{ - uint tid = gl_LocalInvocationID.x; - uint K = 9u * p.ic, oc4 = p.oc >> 2; - for (uint e = tid; e < K * p.oc; e += 256u) ws[e >> 2][e & 3u] = float(w[e]); - barrier(); - - uint groups = p.oc >> 3; - uint idx = gl_GlobalInvocationID.x; - if (idx >= p.ph * p.pw * groups) return; - uint g = idx % groups, pp = idx / groups; - uint pxx = pp % p.pw, pyy = pp / p.pw; - bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u; - int ro[4], co[4]; // window rows (pre-multiplied by iw) / columns - bool rv[4], cv[4]; - [[unroll]] for (uint r = 0u; r < 4u; r++) - { - int iy = int(pyy * 2u + r) - int(p.pt); - bool vy = iy >= 0 && iy < int(p.ih); - if (!vy && reflect) { iy = mirror(iy, int(p.ih)); vy = true; } - rv[r] = vy; ro[r] = vy ? iy * int(p.iw) : 0; - int ix = int(pxx * 2u + r) - int(p.pl); - bool vx = ix >= 0 && ix < int(p.iw); - if (!vx && reflect) { ix = mirror(ix, int(p.iw)); vx = true; } - cv[r] = vx; co[r] = vx ? ix : 0; - } - uint n0 = g * 8u; - vec4 b0 = vec4(0.0), b1 = vec4(0.0); - if ((p.flags & 1u) != 0u) - { - b0 = vec4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]); - b1 = vec4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]); - } - vec4 a0[4], a1[4]; - [[unroll]] for (uint d = 0u; d < 4u; d++) { a0[d] = b0; a1[d] = b1; } - [[unroll]] for (uint r = 0u; r < 4u; r++) - { - float xr[4][8]; - [[unroll]] for (uint c = 0u; c < 4u; c++) - { - bool v = rv[r] && cv[c]; - uint base = uint(ro[r] + co[c]) * p.ic; - [[unroll]] for (uint ci = 0u; ci < 8u; ci++) - xr[c][ci] = ci < p.ic ? (v ? float(x[base + ci]) : p.padv) : 0.0; - } - [[unroll]] for (uint dy = 0u; dy < 2u; dy++) - { - if (r < dy || r - dy > 2u) continue; - uint ky = r - dy; - [[unroll]] for (uint kx = 0u; kx < 3u; kx++) - { - uint kb = (ky * 3u + kx) * p.ic; - [[unroll]] for (uint ci = 0u; ci < 8u; ci++) - { - if (ci >= p.ic) break; - uint wr = (kb + ci) * oc4 + g * 2u; - vec4 w0 = ws[wr], w1 = ws[wr + 1u]; - [[unroll]] for (uint dx = 0u; dx < 2u; dx++) - { - float xv = xr[kx + dx][ci]; - a0[dy * 2u + dx] += xv * w0; - a1[dy * 2u + dx] += xv * w1; - } - } - } - } - } - vec4 m0 = vec4(-3.4e38), m1 = vec4(-3.4e38); - [[unroll]] for (uint d = 0u; d < 4u; d++) - { - if (crelu) { a0[d] = max(a0[d], vec4(0.0)); a1[d] = max(a1[d], vec4(0.0)); } - m0 = max(m0, a0[d]); m1 = max(m1, a1[d]); - } - if ((p.flags & 8u) != 0u) { m0 = max(m0, vec4(0.0)); m1 = max(m1, vec4(0.0)); } - uint ob = pp * p.oc + n0; - [[unroll]] for (uint l = 0u; l < 4u; l++) { o[ob + l] = ET(m0[l]); o[ob + 4u + l] = ET(m1[l]); } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.spv deleted file mode 100644 index 6073da8..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3_h.spv deleted file mode 100644 index fe50d31..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_pi_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_pi_h.spv deleted file mode 100644 index 8153e1e..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_pi_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.comp deleted file mode 100644 index 383d084..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.comp +++ /dev/null @@ -1,57 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// NHWC depthwise conv: out[oy,ox,c] = bias[c] + sum w[tap,c] * x[sy,sx,c] -// (w backend-packed [tap][C]); channel-fastest flat index -> coalesced. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) readonly buffer W { ET w[]; }; -layout(set = 0, binding = 2) readonly buffer B { float b[]; }; -layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl; - uint flags; // bit0 hasBias, bit1 reflect, bit2 relu, bit4 gelu - float padVal; -} p; - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -void main() -{ - uint id = gl_GlobalInvocationID.x; - if (id >= p.c * p.oh * p.ow) return; - uint ch = id % p.c, px = id / p.c; - uint ox = px % p.ow, oy = px / p.ow; - bool refl = (p.flags & 2u) != 0u; - float acc = ((p.flags & 1u) != 0u) ? b[ch] : 0.0; - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - float wv = float(w[(ky * p.kw + kx) * p.c + ch]); - // branch-free border: Adreno 750 mis-executed the former `acc += pad; continue;` form - bool oob = uint(sy) >= p.ih || uint(sx) >= p.iw; - int yy = refl ? mirror(sy, int(p.ih)) : clamp(sy, 0, int(p.ih) - 1); - int xx = refl ? mirror(sx, int(p.iw)) : clamp(sx, 0, int(p.iw) - 1); - float xv = float(x[(uint(yy) * p.iw + uint(xx)) * p.c + ch]); - acc += wv * (oob && !refl ? p.padVal : xv); - } - } - if ((p.flags & 4u) != 0u) acc = max(acc, 0.0); - if ((p.flags & 16u) != 0u) acc = 0.5 * acc * (1.0 + ferf(acc * 0.7071067811865475)); - o[id] = ET(acc); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.spv deleted file mode 100644 index ec4f43b..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.comp deleted file mode 100644 index b158de4..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.comp +++ /dev/null @@ -1,58 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// depthwise.comp with 4 channels per invocation (c % 4 == 0), same per-element math. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET4 x[]; }; -layout(set = 0, binding = 1) readonly buffer W { ET4 w[]; }; -layout(set = 0, binding = 2) readonly buffer B { vec4 b[]; }; -layout(set = 0, binding = 3) writeonly buffer O { ET4 o[]; }; - -layout(push_constant) uniform PC { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl; - uint flags; // bit0 hasBias, bit1 reflect, bit2 relu, bit4 gelu - float padVal; -} p; - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -void main() -{ - uint id = gl_GlobalInvocationID.x; - uint c4 = p.c >> 2; - if (id >= c4 * p.oh * p.ow) return; - uint ch = id % c4, px = id / c4; - uint ox = px % p.ow, oy = px / p.ow; - bool refl = (p.flags & 2u) != 0u; - vec4 acc = ((p.flags & 1u) != 0u) ? b[ch] : vec4(0.0); - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - vec4 wv = vec4(w[(ky * p.kw + kx) * c4 + ch]); - // branch-free border: Adreno 750 mis-executed the former `acc += pad; continue;` form - bool oob = uint(sy) >= p.ih || uint(sx) >= p.iw; - int yy = refl ? mirror(sy, int(p.ih)) : clamp(sy, 0, int(p.ih) - 1); - int xx = refl ? mirror(sx, int(p.iw)) : clamp(sx, 0, int(p.iw) - 1); - vec4 xv = vec4(x[(uint(yy) * p.iw + uint(xx)) * c4 + ch]); - acc += wv * (oob && !refl ? vec4(p.padVal) : xv); - } - } - if ((p.flags & 4u) != 0u) acc = max(acc, vec4(0.0)); - if ((p.flags & 16u) != 0u) - for (int l = 0; l < 4; l++) acc[l] = 0.5 * acc[l] * (1.0 + ferf(acc[l] * 0.7071067811865475)); - o[id] = ET4(acc); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.spv deleted file mode 100644 index ff663e4..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4_h.spv deleted file mode 100644 index ca0dfb6..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise_h.spv deleted file mode 100644 index f476c95..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.comp deleted file mode 100644 index 3eb4856..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.comp +++ /dev/null @@ -1,33 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Elementwise binary + fused activation, general rank<=4 broadcast. -// dst flat index decomposed by out dims (d0..d3); operand index = sum(dk * stride_k). -// op: 0 add 1 sub 2 mul 3 div; act: 0 none 1 relu 2 sigmoid -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer A { ET a[]; }; -layout(set = 0, binding = 1) readonly buffer B { ET b[]; }; -layout(set = 0, binding = 2) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint n, s0, s1, s2; // out strides (s3 = 1 implied) - uint as0, as1, as2, as3; - uint bs0, bs1, bs2, bs3; - uint op, act; -} p; - -void main() -{ - uint i = gl_GlobalInvocationID.x; - if (i >= p.n) return; - uint d0 = i / p.s0, r0 = i % p.s0; - uint d1 = r0 / p.s1, r1 = r0 % p.s1; - uint d2 = r1 / p.s2, d3 = r1 % p.s2; - float va = float(a[d0 * p.as0 + d1 * p.as1 + d2 * p.as2 + d3 * p.as3]); - float vb = float(b[d0 * p.bs0 + d1 * p.bs1 + d2 * p.bs2 + d3 * p.bs3]); - float v = p.op == 0u ? va + vb : p.op == 1u ? va - vb : p.op == 2u ? va * vb : va / vb; - if (p.act == 1u) v = max(v, 0.0); - else if (p.act == 2u) v = 1.0 / (1.0 + exp(-v)); - o[i] = ET(v); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.spv deleted file mode 100644 index fc007c5..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2_h.spv deleted file mode 100644 index 87a2dfe..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.comp deleted file mode 100644 index de8d905..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.comp +++ /dev/null @@ -1,26 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Elementwise binary + fused activation, contiguous same-shape fast path (4 floats/thread). -// op: 0 add 1 sub 2 mul 3 div; act: 0 none 1 relu 2 sigmoid -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer A { ET4 a[]; }; -layout(set = 0, binding = 1) readonly buffer B { ET4 b[]; }; -layout(set = 0, binding = 2) writeonly buffer O { ET4 o[]; }; - -layout(push_constant) uniform PC { uint nv4; uint op, act; } p; - -void main() -{ - uint i = gl_GlobalInvocationID.x; - if (i >= p.nv4) return; - vec4 va = vec4(a[i]), vb = vec4(b[i]), v; - if (p.op == 0u) v = va + vb; - else if (p.op == 1u) v = va - vb; - else if (p.op == 2u) v = va * vb; - else v = va / vb; - if (p.act == 1u) v = max(v, vec4(0.0)); - else if (p.act == 2u) v = 1.0 / (1.0 + exp(-v)); - o[i] = ET4(v); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.spv deleted file mode 100644 index da05819..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v_h.spv deleted file mode 100644 index c108578..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.comp deleted file mode 100644 index e37fd69..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.comp +++ /dev/null @@ -1,88 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Tiled GEMM: C[M,N] = A[M,K] * B + bias[N]; B is [K,N] or [N,K] when transB. -// Tile 64x64, K-chunk 16, 256 threads -> 4x4 each. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer A { ET a[]; }; -layout(set = 0, binding = 1) readonly buffer B { ET b[]; }; -layout(set = 0, binding = 2) readonly buffer BS { float bs[]; }; // bias[N] (dummy if none) -layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint m, n, k; - uint flags; // bit0 transB, bit1 hasBias, bit2 relu, bit3 gelu -} p; - -const uint BM = 64u, BN = 64u, BK = 16u; -shared float As[BK * BM]; -shared float Bs[BK * BN]; - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -void main() -{ - uint m0 = gl_WorkGroupID.y * BM; - uint n0 = gl_WorkGroupID.x * BN; - uint tid = gl_LocalInvocationID.x; - uint ty = tid >> 4, tx = tid & 15u; - bool transB = (p.flags & 1u) != 0u; - - float acc[4][4]; - for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) acc[i][j] = 0.0; - - for (uint k0 = 0u; k0 < p.k; k0 += BK) - { - for (uint e = tid; e < BK * BM; e += 256u) - { - uint kk = e >> 6, mm = e & 63u; - uint gm = m0 + mm, gk = k0 + kk; - As[e] = (gm < p.m && gk < p.k) ? float(a[gm * p.k + gk]) : 0.0; - } - for (uint e = tid; e < BK * BN; e += 256u) - { - uint kk = e >> 6, nn = e & 63u; - uint gn = n0 + nn, gk = k0 + kk; - float v = 0.0; - if (gn < p.n && gk < p.k) - v = float(transB ? b[gn * p.k + gk] : b[gk * p.n + gn]); - Bs[e] = v; - } - barrier(); - for (uint kk = 0u; kk < BK; kk++) - { - float av[4], bv[4]; - for (int i = 0; i < 4; i++) av[i] = As[kk * BM + ty * 4u + uint(i)]; - for (int j = 0; j < 4; j++) bv[j] = Bs[kk * BN + tx * 4u + uint(j)]; - for (int i = 0; i < 4; i++) - for (int j = 0; j < 4; j++) - acc[i][j] += av[i] * bv[j]; - } - barrier(); - } - - bool hasBias = (p.flags & 2u) != 0u; - for (int i = 0; i < 4; i++) - { - uint gm = m0 + ty * 4u + uint(i); - if (gm >= p.m) continue; - for (int j = 0; j < 4; j++) - { - uint gn = n0 + tx * 4u + uint(j); - if (gn >= p.n) continue; - float v = acc[i][j]; - if (hasBias) v += bs[gn]; - if ((p.flags & 4u) != 0u) v = max(v, 0.0); - if ((p.flags & 8u) != 0u) v = 0.5 * v * (1.0 + ferf(v * 0.7071067811865475)); - o[gm * p.n + gn] = ET(v); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.spv deleted file mode 100644 index ff9d30d..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot.comp deleted file mode 100644 index 3ef44c9..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot.comp +++ /dev/null @@ -1,89 +0,0 @@ -#version 450 -#extension GL_EXT_shader_16bit_storage : require -#extension GL_EXT_control_flow_attributes : require -// GEMM C[M,N] = A[M,K] * B[K,N] + bias[N] for the conv_dot devices (wide waves, no -// cooperative matrices): conv_dot.comp's register tiling for a 1x1 "conv" without -// shared memory. Each invocation computes TM rows x TN*4 columns from f16vec4 loads -// (n4 fastest across the wave: A loads broadcast, B loads coalesce). Requires fp16 -// storage, !transB, K % 4 == 0 and N % (4*TN) == 0. fp32 accumulation, K in order. -// Push constants as gemm.comp (flags: bit0 transB (unsupported), bit1 bias, bit2 relu, -// bit3 gelu). -layout(local_size_x = 64) in; - -layout(set = 0, binding = 0) readonly buffer A { f16vec4 a[]; }; -layout(set = 0, binding = 1) readonly buffer B { f16vec4 b[]; }; -layout(set = 0, binding = 2) readonly buffer BS { float bs[]; }; -layout(set = 0, binding = 3) writeonly buffer O { f16vec4 o[]; }; - -layout(push_constant) uniform PC { - uint m, n, k; - uint flags; -} p; - -#ifndef TM -#define TM 4 -#endif -#ifndef TN -#define TN 2 -#endif - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -void main() -{ - uint ng = p.n / uint(4 * TN); - uint gid = gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * gl_WorkGroupSize.x; - uint mg = (p.m + uint(TM) - 1u) / uint(TM); - if (gid >= ng * mg) return; - uint g = gid % ng, m0 = (gid / ng) * uint(TM); - uint k4 = p.k >> 2, n4s = p.n >> 2, n40 = g * uint(TN); - - vec4 acc[TM * TN]; - [[unroll]] for (int i = 0; i < TM * TN; i++) acc[i] = vec4(0.0); - uint arow[TM]; - [[unroll]] for (int i = 0; i < TM; i++) arow[i] = min(m0 + uint(i), p.m - 1u) * k4; - - for (uint kk = 0u; kk < k4; kk++) - { - vec4 av[TM]; - [[unroll]] for (int i = 0; i < TM; i++) av[i] = vec4(a[arow[i] + kk]); - uint brow = kk * 4u * n4s + n40; - [[unroll]] for (int j = 0; j < TN; j++) - { - vec4 w0 = vec4(b[brow + uint(j)]); - vec4 w1 = vec4(b[brow + n4s + uint(j)]); - vec4 w2 = vec4(b[brow + 2u * n4s + uint(j)]); - vec4 w3 = vec4(b[brow + 3u * n4s + uint(j)]); - [[unroll]] for (int i = 0; i < TM; i++) - acc[i * TN + j] += av[i].x * w0 + av[i].y * w1 + av[i].z * w2 + av[i].w * w3; - } - } - - bool hasBias = (p.flags & 2u) != 0u, relu = (p.flags & 4u) != 0u, gelu = (p.flags & 8u) != 0u; - [[unroll]] for (int j = 0; j < TN; j++) - { - uint n4 = n40 + uint(j), c0 = n4 * 4u; - vec4 bias = hasBias ? vec4(bs[c0], bs[c0 + 1u], bs[c0 + 2u], bs[c0 + 3u]) : vec4(0.0); - [[unroll]] for (int i = 0; i < TM; i++) - { - uint row = m0 + uint(i); - if (row < p.m) - { - vec4 v = acc[i * TN + j] + bias; - if (relu) v = max(v, vec4(0.0)); - if (gelu) v = vec4(0.5 * v.x * (1.0 + ferf(v.x * 0.7071067811865475)), - 0.5 * v.y * (1.0 + ferf(v.y * 0.7071067811865475)), - 0.5 * v.z * (1.0 + ferf(v.z * 0.7071067811865475)), - 0.5 * v.w * (1.0 + ferf(v.w * 0.7071067811865475))); - o[row * n4s + n4] = f16vec4(v); - } - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot_h.spv deleted file mode 100644 index c20ca7c..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_h.spv deleted file mode 100644 index 2aecca4..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.comp deleted file mode 100644 index 09e4b94..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.comp +++ /dev/null @@ -1,41 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// LayerNorm over last dim: one workgroup (256 threads) per row. -// o[r,i] = (x[r,i] - mean) * rsqrt(var + eps) * w[i] + b[i] -// Single reduction pass: sum and sum-of-squares accumulate together in vec2 lanes. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) readonly buffer W { float w[]; }; -layout(set = 0, binding = 2) readonly buffer B { float b[]; }; -layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { uint len; float eps; } p; - -shared vec2 red[256]; - -void main() -{ - uint row = gl_WorkGroupID.x; - uint tid = gl_LocalInvocationID.x; - uint base = row * p.len; - - float s = 0.0, s2 = 0.0; - for (uint i = tid; i < p.len; i += 256u) - { - float v = float(x[base + i]); - s += v; s2 += v * v; - } - red[tid] = vec2(s, s2); - barrier(); - for (uint st = 128u; st > 0u; st >>= 1) - { - if (tid < st) red[tid] += red[tid + st]; - barrier(); - } - float mean = red[0].x / float(p.len); - float inv = inversesqrt(red[0].y / float(p.len) - mean * mean + p.eps); - for (uint i = tid; i < p.len; i += 256u) - o[base + i] = ET((float(x[base + i]) - mean) * inv * w[i] + b[i]); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.spv deleted file mode 100644 index 6f5d4a9..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm_h.spv deleted file mode 100644 index 38e49d0..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.comp deleted file mode 100644 index 96dce91..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.comp +++ /dev/null @@ -1,37 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// NHWC maxpool + optional relu (channel-fastest flat index). -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl; - uint relu; -} p; - -void main() -{ - uint id = gl_GlobalInvocationID.x; - if (id >= p.c * p.oh * p.ow) return; - uint ch = id % p.c, px = id / p.c; - uint ox = px % p.ow, oy = px / p.ow; - float m = -3.4e38; - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - if (uint(sy) >= p.ih) continue; - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - if (uint(sx) >= p.iw) continue; - m = max(m, float(x[(uint(sy) * p.iw + uint(sx)) * p.c + ch])); - } - } - if (m == -3.4e38) m = 0.0; - if (p.relu != 0u && m < 0.0) m = 0.0; - o[id] = ET(m); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.spv deleted file mode 100644 index 38f34c4..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.comp deleted file mode 100644 index 649b334..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.comp +++ /dev/null @@ -1,39 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// maxpool.comp with 4 channels per invocation (c % 4 == 0): same per-element math, -// a quarter of the index arithmetic and load instructions (bandwidth-bound iGPUs). -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET4 x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET4 o[]; }; - -layout(push_constant) uniform PC { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl; - uint relu; -} p; - -void main() -{ - uint id = gl_GlobalInvocationID.x; - uint c4 = p.c >> 2; - if (id >= c4 * p.oh * p.ow) return; - uint ch = id % c4, px = id / c4; - uint ox = px % p.ow, oy = px / p.ow; - vec4 m = vec4(-3.4e38); - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - if (uint(sy) >= p.ih) continue; - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - if (uint(sx) >= p.iw) continue; - m = max(m, vec4(x[(uint(sy) * p.iw + uint(sx)) * c4 + ch])); - } - } - if (m.x == -3.4e38) m = vec4(0.0); - if (p.relu != 0u) m = mix(m, vec4(0.0), lessThan(m, vec4(0.0))); // as the scalar m < 0 test - o[id] = ET4(m); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.spv deleted file mode 100644 index 59aff0e..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4_h.spv deleted file mode 100644 index e577d02..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool_h.spv deleted file mode 100644 index f8fa66d..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.comp deleted file mode 100644 index c015c7a..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.comp +++ /dev/null @@ -1,47 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Rank-4 pad on all dims (spatial in practice), constant or reflect. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint id0, id1, id2, id3; // in dims - uint od0, od1, od2, od3; // out dims - uint pb0, pb1, pb2, pb3; // pad before per dim - uint mode; // 0 const 1 reflect - float value; -} p; - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } -int coord(uint oidx, uint pb, uint idim, out bool miss) -{ - int s = int(oidx) - int(pb); - if (uint(s) >= idim) - { - if (p.mode == 0u) { miss = true; return 0; } - s = mirror(s, int(idim)); - } - miss = false; return s; -} - -void main() -{ - uint i = gl_GlobalInvocationID.x; - uint n = p.od0 * p.od1 * p.od2 * p.od3; - if (i >= n) return; - uint s2 = p.od3, s1 = p.od2 * s2, s0 = p.od1 * s1; - uint d0 = i / s0, r0 = i % s0; - uint d1 = r0 / s1, r1 = r0 % s1; - uint d2 = r1 / s2, d3 = r1 % s2; - bool m0, m1, m2, m3; - int c0 = coord(d0, p.pb0, p.id0, m0); - int c1 = coord(d1, p.pb1, p.id1, m1); - int c2 = coord(d2, p.pb2, p.id2, m2); - int c3 = coord(d3, p.pb3, p.id3, m3); - if (m0 || m1 || m2 || m3) { o[i] = ET(p.value); return; } - uint si2 = p.id3, si1 = p.id2 * si2, si0 = p.id1 * si1; - o[i] = x[uint(c0) * si0 + uint(c1) * si1 + uint(c2) * si2 + uint(c3)]; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.spv deleted file mode 100644 index b5e56ec..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad_h.spv deleted file mode 100644 index 683332e..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.comp deleted file mode 100644 index 56a9127..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.comp +++ /dev/null @@ -1,13 +0,0 @@ -#version 450 -#extension GL_KHR_shader_subgroup_basic : require - -// Device probe for pipelines created with a requiredSubgroupSize (32, or 16 for SG16): every -// invocation reports subgroup size | count << 8 | id << 16 | lane << 24 (VkDevice.ProbeSubgroup). -layout(local_size_x = 256) in; -layout(set = 0, binding = 0) writeonly buffer O { uint o[]; }; - -void main() -{ - o[gl_LocalInvocationID.x] = gl_SubgroupSize | (gl_NumSubgroups << 8) - | (gl_SubgroupID << 16) | (gl_SubgroupInvocationID << 24); -} \ No newline at end of file diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.spv deleted file mode 100644 index 913cb72..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.comp deleted file mode 100644 index 484e0ac..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.comp +++ /dev/null @@ -1,27 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Generic rank<=4 strided slice: dst[k0,k1,k2,k3] = src[start_k + k*step_k]. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint n; - uint os0, os1, os2; // out strides (os3 = 1) - uint st0, st1, st2, st3; // start per out dim - uint sp0, sp1, sp2, sp3; // step per out dim - uint is0, is1, is2, is3; // in strides -} p; - -void main() -{ - uint i = gl_GlobalInvocationID.x; - if (i >= p.n) return; - uint d0 = i / p.os0, r0 = i % p.os0; - uint d1 = r0 / p.os1, r1 = r0 % p.os1; - uint d2 = r1 / p.os2, d3 = r1 % p.os2; - o[i] = x[(p.st0 + d0 * p.sp0) * p.is0 + (p.st1 + d1 * p.sp1) * p.is1 - + (p.st2 + d2 * p.sp2) * p.is2 + (p.st3 + d3 * p.sp3) * p.is3]; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.spv deleted file mode 100644 index 459b1b8..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice_h.spv deleted file mode 100644 index 6a0e3a4..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.comp deleted file mode 100644 index 2448009..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.comp +++ /dev/null @@ -1,24 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Generic rank<=4 permute: dst flat idx -> src idx via permuted in-strides. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint n; - uint os0, os1, os2; // out strides (os3 = 1) - uint ps0, ps1, ps2, ps3; // in strides in out-dim order: inStr[perm[k]] -} p; - -void main() -{ - uint i = gl_GlobalInvocationID.x; - if (i >= p.n) return; - uint d0 = i / p.os0, r0 = i % p.os0; - uint d1 = r0 / p.os1, r1 = r0 % p.os1; - uint d2 = r1 / p.os2, d3 = r1 % p.os2; - o[i] = x[d0 * p.ps0 + d1 * p.ps1 + d2 * p.ps2 + d3 * p.ps3]; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.spv deleted file mode 100644 index 7d17649..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose_h.spv deleted file mode 100644 index be61c6c..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.comp deleted file mode 100644 index ade3618..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.comp +++ /dev/null @@ -1,44 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Elementwise unary: mode 0 relu, 1 sigmoid, 2 erf, 3 gelu, 4 copy -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { uint n, mode; } p; - -float fexp(float v) -{ - v = clamp(v, -87.3, 88.7); - float fx = v * 1.4426950408889634 + 0.5; - float nf = floor(fx); - float r = v - nf * 0.693359375 - nf * -0.00021219444005469058; - float poly = (((((1.9875691500e-4 * r + 1.3981999507e-3) * r + 8.3334519073e-3) * r - + 4.1665795894e-2) * r + 1.6666665459e-1) * r + 5.0000001201e-1) * r * r + r + 1.0; - return poly * uintBitsToFloat((uint(int(nf)) + 127u) << 23); -} - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * fexp(-ax * ax)) * s; -} - -void main() -{ - uint i = gl_GlobalInvocationID.x; - if (i >= p.n) return; - float v = float(x[i]); - float r; - if (p.mode == 0u) r = max(v, 0.0); - else if (p.mode == 1u) r = 1.0 / (1.0 + fexp(-v)); - else if (p.mode == 2u) r = ferf(v); - else if (p.mode == 3u) r = 0.5 * v * (1.0 + ferf(v * 0.7071067811865475)); - else r = v; - o[i] = ET(r); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.spv deleted file mode 100644 index 72742dd..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary_h.spv deleted file mode 100644 index b6a9285..0000000 Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary_h.spv and /dev/null differ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Vk.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Vk.cs deleted file mode 100644 index 98f3a54..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Vk.cs +++ /dev/null @@ -1,589 +0,0 @@ -// Minimal Vulkan 1.1 P/Invoke for compute-only use (SimdPaddleOCR Vulkan backend). -// Loads vulkan-1.dll (Windows) / libvulkan.so.1 (Linux) / libvulkan.so (Android) -// via SetDllImportResolver. -// Only the entry points and structs we need; all structs are blittable. -using System.Runtime.InteropServices; - -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -internal static unsafe partial class Vk -{ - public const string LibName = "vulkan-1"; - - private static int _resolverRegistered; - - // Android ships only the unversioned NDK name (/system/lib64/libvulkan.so) - private static readonly string[] s_candidates = OperatingSystem.IsWindows() - ? ["vulkan-1.dll"] : ["libvulkan.so.1", "libvulkan.so"]; - - public static void RegisterResolver() - { - if (Interlocked.Exchange(ref _resolverRegistered, 1) != 0) - return; - NativeLibrary.SetDllImportResolver(typeof(Vk).Assembly, (name, asm, path) => - { - if (name != LibName) - return IntPtr.Zero; - foreach (string candidate in s_candidates) - if (NativeLibrary.TryLoad(candidate, out IntPtr h)) - { - LoadedLibrary = candidate; - return h; - } - throw new DllNotFoundException( - $"Vulkan loader not found (tried {string.Join(", ", s_candidates)})"); - }); - } - - /// File name the resolver loaded, or null before the first call. - public static string? LoadedLibrary { get; private set; } - - public const ulong WholeSize = ~0UL; - public const uint QueueFamilyIgnored = ~0u; - - public static void Check(VkResult r, string what) - { - if (r != VkResult.Success) - throw new InvalidOperationException($"Vulkan {what} failed: {r}"); - } - - // ---- structs (field order must match the Vulkan spec) ---- - - [StructLayout(LayoutKind.Sequential)] - public struct VkApplicationInfo - { - public uint SType; public void* PNext; public void* PAppName; - public uint AppVersion; public void* PEngineName; public uint EngineVersion; public uint ApiVersion; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkInstanceCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public VkApplicationInfo* PAppInfo; - public uint EnabledLayerCount; public void* PpEnabledLayerNames; - public uint EnabledExtensionCount; public void* PpEnabledExtensionNames; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceProperties - { - public uint ApiVersion, DriverVersion, VendorID, DeviceID, DeviceType; - public fixed byte DeviceName[256]; - public fixed byte PipelineCacheUUID[16]; - public fixed byte LimitsAndSparse[1024]; // VkPhysicalDeviceLimits then Sparse; timestampPeriod float @428 - public float TimestampPeriodNs - { - get { fixed (byte* p = LimitsAndSparse) return *(float*)(p + 428); } - } - // limits start 4 bytes in (8-byte aligned natively); maxComputeSharedMemorySize @216 - public uint MaxComputeSharedMemorySize - { - get { fixed (byte* p = LimitsAndSparse) return *(uint*)(p + 4 + 216); } - } - public uint MaxStorageBufferRange - { - get { fixed (byte* p = LimitsAndSparse) return *(uint*)(p + 4 + 28); } - } - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceProperties2 - { - public uint SType; public void* PNext; - public VkPhysicalDeviceProperties Properties; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceSubgroupSizeControlProperties - { - public uint SType; public void* PNext; - public uint MinSubgroupSize, MaxSubgroupSize; - public uint MaxComputeWorkgroupSubgroups, RequiredSubgroupSizeStages; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceSubgroupProperties - { - public uint SType; public void* PNext; - public uint SubgroupSize; - public uint SupportedStages; // VkShaderStageFlags - public uint SupportedOperations; // VkSubgroupFeatureFlags - public uint QuadOperationsInAllStages; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMemoryType { public uint PropertyFlags; public uint HeapIndex; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMemoryHeap { public ulong Size; public uint Flags; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceMemoryProperties - { - public uint MemoryTypeCount; - public fixed byte MemoryTypes[32 * 8]; // VkMemoryType[32] - public uint MemoryHeapCount; - public fixed byte MemoryHeaps[16 * 16]; // VkMemoryHeap is 16 bytes (8-aligned), not 12 - - public VkMemoryType TypeAt(int i) - { - fixed (byte* p = MemoryTypes) return ((VkMemoryType*)p)[i]; - } - public VkMemoryHeap HeapAt(int i) - { - // native VkMemoryHeap: { VkDeviceSize size; VkMemoryHeapFlags flags; } = 16 bytes - fixed (byte* p = MemoryHeaps) - { - byte* h = p + i * 16; - return new() { Size = *(ulong*)h, Flags = *(uint*)(h + 8) }; - } - } - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkQueueFamilyProperties - { - public uint QueueFlags, QueueCount, TimestampValidBits; - public uint MinImageTransferGranularityX, MinImageTransferGranularityY, MinImageTransferGranularityZ; - } - - /// VkPhysicalDeviceFeatures as raw uint[55]; shaderInt16 is index 41. - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceFeatures - { - public fixed uint F[55]; - public uint ShaderInt16 { get => F[41]; set => F[41] = value; } - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceFeatures2 - { - public uint SType; public void* PNext; - public VkPhysicalDeviceFeatures Features; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDevice16BitStorageFeatures - { - public uint SType; public void* PNext; - public uint StorageBuffer16BitAccess, UniformAndStorageBuffer16BitAccess, StoragePushConstant16, StorageInputOutput16; - } - - // VK_KHR_shader_float16_int8 (Vulkan 1.2: shaderFloat16/shaderInt8) - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceVulkanMemoryModelFeatures - { - public uint SType; public void* PNext; - public uint VulkanMemoryModel, VulkanMemoryModelDeviceScope, VulkanMemoryModelAvailabilityVisibilityChains; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceShaderFloat16Int8Features - { - public uint SType; public void* PNext; - public uint ShaderFloat16, ShaderInt8; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDeviceQueueCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint QueueFamilyIndex, QueueCount; public float* PQueuePriorities; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDeviceCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint QueueCreateInfoCount; public VkDeviceQueueCreateInfo* PQueueCreateInfos; - public uint EnabledLayerCount; public void* PpEnabledLayerNames; - public uint EnabledExtensionCount; public void* PpEnabledExtensionNames; - public void* PEnabledFeatures; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkBufferCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public ulong Size; public uint Usage; public uint SharingMode; - public uint QueueFamilyIndexCount; public uint* PQueueFamilyIndices; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMemoryRequirements { public ulong Size; public ulong Alignment; public uint MemoryTypeBits; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMemoryAllocateInfo - { - public uint SType; public void* PNext; - public ulong AllocationSize; public uint MemoryTypeIndex; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMappedMemoryRange - { - public uint SType; public void* PNext; - public IntPtr Memory; public ulong Offset; public ulong Size; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkShaderModuleCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public nuint CodeSize; public uint* PCode; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPushConstantRange { public uint StageFlags; public uint Offset; public uint Size; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPipelineLayoutCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint SetLayoutCount; public IntPtr* PSetLayouts; - public uint PushConstantRangeCount; public VkPushConstantRange* PPushConstantRanges; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPipelineShaderStageCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint Stage; public IntPtr Module; public byte* PName; public void* PSpecializationInfo; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceCooperativeMatrixFeaturesKHR - { - public uint SType; public void* PNext; - public uint CooperativeMatrix; public uint CooperativeMatrixRobustBufferAccess; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkCooperativeMatrixPropertiesKHR - { - public uint SType; public void* PNext; - public uint MSize, NSize, KSize; - public uint AType, BType, CType, ResultType; - public uint SaturatingAccumulation, Scope; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceSubgroupSizeControlFeaturesEXT - { - public uint SType; public void* PNext; - public uint SubgroupSizeControl; public uint ComputeFullSubgroups; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPipelineShaderStageRequiredSubgroupSizeCreateInfo - { - public uint SType; public void* PNext; public uint RequiredSubgroupSize; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkComputePipelineCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public VkPipelineShaderStageCreateInfo Stage; - public IntPtr Layout; public IntPtr BasePipelineHandle; public int BasePipelineIndex; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorSetLayoutBinding - { - public uint Binding; public uint DescriptorType; public uint DescriptorCount; - public uint StageFlags; public IntPtr* PImmutableSamplers; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorSetLayoutCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint BindingCount; public VkDescriptorSetLayoutBinding* PBindings; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorPoolSize { public uint Type; public uint DescriptorCount; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorPoolCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint MaxSets; public uint PoolSizeCount; public VkDescriptorPoolSize* PPoolSizes; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorSetAllocateInfo - { - public uint SType; public void* PNext; public IntPtr DescriptorPool; - public uint DescriptorSetCount; public IntPtr* PSetLayouts; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorBufferInfo { public IntPtr Buffer; public ulong Offset; public ulong Range; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkExtensionProperties - { - public unsafe fixed byte ExtensionName[256]; - public uint SpecVersion; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkWriteDescriptorSet - { - public uint SType; public void* PNext; public IntPtr DstSet; - public uint DstBinding; public uint DstArrayElement; public uint DescriptorCount; - public uint DescriptorType; public void* PImageInfo; public VkDescriptorBufferInfo* PBufferInfo; - public void* PTexelBufferView; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDeviceQueueGlobalPriorityCreateInfo - { - public uint SType; public void* PNext; public uint GlobalPriority; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkCommandPoolCreateInfo - { - public uint SType; public void* PNext; public uint Flags; public uint QueueFamilyIndex; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkCommandBufferAllocateInfo - { - public uint SType; public void* PNext; public IntPtr CommandPool; - public uint Level; public uint CommandBufferCount; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkCommandBufferBeginInfo - { - public uint SType; public void* PNext; public uint Flags; public void* PInheritanceInfo; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkSubmitInfo - { - public uint SType; public void* PNext; - public uint WaitSemaphoreCount; public IntPtr* PWaitSemaphores; public uint* PWaitDstStageMask; - public uint CommandBufferCount; public IntPtr* PCommandBuffers; - public uint SignalSemaphoreCount; public IntPtr* PSignalSemaphores; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkFenceCreateInfo { public uint SType; public void* PNext; public uint Flags; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMemoryBarrier - { - public uint SType; public void* PNext; public uint SrcAccessMask; public uint DstAccessMask; - } - - // ---- entry points ---- - - [LibraryImport(LibName)] public static partial VkResult vkCreateInstance(VkInstanceCreateInfo* ci, void* alloc, out IntPtr instance); - [LibraryImport(LibName)] public static partial void vkDestroyInstance(IntPtr instance, void* alloc); - [LibraryImport(LibName)] public static partial VkResult vkEnumeratePhysicalDevices(IntPtr instance, uint* count, IntPtr* devices); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceProperties(IntPtr device, VkPhysicalDeviceProperties* props); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceProperties2(IntPtr device, VkPhysicalDeviceProperties2* props); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceMemoryProperties(IntPtr device, VkPhysicalDeviceMemoryProperties* props); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceQueueFamilyProperties(IntPtr device, uint* count, VkQueueFamilyProperties* props); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceFeatures(IntPtr device, VkPhysicalDeviceFeatures* features); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceFeatures2(IntPtr device, VkPhysicalDeviceFeatures2* features); - [LibraryImport(LibName)] public static partial VkResult vkCreateDevice(IntPtr physicalDevice, VkDeviceCreateInfo* ci, void* alloc, out IntPtr device); - [LibraryImport(LibName)] public static partial void vkDestroyDevice(IntPtr device, void* alloc); - [LibraryImport(LibName)] public static partial void vkGetDeviceQueue(IntPtr device, uint family, uint index, out IntPtr queue); - - [LibraryImport(LibName)] public static partial VkResult vkCreateBuffer(IntPtr device, VkBufferCreateInfo* ci, void* alloc, out IntPtr buffer); - [LibraryImport(LibName)] public static partial void vkDestroyBuffer(IntPtr device, IntPtr buffer, void* alloc); - [LibraryImport(LibName)] public static partial void vkFreeCommandBuffers(IntPtr device, IntPtr pool, uint count, IntPtr* buffers); - [LibraryImport(LibName)] public static partial VkResult vkFreeDescriptorSets(IntPtr device, IntPtr pool, uint count, IntPtr* sets); - [LibraryImport(LibName)] public static partial void vkDestroyCommandPool(IntPtr device, IntPtr pool, void* alloc); - [LibraryImport(LibName)] public static partial void vkGetBufferMemoryRequirements(IntPtr device, IntPtr buffer, VkMemoryRequirements* req); - [LibraryImport(LibName)] public static partial VkResult vkAllocateMemory(IntPtr device, VkMemoryAllocateInfo* ai, void* alloc, out IntPtr memory); - [LibraryImport(LibName)] public static partial void vkFreeMemory(IntPtr device, IntPtr memory, void* alloc); - [LibraryImport(LibName)] public static partial VkResult vkBindBufferMemory(IntPtr device, IntPtr buffer, IntPtr memory, ulong offset); - [LibraryImport(LibName)] public static partial VkResult vkMapMemory(IntPtr device, IntPtr memory, ulong offset, ulong size, uint flags, void** data); - [LibraryImport(LibName)] public static partial void vkUnmapMemory(IntPtr device, IntPtr memory); - [LibraryImport(LibName)] public static partial VkResult vkFlushMappedMemoryRanges(IntPtr device, uint count, VkMappedMemoryRange* ranges); - [LibraryImport(LibName)] public static partial VkResult vkInvalidateMappedMemoryRanges(IntPtr device, uint count, VkMappedMemoryRange* ranges); - - [LibraryImport(LibName)] public static partial VkResult vkCreateShaderModule(IntPtr device, VkShaderModuleCreateInfo* ci, void* alloc, out IntPtr module); - [LibraryImport(LibName)] public static partial void vkDestroyShaderModule(IntPtr device, IntPtr module, void* alloc); - [LibraryImport(LibName)] public static partial VkResult vkCreatePipelineLayout(IntPtr device, VkPipelineLayoutCreateInfo* ci, void* alloc, out IntPtr layout); - [LibraryImport(LibName)] public static partial VkResult vkCreateComputePipelines(IntPtr device, IntPtr cache, uint count, VkComputePipelineCreateInfo* ci, void* alloc, IntPtr* pipelines); - [LibraryImport(LibName)] public static partial void vkDestroyPipeline(IntPtr device, IntPtr pipeline, void* alloc); - - [LibraryImport(LibName)] public static partial VkResult vkCreateDescriptorSetLayout(IntPtr device, VkDescriptorSetLayoutCreateInfo* ci, void* alloc, out IntPtr setLayout); - [LibraryImport(LibName)] public static partial void vkDestroyPipelineLayout(IntPtr device, IntPtr layout, void* alloc); - [LibraryImport(LibName)] public static partial void vkDestroyDescriptorSetLayout(IntPtr device, IntPtr setLayout, void* alloc); - [LibraryImport(LibName)] public static partial VkResult vkCreateDescriptorPool(IntPtr device, VkDescriptorPoolCreateInfo* ci, void* alloc, out IntPtr pool); - [LibraryImport(LibName)] public static partial VkResult vkAllocateDescriptorSets(IntPtr device, VkDescriptorSetAllocateInfo* ai, IntPtr* sets); - [LibraryImport(LibName)] public static partial VkResult vkResetDescriptorPool(IntPtr device, IntPtr pool, uint flags); - [LibraryImport(LibName)] public static partial void vkDestroyDescriptorPool(IntPtr device, IntPtr pool, void* alloc); - [LibraryImport(LibName)] public static partial void vkUpdateDescriptorSets(IntPtr device, uint writeCount, VkWriteDescriptorSet* writes, uint copyCount, void* copies); - [LibraryImport(LibName)] public static partial VkResult vkEnumerateDeviceExtensionProperties(IntPtr physicalDevice, byte* layerName, uint* count, VkExtensionProperties* props); - [LibraryImport(LibName)] public static partial IntPtr vkGetDeviceProcAddr(IntPtr device, byte* name); - [LibraryImport(LibName)] public static partial IntPtr vkGetInstanceProcAddr(IntPtr instance, byte* name); - - [LibraryImport(LibName)] public static partial VkResult vkCreateCommandPool(IntPtr device, VkCommandPoolCreateInfo* ci, void* alloc, out IntPtr pool); - [LibraryImport(LibName)] public static partial VkResult vkAllocateCommandBuffers(IntPtr device, VkCommandBufferAllocateInfo* ai, IntPtr* buffers); - [LibraryImport(LibName)] public static partial VkResult vkResetCommandBuffer(IntPtr cmd, uint flags); - [LibraryImport(LibName)] public static partial VkResult vkBeginCommandBuffer(IntPtr cmd, VkCommandBufferBeginInfo* bi); - [LibraryImport(LibName)] public static partial VkResult vkEndCommandBuffer(IntPtr cmd); - [LibraryImport(LibName)] public static partial void vkCmdBindPipeline(IntPtr cmd, uint bindPoint, IntPtr pipeline); - [LibraryImport(LibName)] public static partial void vkCmdBindDescriptorSets(IntPtr cmd, uint bindPoint, IntPtr layout, uint firstSet, uint count, IntPtr* sets, uint dynCount, uint* dynOffsets); - [LibraryImport(LibName)] public static partial void vkCmdPushConstants(IntPtr cmd, IntPtr layout, uint stageFlags, uint offset, uint size, void* values); - [LibraryImport(LibName)] public static partial void vkCmdDispatch(IntPtr cmd, uint x, uint y, uint z); - [LibraryImport(LibName)] public static partial void vkCmdPipelineBarrier(IntPtr cmd, uint srcStage, uint dstStage, uint depFlags, - uint memBarrierCount, VkMemoryBarrier* memBarriers, uint bufBarrierCount, void* bufBarriers, uint imgBarrierCount, void* imgBarriers); - [LibraryImport(LibName)] public static partial void vkCmdFillBuffer(IntPtr cmd, IntPtr buffer, ulong offset, ulong size, uint data); - [LibraryImport(LibName)] public static partial void vkCmdCopyBuffer(IntPtr cmd, IntPtr src, IntPtr dst, uint count, VkBufferCopy* regions); - - [StructLayout(LayoutKind.Sequential)] - public struct VkQueryPoolCreateInfo { public uint SType; public void* PNext; public uint Flags; public uint QueryType; public uint QueryCount; public uint PipelineStatistics; } - [LibraryImport(LibName)] public static partial VkResult vkCreateQueryPool(IntPtr device, VkQueryPoolCreateInfo* ci, void* alloc, out IntPtr pool); - [LibraryImport(LibName)] public static partial void vkDestroyQueryPool(IntPtr device, IntPtr pool, void* alloc); - [LibraryImport(LibName)] public static partial void vkCmdResetQueryPool(IntPtr cmd, IntPtr pool, uint first, uint count); - [LibraryImport(LibName)] public static partial void vkCmdWriteTimestamp(IntPtr cmd, uint stage, IntPtr pool, uint query); - [LibraryImport(LibName)] public static partial VkResult vkGetQueryPoolResults(IntPtr device, IntPtr pool, uint first, uint count, nuint dataSize, void* data, ulong stride, uint flags); - - [StructLayout(LayoutKind.Sequential)] - public struct VkBufferCopy { public ulong SrcOffset, DstOffset, Size; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkBufferMemoryBarrier - { - public uint SType; public void* PNext; - public uint SrcAccessMask, DstAccessMask; - public uint SrcQueueFamilyIndex, DstQueueFamilyIndex; - public IntPtr Buffer; public ulong Offset, Size; - } - - [LibraryImport(LibName)] public static partial VkResult vkQueueSubmit(IntPtr queue, uint submitCount, VkSubmitInfo* submits, IntPtr fence); - [LibraryImport(LibName)] public static partial VkResult vkCreateFence(IntPtr device, VkFenceCreateInfo* ci, void* alloc, out IntPtr fence); - [LibraryImport(LibName)] public static partial VkResult vkWaitForFences(IntPtr device, uint count, IntPtr* fences, uint waitAll, ulong timeout); - [LibraryImport(LibName)] public static partial VkResult vkResetFences(IntPtr device, uint count, IntPtr* fences); - [LibraryImport(LibName)] public static partial void vkDestroyFence(IntPtr device, IntPtr fence, void* alloc); - [LibraryImport(LibName)] public static partial VkResult vkGetFenceStatus(IntPtr device, IntPtr fence); - [LibraryImport(LibName)] public static partial VkResult vkQueueWaitIdle(IntPtr queue); - [LibraryImport(LibName)] public static partial VkResult vkDeviceWaitIdle(IntPtr device); -} - -internal enum VkResult : int -{ - Success = 0, - NotReady = 1, - Timeout = 2, - ErrorOutOfHostMemory = -1, - ErrorOutOfDeviceMemory = -2, - ErrorInitializationFailed = -3, - ErrorDeviceLost = -4, - ErrorExtensionNotPresent = -7, - ErrorFeatureNotPresent = -8, - ErrorIncompatibleDriver = -9, -} - -internal static class VkConst -{ - // VkStructureType - public const uint StApplicationInfo = 0; - public const uint StInstanceCreateInfo = 1; - public const uint StDeviceQueueCreateInfo = 2; - public const uint StDeviceCreateInfo = 3; - public const uint StSubmitInfo = 4; - public const uint StMemoryAllocateInfo = 5; - public const uint StMappedMemoryRange = 6; - public const uint StFenceCreateInfo = 8; - public const uint StQueryPoolCreateInfo = 11; - public const uint StBufferCreateInfo = 12; - public const uint StShaderModuleCreateInfo = 16; - public const uint StPipelineShaderStageCreateInfo = 18; - public const uint StComputePipelineCreateInfo = 29; - public const uint StPipelineLayoutCreateInfo = 30; - public const uint StDescriptorSetLayoutCreateInfo = 32; - public const uint StDescriptorPoolCreateInfo = 33; - public const uint StDescriptorSetAllocateInfo = 34; - public const uint StWriteDescriptorSet = 35; - public const uint StCommandPoolCreateInfo = 39; - public const uint StCommandBufferAllocateInfo = 40; - public const uint StCommandBufferBeginInfo = 42; - public const uint StMemoryBarrier = 46; - public const uint StBufferMemoryBarrier = 44; - public const uint QueueFamilyIgnored = 0xFFFFFFFFu; - public const uint StPhysicalDeviceFeatures2 = 49; - public const uint StPhysicalDevice16BitStorageFeatures = 1000083000u; - public const uint StPhysicalDeviceShaderFloat16Int8Features = 1000082000u; - public const uint StPhysicalDeviceVulkanMemoryModelFeatures = 1000211000u; - public const uint StPhysicalDeviceCooperativeMatrixFeaturesKHR = 1000506000u; - public const uint StCooperativeMatrixPropertiesKHR = 1000506001u; - public const uint StPhysicalDeviceSubgroupProperties = 1000094000u; - public const uint StPhysicalDeviceSubgroupSizeControlPropertiesEXT = 1000225000u; - public const uint StPhysicalDeviceSubgroupSizeControlFeaturesEXT = 1000225002u; - public const uint StPhysicalDeviceProperties2 = 1000059000u; - public const uint StPipelineShaderStageRequiredSubgroupSizeCreateInfo = 1000225001u; - - // VkPhysicalDeviceType - public const uint PhysDeviceIntegrated = 1; - public const uint PhysDeviceDiscrete = 2; - public const uint PhysDeviceVirtualGpu = 3; - public const uint PhysDeviceCpu = 4; - - // VkQueueFlagBits - public const uint QueueGraphics = 0x1; - public const uint QueueCompute = 0x2; - public const uint QueueTransfer = 0x4; - - // VkBufferUsageFlagBits - public const uint BufferUsageTransferSrc = 0x1; - public const uint BufferUsageTransferDst = 0x2; - public const uint BufferUsageStorageBuffer = 0x20; - - // VkMemoryPropertyFlagBits - public const uint MemDeviceLocal = 0x1; - public const uint MemHostVisible = 0x2; - public const uint MemHostCoherent = 0x4; - public const uint MemHostCached = 0x8; - - // VkDescriptorType - public const uint DescStorageBuffer = 7; - public const uint DescUniformBuffer = 6; - - // VkShaderStageFlagBits / VkPipelineStageFlagBits - public const uint StageComputeShader = 0x20; - public const uint PipelineStageTransfer = 0x1000; - public const uint PipelineStageComputeShader = 0x800; - public const uint PipelineStageHost = 0x4000; - public const uint PipelineStageTopOfPipe = 0x1; - public const uint PipelineStageBottomOfPipe = 0x2000; - - // VkAccessFlagBits - public const uint AccessShaderRead = 0x20; - public const uint AccessShaderWrite = 0x40; - public const uint AccessTransferRead = 0x800; - public const uint AccessTransferWrite = 0x1000; - public const uint AccessHostRead = 0x2000; - public const uint AccessHostWrite = 0x4000; - public const uint StageTopOfPipe = 0x1; - public const uint StageAllCommands = 0x10000; - - // VkPipelineBindPoint - public const uint BindPointCompute = 1; - - // VkCommandBufferLevel - public const uint CmdBufferLevelPrimary = 0; - - // VkCommandPoolCreateFlagBits - public const uint CmdPoolResetCommandBuffer = 0x2; - - // VkSharingMode - public const uint SharingExclusive = 0; - - // VkQueryType / VkQueryResultFlagBits - public const uint QueryTypeTimestamp = 1; - public const uint QueryResult64 = 0x1; - public const uint QueryResultWait = 0x2; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkBackend.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkBackend.cs deleted file mode 100644 index 661321e..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkBackend.cs +++ /dev/null @@ -1,91 +0,0 @@ -using System.Collections.Concurrent; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; - -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -/// -/// Process-wide Vulkan device + per-(model,size) compiled graph cache. -/// Constructed lazily on first Vulkan use. -/// -internal sealed unsafe class VkBackend : IDisposable -{ - internal readonly VkDevice Dev; - // (model, c, h, w) -> compiled graph (weights upload + dispatch list) - private readonly ConcurrentDictionary<(OnnxModel m, int c, int h, int w, bool physIo), VkGraphModel> _graphs = new(); - - private static readonly Lazy s_default = new(() => new VkBackend()); - internal static VkBackend Default => s_default.Value; - - /// - /// Plan options for GPU compiles: always channels-last (shaders address NHWC). - /// Standalone sessions keep logical NCHW I/O (boundary relayouts in the plan); - /// the fused pipeline keeps I/O physical and its glue shaders speak NHWC. - /// - internal static CompileOptions PlanOptions => new(Nhwc: true, PhysicalIo: false, Gpu: true); - internal static CompileOptions PipeOptions => new(Nhwc: true, PhysicalIo: true, Gpu: true); - - /// fp16 activation/weight storage (needs storageBuffer16BitAccess; GCDRNET_VK_F16=0 forces fp32). - internal bool F16 => Dev.StorageBuffer16 && Environment.GetEnvironmentVariable("GCDRNET_VK_F16") != "0"; - - internal VkBackend() - { - Dev = VkDevice.Create(); - } - - /// Standalone session for one model at [1,c,h,w] (test path). - public VkInferenceSession NewSession(OnnxModel m, int[] shape) - { - VkGraphModel g = AcquireGraph(m, shape, physIo: false); // caller ref: later evictions can't free it under us - try { return new VkInferenceSession(Dev, g.Plan, g); } - finally { g.Release(); } - } - - /// Compiled GPU graph for a model+shape (cached, bounded). - internal VkGraphModel GraphFor(OnnxModel m, int[] shape, bool physIo = false) - { - var key = (m, shape[1], shape[2], shape[3], physIo); - if (_graphs.TryGetValue(key, out var g)) return g; - // weights live in a per-model const store shared across sizes, so a graph is - // just a plan + dispatch list (no per-size device memory) - g = new VkGraphModel(Dev, GraphCompiler.Compile(m, [1, shape[1], shape[2], shape[3]], - physIo ? PipeOptions : PlanOptions), F16, constKey: m); - if (!_graphs.TryAdd(key, g)) { g.Dispose(); return _graphs[key]; } - // bound the cache: evict an arbitrary other graph beyond the cap - while (_graphs.Count > MaxGraphs) - { - var victim = _graphs.Keys.FirstOrDefault(k => !k.Equals(key)); - if (victim.Equals(default)) break; - if (_graphs.TryRemove(victim, out var old)) old.Release(); - } - return g; - } - - // graphs hold no device memory beyond the shared const store; the bound is for - // the managed plans (each holds its own copy of the model's const floats) - private const int MaxGraphs = 32; - - /// plus a caller reference. Retries when a concurrent - /// eviction freed the cached graph between lookup and acquire (never resurrects it). - internal VkGraphModel AcquireGraph(OnnxModel m, int[] shape, bool physIo) - { - while (true) - { - VkGraphModel g = GraphFor(m, shape, physIo); - if (g.TryAcquire()) return g; - } - } - - /// New size-agnostic GPU worker for a (gc, dr) model pair (stateful; callers pool them). - internal VkGcdrPipeline NewWorker(OnnxModel gc, OnnxModel dr) => new(this, gc, dr); - - /// New GPU worker pre-bound to one source size (debug/bench helper). - internal VkGcdrPipeline NewPipe(OnnxModel gc, OnnxModel dr, int ih, int iw) - { - var p = new VkGcdrPipeline(this, gc, dr); - try { p.Bind(ih, iw); return p; } - catch { p.Dispose(); throw; } - } - - public void Dispose() => Dev.Dispose(); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkConstStore.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkConstStore.cs deleted file mode 100644 index 5c46aeb..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkConstStore.cs +++ /dev/null @@ -1,91 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -/// -/// Device-side const tensors (backend-packed weights) plus a 64B zero buffer, -/// shared by every of one model: per-size plans copy -/// the same initializers, so instead of re-packing and re-uploading ~20 MB per -/// input size, a new graph whose const list is element-for-element identical -/// (same consumer packing, same contents) reuses the existing buffer. A graph -/// whose consts differ gets a private, unshared store. Ref-counted; the -/// buffers are freed when the last graph releases. -/// -internal sealed class VkConstStore -{ - private static readonly Dictionary<(VkDevice dev, object key, bool f16), VkConstStore> s_shared = new(); - - private readonly (VkDevice, object, bool)? _key; - private readonly float[][] _src; - private readonly int[] _kinds; - private int _refs = 1; - internal readonly VkBuffer Consts, Dummy; - internal readonly ulong[] Offsets; - internal readonly ulong Bytes; - - private VkConstStore(VkDevice dev, (VkDevice, object, bool)? key, float[][] src, int[] kinds, - ulong[] offsets, byte[] host) - { - _key = key; - _src = src; - _kinds = kinds; - Offsets = offsets; - Bytes = (ulong)host.LongLength; - Consts = dev.NewStorageBuffer(Bytes, hostVisible: dev.CoherentDeviceLocal); - Dummy = dev.NewStorageBuffer(64, hostVisible: dev.CoherentDeviceLocal); - unsafe - { - fixed (byte* hp = host) dev.Upload(Consts, hp, Bytes); - byte* z = stackalloc byte[64]; - new Span(z, 64).Clear(); - dev.Upload(Dummy, z, 64); - } - } - - /// - /// Returns a store holding (kinds = consumer packing ids), - /// shared under when an identical list was uploaded before. - /// (offsets, host bytes) only runs when a new store is built. - /// - internal static VkConstStore Acquire(VkDevice dev, object? key, bool f16, float[][] src, int[] kinds, - Func<(ulong[] offsets, byte[] host)> pack) - { - if (key is null) - { - var (o, h) = pack(); - return new VkConstStore(dev, null, src, kinds, o, h); - } - var k = (dev, key, f16); - lock (s_shared) - { - if (s_shared.TryGetValue(k, out VkConstStore? s) && s.Matches(src, kinds)) - { - s._refs++; - return s; - } - var (offs, host) = pack(); - bool share = s is null; // keep the first registered store on a mismatch - var ns = new VkConstStore(dev, share ? k : null, src, kinds, offs, host); - if (share) s_shared[k] = ns; - return ns; - } - } - - private bool Matches(float[][] src, int[] kinds) - { - if (src.Length != _src.Length || !kinds.AsSpan().SequenceEqual(_kinds)) return false; - for (int i = 0; i < src.Length; i++) - if (!ReferenceEquals(src[i], _src[i]) && !src[i].AsSpan().SequenceEqual(_src[i])) return false; - return true; - } - - internal void Release() - { - lock (s_shared) - { - if (--_refs > 0) return; - if (_key is { } k && s_shared.TryGetValue(k, out VkConstStore? cur) && cur == this) - s_shared.Remove(k); - } - Consts.Free(); - Dummy.Free(); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkDevice.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkDevice.cs deleted file mode 100644 index 34f6d51..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkDevice.cs +++ /dev/null @@ -1,938 +0,0 @@ -// Thin object wrappers over Vk.cs: instance/device/queue, buffers with -// memory-type selection (ReBAR fast path vs host-visible staging), shader -// modules from embedded SPIR-V, compute pipelines, descriptor sets, and -// one-shot command submission. Adapted from Sdcb.SimdPaddleOCR. -using System.Runtime.InteropServices; - -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -internal unsafe sealed class VkDevice : IDisposable -{ - public IntPtr Instance, PhysDevice, Device, Queue; - public string DeviceName = ""; - public uint VendorId; - public uint SubgroupSize; - public uint QueueFamily; - public Vk.VkPhysicalDeviceMemoryProperties MemProps; - public bool CoherentDeviceLocal; // DEVICE_LOCAL|HOST_VISIBLE|HOST_COHERENT exists (ReBAR) - public bool StorageBuffer16; // VK_KHR_16bit_storage: activations may be stored as float16 - public uint MaxSharedMemory; - public ulong MaxStorageRange = uint.MaxValue; - public bool PushDescriptors; - /// Tensor-core convs (conv_cm.comp; GCDRNET_VK_COOPMAT=0 disables): VK_KHR_cooperative_matrix - /// plus shaderFloat16, in one of two shapes. Default: a 16x16x16 fp16 x fp16 -> fp32 subgroup config - /// on 32-lane subgroups; devices whose default compute subgroup is not 32 lanes (RDNA wave64) qualify - /// only when VK_EXT_subgroup_size_control can pin it to 32 (). - /// Otherwise : the 8x16x16 config on subgroups pinned to 16 lanes (Intel Xe2). - /// A probe dispatch must confirm the pinned width; failing that the convs stay on conv_tile. - public bool CoopMat16; - /// The coopmat convs use the SG16 build (8x16x16 fragments, 16-lane subgroups; Intel Arc - /// XMX): taken only when the 16x16x16 / 32-lane path is unavailable (GCDRNET_VK_SG16=0 disables). - public bool CoopM8; - /// requiredSubgroupSize for the coopmat pipelines: 0 = device default (already 32), - /// 32 / 16 = pinned through VK_EXT_subgroup_size_control. - public uint CoopSubgroupSize; - /// Coopmat convs use the LITE variants (single-buffered 28 KB staging, scalar epilogue, - /// grouped raster): wave64-capable parts or less than the 48 KB shared memory the - /// double-buffered kernels take. - public bool CoopLite; - /// VK_EXT_subgroup_size_control range (1..128 when the extension is absent). - public uint SubgroupMin = 1, SubgroupMax = 128; - /// Memory-bound NHWC shaders (maxpool, bilinear, depthwise, shift) take 4 channels per - /// invocation when c % 4 == 0 — integrated GPUs, where they are bandwidth/issue bound, and Intel - /// GPUs (Arc B580: 10.4 -> 6.7 ms for the four; per-element index math is costly on SIMD16) - /// (GCDRNET_VK_VEC4=0 disables; other discrete GPUs keep the per-element shaders). - public bool Vec4Eltwise; - /// 3x3 stem conv+pool uses convpool_k3 (input window rows in registers): Intel GPUs - /// (Arc B580: 7.5 -> 4.9 ms for the two stems) (GCDRNET_VK_CONVPOOL3=0 disables). - public bool ConvPoolK3; - /// Register-tiled kernels for GPUs without cooperative matrices whose compute subgroups - /// are at least 64 lanes (Adreno 750: 64/128, 32 KB shared memory), where conv_tile / gemm's - /// shared-memory staging is load-bound: 3x3 convs on conv_dot.comp, Gemm on gemm_dot.comp, - /// the stem conv+pool with the four pool pixels per weight load (convpool_pi) and the UpHead - /// tail two pixels per invocation (uphead_sc2). fp16 storage only (GCDRNET_VK_DOT=0 disables). - public bool ConvDot; - private const uint VendorIntel = 0x8086; - /// Largest DEVICE_LOCAL heap (bytes). - public ulong DeviceLocalHeap; - /// Integrated GPU: "device-local" memory is system RAM shared with the CPU. - public bool UnifiedMemory; - private unsafe delegate* unmanaged[Cdecl] _pushDesc; - public double TimestampPeriodNs = 1; - private IntPtr _commandPool; - private IntPtr _descPool; - - // live vkAllocateMemory bytes across the process (diagnostic — buffers are pooled) - private static long _liveBytes, _peakBytes; - public static long LiveDeviceBytes => Interlocked.Read(ref _liveBytes); - public static long PeakDeviceBytes => Interlocked.Read(ref _peakBytes); - public static void ResetPeakDeviceBytes() => Interlocked.Exchange(ref _peakBytes, _liveBytes); - private static void TrackAlloc(long bytes) - { - long now = Interlocked.Add(ref _liveBytes, bytes), p; - while ((p = Interlocked.Read(ref _peakBytes)) < now) - Interlocked.CompareExchange(ref _peakBytes, now, p); - } - internal static void TrackFree(long bytes) => Interlocked.Add(ref _liveBytes, -bytes); - /// Guards the device-level command pool / descriptor pool (uploads). - public readonly object Sync = new(); - /// vkQueueSubmit on the single compute queue is externally synchronized. - public readonly object QueueLock = new(); - private readonly Dictionary<(string name, int bindings, int pcBytes, uint sg), VkPipeline> _pipes = new(); - - /// Process-wide pipeline cache: every graph/session shares one - /// VkPipeline per (shader, binding-count, push-const-size). Set layouts carry - /// PUSH_DESCRIPTOR_BIT when the extension is available. - public VkPipeline GetPipeline(string name, int bindings, int pcBytes, uint requiredSubgroupSize = 0) - { - var key = (name, bindings, pcBytes, requiredSubgroupSize); - lock (_pipes) - { - if (_pipes.TryGetValue(key, out VkPipeline? p)) return p; - p = NewPipeline(NewShaderModule(VkShaders.Load(name)), bindings, pcBytes, PushDescriptors, requiredSubgroupSize); - p.Name = name; - p.Bindings = bindings; - _pipes[key] = p; - return p; - } - } - - /// Pipeline of a conv_cm variant ("conv_cm", "conv_cm_n64", ...; fp16 storage): the - /// LITE build on devices (or when forces it), - /// the SG16 build (always LITE-style) on devices, with the subgroup pinned - /// to . Every variant keeps BM = 128 rows per workgroup. - public VkPipeline GetCoopPipeline(string variant, int bindings, int pcBytes, bool? lite = null) - { - if (!variant.StartsWith("conv_cm", StringComparison.Ordinal)) throw new ArgumentException(variant); - string suffix = variant["conv_cm".Length..]; - string name = CoopM8 ? "conv_cm_sg16" + suffix : (lite ?? CoopLite) ? "conv_cm_l" + suffix : variant; - return GetPipeline(name + "_h", bindings, pcBytes, CoopSubgroupSize); - } - - public static VkDevice Create(uint deviceIndex = 0) - { - Vk.RegisterResolver(); - var d = new VkDevice(); - - Vk.VkApplicationInfo app = new() - { - SType = VkConst.StApplicationInfo, - AppVersion = 1, EngineVersion = 1, ApiVersion = (1u << 22) | (1u << 12), // Vulkan 1.1 - }; - Vk.VkInstanceCreateInfo ici = new() { SType = VkConst.StInstanceCreateInfo, PAppInfo = &app }; - Vk.Check(Vk.vkCreateInstance(&ici, null, out d.Instance), "vkCreateInstance"); - - uint ndev = 0; - Vk.Check(Vk.vkEnumeratePhysicalDevices(d.Instance, &ndev, null), "enum physical devices"); - if (ndev == 0) throw new PlatformNotSupportedException("Vulkan: no physical devices (no Vulkan-capable GPU/driver)"); - IntPtr* devs = stackalloc IntPtr[(int)ndev]; - Vk.Check(Vk.vkEnumeratePhysicalDevices(d.Instance, &ndev, devs), "enum physical devices"); - - // Prefer discrete GPU; honor deviceIndex as "the nth discrete first, else nth overall". - int chosen = -1; - for (int pass = 0; pass < 2 && chosen < 0; pass++) - { - uint seen = 0; - for (int i = 0; i < (int)ndev; i++) - { - Vk.VkPhysicalDeviceProperties p; - Vk.vkGetPhysicalDeviceProperties(devs[i], &p); - bool want = pass == 0 ? p.DeviceType == VkConst.PhysDeviceDiscrete : p.DeviceType != VkConst.PhysDeviceCpu; - if (!want) continue; - if (seen++ == deviceIndex) { chosen = i; break; } - } - } - if (chosen < 0) throw new PlatformNotSupportedException($"Vulkan: device index {deviceIndex} out of range ({ndev} devices)"); - d.PhysDevice = devs[chosen]; - - Vk.VkPhysicalDeviceProperties props; - Vk.vkGetPhysicalDeviceProperties(d.PhysDevice, &props); - d.DeviceName = Marshal.PtrToStringAnsi((IntPtr)props.DeviceName) ?? "?"; - d.VendorId = props.VendorID; - d.TimestampPeriodNs = props.TimestampPeriodNs; - d.MaxSharedMemory = props.MaxComputeSharedMemorySize; - d.MaxStorageRange = props.MaxStorageBufferRange; - - uint nqf = 0; - Vk.vkGetPhysicalDeviceQueueFamilyProperties(d.PhysDevice, &nqf, null); - Vk.VkQueueFamilyProperties* qf = stackalloc Vk.VkQueueFamilyProperties[(int)nqf]; - Vk.vkGetPhysicalDeviceQueueFamilyProperties(d.PhysDevice, &nqf, qf); - if (Environment.GetEnvironmentVariable("GCDRNET_VK_DBG") == "1") - for (uint i = 0; i < nqf; i++) - Console.Error.WriteLine($"[vk] qf{i}: flags={qf[i].QueueFlags:x} count={qf[i].QueueCount} tsbits={qf[i].TimestampValidBits}"); - // Prefer a compute-only family (no GRAPHICS bit) for async-compute — - // but only if it supports timestamps; otherwise fall back to graphics+compute. - d.QueueFamily = uint.MaxValue; - for (uint i = 0; i < nqf; i++) - if ((qf[i].QueueFlags & VkConst.QueueCompute) != 0 && (qf[i].QueueFlags & 1u) == 0 - && qf[i].TimestampValidBits > 0) - { d.QueueFamily = i; break; } - for (uint i = 0; i < nqf && d.QueueFamily == uint.MaxValue; i++) - if ((qf[i].QueueFlags & VkConst.QueueCompute) != 0 && qf[i].TimestampValidBits > 0) - { d.QueueFamily = i; break; } - for (uint i = 0; i < nqf && d.QueueFamily == uint.MaxValue; i++) - if ((qf[i].QueueFlags & VkConst.QueueCompute) != 0) { d.QueueFamily = i; break; } - if (d.QueueFamily == uint.MaxValue) - throw new PlatformNotSupportedException("Vulkan: no compute queue family"); - - // Extension probe: VK_KHR_push_descriptor (inline descriptor writes, - // skips per-dispatch descriptor-set binds). - uint next = 0; - Vk.vkEnumerateDeviceExtensionProperties(d.PhysDevice, null, &next, null); - Vk.VkExtensionProperties* exts = stackalloc Vk.VkExtensionProperties[(int)next]; - Vk.vkEnumerateDeviceExtensionProperties(d.PhysDevice, null, &next, exts); - byte* wantPush = stackalloc byte[] { (byte)'V', (byte)'K', (byte)'_', (byte)'K', (byte)'H', (byte)'R', - (byte)'_', (byte)'p', (byte)'u', (byte)'s', (byte)'h', (byte)'_', (byte)'d', (byte)'e', (byte)'s', (byte)'c', - (byte)'r', (byte)'i', (byte)'p', (byte)'t', (byte)'o', (byte)'r', 0 }; - bool hasPush = false, hasCoop = false, hasF16Int8 = false, hasVmm = false, hasSgc = false; - for (int i = 0; i < (int)next; i++) - { - string en = new((sbyte*)exts[i].ExtensionName); - if (en == "VK_KHR_push_descriptor") hasPush = true; - else if (en == "VK_KHR_cooperative_matrix") hasCoop = true; - else if (en == "VK_KHR_shader_float16_int8") hasF16Int8 = true; - else if (en == "VK_KHR_vulkan_memory_model") hasVmm = true; - else if (en == "VK_EXT_subgroup_size_control") hasSgc = true; - } - if (Environment.GetEnvironmentVariable("GCDRNET_VK_NOPUSH") == "1") hasPush = false; - - float prio = 1f; - Vk.VkDeviceQueueCreateInfo qci = new() - { - SType = VkConst.StDeviceQueueCreateInfo, - QueueFamilyIndex = d.QueueFamily, QueueCount = 1, PQueuePriorities = &prio, - }; - Vk.VkPhysicalDeviceSubgroupProperties sgP = new() { SType = VkConst.StPhysicalDeviceSubgroupProperties }; - Vk.VkPhysicalDeviceSubgroupSizeControlProperties sgcP = new() { SType = VkConst.StPhysicalDeviceSubgroupSizeControlPropertiesEXT }; - if (hasSgc) sgP.PNext = &sgcP; - Vk.VkPhysicalDeviceProperties2 p2 = new() { SType = VkConst.StPhysicalDeviceProperties2, PNext = &sgP }; - Vk.vkGetPhysicalDeviceProperties2(d.PhysDevice, &p2); - d.SubgroupSize = sgP.SubgroupSize; - bool sgcCompute = false; - if (hasSgc) - { - d.SubgroupMin = sgcP.MinSubgroupSize; - d.SubgroupMax = sgcP.MaxSubgroupSize; - sgcCompute = (sgcP.RequiredSubgroupSizeStages & VkConst.StageComputeShader) != 0; - } - - // Query storageBuffer16BitAccess (VK 1.1 core) for fp16 activation storage. - Vk.VkPhysicalDevice16BitStorageFeatures feat16 = new() - { - SType = VkConst.StPhysicalDevice16BitStorageFeatures, - }; - Vk.VkPhysicalDeviceFeatures2 feats2 = new() - { - SType = VkConst.StPhysicalDeviceFeatures2, - PNext = &feat16, - }; - Vk.vkGetPhysicalDeviceFeatures2(d.PhysDevice, &feats2); - d.StorageBuffer16 = feat16.StorageBuffer16BitAccess != 0; - - // tensor-core path: coopmat + shaderFloat16 on top of 16-bit storage, 32-lane subgroups; - // coopmat SPIR-V uses the Vulkan memory model, so that feature is required as well - Vk.VkPhysicalDeviceShaderFloat16Int8Features f16q = new() { SType = VkConst.StPhysicalDeviceShaderFloat16Int8Features }; - Vk.VkPhysicalDeviceCooperativeMatrixFeaturesKHR coopQ = new() { SType = VkConst.StPhysicalDeviceCooperativeMatrixFeaturesKHR }; - Vk.VkPhysicalDeviceVulkanMemoryModelFeatures vmmQ = new() { SType = VkConst.StPhysicalDeviceVulkanMemoryModelFeatures }; - Vk.VkPhysicalDeviceSubgroupSizeControlFeaturesEXT sgcQ = new() { SType = VkConst.StPhysicalDeviceSubgroupSizeControlFeaturesEXT }; - // a default other than 32 lanes (RDNA: wave64) needs the compute subgroup pinned to 32 - bool pinSg32 = d.SubgroupSize != 32 && hasSgc && sgcCompute && d.SubgroupMin <= 32 && d.SubgroupMax >= 32 - && Environment.GetEnvironmentVariable("GCDRNET_VK_SG32") != "0"; - // the SG16 build (8x16x16, Intel Xe2 XMX) needs 16 lanes in the pinnable compute range - bool canSg16 = hasSgc && sgcCompute && d.SubgroupMin <= 16 && d.SubgroupMax >= 16 - && Environment.GetEnvironmentVariable("GCDRNET_VK_SG16") != "0"; - bool wantCm = hasCoop && hasF16Int8 && hasVmm && d.StorageBuffer16 && (d.SubgroupSize == 32 || pinSg32 || canSg16) - && Environment.GetEnvironmentVariable("GCDRNET_VK_COOPMAT") != "0"; - bool pinSg16 = false; - if (wantCm) - { - f16q.PNext = &coopQ; - coopQ.PNext = &vmmQ; - if (pinSg32 || canSg16) vmmQ.PNext = &sgcQ; - Vk.VkPhysicalDeviceFeatures2 fq = new() { SType = VkConst.StPhysicalDeviceFeatures2, PNext = &f16q }; - Vk.vkGetPhysicalDeviceFeatures2(d.PhysDevice, &fq); - bool feats = f16q.ShaderFloat16 != 0 && coopQ.CooperativeMatrix != 0 && vmmQ.VulkanMemoryModel != 0; - CoopShapes(d, out bool m16, out bool m8); - // 16x16x16 on 32 lanes first (NVIDIA, RDNA pinned to 32); 8x16x16 on 16 lanes only without it - bool cm32 = feats && (d.SubgroupSize == 32 || pinSg32) && (!pinSg32 || sgcQ.SubgroupSizeControl != 0) && m16; - pinSg16 = !cm32 && feats && canSg16 && sgcQ.SubgroupSizeControl != 0 && m8; - wantCm = cm32 || pinSg16; - pinSg32 &= cm32; - } - else pinSg32 = false; - bool pinSg = pinSg32 || pinSg16; - - byte** extsToEnable = stackalloc byte*[5]; - uint nExt = 0; - if (hasPush) extsToEnable[nExt++] = wantPush; - byte* pCm = stackalloc byte[32], pF16 = stackalloc byte[32], pVmm = stackalloc byte[32], pSgc = stackalloc byte[32]; - System.Text.Encoding.ASCII.GetBytes("VK_KHR_cooperative_matrix\0", new Span(pCm, 32)); - System.Text.Encoding.ASCII.GetBytes("VK_KHR_shader_float16_int8\0", new Span(pF16, 32)); - System.Text.Encoding.ASCII.GetBytes("VK_KHR_vulkan_memory_model\0", new Span(pVmm, 32)); - System.Text.Encoding.ASCII.GetBytes("VK_EXT_subgroup_size_control\0", new Span(pSgc, 32)); - if (wantCm) { extsToEnable[nExt++] = pCm; extsToEnable[nExt++] = pF16; extsToEnable[nExt++] = pVmm; } - if (pinSg) extsToEnable[nExt++] = pSgc; - Vk.VkPhysicalDeviceShaderFloat16Int8Features f16en = new() - { - SType = VkConst.StPhysicalDeviceShaderFloat16Int8Features, ShaderFloat16 = 1, - }; - Vk.VkPhysicalDeviceCooperativeMatrixFeaturesKHR coopEn = new() - { - SType = VkConst.StPhysicalDeviceCooperativeMatrixFeaturesKHR, CooperativeMatrix = 1, - }; - Vk.VkPhysicalDeviceVulkanMemoryModelFeatures vmmEn = new() - { - SType = VkConst.StPhysicalDeviceVulkanMemoryModelFeatures, VulkanMemoryModel = 1, - }; - Vk.VkPhysicalDeviceSubgroupSizeControlFeaturesEXT sgcEn = new() - { - SType = VkConst.StPhysicalDeviceSubgroupSizeControlFeaturesEXT, SubgroupSizeControl = 1, - ComputeFullSubgroups = sgcQ.ComputeFullSubgroups, - }; - if (pinSg) vmmEn.PNext = &sgcEn; - coopEn.PNext = &vmmEn; - f16en.PNext = &coopEn; - feat16.PNext = wantCm ? &f16en : null; - - // To enable 16-bit storage we chain feat16 into VkDeviceCreateInfo.pNext - // (VK 1.1 core: no extension needed). PEnabledFeatures must be null when - // using Features2 pNext; we inline a zero VkPhysicalDeviceFeatures in feat2. - Vk.VkPhysicalDeviceFeatures2 feat2enable = new() - { - SType = VkConst.StPhysicalDeviceFeatures2, - PNext = d.StorageBuffer16 ? &feat16 : null, - }; - Vk.VkDeviceCreateInfo dci = new() - { - SType = VkConst.StDeviceCreateInfo, - QueueCreateInfoCount = 1, PQueueCreateInfos = &qci, - PEnabledFeatures = null, - PNext = &feat2enable, - EnabledExtensionCount = nExt, - PpEnabledExtensionNames = nExt > 0 ? extsToEnable : null, - }; - Vk.Check(Vk.vkCreateDevice(d.PhysDevice, &dci, null, out d.Device), "vkCreateDevice"); - d.CoopMat16 = wantCm; - d.CoopM8 = pinSg16; - d.CoopSubgroupSize = pinSg32 ? 32u : pinSg16 ? 16u : 0u; - d._fullSubgroups = pinSg && sgcQ.ComputeFullSubgroups != 0; - d.CoopLite = wantCm && !pinSg16 && ((hasSgc ? d.SubgroupMax : d.SubgroupSize) >= 64 || d.MaxSharedMemory < 48 * 1024) - && Environment.GetEnvironmentVariable("GCDRNET_VK_LITE") != "0"; - Vk.vkGetDeviceQueue(d.Device, d.QueueFamily, 0, out d.Queue); - - if (hasPush) - { - byte* fnName = stackalloc byte[] { (byte)'v', (byte)'k', (byte)'C', (byte)'m', (byte)'d', - (byte)'P', (byte)'u', (byte)'s', (byte)'h', (byte)'D', (byte)'e', (byte)'s', (byte)'c', - (byte)'r', (byte)'i', (byte)'p', (byte)'t', (byte)'o', (byte)'r', (byte)'S', (byte)'e', - (byte)'t', (byte)'K', (byte)'H', (byte)'R', 0 }; - IntPtr fn = Vk.vkGetDeviceProcAddr(d.Device, fnName); - if (fn != IntPtr.Zero) - { - d.PushDescriptors = true; - d._pushDesc = (delegate* unmanaged[Cdecl])fn; - } - } - - fixed (Vk.VkPhysicalDeviceMemoryProperties* mp = &d.MemProps) - Vk.vkGetPhysicalDeviceMemoryProperties(d.PhysDevice, mp); - for (int i = 0; i < (int)d.MemProps.MemoryTypeCount; i++) - { - uint f = d.MemProps.TypeAt(i).PropertyFlags; - if ((f & (VkConst.MemDeviceLocal | VkConst.MemHostVisible | VkConst.MemHostCoherent)) == (VkConst.MemDeviceLocal | VkConst.MemHostVisible | VkConst.MemHostCoherent)) - { - d.CoherentDeviceLocal = true; - break; - } - } - d.UnifiedMemory = props.DeviceType == VkConst.PhysDeviceIntegrated; - d.Vec4Eltwise = (d.UnifiedMemory || d.VendorId == VendorIntel) && Environment.GetEnvironmentVariable("GCDRNET_VK_VEC4") != "0"; - d.ConvPoolK3 = d.VendorId == VendorIntel && Environment.GetEnvironmentVariable("GCDRNET_VK_CONVPOOL3") != "0"; - for (int i = 0; i < (int)d.MemProps.MemoryHeapCount; i++) - { - var hp = d.MemProps.HeapAt(i); - if ((hp.Flags & 1u) != 0) d.DeviceLocalHeap = Math.Max(d.DeviceLocalHeap, hp.Size); // HEAP_DEVICE_LOCAL - } - - Vk.VkCommandPoolCreateInfo cpci = new() - { - SType = VkConst.StCommandPoolCreateInfo, - Flags = VkConst.CmdPoolResetCommandBuffer, - QueueFamilyIndex = d.QueueFamily, - }; - Vk.Check(Vk.vkCreateCommandPool(d.Device, &cpci, null, out d._commandPool), "vkCreateCommandPool"); - - Vk.VkDescriptorPoolSize psz = new() { Type = VkConst.DescStorageBuffer, DescriptorCount = 1 << 20 }; - Vk.VkDescriptorPoolCreateInfo dpci = new() - { - SType = VkConst.StDescriptorPoolCreateInfo, - Flags = 1, MaxSets = 1 << 16, PoolSizeCount = 1, PPoolSizes = &psz, - }; - Vk.Check(Vk.vkCreateDescriptorPool(d.Device, &dpci, null, out d._descPool), "vkCreateDescriptorPool"); - - // never trust a pinned subgroup size blindly: the coopmat lane mapping is hard-wired to - // the pinned width (32, or 16 for SG16), so anything else would silently compute garbage - if (pinSg && !d.ProbeSubgroup(d.CoopSubgroupSize)) - { - d.CoopMat16 = false; - d.CoopM8 = false; - d.CoopLite = false; - d.CoopSubgroupSize = 0; - } - d.ConvDot = !d.CoopMat16 && hasSgc && d.SubgroupMin >= 64 && d.StorageBuffer16 - && Environment.GetEnvironmentVariable("GCDRNET_VK_DOT") != "0"; - if (Environment.GetEnvironmentVariable("GCDRNET_VK_VERBOSE") == "1") - { - Console.Error.WriteLine($"[vk] device={d.DeviceName} type={props.DeviceType} vendor=0x{d.VendorId:x4} " + - $"api={props.ApiVersion >> 22}.{(props.ApiVersion >> 12) & 0x3ff}.{props.ApiVersion & 0xfff} driver=0x{props.DriverVersion:x8} qfamily={d.QueueFamily} " + - $"sg={d.SubgroupSize} sgRange={d.SubgroupMin}-{d.SubgroupMax} sgcCompute={sgcCompute} push={d.PushDescriptors} " + - $"coopmat={d.CoopMat16} coopSg={(d.CoopSubgroupSize == 0 ? "default" : d.CoopSubgroupSize.ToString())} " + - $"m8={d.CoopM8} fullSg={d._fullSubgroups} lite={d.CoopLite} vec4={d.Vec4Eltwise} cp3={d.ConvPoolK3} " + - $"dot={d.ConvDot} lds={d.MaxSharedMemory} sb16={d.StorageBuffer16}"); - for (uint i = 0; i < nqf; i++) - Console.Error.WriteLine($"[vk] qf{i}: flags=0x{qf[i].QueueFlags:x} count={qf[i].QueueCount} tsbits={qf[i].TimestampValidBits}" - + (i == d.QueueFamily ? " <- selected" : "")); - if (hasCoop) - foreach (var cp in CoopProps(d)) - Console.Error.WriteLine($"[vk] coopmat {cp.MSize}x{cp.NSize}x{cp.KSize} a={cp.AType} b={cp.BType} c={cp.CType} r={cp.ResultType} scope={cp.Scope}"); - for (int i = 0; i < (int)d.MemProps.MemoryHeapCount; i++) - { - var hp = d.MemProps.HeapAt(i); - Console.Error.WriteLine($"[vk] heap{i}: {hp.Size >> 20} MB flags={hp.Flags:x}"); - } - for (int i = 0; i < (int)d.MemProps.MemoryTypeCount; i++) - Console.Error.WriteLine($"[vk] type{i}: heap{d.MemProps.TypeAt(i).HeapIndex} flags={d.MemProps.TypeAt(i).PropertyFlags:x}"); - } - return d; - } - - private bool _fullSubgroups; - - /// Runs sgprobe.comp (256 threads) with requiredSubgroupSize = and - /// checks every invocation reports -lane subgroups, 256 / lanes per workgroup. - private bool ProbeSubgroup(uint lanes) - { - VkPipeline? pipe = null; - VkBuffer? buf = null; - uint count = 256 / lanes; - try - { - pipe = NewPipeline(NewShaderModule(VkShaders.Load("sgprobe")), 1, 0, PushDescriptors, lanes); - buf = NewStorageBuffer(256 * 4, hostVisible: true, preferHost: true); - IntPtr set = IntPtr.Zero, pool = IntPtr.Zero; - lock (Sync) - { - IntPtr cmd = NewCommandBuffer(); - IntPtr fence = NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, pipe.Pipeline); - Vk.VkDescriptorBufferInfo bi = new() { Buffer = buf.Buffer, Offset = 0, Range = 256 * 4 }; - Vk.VkWriteDescriptorSet w = new() - { - SType = VkConst.StWriteDescriptorSet, DstBinding = 0, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, PBufferInfo = &bi, - }; - if (PushDescriptors) CmdPushDescriptors(cmd, pipe.Layout, &w, 1); - else - { - pool = NewDescriptorPool(1); - set = AllocateDescriptorSet(pool, pipe.SetLayout); - BindBuffer(set, 0, buf, 0, 256 * 4); - Vk.vkCmdBindDescriptorSets(cmd, VkConst.BindPointCompute, pipe.Layout, 0, 1, &set, 0, null); - } - Vk.vkCmdDispatch(cmd, 1, 1, 1); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - Submit(cmd, fence); - WaitFence(fence); - Vk.vkDestroyFence(Device, fence, null); - FreeCommandBuffer(cmd); - if (pool != IntPtr.Zero) Vk.vkDestroyDescriptorPool(Device, pool, null); - } - uint* r = (uint*)buf.Map(); - buf.Invalidate(0, 256 * 4); - bool ok = true; - for (int i = 0; i < 256; i++) - ok &= (r[i] & 0xff) == lanes && ((r[i] >> 8) & 0xff) == count && ((r[i] >> 16) & 0xff) < count && (r[i] >> 24) < lanes; - buf.Unmap(); - return ok; - } - catch (InvalidOperationException) { return false; } - finally - { - buf?.Free(); - if (pipe is not null) - { - Vk.vkDestroyPipeline(Device, pipe.Pipeline, null); - Vk.vkDestroyPipelineLayout(Device, pipe.Layout, null); - Vk.vkDestroyDescriptorSetLayout(Device, pipe.SetLayout, null); - } - } - } - - /// Which fp16 x fp16 -> fp32 (C and result) subgroup coopmat shapes the device offers: - /// 16x16x16 (the default conv_cm builds) and 8x16x16 (SG16). - private static void CoopShapes(VkDevice d, out bool m16, out bool m8) - { - m16 = m8 = false; - // component types: 0 = float16, 1 = float32; scope 3 = subgroup - foreach (var p in CoopProps(d)) - if (p.NSize == 16 && p.KSize == 16 && p.AType == 0 && p.BType == 0 - && p.CType == 1 && p.ResultType == 1 && p.Scope == 3) - { - m16 |= p.MSize == 16; - m8 |= p.MSize == 8; - } - } - - /// vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR (empty when unavailable). - private static Vk.VkCooperativeMatrixPropertiesKHR[] CoopProps(VkDevice d) - { - byte[] name = System.Text.Encoding.ASCII.GetBytes("vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR\0"); - IntPtr fn; - fixed (byte* pn = name) fn = Vk.vkGetInstanceProcAddr(d.Instance, pn); - if (fn == IntPtr.Zero) return []; - var query = (delegate* unmanaged[Cdecl])fn; - uint np = 0; - if (query(d.PhysDevice, &np, null) != VkResult.Success || np == 0) return []; - var props = new Vk.VkCooperativeMatrixPropertiesKHR[np]; - for (int i = 0; i < props.Length; i++) props[i].SType = VkConst.StCooperativeMatrixPropertiesKHR; - fixed (Vk.VkCooperativeMatrixPropertiesKHR* pp = props) - if (query(d.PhysDevice, &np, pp) != VkResult.Success) return []; - return props; - } - - /// Allocate a storage buffer. hostVisible=false prefers DEVICE_LOCAL; - /// hostVisible=true prefers DEVICE_LOCAL|HOST_VISIBLE|HOST_COHERENT (ReBAR zero-copy), falling - /// back to HOST_VISIBLE|HOST_COHERENT. preferHost skips DEVICE_LOCAL types — for buffers the - /// CPU reads (uncached WC reads from VRAM are ~15MB/s, catastrophic). - public VkBuffer NewStorageBuffer(ulong bytes, bool hostVisible, bool preferHost = false) - { - var b = new VkBuffer { Dev = Device, Size = bytes }; - Vk.VkBufferCreateInfo bci = new() - { - SType = VkConst.StBufferCreateInfo, - Size = bytes, - Usage = VkConst.BufferUsageStorageBuffer | VkConst.BufferUsageTransferSrc | VkConst.BufferUsageTransferDst, - SharingMode = VkConst.SharingExclusive, - }; - Vk.Check(Vk.vkCreateBuffer(Device, &bci, null, out b.Buffer), "vkCreateBuffer"); - Vk.VkMemoryRequirements req; - Vk.vkGetBufferMemoryRequirements(Device, b.Buffer, &req); - - uint memType = uint.MaxValue; - for (int i = 0; i < (int)MemProps.MemoryTypeCount; i++) - { - if ((req.MemoryTypeBits & (1u << i)) == 0) continue; - uint f = MemProps.TypeAt(i).PropertyFlags; - bool ok; - if (!hostVisible) - ok = (f & VkConst.MemDeviceLocal) != 0; - else if (preferHost) - ok = (f & (VkConst.MemHostVisible | VkConst.MemHostCoherent)) == (VkConst.MemHostVisible | VkConst.MemHostCoherent) - && (f & VkConst.MemDeviceLocal) == 0 - && (f & VkConst.MemHostCached) != 0; - else - ok = (f & VkConst.MemHostVisible) != 0 && (f & VkConst.MemHostCoherent) != 0; - if (ok && (memType == uint.MaxValue || (!preferHost && (f & VkConst.MemDeviceLocal) != 0))) - { - memType = (uint)i; - if (preferHost || (f & VkConst.MemDeviceLocal) != 0) break; - } - } - if (memType == uint.MaxValue && preferHost) - for (int i = 0; i < (int)MemProps.MemoryTypeCount && memType == uint.MaxValue; i++) - { - if ((req.MemoryTypeBits & (1u << i)) == 0) continue; - uint f = MemProps.TypeAt(i).PropertyFlags; - if ((f & (VkConst.MemHostVisible | VkConst.MemHostCoherent)) - == (VkConst.MemHostVisible | VkConst.MemHostCoherent) - && (f & VkConst.MemDeviceLocal) == 0) - memType = (uint)i; - } - if (memType == uint.MaxValue && preferHost) - { - for (int i = 0; i < (int)MemProps.MemoryTypeCount && memType == uint.MaxValue; i++) - { - if ((req.MemoryTypeBits & (1u << i)) == 0) continue; - uint f = MemProps.TypeAt(i).PropertyFlags; - if ((f & (VkConst.MemHostVisible | VkConst.MemHostCoherent | VkConst.MemHostCached)) - == (VkConst.MemHostVisible | VkConst.MemHostCoherent | VkConst.MemHostCached)) - memType = (uint)i; - } - for (int i = 0; i < (int)MemProps.MemoryTypeCount && memType == uint.MaxValue; i++) - { - if ((req.MemoryTypeBits & (1u << i)) == 0) continue; - uint f = MemProps.TypeAt(i).PropertyFlags; - if ((f & (VkConst.MemHostVisible | VkConst.MemHostCoherent)) - == (VkConst.MemHostVisible | VkConst.MemHostCoherent)) - memType = (uint)i; - } - } - if (memType == uint.MaxValue) - throw new PlatformNotSupportedException($"Vulkan: no suitable memory type for {bytes}-byte buffer (hostVisible={hostVisible})"); - - Vk.VkMemoryAllocateInfo mai = new() - { - SType = VkConst.StMemoryAllocateInfo, - AllocationSize = req.Size, MemoryTypeIndex = memType, - }; - Vk.Check(Vk.vkAllocateMemory(Device, &mai, null, out b.Memory), "vkAllocateMemory"); - b.AllocSize = req.Size; - TrackAlloc((long)req.Size); - Vk.Check(Vk.vkBindBufferMemory(Device, b.Buffer, b.Memory, 0), "vkBindBufferMemory"); - b.Flags = MemProps.TypeAt((int)memType).PropertyFlags; - return b; - } - - public IntPtr NewShaderModule(byte[] spirv) - { - if ((spirv.Length & 3) != 0) throw new ArgumentException("SPIR-V size must be a multiple of 4"); - fixed (byte* p = spirv) - { - Vk.VkShaderModuleCreateInfo ci = new() - { - SType = VkConst.StShaderModuleCreateInfo, - CodeSize = (nuint)spirv.Length, PCode = (uint*)p, - }; - Vk.Check(Vk.vkCreateShaderModule(Device, &ci, null, out IntPtr m), "vkCreateShaderModule"); - return m; - } - } - - /// One set layout with `bindings` storage buffers (0..bindings-1) + push constants. - /// requiredSubgroupSize != 0 pins the compute subgroup width (VK_EXT_subgroup_size_control). - public VkPipeline NewPipeline(IntPtr shaderModule, int bindings, int pushConstBytes, bool pushLayout, - uint requiredSubgroupSize = 0) - { - var dslb = stackalloc Vk.VkDescriptorSetLayoutBinding[bindings]; - for (int i = 0; i < bindings; i++) - dslb[i] = new Vk.VkDescriptorSetLayoutBinding - { - Binding = (uint)i, DescriptorType = VkConst.DescStorageBuffer, - DescriptorCount = 1, StageFlags = VkConst.StageComputeShader, - }; - IntPtr setLayout; - Vk.VkDescriptorSetLayoutCreateInfo dslci = new() - { - SType = VkConst.StDescriptorSetLayoutCreateInfo, - BindingCount = (uint)bindings, PBindings = dslb, - Flags = pushLayout ? 1u : 0u, // VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR - }; - Vk.Check(Vk.vkCreateDescriptorSetLayout(Device, &dslci, null, out setLayout), "vkCreateDescriptorSetLayout"); - - Vk.VkPushConstantRange pcr = new() - { - StageFlags = VkConst.StageComputeShader, Offset = 0, Size = (uint)pushConstBytes, - }; - IntPtr layout; - Vk.VkPipelineLayoutCreateInfo plci = new() - { - SType = VkConst.StPipelineLayoutCreateInfo, - SetLayoutCount = 1, PSetLayouts = &setLayout, - PushConstantRangeCount = pushConstBytes > 0 ? 1u : 0u, - PPushConstantRanges = pushConstBytes > 0 ? &pcr : null, - }; - Vk.Check(Vk.vkCreatePipelineLayout(Device, &plci, null, out layout), "vkCreatePipelineLayout"); - - byte* name = stackalloc byte[5] { (byte)'m', (byte)'a', (byte)'i', (byte)'n', 0 }; - Vk.VkPipelineShaderStageRequiredSubgroupSizeCreateInfo sgc = new() - { - SType = VkConst.StPipelineShaderStageRequiredSubgroupSizeCreateInfo, - RequiredSubgroupSize = requiredSubgroupSize, - }; - Vk.VkComputePipelineCreateInfo cpci = new() - { - SType = VkConst.StComputePipelineCreateInfo, - Stage = new Vk.VkPipelineShaderStageCreateInfo - { - SType = VkConst.StPipelineShaderStageCreateInfo, - Stage = VkConst.StageComputeShader, Module = shaderModule, PName = name, - PNext = requiredSubgroupSize != 0 ? &sgc : null, - // REQUIRE_FULL_SUBGROUPS: every subgroup of a pinned pipeline has all its lanes - Flags = requiredSubgroupSize != 0 && _fullSubgroups ? 2u : 0u, - }, - Layout = layout, - }; - IntPtr pipeline; - Vk.Check(Vk.vkCreateComputePipelines(Device, IntPtr.Zero, 1, &cpci, null, &pipeline), "vkCreateComputePipelines"); - return new VkPipeline { Pipeline = pipeline, Layout = layout, SetLayout = setLayout, PushLayout = pushLayout }; - } - - public IntPtr NewDescriptorPool(uint maxSets) - { - Vk.VkDescriptorPoolSize psz = new() { Type = VkConst.DescStorageBuffer, DescriptorCount = maxSets * 16 }; - Vk.VkDescriptorPoolCreateInfo dpci = new() - { - SType = VkConst.StDescriptorPoolCreateInfo, - Flags = 0, MaxSets = maxSets, PoolSizeCount = 1, PPoolSizes = &psz, - }; - Vk.Check(Vk.vkCreateDescriptorPool(Device, &dpci, null, out IntPtr pool), "vkCreateDescriptorPool"); - return pool; - } - - public IntPtr AllocateDescriptorSet(IntPtr pool, IntPtr setLayout) - { - IntPtr set; - Vk.VkDescriptorSetAllocateInfo ai = new() - { - SType = VkConst.StDescriptorSetAllocateInfo, - DescriptorPool = pool, DescriptorSetCount = 1, PSetLayouts = &setLayout, - }; - Vk.Check(Vk.vkAllocateDescriptorSets(Device, &ai, &set), "vkAllocateDescriptorSets"); - return set; - } - - public IntPtr NewDescriptorSet(IntPtr setLayout) - { - IntPtr set; - Vk.VkDescriptorSetAllocateInfo ai = new() - { - SType = VkConst.StDescriptorSetAllocateInfo, - DescriptorPool = _descPool, DescriptorSetCount = 1, PSetLayouts = &setLayout, - }; - Vk.Check(Vk.vkAllocateDescriptorSets(Device, &ai, &set), "vkAllocateDescriptorSets"); - return set; - } - - /// VK_KHR_push_descriptor: write bindings inline into the command buffer - /// (DstSet in each write is ignored). - public void CmdPushDescriptors(IntPtr cmd, IntPtr layout, Vk.VkWriteDescriptorSet* writes, uint count) - => _pushDesc(cmd, VkConst.BindPointCompute, layout, 0, count, writes); - - /// Record the standard write->read barrier between two compute dispatches. - public void CmdComputeBarrier(IntPtr cmd) - { - Vk.VkMemoryBarrier mb = new() - { - SType = VkConst.StMemoryBarrier, - SrcAccessMask = VkConst.AccessShaderWrite, DstAccessMask = VkConst.AccessShaderRead, - }; - Vk.vkCmdPipelineBarrier(cmd, VkConst.PipelineStageComputeShader, VkConst.PipelineStageComputeShader, 0, - 1, &mb, 0, null, 0, null); - } - - /// Upload raw bytes into a buffer (host memcpy when mappable, else staging + copy cmd). - public void Upload(VkBuffer buf, void* src, ulong bytes) - { - if ((buf.Flags & VkConst.MemHostVisible) != 0) - { - void* p = buf.Map(); - Buffer.MemoryCopy(src, p, bytes, bytes); - buf.Flush(0, bytes); - buf.Unmap(); - return; - } - VkBuffer staging = NewStorageBuffer(bytes, hostVisible: true); - void* sp = staging.Map(); - Buffer.MemoryCopy(src, sp, bytes, bytes); - staging.Flush(0, bytes); - staging.Unmap(); - lock (Sync) - { - IntPtr cmd = NewCommandBuffer(); - IntPtr fence = NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - Vk.VkBufferCopy r = new() { Size = bytes }; - Vk.vkCmdCopyBuffer(cmd, staging.Buffer, buf.Buffer, 1, &r); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - Submit(cmd, fence); - WaitFence(fence); - Vk.vkDestroyFence(Device, fence, null); - FreeCommandBuffer(cmd); - } - staging.Free(); - } - - /// Download raw bytes from a buffer into dst (host memcpy or staging). - public void Download(VkBuffer buf, void* dst, ulong bytes, ulong srcOffset = 0) - { - if ((buf.Flags & VkConst.MemHostVisible) != 0) - { - void* p = buf.Map(); - buf.Invalidate(srcOffset, bytes); - Buffer.MemoryCopy((byte*)p + srcOffset, dst, bytes, bytes); - buf.Unmap(); - return; - } - VkBuffer staging = NewStorageBuffer(bytes, hostVisible: true, preferHost: true); - lock (Sync) - { - IntPtr cmd = NewCommandBuffer(); - IntPtr fence = NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - Vk.VkBufferCopy r = new() { Size = bytes, SrcOffset = srcOffset }; - Vk.vkCmdCopyBuffer(cmd, buf.Buffer, staging.Buffer, 1, &r); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - Submit(cmd, fence); - WaitFence(fence); - Vk.vkDestroyFence(Device, fence, null); - FreeCommandBuffer(cmd); - } - void* sp = staging.Map(); - staging.Invalidate(0, bytes); - Buffer.MemoryCopy(sp, dst, bytes, bytes); - staging.Unmap(); - staging.Free(); - } - - public void BindBuffer(IntPtr set, uint binding, VkBuffer buf, ulong offset, ulong size) - { - Vk.VkDescriptorBufferInfo bi = new() { Buffer = buf.Buffer, Offset = offset, Range = size }; - Vk.VkWriteDescriptorSet w = new() - { - SType = VkConst.StWriteDescriptorSet, DstSet = set, DstBinding = binding, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, PBufferInfo = &bi, - }; - Vk.vkUpdateDescriptorSets(Device, 1, &w, 0, null); - } - - public IntPtr NewCommandBuffer() - { - Vk.VkCommandBufferAllocateInfo ai = new() - { - SType = VkConst.StCommandBufferAllocateInfo, - CommandPool = _commandPool, Level = VkConst.CmdBufferLevelPrimary, CommandBufferCount = 1, - }; - IntPtr cmd; - Vk.Check(Vk.vkAllocateCommandBuffers(Device, &ai, &cmd), "vkAllocateCommandBuffers"); - return cmd; - } - - public void FreeCommandBuffer(IntPtr cmd) - { - if (cmd != IntPtr.Zero) Vk.vkFreeCommandBuffers(Device, _commandPool, 1, &cmd); - } - - /// Private resettable command pool (command pools are externally synchronized: - /// a worker that re-records on its own thread must not share the device pool). - public IntPtr NewCommandPool() - { - Vk.VkCommandPoolCreateInfo ci = new() - { - SType = VkConst.StCommandPoolCreateInfo, - Flags = VkConst.CmdPoolResetCommandBuffer, - QueueFamilyIndex = QueueFamily, - }; - Vk.Check(Vk.vkCreateCommandPool(Device, &ci, null, out IntPtr pool), "vkCreateCommandPool"); - return pool; - } - - public IntPtr AllocateCommandBuffer(IntPtr pool) - { - Vk.VkCommandBufferAllocateInfo ai = new() - { - SType = VkConst.StCommandBufferAllocateInfo, - CommandPool = pool, Level = VkConst.CmdBufferLevelPrimary, CommandBufferCount = 1, - }; - IntPtr cmd; - Vk.Check(Vk.vkAllocateCommandBuffers(Device, &ai, &cmd), "vkAllocateCommandBuffers"); - return cmd; - } - - /// Destroys a private command pool (frees its command buffers). - public void DestroyCommandPool(IntPtr pool) - { - if (pool != IntPtr.Zero) Vk.vkDestroyCommandPool(Device, pool, null); - } - - public IntPtr NewFence(bool signaled = false) - { - Vk.VkFenceCreateInfo ci = new() { SType = VkConst.StFenceCreateInfo, Flags = signaled ? 1u : 0u }; - Vk.Check(Vk.vkCreateFence(Device, &ci, null, out IntPtr f), "vkCreateFence"); - return f; - } - - public void Submit(IntPtr cmd, IntPtr fence) - { - Vk.VkSubmitInfo si = new() - { - SType = VkConst.StSubmitInfo, - CommandBufferCount = 1, PCommandBuffers = &cmd, - }; - VkResult r; - lock (QueueLock) r = Vk.vkQueueSubmit(Queue, 1, &si, fence); - Vk.Check(r, "vkQueueSubmit"); - } - - public void DestroyFence(IntPtr fence) - { - if (fence != IntPtr.Zero) Vk.vkDestroyFence(Device, fence, null); - } - - public void WaitFence(IntPtr fence) - { - Vk.Check(Vk.vkWaitForFences(Device, 1, &fence, 1, ulong.MaxValue), "vkWaitForFences"); - Vk.Check(Vk.vkResetFences(Device, 1, &fence), "vkResetFences"); - } - - public void Dispose() - { - if (Device != IntPtr.Zero) - { - Vk.vkDeviceWaitIdle(Device); - if (_descPool != IntPtr.Zero) { Vk.vkDestroyDescriptorPool(Device, _descPool, null); _descPool = IntPtr.Zero; } - if (_commandPool != IntPtr.Zero) { Vk.vkDestroyCommandPool(Device, _commandPool, null); _commandPool = IntPtr.Zero; } - lock (_pipes) - foreach (VkPipeline p in _pipes.Values) - { - Vk.vkDestroyPipeline(Device, p.Pipeline, null); - Vk.vkDestroyPipelineLayout(Device, p.Layout, null); - Vk.vkDestroyDescriptorSetLayout(Device, p.SetLayout, null); - } - Vk.vkDestroyDevice(Device, null); Device = IntPtr.Zero; - } - if (Instance != IntPtr.Zero) { Vk.vkDestroyInstance(Instance, null); Instance = IntPtr.Zero; } - } -} - -internal sealed class VkPipeline -{ - public IntPtr Pipeline, Layout, SetLayout; - public string Name = ""; - public int Bindings; - public bool PushLayout; -} - -internal unsafe sealed class VkBuffer -{ - public IntPtr Dev, Buffer, Memory; - public ulong Size, AllocSize; - public uint Flags; - - public bool HostCoherent => (Flags & VkConst.MemHostCoherent) != 0; - - public void* Map() - { - void* p; - Vk.Check(Vk.vkMapMemory(Dev, Memory, 0, Size, 0, &p), "vkMapMemory"); - return p; - } - public void Unmap() => Vk.vkUnmapMemory(Dev, Memory); - - public void Flush(ulong offset, ulong size) - { - if (HostCoherent) return; - Vk.VkMappedMemoryRange r = new() { SType = VkConst.StMappedMemoryRange, Memory = Memory, Offset = offset, Size = size }; - Vk.vkFlushMappedMemoryRanges(Dev, 1, &r); - } - public void Invalidate(ulong offset, ulong size) - { - if (HostCoherent) return; - Vk.VkMappedMemoryRange r = new() { SType = VkConst.StMappedMemoryRange, Memory = Memory, Offset = offset, Size = size }; - Vk.vkInvalidateMappedMemoryRanges(Dev, 1, &r); - } - - public void Free() - { - if (Buffer != IntPtr.Zero) { Vk.vkDestroyBuffer(Dev, Buffer, null); Buffer = IntPtr.Zero; } - if (Memory != IntPtr.Zero) - { - Vk.vkFreeMemory(Dev, Memory, null); Memory = IntPtr.Zero; - VkDevice.TrackFree((long)AllocSize); - AllocSize = 0; - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkGraph.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkGraph.cs deleted file mode 100644 index d686644..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkGraph.cs +++ /dev/null @@ -1,738 +0,0 @@ -using System.Runtime.InteropServices; -using Sdcb.SimdGcdrNet.Engine; - -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -/// Binding slots resolved per-session: which underlying buffer. -internal enum VkSlot : int { Arena = 0, Consts = 1, Dummy = 2, StageIn = 3, StageOut = 4, Direct = 5 } - -internal readonly struct VkBind -{ - public readonly VkSlot Slot; - public readonly VkBuffer? Buf; // when Slot == Direct - public readonly ulong Offset, Size; - public VkBind(VkSlot s, ulong off, ulong size) { Slot = s; Offset = off; Size = size; } - public VkBind(VkBuffer b, ulong off, ulong size) { Slot = VkSlot.Direct; Buf = b; Offset = off; Size = size; } -} - -/// One recorded compute dispatch. -internal sealed class VkDispatch -{ - public required VkPipeline Pipe; - public required VkBind[] Binds; - public required byte[] Pc; - public uint Gx, Gy = 1, Gz = 1; - public string Name = ""; -} - -/// -/// Per-(device, ExecPlan) half of the Vulkan backend: backend-private constant -/// packing/upload and the node list compiled to records. -/// Sessions own the activation arena and resolve slot ids to live buffers. -/// -/// Plans are layout-planned (NHWC, see ); every -/// addressing decision reads . With -/// activations and weights are stored as float16 (16-bit storage only — all -/// shader math, reductions included, is fp32); conv bias and LayerNorm params -/// stay fp32. Const bytes therefore differ per backend and per precision. -/// -internal sealed class VkGraphModel : IDisposable -{ - private readonly VkDevice _dev; - private readonly ExecPlan _plan; - private int _refs = 1; // the graph cache holds the first reference - private readonly VkConstStore _store; - internal readonly VkBuffer Consts; // all const tensors, backend-packed (shared store) - private readonly Dictionary<(int v, CK k), ulong> _constOff = new(); - internal readonly VkBuffer Dummy; // 64B zero buffer for unused bindings - internal readonly VkDispatch[] Dispatches; - internal readonly ulong ConstBytes; - /// Activations + weights stored as float16 (fp32 math in every shader). - internal readonly bool F16; - /// Bytes per stored activation element. - internal int Es => F16 ? 2 : 4; - - /// How a const is laid out on the device (decided by its consumer). - private enum CK { Raw, F32, ConvW, DwW, ConvWT, GemmNK } - - /// Vulkan device the graph is lowered for. - /// Compiled execution plan to lower. - /// Store activations and (non-fp32) weights as fp16; math stays fp32. - /// Sharing key for the const store (the source model): graphs of the - /// same model at other sizes reuse its uploaded weights. Null = private store. - internal VkGraphModel(VkDevice dev, ExecPlan plan, bool f16, object? constKey = null) - { - _dev = dev; - _plan = plan; - F16 = f16; - if (!plan.Nhwc) throw new NotSupportedException("Vulkan backend requires a layout-planned (NHWC) plan."); - - // ---- consts: per-consumer packing ---- - var need = new List<(int v, CK k)>(); - void Need(int v, CK k) - { - if (v < 0 || plan.Values[v].Floats is null) return; - if (!need.Contains((v, k))) need.Add((v, k)); - } - foreach (ExecNode n in plan.Nodes) - { - switch (n.P) - { - case ConvP cp: - Need(n.In[0], CK.Raw); - Need(n.In[1], cp.Group > 1 ? CK.DwW : UseCoopConv(dev, f16, plan, n, cp) ? CK.ConvWT : CK.ConvW); - Need(cp.BiasValue, CK.F32); - Need(cp.AddValue, CK.Raw); - break; - case GemmP gp: - Need(n.In[0], CK.Raw); Need(n.In[1], UseCoopGemm(dev, f16, plan, n, gp) ? CK.GemmNK : CK.Raw); - Need(gp.BiasValue, CK.F32); - break; - case LnP: - Need(n.In[0], CK.Raw); Need(n.In[1], CK.F32); Need(n.In[2], CK.F32); - break; - case UpHeadP up: - Need(n.In[2], CK.Raw); - if (n.In.Length > 3) Need(n.In[3], CK.F32); - if (up.SkipConv is not null) { Need(up.SkipW, CK.ConvW); Need(up.SkipB, CK.F32); } - break; - default: - foreach (int iv in n.In) Need(iv, CK.Raw); - break; - } - } - foreach (var (v, k) in need) - { - CValue cv = plan.Values[v]; - if (k == CK.Raw && cv.Layout is { } l && !Engine.Layout.IsIdentity(l)) - throw new NotSupportedException($"Vulkan: const '{cv.Name}' with non-identity layout"); - } - float[][] srcs = need.Select(x => plan.Values[x.v].Floats!).ToArray(); - int[] kinds = need.Select(x => (int)x.k).ToArray(); - _store = VkConstStore.Acquire(dev, constKey, f16, srcs, kinds, () => - { - ulong total = 0; - var offs = new ulong[need.Count]; - var items = new List<(ulong off, float[] data, bool half)>(); - for (int i = 0; i < need.Count; i++) - { - var (v, k) = need[i]; - CValue cv = plan.Values[v]; - float[] data = k switch - { - CK.ConvW => PackConv(cv), - CK.ConvWT => PackConvT(cv), - CK.GemmNK => PackGemmNK(cv, plan.Nodes.First(m => m.P is GemmP && m.In[1] == v)), - CK.DwW => PackDw(cv), - _ => cv.Floats!, - }; - bool half = f16 && k != CK.F32; - total = (total + 63) & ~63UL; - offs[i] = total; - items.Add((total, data, half)); - total += (ulong)data.Length * (half ? 2UL : 4UL); - } - byte[] host = new byte[Math.Max((total + 63) & ~63UL, 64)]; - foreach (var (off, data, half) in items) - { - if (half) - { - Span dst = MemoryMarshal.Cast(host.AsSpan(checked((int)off), data.Length * 2)); - for (int i = 0; i < data.Length; i++) dst[i] = (Half)data[i]; - } - else - MemoryMarshal.AsBytes(data.AsSpan()).CopyTo(host.AsSpan(checked((int)off))); - } - return (offs, host); - }); - for (int i = 0; i < need.Count; i++) _constOff[need[i]] = _store.Offsets[i]; - Consts = _store.Consts; - Dummy = _store.Dummy; - ConstBytes = _store.Bytes; - - var disps = new List(); - NodeRanges = new (int start, int count)[plan.Nodes.Length]; - for (int i = 0; i < plan.Nodes.Length; i++) - { - int start = disps.Count; - disps.AddRange(Emit(plan.Nodes[i])); - NodeRanges[i] = (start, disps.Count - start); - } - Dispatches = disps.ToArray(); - } - - // ONNX [OC][IC][KH][KW] -> [K][OC], k = (ky*KW + kx)*IC + ic (matches conv_tile.comp) - private static float[] PackConv(CValue w) - { - int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3]; - float[] src = w.Floats!, dst = new float[src.Length]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < taps; t++) - dst[(t * ic + c) * oc + o] = src[(o * ic + c) * taps + t]; - return dst; - } - - // ONNX [OC][IC][KH][KW] -> [OC][Kpad], k = (ky*KW + kx)*IC + ic, Kpad = K rounded up to 8 - // (zero-filled; matches conv_cm.comp) - private static float[] PackConvT(CValue w) - { - int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3], k = (taps * ic + 7) & ~7; - float[] src = w.Floats!, dst = new float[oc * k]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < taps; t++) - dst[o * k + t * ic + c] = src[(o * ic + c) * taps + t]; - return dst; - } - - /// Tensor-core conv eligibility (conv_cm.comp): fp16 storage on a coopmat device, - /// dense, channel counts in 16 B quads, an epilogue it implements, M within the grid limit. - private static bool UseCoopConv(VkDevice dev, bool f16, ExecPlan plan, ExecNode n, ConvP cp) - { - if (!dev.CoopMat16 || dev.ConvDot || !f16 || cp.Group != 1 || cp.Pool is not null || cp.Dilations.Any(d => d != 1)) return false; - if (cp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false; - if (cp.PadMode == PadMode.Constant && cp.PadValue != 0f) return false; - int[] si = plan.Values[n.In[0]].PhysShape, so = plan.Values[n.Out[0]].PhysShape; // [1,h,w,c] - long m = (long)so[1] * so[2]; - return so[3] % 8 == 0 && (m + 127) / 128 <= 65535; // cin % 8 != 0 -> scalar-gather stem variant - } - - // Gemm B -> [N][K] (k fastest), the conv_cm weight layout - private static float[] PackGemmNK(CValue b, ExecNode n) - { - var gp = (GemmP)n.P!; - if (gp.TransB) return b.Floats!; // already [N][K] - int k = b.Shape[0], nn = b.Shape[1]; - float[] src = b.Floats!, dst = new float[src.Length]; - for (int i = 0; i < k; i++) - for (int j = 0; j < nn; j++) dst[j * k + i] = src[i * nn + j]; - return dst; - } - - /// Tensor-core Gemm (a 1x1 conv over M "pixels" through conv_cm.comp): fp16 storage - /// on a coopmat device, const B, K/N in 16 B quads, relu/gelu/none epilogue. - private static bool UseCoopGemm(VkDevice dev, bool f16, ExecPlan plan, ExecNode n, GemmP gp) - { - if (!dev.CoopMat16 || dev.ConvDot || !f16 || plan.Values[n.In[1]].Floats is null || plan.Values[n.In[1]].Shape.Length != 2) return false; - // B packed per value: only when no other node reads it (its transB could differ) - if (plan.Nodes.Count(m => m.In.Contains(n.In[1])) != 1) return false; - if (gp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false; - CValue a = plan.Values[n.In[0]]; - int k = a.Shape[^1], nn = plan.Values[n.Out[0]].Shape[^1]; - if (a.PhysShape[^1] != k) return false; - long m = a.ElementCount / k; - return k % 8 == 0 && nn % 8 == 0 && (m + 127) / 128 <= 65535; - } - - // ONNX [C][1][KH][KW] -> [tap][C] - private static float[] PackDw(CValue w) - { - int c = w.Shape[0], taps = w.Shape[2] * w.Shape[3]; - float[] src = w.Floats!, dst = new float[src.Length]; - for (int ch = 0; ch < c; ch++) - for (int t = 0; t < taps; t++) dst[t * c + ch] = src[ch * taps + t]; - return dst; - } - - /// Dispatch index range per source ExecNode (for per-node debug runs). - internal readonly (int start, int count)[] NodeRanges; - - // ---- value → bind resolution (byte offsets/sizes: all long math) ---- - private VkBind Bind(int valueId, CK kind = CK.Raw) - { - CValue v = _plan.Values[valueId]; - if (v.Buf >= 0) - return new VkBind(VkSlot.Arena, (ulong)_plan.BufOffset[v.Buf] * (ulong)Es, - (ulong)Math.Max(v.ElementCount, 1) * (ulong)Es); - if (_constOff.TryGetValue((valueId, kind), out ulong off)) - { - ulong es = F16 && kind != CK.F32 ? 2UL : 4UL; - // ConvWT rows are K-padded to 8: bind the packed length, not the logical one - long elems = kind == CK.ConvWT - ? (long)v.Shape[0] * ((v.Shape[1] * v.Shape[2] * v.Shape[3] + 7) & ~7) - : Math.Max(v.ElementCount, 1); - return new VkBind(VkSlot.Consts, off, (ulong)elems * es); - } - if (v.Floats != null) - throw new InvalidOperationException($"const value '{v.Name}' not packed as {kind}"); - throw new InvalidOperationException($"value '{v.Name}' has no GPU storage (ints must be folded)"); - } - - private static byte[] PcOf(params uint[] ints) - { - byte[] b = new byte[ints.Length * 4]; - Buffer.BlockCopy(ints, 0, b, 0, b.Length); - return b; - } - - private static byte[] PcOfF(uint[] ints, params float[] floats) - { - byte[] b = new byte[(ints.Length + floats.Length) * 4]; - Buffer.BlockCopy(ints, 0, b, 0, ints.Length * 4); - Buffer.BlockCopy(floats, 0, b, ints.Length * 4, floats.Length * 4); - return b; - } - - private static uint[] U(params int[] v) { uint[] u = new uint[v.Length]; for (int i = 0; i < v.Length; i++) u[i] = (uint)v[i]; return u; } - - private static uint G(uint n) => (n + 255) / 256; - - /// - /// Broadcast strides (padded rank-4, storage order of the output) of operand - /// onto output . Output physical dim i - /// holds logical dim o.Layout[i]; the operand's logical dim is that minus the - /// numpy right-alignment offset. - /// - private static int[] PhysBroadcastStrides(CValue a, CValue o) - { - int rank = o.Shape.Length, off = rank - a.Shape.Length; - if (rank > 4) throw new NotSupportedException("Vulkan elementwise: rank > 4"); - int[] lo = o.Layout ?? Engine.Layout.Identity(rank); - int[] la = a.Layout ?? Engine.Layout.Identity(a.Shape.Length); - int[] ap = a.PhysShape; - int[] aStr = new int[ap.Length]; - for (int i = ap.Length - 1, acc = 1; i >= 0; i--) { aStr[i] = acc; acc *= ap[i]; } - int[] r4 = new int[4]; - for (int i = 0; i < rank; i++) - { - int d = lo[i] - off; - r4[4 - rank + i] = d < 0 || a.Shape[d] == 1 ? 0 : aStr[Array.IndexOf(la, d)]; - } - return r4; - } - - private string Sh(string name) => F16 ? name + "_h" : name; - - private IEnumerable Emit(ExecNode n) - { - CValue[] V = _plan.Values; - // all addressing on storage-order shapes - int[] shapeIn(int k) => V[n.In[k]].PhysShape; - int[] shapeOut() => V[n.Out[0]].PhysShape; - int nOut() => checked((int)V[n.Out[0]].ElementCount); - - switch (n.Op) - { - case ExecOp.Copy: - yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)nOut(), 4), G((uint)nOut())); - break; - - case ExecOp.Relu or ExecOp.Sigmoid or ExecOp.Erf or ExecOp.Gelu: - { - uint mode = n.Op switch - { - ExecOp.Relu => 0, ExecOp.Sigmoid => 1, ExecOp.Erf => 2, _ => 3, - }; - yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)nOut(), mode), G((uint)nOut())); - break; - } - - case ExecOp.Elementwise: - { - var p = (EwP)n.P!; - CValue a = V[n.In[0]], bv = V[n.In[1]], o = V[n.Out[0]]; - int[] os = Pad4(shapeOut()); - int[] ostr = Strides(os); - int[] astr = PhysBroadcastStrides(a, o); - int[] bstr = PhysBroadcastStrides(bv, o); - uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, _ => 0u }; - // same-storage fast path: vec4 (n % 4 guard — no OOB tail writes) - bool dense = astr.SequenceEqual(ostr) && bstr.SequenceEqual(ostr); - if (nOut() % 4 == 0 && dense) - { - yield return ND("ew2v", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])], - PcOf((uint)(nOut() / 4), (uint)p.Op, act), G((uint)(nOut() / 4))); - break; - } - yield return ND("ew2", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - astr[0], astr[1], astr[2], astr[3], - bstr[0], bstr[1], bstr[2], bstr[3], (int)p.Op, (int)act)), - G((uint)nOut())); - break; - } - - case ExecOp.Conv: - { - var p = (ConvP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - if (p.Pool is { } pool) - { - // small-ic stem conv + 2x2/2 max pool (GraphCompiler.GpuSmallConv) - int[] ci = shapeIn(0), co = shapeOut(), cw = V[n.In[1]].Shape; - if (!GraphCompiler.GpuSmallConv(p, cw, ci[3]) || p.AddValue >= 0 || ci[1] != co[1] * 2 || ci[2] != co[2] * 2) - throw new NotSupportedException("Vulkan convpool: unsupported conv"); - uint pf = (p.BiasValue >= 0 ? 1u : 0u) | (p.PadMode == PadMode.Reflect ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) | (pool.FuseRelu ? 8u : 0u); - bool k3 = _dev.ConvPoolK3 && p.Kernel[0] == 3 && p.Kernel[1] == 3; - string cpName = k3 ? "convpool_k3" : _dev.ConvDot && F16 && cw[0] % 8 == 0 ? "convpool_pi" : "convpool"; - yield return ND(cpName, - [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOfF(U(ci[3], ci[1], ci[2], cw[0], co[1], co[2], p.Kernel[0], p.Kernel[1], p.Pads[0], p.Pads[1], (int)pf), - p.PadMode == PadMode.Constant ? p.PadValue : 0f), - G((uint)co[1] * (uint)co[2] * (uint)(cw[0] / 8)), name: "convpool"); - break; - } - if (p.AddValue >= 0) RequireNhwc(p.AddValue); - int[] si = shapeIn(0), sw_ = V[n.In[1]].Shape, so = shapeOut(); // [1,h,w,c] - uint ih = (uint)si[1], iw = (uint)si[2], ic = (uint)si[3]; - uint oc = (uint)sw_[0], oh = (uint)so[1], ow = (uint)so[2]; - if (p.PadMode == PadMode.Constant && p.PadValue != 0f) - throw new NotSupportedException("Vulkan conv: nonzero constant pad"); - if (p.Dilations.Any(d => d != 1)) - throw new NotSupportedException("Vulkan conv: dilation"); - if (p.Act == ActivationKind.Sigmoid) - throw new NotSupportedException("Vulkan conv: sigmoid epilogue"); - uint flags = (p.BiasValue >= 0 ? 1u : 0u) - | (p.PadMode == PadMode.Reflect ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) - | (p.AddValue >= 0 ? 8u : 0u) - | (p.Act == ActivationKind.Gelu ? 16u : 0u); - if (p.Group > 1) - { - if (p.Group != ic || oc != ic) throw new NotSupportedException("grouped non-depthwise conv"); - if (p.AddValue >= 0) throw new NotSupportedException("depthwise conv + residual"); - bool v4 = _dev.Vec4Eltwise && ic % 4 == 0; - yield return ND(v4 ? "depthwise4" : "depthwise", - [Bind(n.In[0]), Bind(n.In[1], CK.DwW), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOfF(U((int)ic, (int)ih, (int)iw, (int)oh, (int)ow, p.Kernel[0], p.Kernel[1], - p.Strides[0], p.Strides[1], p.Pads[0], p.Pads[1], (int)(flags & ~8u)), p.PadValue), - G(ic * oh * ow / (v4 ? 4u : 1u)), name: "depthwise"); - break; - } - if (UseCoopConv(_dev, F16, _plan, n, p)) - { - // BM = 128 rows (pixels) in every variant; BN by oc divisibility - (string cm, uint bn) = ic % 8 != 0 ? ("conv_cm_n32s", 32u) - : oc % 128 == 0 ? ("conv_cm", 128u) : oc % 64 == 0 ? ("conv_cm_n64", 64u) : ("conv_cm_n32", 32u); - uint m = oh * ow, kr = (uint)(p.Kernel[0] * p.Kernel[1]) * ic, kd = (kr + 7u) & ~7u; - uint cflags = (p.BiasValue >= 0 ? 1u : 0u) | (p.AddValue >= 0 ? 2u : 0u) - | (p.PadMode == PadMode.Reflect ? 4u : 0u) - | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4); - yield return ND(cm, - [Bind(n.In[0]), Bind(n.In[1], CK.ConvWT), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])], - PcOf(m, oc, kd, cflags, ow, iw, ih, ic, (uint)p.Kernel[1], - (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], kr), - (m + 127) / 128, (oc + bn - 1) / bn, name: "conv_cm"); - break; - } - string convShader = - p.Kernel[0] == 3 && p.Kernel[1] == 3 && p.Strides[0] == 1 && p.Strides[1] == 1 ? "conv_k33s1" : - p.Kernel[0] == 3 && p.Kernel[1] == 3 && p.Strides[0] == 2 && p.Strides[1] == 2 ? "conv_k33s2" : - p.Kernel[0] == 1 && p.Kernel[1] == 1 && p.Strides[0] == 1 && p.Strides[1] == 1 ? "conv_k11s1" : - "conv_gen"; - if (_dev.ConvDot && F16 && convShader is "conv_k33s1" or "conv_k33s2" && ic % 4 == 0 && oc % 4 == 0) - { - // Adreno 750 sweep (ms, 1600x1152 gc+dr): stride 1 at 4 px x 8 oc, stride 2 at 2 px x 8 oc - uint tm = p.Strides[1] == 1 ? 4u : 2u, tn = oc % 8 == 0 ? 2u : 1u; - uint threads = oc / (4 * tn) * ((ow + tm - 1) / tm) * oh; - uint wgs = (threads + 63) / 64, gx = Math.Min(wgs, 32768u); - yield return ND($"conv_dot_s{p.Strides[1]}_{tm}x{tn}", - [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])], - PcOf(ic, ih, iw, oc, oh, ow, (uint)p.Kernel[0], (uint)p.Kernel[1], - (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], - flags, 0), - gx, (wgs + gx - 1) / gx, name: "conv"); - break; - } - yield return ND(convShader, - [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])], - PcOf(ic, ih, iw, oc, oh, ow, (uint)p.Kernel[0], (uint)p.Kernel[1], - (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], - flags, 0), - (ow + 15) / 16, (oh + 7) / 8, (oc + 63) / 64, name: "conv"); - break; - } - - case ExecOp.MaxPool: - { - var p = (PoolP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0), so = shapeOut(); - bool v4 = _dev.Vec4Eltwise && si[3] % 4 == 0; - yield return ND(v4 ? "maxpool4" : "maxpool", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], - (uint)p.Kernel[0], (uint)p.Kernel[1], (uint)p.Strides[0], (uint)p.Strides[1], - (uint)p.Pads[0], (uint)p.Pads[1], p.FuseRelu ? 1u : 0u), - G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "maxpool"); - break; - } - - case ExecOp.LayerNorm: - { - var p = (LnP)n.P!; - uint len = (uint)V[n.In[0]].Shape[^1]; - if ((uint)shapeIn(0)[^1] != len) throw new NotSupportedException("Vulkan LayerNorm: last dim not innermost"); - uint rows = (uint)(V[n.In[0]].ElementCount / len); - yield return ND("layernorm", - [Bind(n.In[0]), Bind(n.In[1], CK.F32), Bind(n.In[2], CK.F32), Bind(n.Out[0])], - PcOfF([len], p.Eps), rows, name: "layernorm"); - break; - } - - case ExecOp.Gemm: - { - var p = (GemmP)n.P!; - uint k = (uint)V[n.In[0]].Shape[^1]; - if ((uint)shapeIn(0)[^1] != k) throw new NotSupportedException("Vulkan Gemm: K not innermost"); - uint m = (uint)(V[n.In[0]].ElementCount / k); - uint nn = (uint)V[n.Out[0]].Shape[^1]; - if (UseCoopGemm(_dev, F16, _plan, n, p)) - { - (string cm, uint bn) = nn % 128 == 0 ? ("conv_cm", 128u) : nn % 64 == 0 ? ("conv_cm_n64", 64u) : ("conv_cm_n32", 32u); - uint cflags = (p.BiasValue >= 0 ? 1u : 0u) - | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4); - // [M,K] x [N,K]^T as a 1x1 conv over a 1 x M image - yield return ND(cm, - [Bind(n.In[0]), Bind(n.In[1], CK.GemmNK), B(p.BiasValue, CK.F32), B(-1), Bind(n.Out[0])], - PcOf(m, nn, k, cflags, m, m, 1u, k, 1u, 1u, 1u, 0u, 0u, k), - (m + 127) / 128, (nn + bn - 1) / bn, name: "gemm_cm"); - break; - } - uint gflags = (p.TransB ? 1u : 0u) | (p.BiasValue >= 0 ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) - | (p.Act == ActivationKind.Gelu ? 8u : 0u); - if (_dev.ConvDot && F16 && !p.TransB && k % 4 == 0 && nn % 8 == 0) - { - // 4 rows x 8 columns per invocation - uint gth = nn / 8 * ((m + 3) / 4); - uint gwg = (gth + 63) / 64, ggx = Math.Min(gwg, 32768u); - yield return ND("gemm_dot", - [Bind(n.In[0]), Bind(n.In[1]), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOf(m, nn, k, gflags), ggx, (gwg + ggx - 1) / ggx, name: "gemm"); - break; - } - yield return ND("gemm", - [Bind(n.In[0]), Bind(n.In[1]), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOf(m, nn, k, gflags), - (nn + 63) / 64, (m + 63) / 64, name: "gemm"); - break; - } - - case ExecOp.ResizeBilinear: - { - var p = (ResizeP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - bool hasAdd = n.In.Length > 1; - if (hasAdd) RequireNhwc(n.In[1]); - int[] si = shapeIn(0), so = shapeOut(); - uint flags = (p.FuseRelu ? 1u : 0u) | (hasAdd ? 2u : 0u) | (p.AddRelu ? 4u : 0u); - bool v4 = _dev.Vec4Eltwise && si[3] % 4 == 0; - yield return ND(v4 ? "bilinear4" : "bilinear", [Bind(n.In[0]), hasAdd ? Bind(n.In[1]) : B(-1), Bind(n.Out[0])], - PcOfF([(uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], flags], - p.Scales[2], p.Scales[3]), - G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "bilinear"); - break; - } - - case ExecOp.UpHead: - { - var p = (UpHeadP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.In[1]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0), so = shapeOut(); - uint c = (uint)si[3], oc = (uint)so[3]; - if (oc > 8 || c > 128 || c % 4 != 0) throw new NotSupportedException($"Vulkan UpHead: c={c} oc={oc}"); - bool hasBias = n.In.Length > 3; - uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, ActivationKind.None => 0u, - _ => throw new NotSupportedException($"Vulkan UpHead: {p.Act}") }; - uint flags = (p.ResizeRelu ? 1u : 0u) | (p.SkipRelu ? 2u : 0u) | (hasBias ? 4u : 0u) | (act << 3); - if (p.SkipConv is { } sc) - { - // skip recomputed from the stem conv's input (GraphCompiler.GpuSmallConv, c <= 32) - int[] xi = shapeIn(1), cw = V[p.SkipW].Shape; - if (!GraphCompiler.GpuSmallConv(sc, cw, xi[3]) || cw[0] != c || c > 32 || xi[1] != so[1] || xi[2] != so[2]) - throw new NotSupportedException("Vulkan UpHead: unsupported skip conv"); - uint cf = (sc.BiasValue >= 0 ? 1u : 0u) | (sc.PadMode == PadMode.Reflect ? 2u : 0u) - | (sc.Act == ActivationKind.Relu ? 4u : 0u); - uint np = _dev.ConvDot && F16 ? 2u : 1u; - yield return ND(np > 1 ? $"uphead_sc{np}" : "uphead_sc", - [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0]), - Bind(p.SkipW, CK.ConvW), B(p.SkipB, CK.F32)], - PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], - p.Scales[2], p.Scales[3]).Concat(PcOfF(U(xi[3], sc.Kernel[0], sc.Kernel[1], sc.Pads[0], sc.Pads[1], (int)cf), - sc.PadMode == PadMode.Constant ? sc.PadValue : 0f)).ToArray(), - G((uint)so[1] * (((uint)so[2] + np - 1) / np)), name: "uphead_sc"); - break; - } - yield return ND("uphead", - [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0])], - PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], - p.Scales[2], p.Scales[3]), - G((uint)so[1] * (uint)so[2])); - break; - } - - case ExecOp.Pad: - { - // generic rank-4 pad in storage order (pads permuted to physical dims) - var p = (PadP)n.P!; - CValue x = V[n.In[0]]; - if (x.Shape.Length != 4) throw new NotSupportedException("Vulkan pad: rank != 4"); - int[] l = x.Layout ?? Engine.Layout.Identity(4); - int[] id4 = shapeIn(0), od4 = shapeOut(); - int[] pb4 = new int[4]; - for (int d = 0; d < 4; d++) pb4[d] = p.Pads[l[d]]; - yield return ND("pad", [Bind(n.In[0]), Bind(n.Out[0])], - PcOfF(U(id4[0], id4[1], id4[2], id4[3], od4[0], od4[1], od4[2], od4[3], - pb4[0], pb4[1], pb4[2], pb4[3], p.Mode == PadMode.Reflect ? 1 : 0), - p.Value), - G((uint)V[n.Out[0]].ElementCount)); - break; - } - - case ExecOp.Transpose: - { - // only planner relayouts reach here; the perm is physical - var p = (TransposeP)n.P!; - int[] is4 = Pad4(shapeIn(0)); - int[] os4 = Pad4(shapeOut()); - int[] ostr = Strides(os4), istr = Strides(is4); - int[] perm4 = PermuteTo4(p.Perm, shapeIn(0).Length); - yield return ND("transpose", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - istr[perm4[0]], istr[perm4[1]], istr[perm4[2]], istr[perm4[3]])), - G((uint)nOut())); - break; - } - - case ExecOp.Slice: - { - // SliceP.Axes are physical after the layout pass - var p = (SliceP)n.P!; - int[] is4 = Pad4(shapeIn(0)), os4 = Pad4(shapeOut()); - int[] ostr = Strides(os4), istr = Strides(is4); - uint[] st = [0, 0, 0, 0], sp = [1, 1, 1, 1]; - int shift = 4 - shapeIn(0).Length; - for (int i = 0; i < p.Axes.Length; i++) - { - st[p.Axes[i] + shift] = (uint)p.Starts[i]; - sp[p.Axes[i] + shift] = (uint)p.Steps[i]; - } - yield return ND("slice", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - (int)st[0], (int)st[1], (int)st[2], (int)st[3], - (int)sp[0], (int)sp[1], (int)sp[2], (int)sp[3], - istr[0], istr[1], istr[2], istr[3])), - G((uint)nOut())); - break; - } - - case ExecOp.Concat: - { - // ConcatP.Axis is physical after the layout pass - var p = (ConcatP)n.P!; - int[] os = shapeOut(); - int axis = p.Axis; - int inner = 1, dstAxis = os[axis]; - for (int d = axis + 1; d < os.Length; d++) inner *= os[d]; - int axisOff = 0; - foreach (int iv in n.In) - { - int[] ishape = V[iv].PhysShape; - int cnt = checked((int)V[iv].ElementCount); - if (cnt > 0) - { - yield return ND("concat_ax", [Bind(iv), Bind(n.Out[0])], - PcOf((uint)cnt, (uint)inner, (uint)ishape[axis], (uint)dstAxis, (uint)axisOff), - G((uint)cnt)); - } - axisOff += ishape[axis]; - } - break; - } - - case ExecOp.ChannelShift: - { - var p = (ShiftP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0); // [1,h,w,c] - var ge = p.GroupEnds; var of = p.Offsets; - if (ge.Length > 5) throw new NotSupportedException("Vulkan shift: > 5 groups"); - var pc = new List { (uint)si[3], (uint)si[1], (uint)si[2], (uint)p.Axis }; - for (int i = 0; i < 5; i++) pc.Add((uint)(i < ge.Length ? ge[i] : si[3])); - for (int i = 0; i < 5; i++) pc.Add((uint)(i < of.Length ? of[i] : 0)); - bool v4 = _dev.Vec4Eltwise && si[3] % 4 == 0 && ge.All(e => e % 4 == 0); - yield return ND(v4 ? "shift4" : "shift", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(pc.ToArray()), G((uint)nOut() / (v4 ? 4u : 1u)), name: "shift"); - break; - } - - default: - throw new NotSupportedException($"Vulkan: ExecOp {n.Op} not implemented"); - } - - VkDispatch ND(string shader, VkBind[] binds, byte[] pc, uint gx, uint gy = 1, uint gz = 1, string? name = null) - { - return new VkDispatch - { - Pipe = shader.StartsWith("conv_cm", StringComparison.Ordinal) - ? _dev.GetCoopPipeline(shader, binds.Length, pc.Length) - : _dev.GetPipeline(Sh(shader), binds.Length, pc.Length), - Pc = pc, Binds = binds, Gx = gx, Gy = gy, Gz = gz, - Name = $"{shader}({n.Name})", - }; - } - - VkBind B(int valueId, CK kind = CK.Raw) => valueId >= 0 ? Bind(valueId, kind) : new VkBind(VkSlot.Dummy, 0, 64); - } - - private void RequireNhwc(int v) - { - CValue cv = _plan.Values[v]; - if (cv.Shape.Length != 4 || !Engine.Layout.Equivalent(cv.Shape, cv.Layout ?? Engine.Layout.Identity(4), Engine.Layout.Nhwc)) - throw new NotSupportedException($"Vulkan: '{cv.Name}' expected NHWC storage, got {Engine.Layout.Str(cv.Layout)}"); - } - - // pad a shape to rank 4 with leading 1s - private static int[] Pad4(int[] s) - { - if (s.Length == 4) return s; - int[] r = [1, 1, 1, 1]; - s.CopyTo(r, 4 - s.Length); - return r; - } - - private static int[] Strides(int[] shape4) - { - int[] s = new int[4]; s[3] = 1; - for (int i = 2; i >= 0; i--) s[i] = s[i + 1] * shape4[i + 1]; - return s; - } - - // lift an original-rank perm into padded rank-4 (leading dims map to themselves) - private static int[] PermuteTo4(int[] perm, int rank) - { - int shift = 4 - rank; - int[] p4 = [0, 1, 2, 3]; - for (int i = 0; i < rank; i++) p4[i + shift] = perm[i] + shift; - return p4; - } - - /// Sessions hold a reference so cache eviction can't free live const/dummy buffers. - public void Acquire() - { - if (Interlocked.Increment(ref _refs) <= 1) - throw new ObjectDisposedException(nameof(VkGraphModel), "const buffers already freed"); - } - /// Acquire unless already freed (no resurrection; for racing cache readers). - public bool TryAcquire() - { - while (true) - { - int r = Volatile.Read(ref _refs); - if (r <= 0) return false; - if (Interlocked.CompareExchange(ref _refs, r + 1, r) == r) return true; - } - } - public void Release() - { - if (Interlocked.Decrement(ref _refs) == 0) _store.Release(); - } - - /// The plan this graph was compiled from (arena offsets, I/O values). - internal ExecPlan Plan => _plan; - public void Dispose() => Release(); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs index dd6a5d7..154a0a3 100644 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs +++ b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs @@ -1,5 +1,7 @@ -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Backends.Vulkan; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Onnx; namespace Sdcb.SimdGcdrNet.Backends.Vulkan; @@ -45,9 +47,9 @@ internal VkGcdrPipeline(VkBackend be, OnnxModel gc, OnnxModel dr) _drModel = dr; _f16 = be.F16; string sfx = _f16 ? "_h" : ""; - _pPre = _dev.GetPipeline("bgr2nhwc" + sfx, 2, 9 * 4); - _pDiv6 = _dev.GetPipeline("div6" + sfx, 3, 4); - _pPost = _dev.GetPipeline("nhwc2bgr" + sfx, 3, 9 * 4); + _pPre = _dev.GetPipeline("bgr2nhwc" + sfx, 2, 9 * 4, owner: GcdrExtensions.Owner); + _pDiv6 = _dev.GetPipeline("div6" + sfx, 3, 4, owner: GcdrExtensions.Owner); + _pPost = _dev.GetPipeline("nhwc2bgr" + sfx, 3, 9 * 4, owner: GcdrExtensions.Owner); _cmdPool = _dev.NewCommandPool(); try { @@ -80,9 +82,9 @@ internal void Bind(int ih, int iw, GcdrPixelFormat fmt = GcdrPixelFormat.Bgr24) if (_gcG is null || h != _h || w != _w) { // a new size compiles two plans + graphs: build them side by side - var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], physIo: true)); + var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], GraphOptions)); VkGraphModel gd; - try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], physIo: true); } + try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], GraphOptions); } catch { try { tg.GetAwaiter().GetResult().Release(); } catch { } throw; } VkGraphModel gg; try { gg = tg.GetAwaiter().GetResult(); } @@ -100,7 +102,9 @@ internal void Bind(int ih, int iw, GcdrPixelFormat fmt = GcdrPixelFormat.Bgr24) ulong es = (ulong)Es; // gc base aligned to 256 B (any minStorageBufferOffsetAlignment) - var (gcOff, total) = GcdrEnhancer.StackArenas(GcPlan, DrPlan, (long)(256 / es)); + // the graphs' full arenas: the plan's values plus scratch / tile-read slack past them + var (gcOff, total) = GcdrEnhancer.StackArenas((long)((_gcG!.ArenaBytes + es - 1) / es), DrPlan, + (long)((_drG!.ArenaBytes + es - 1) / es), (long)(256 / es)); _gcBase = (ulong)gcOff * es; ulong need = (ulong)total * es + 64; // the arena only grows while within ArenaBudget; an over-budget one (huge image) @@ -118,6 +122,12 @@ internal void Bind(int ih, int iw, GcdrPixelFormat fmt = GcdrPixelFormat.Bgr24) _recorded = true; } + /// Fused-pipeline graphs: physical I/O, the GcdrNet extension ops, and the staged + /// tensor-core convs on SG16 (Intel Xe2) devices — twice as fast here as the direct family and + /// stress-clean on this model (the OCR models are not; see CompileOptions.StagedCoopConv). + private static CompileOptions GraphOptions { get; } = + VkBackend.PipeOptions with { Extensions = GcdrExtensions.Instance, StagedCoopConv = true }; + /// Per-worker device-arena budget (gc + dr). Within it the arena is grow-only /// (12.5% headroom on the first allocation, 25% on regrowth) so mixed sizes don't reallocate. /// 3 GB; on integrated GPUs the "device-local" heap is system RAM shared with the CPU, so diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkSession.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkSession.cs deleted file mode 100644 index edfc0ae..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkSession.cs +++ /dev/null @@ -1,394 +0,0 @@ -using Sdcb.SimdGcdrNet.Engine; - -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -/// -/// GPU counterpart of : one activation arena in -/// device-local memory, persistent host-visible staging buffers, and a single -/// pre-recorded command buffer (staging copy-in → all dispatches → copy-out). -/// Not thread-safe; one per thread or serialize externally. -/// -public sealed unsafe class VkInferenceSession : IDisposable -{ - private readonly VkDevice _dev; - private readonly VkGraphModel _graph; - private readonly ExecPlan _plan; - - internal readonly VkBuffer Arena; // device-local activation arena - private readonly VkBuffer _stageIn; // host-visible float input staging - private readonly VkBuffer _stageOut; // host-visible float output staging - private float* _stageInMap, _stageOutMap; - private readonly float[] _inHost, _outHost; - private readonly IntPtr _cmd, _fence; - private readonly IntPtr[] _descSets; // per-dispatch sets when !PushDescriptors - private readonly ulong _inOff, _outOff; - private readonly int _inFloats, _outFloats; - private readonly int _es; // bytes per stored element (2 = fp16 storage) - internal bool F16 => _es == 2; - - internal VkInferenceSession(VkDevice dev, ExecPlan plan, VkGraphModel graph) - { - _dev = dev; - _plan = plan; - _graph = graph; - _graph.Acquire(); - _es = graph.Es; - - Arena = dev.NewStorageBuffer((ulong)plan.ArenaFloats * (ulong)_es + 64, hostVisible: false); - _inFloats = checked((int)plan.Values[plan.InputValue].ElementCount); - _outFloats = checked((int)plan.Values[plan.OutputValue].ElementCount); - _inOff = (ulong)plan.BufOffset[plan.Values[plan.InputValue].Buf] * (ulong)_es; - _outOff = (ulong)plan.BufOffset[plan.Values[plan.OutputValue].Buf] * (ulong)_es; - _stageIn = dev.NewStorageBuffer(InputBytes, hostVisible: true, preferHost: true); - _stageOut = dev.NewStorageBuffer(OutputBytes, hostVisible: true, preferHost: true); - _stageInMap = (float*)_stageIn.Map(); - _stageOutMap = (float*)_stageOut.Map(); - _inHost = new float[_inFloats]; - _outHost = new float[_outFloats]; - _fence = dev.NewFence(); - - if (!dev.PushDescriptors) - { - _descSets = new IntPtr[_graph.Dispatches.Length]; - for (int i = 0; i < _graph.Dispatches.Length; i++) - { - var d = _graph.Dispatches[i]; - IntPtr set = dev.NewDescriptorSet(d.Pipe.SetLayout); - for (uint b = 0; b < d.Binds.Length; b++) - { - (VkBuffer buf, _, _) = Resolve(d.Binds[b]); - dev.BindBuffer(set, b, buf, d.Binds[b].Offset, d.Binds[b].Size); - } - _descSets[i] = set; - } - } - else _descSets = []; - - _cmd = dev.NewCommandBuffer(); - Record(); - } - - internal VkBuffer ResolveBuf(VkBind b) => Resolve(b).buf; - internal ulong ResolveOff(VkBind b) => b.Offset; - - internal (VkBuffer buf, ulong off, ulong size) Resolve(VkBind b) => b.Slot switch - { - VkSlot.Arena => (Arena, b.Offset, b.Size), - VkSlot.Consts => (_graph.Consts, b.Offset, b.Size), - VkSlot.StageIn => (_stageIn, b.Offset, b.Size), - VkSlot.StageOut => (_stageOut, b.Offset, b.Size), - VkSlot.Direct => (b.Buf!, b.Offset, b.Size), - _ => (_graph.Dummy, b.Offset, b.Size), - }; - - /// Input tensor shape (e.g. [1,3,H,W]). - public int[] InputShape => _plan.Values[_plan.InputValue].Shape; - /// Output tensor shape (e.g. [1,3,H,W]). - public int[] OutputShape => _plan.Values[_plan.OutputValue].Shape; - - /// The writable input window (host copy staged on ). - public Span Input => _inHost; - /// The output tensor after . - public ReadOnlySpan Output => _outHost; - - internal ulong InputOffset => _inOff; - internal ulong OutputOffset => _outOff; - internal ulong InputBytes => (ulong)_inFloats * (ulong)_es; - internal ulong OutputBytes => (ulong)_outFloats * (ulong)_es; - internal int ElemBytes => _es; - - /// Host float -> staged storage elements (fp16 narrowing when F16). - internal static void ToStorage(ReadOnlySpan src, void* dst, bool f16) - { - if (!f16) { src.CopyTo(new Span(dst, src.Length)); return; } - var d = new Span(dst, src.Length); - for (int i = 0; i < src.Length; i++) d[i] = (Half)src[i]; - } - - /// Stored elements -> host float (fp16 widening when F16). - internal static void FromStorage(void* src, Span dst, bool f16) - { - if (!f16) { new ReadOnlySpan(src, dst.Length).CopyTo(dst); return; } - var sp = new ReadOnlySpan(src, dst.Length); - for (int i = 0; i < dst.Length; i++) dst[i] = (float)sp[i]; - } - - /// Downloads dst.Length stored elements at byte offset . - internal void DownloadElems(VkBuffer buf, ulong off, Span dst) - { - ulong bytes = (ulong)dst.Length * (ulong)_es; - byte[] tmp = new byte[bytes]; - fixed (byte* t = tmp) - { - _dev.Download(buf, t, bytes, off); - FromStorage(t, dst, F16); - } - } - - /// Record one dispatch (push-descriptor path only; caller emits barriers). - internal static void RecordOne(VkDevice dev, IntPtr cmd, VkPipeline pipe, - (VkBuffer buf, ulong off, ulong size)[] binds, byte[] pc, uint gx, uint gy, uint gz = 1) - { - Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, pipe.Pipeline); - var writes = stackalloc Vk.VkWriteDescriptorSet[8]; - var infos = stackalloc Vk.VkDescriptorBufferInfo[8]; - for (uint b = 0; b < binds.Length; b++) - { - infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = binds[b].buf.Buffer, Offset = binds[b].off, Range = binds[b].size }; - writes[b] = new Vk.VkWriteDescriptorSet - { - SType = VkConst.StWriteDescriptorSet, DstBinding = b, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, - PBufferInfo = infos + b, - }; - } - dev.CmdPushDescriptors(cmd, pipe.Layout, writes, (uint)binds.Length); - if (pc.Length > 0) - fixed (byte* p = pc) - Vk.vkCmdPushConstants(cmd, pipe.Layout, VkConst.StageComputeShader, 0, (uint)pc.Length, p); - Vk.vkCmdDispatch(cmd, gx, gy, gz); - } - - /// Record all dispatches with barriers into an open command buffer. - internal void RecordCompute(IntPtr cmd) - { - var writes = stackalloc Vk.VkWriteDescriptorSet[8]; - var infos = stackalloc Vk.VkDescriptorBufferInfo[8]; - bool first = true; - for (int i = 0; i < _graph.Dispatches.Length; i++) - { - var d = _graph.Dispatches[i]; - if (!first) _dev.CmdComputeBarrier(cmd); - first = false; - Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, d.Pipe.Pipeline); - if (_dev.PushDescriptors) - { - for (uint b = 0; b < d.Binds.Length; b++) - { - (VkBuffer buf, ulong off, ulong size) = Resolve(d.Binds[b]); - infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = buf.Buffer, Offset = off, Range = size }; - writes[b] = new Vk.VkWriteDescriptorSet - { - SType = VkConst.StWriteDescriptorSet, DstBinding = b, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, - PBufferInfo = infos + b, - }; - } - _dev.CmdPushDescriptors(cmd, d.Pipe.Layout, writes, (uint)d.Binds.Length); - } - else - { - IntPtr set = _descSets[i]; - Vk.vkCmdBindDescriptorSets(cmd, VkConst.BindPointCompute, d.Pipe.Layout, 0, 1, &set, 0, null); - } - if (d.Pc.Length > 0) - fixed (byte* pc = d.Pc) - Vk.vkCmdPushConstants(cmd, d.Pipe.Layout, VkConst.StageComputeShader, 0, (uint)d.Pc.Length, pc); - Vk.vkCmdDispatch(cmd, d.Gx, d.Gy, d.Gz); - } - } - - private void Record() - { - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(_cmd, &begin), "vkBeginCommandBuffer"); - // staging -> arena(input) - Vk.VkBufferCopy cin = new() { Size = InputBytes, DstOffset = _inOff }; - Vk.vkCmdCopyBuffer(_cmd, _stageIn.Buffer, Arena.Buffer, 1, &cin); - // barrier: transfer-write -> shader-read - Vk.VkMemoryBarrier mb = new() - { - SType = VkConst.StMemoryBarrier, - SrcAccessMask = VkConst.AccessTransferWrite, DstAccessMask = VkConst.AccessShaderRead, - }; - Vk.vkCmdPipelineBarrier(_cmd, VkConst.PipelineStageTransfer, VkConst.PipelineStageComputeShader, 0, - 1, &mb, 0, null, 0, null); - RecordCompute(_cmd); - // barrier: shader-write -> transfer-read - Vk.VkMemoryBarrier mb2 = new() - { - SType = VkConst.StMemoryBarrier, - SrcAccessMask = VkConst.AccessShaderWrite, DstAccessMask = VkConst.AccessTransferRead, - }; - Vk.vkCmdPipelineBarrier(_cmd, VkConst.PipelineStageComputeShader, VkConst.PipelineStageTransfer, 0, - 1, &mb2, 0, null, 0, null); - Vk.VkBufferCopy cout = new() { Size = OutputBytes, SrcOffset = _outOff }; - Vk.vkCmdCopyBuffer(_cmd, Arena.Buffer, _stageOut.Buffer, 1, &cout); - Vk.Check(Vk.vkEndCommandBuffer(_cmd), "vkEndCommandBuffer"); - } - - /// Debug/test: record a single dispatch list slice and run it. - internal void DebugRunRange(int dStart, int dCount) - { - IntPtr cmd = _dev.NewCommandBuffer(); - IntPtr fence = _dev.NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - var writes = stackalloc Vk.VkWriteDescriptorSet[8]; - var infos = stackalloc Vk.VkDescriptorBufferInfo[8]; - for (int i = dStart; i < dStart + dCount; i++) - { - var d = _graph.Dispatches[i]; - if (i > dStart) _dev.CmdComputeBarrier(cmd); - Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, d.Pipe.Pipeline); - for (uint b = 0; b < d.Binds.Length; b++) - { - (VkBuffer buf, ulong off, ulong size) = Resolve(d.Binds[b]); - infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = buf.Buffer, Offset = off, Range = size }; - writes[b] = new Vk.VkWriteDescriptorSet - { - SType = VkConst.StWriteDescriptorSet, DstBinding = b, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, - PBufferInfo = infos + b, - }; - } - _dev.CmdPushDescriptors(cmd, d.Pipe.Layout, writes, (uint)d.Binds.Length); - if (d.Pc.Length > 0) - fixed (byte* pc = d.Pc) - Vk.vkCmdPushConstants(cmd, d.Pipe.Layout, VkConst.StageComputeShader, 0, (uint)d.Pc.Length, pc); - Vk.vkCmdDispatch(cmd, d.Gx, d.Gy, d.Gz); - } - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - _dev.Submit(cmd, fence); - _dev.WaitFence(fence); - _dev.DestroyFence(fence); - _dev.FreeCommandBuffer(cmd); - } - - /// - /// Debug/test: time each dispatch by submitting it alone (barrier + dispatch) and - /// wall-timing the fenced submission. Includes ~20-40us launch overhead per dispatch; - /// accurate enough to rank ms-scale kernels. Returns name→best-of-2 ms. - /// - internal Dictionary DebugProfile() - { - int n = _graph.Dispatches.Length; - IntPtr cmd = _dev.NewCommandBuffer(); - IntPtr fence = _dev.NewFence(); - var writes = stackalloc Vk.VkWriteDescriptorSet[8]; - var infos = stackalloc Vk.VkDescriptorBufferInfo[8]; - var best = new double[n]; - Array.Fill(best, double.MaxValue); - void RecordDispatch(VkDispatch d) - { - _dev.CmdComputeBarrier(cmd); - Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, d.Pipe.Pipeline); - for (uint b = 0; b < d.Binds.Length; b++) - { - (VkBuffer buf, ulong off, ulong size) = Resolve(d.Binds[b]); - infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = buf.Buffer, Offset = off, Range = size }; - writes[b] = new Vk.VkWriteDescriptorSet - { - SType = VkConst.StWriteDescriptorSet, DstBinding = b, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, - PBufferInfo = infos + b, - }; - } - _dev.CmdPushDescriptors(cmd, d.Pipe.Layout, writes, (uint)d.Binds.Length); - if (d.Pc.Length > 0) - fixed (byte* pc = d.Pc) - Vk.vkCmdPushConstants(cmd, d.Pipe.Layout, VkConst.StageComputeShader, 0, (uint)d.Pc.Length, pc); - Vk.vkCmdDispatch(cmd, d.Gx, d.Gy, d.Gz); - } - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - // warm pass to keep state identical to production (input already in arena) - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - foreach (var d in _graph.Dispatches) RecordDispatch(d); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - _dev.Submit(cmd, fence); - _dev.WaitFence(fence); - // timed passes: one dispatch per submit, min of 2 - var sw = new System.Diagnostics.Stopwatch(); - for (int rep = 0; rep < 2; rep++) - for (int i = 0; i < n; i++) - { - var d = _graph.Dispatches[i]; - Vk.vkResetCommandBuffer(cmd, 0); - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - RecordDispatch(d); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - sw.Restart(); - _dev.Submit(cmd, fence); - _dev.WaitFence(fence); - double ms = sw.Elapsed.TotalMilliseconds; - if (ms < best[i]) best[i] = ms; - } - _dev.DestroyFence(fence); - _dev.FreeCommandBuffer(cmd); - var agg = new Dictionary(); - PerDispatch = new (string, double)[n]; - for (int i = 0; i < n; i++) - { - string name = _graph.Dispatches[i].Name ?? $"d{i}"; - PerDispatch[i] = (name, best[i]); - name = name[..(name.IndexOf('(') is int cut && cut > 0 ? cut : name.Length)]; - agg[name] = agg.TryGetValue(name, out double v) ? v + best[i] : best[i]; - } - return agg; - } - - /// Per-dispatch timings from the last call. - internal (string Name, double Ms)[]? PerDispatch; - - /// Debug: the compiled plan backing this session. - internal ExecPlan DebugPlan => _plan; - - /// Debug: download the whole arena (ArenaFloats floats) into dst. - internal void DebugArena(Span dst) - { - DownloadElems(Arena, 0, dst.Slice(0, _plan.ArenaFloats)); - } - - /// Debug: graph consts buffer + the session's arena buffer. - internal (VkBuffer consts, VkBuffer arena) DebugBuffers => (_graph.Consts, Arena); - - /// Debug/test: download an activation value's region into dst. - internal void DebugDownload(int valueId, float[] dst) - { - CValue v = _plan.Values[valueId]; - DownloadElems(Arena, (ulong)_plan.BufOffset[v.Buf] * (ulong)_es, dst.AsSpan(0, checked((int)v.ElementCount))); - } - - /// Debug/test: copy host input into the arena's input region. - internal void DebugUploadInput() - { - ToStorage(_inHost, _stageInMap, F16); - _stageIn.Flush(0, InputBytes); - IntPtr cmd = _dev.NewCommandBuffer(); - IntPtr fence = _dev.NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - Vk.VkBufferCopy cin = new() { Size = InputBytes, DstOffset = _inOff }; - Vk.vkCmdCopyBuffer(cmd, _stageIn.Buffer, Arena.Buffer, 1, &cin); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - _dev.Submit(cmd, fence); - _dev.WaitFence(fence); - _dev.DestroyFence(fence); - _dev.FreeCommandBuffer(cmd); - } - - /// Copy host input into staging and run the recorded pipeline. - public void Run() - { - ToStorage(_inHost, _stageInMap, F16); - _stageIn.Flush(0, InputBytes); - _dev.Submit(_cmd, _fence); - _dev.WaitFence(_fence); - _stageOut.Invalidate(0, OutputBytes); - FromStorage(_stageOutMap, _outHost, F16); - } - - /// Releases the session's device buffers, fence and command buffer, and its - /// reference to the shared graph. - public void Dispose() - { - if (_stageInMap != null) { _stageIn.Unmap(); _stageInMap = null; } - if (_stageOutMap != null) { _stageOut.Unmap(); _stageOutMap = null; } - _dev.DestroyFence(_fence); - _dev.FreeCommandBuffer(_cmd); - // descriptor sets live in the device pool; pool teardown frees them - Arena.Free(); _stageIn.Free(); _stageOut.Free(); - _graph.Release(); - } -} - diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkShaders.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkShaders.cs deleted file mode 100644 index 28c5b78..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkShaders.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Reflection; - -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -/// Loads SPIR-V blobs embedded next to their .comp sources. -internal static class VkShaders -{ - private static readonly Assembly s_asm = typeof(VkShaders).Assembly; - private const string Prefix = "Sdcb.SimdGcdrNet.Backends.Vulkan.Shaders."; - - public static byte[] Load(string name) - { - using Stream? s = s_asm.GetManifestResourceStream(Prefix + name + ".spv"); - if (s == null) - throw new InvalidOperationException( - $"Embedded shader '{name}' not found. Available: {string.Join(", ", s_asm.GetManifestResourceNames().Where(n => n.StartsWith(Prefix)))}"); - byte[] b = new byte[s.Length]; - s.ReadExactly(b); - return b; - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat index 017cbf1..914ae88 100644 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat +++ b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat @@ -1,8 +1,6 @@ @echo off -rem Dev-time only: compile .comp GLSL -> .spv next to each source (committed). -rem Not part of `dotnet build` - SPIR-V is embedded as a resource. -rem Every shader is built twice: name.spv (fp32 storage) and name_h.spv -rem (-DF16: 16-bit storage, fp32 math; see Shaders/et.glsl). +rem Dev-time only: compile the GCDR-specific .comp GLSL -> .spv next to each source (committed). +rem Every shader is built twice: name.spv (fp32 storage) and name_h.spv (-DF16, see Shaders/et.glsl). setlocal set "GLSLC=C:\VulkanSDK\1.4.357.0\Bin\glslc.exe" if not exist "%GLSLC%" for %%i in (glslc.exe) do set "GLSLC=%%~$PATH:i" @@ -13,53 +11,9 @@ if not exist "%GLSLC%" ( set "DIR=%~dp0..\Shaders" set "O=-O --target-env=vulkan1.1" for %%f in ("%DIR%\*.comp") do ( - if not "%%~nxf"=="conv_tile.comp" if not "%%~nxf"=="conv_cm.comp" if not "%%~nxf"=="sgprobe.comp" if not "%%~nxf"=="conv_dot.comp" if not "%%~nxf"=="gemm_dot.comp" ( - echo glslc %%~nxf - "%GLSLC%" %O% "%%f" -o "%%~dpnf.spv" || exit /b 1 - "%GLSLC%" %O% -DF16 "%%f" -o "%%~dpnf_h.spv" || exit /b 1 - ) + echo glslc %%~nxf + "%GLSLC%" %O% "%%f" -o "%%~dpnf.spv" || exit /b 1 + "%GLSLC%" %O% -DF16 "%%f" -o "%%~dpnf_h.spv" || exit /b 1 ) -rem conv_tile specialization variants (constant kernel/stride -> fast index math) -call :conv "" "" || exit /b 1 -call :conv -DF16 _h || exit /b 1 -rem tensor-core conv (fp16 storage only): BN=128 / 64 / 32 output-channel tiles -set "CM=%DIR%\conv_cm.comp" -"%GLSLC%" %O% "%CM%" -o "%DIR%\conv_cm_h.spv" || exit /b 1 -"%GLSLC%" %O% -DSGX=2u -DSGY=2u -DTSM=4u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_n64_h.spv" || exit /b 1 -"%GLSLC%" %O% -DSGX=1u -DSGY=4u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_n32_h.spv" || exit /b 1 -rem any-cin stems (k elements gathered one by one) -"%GLSLC%" %O% -DSCALARK -DSGX=1u -DSGY=4u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_n32s_h.spv" || exit /b 1 -rem LITE: the same four for wave64-capable / 32 KB shared-memory parts (subgroup pinned to 32); -rem more, narrower (64x16) subgroup strips per tile measured fastest on the Radeon 880M -"%GLSLC%" %O% -DLITE -DSGX=8u -DSGY=2u -DTSM=4u -DTSN=1u -DNTHR=512 "%CM%" -o "%DIR%\conv_cm_l_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSGX=4u -DSGY=2u -DTSM=4u -DTSN=1u -DNTHR=256 "%CM%" -o "%DIR%\conv_cm_l_n64_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSGX=2u -DSGY=2u -DTSM=4u -DTSN=1u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_l_n32_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSCALARK -DSGX=1u -DSGY=4u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_l_n32s_h.spv" || exit /b 1 -rem SG16: 8x16x16 fragments on 16-lane subgroups (Intel Xe2 XMX), LITE staging, BM = 128; -rem a 16x32 strip per subgroup, eight stacked along M, measured fastest on the Arc B580 -"%GLSLC%" %O% -DLITE -DSG16 -DSGX=4u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=512 "%CM%" -o "%DIR%\conv_cm_sg16_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSG16 -DSGX=2u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=256 "%CM%" -o "%DIR%\conv_cm_sg16_n64_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSG16 -DSGX=1u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_sg16_n32_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSG16 -DSCALARK -DSGX=1u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_sg16_n32s_h.spv" || exit /b 1 -rem wide-wave no-coopmat GPUs (Adreno): register-tiled direct kernels, fp16 storage only -set "CD=%DIR%\conv_dot.comp" -"%GLSLC%" %O% -DSW=1u -DTM=4 -DTN=2 "%CD%" -o "%DIR%\conv_dot_s1_4x2_h.spv" || exit /b 1 -"%GLSLC%" %O% -DSW=1u -DTM=4 -DTN=1 "%CD%" -o "%DIR%\conv_dot_s1_4x1_h.spv" || exit /b 1 -"%GLSLC%" %O% -DSW=2u -DTM=2 -DTN=2 "%CD%" -o "%DIR%\conv_dot_s2_2x2_h.spv" || exit /b 1 -"%GLSLC%" %O% -DSW=2u -DTM=2 -DTN=1 "%CD%" -o "%DIR%\conv_dot_s2_2x1_h.spv" || exit /b 1 -"%GLSLC%" %O% "%DIR%\gemm_dot.comp" -o "%DIR%\gemm_dot_h.spv" || exit /b 1 -"%GLSLC%" %O% -DF16 -DPIXELS_INNER "%DIR%\convpool.comp" -o "%DIR%\convpool_pi_h.spv" || exit /b 1 "%GLSLC%" %O% -DF16 -DNP=2u "%DIR%\uphead_sc.comp" -o "%DIR%\uphead_sc2_h.spv" || exit /b 1 -rem subgroup-size probe for pinned (requiredSubgroupSize = 32 / 16) pipelines -"%GLSLC%" %O% "%DIR%\sgprobe.comp" -o "%DIR%\sgprobe.spv" || exit /b 1 -exit /b 0 - -:conv -set "CT=%DIR%\conv_tile.comp" -set "F=%~1" -set "S=%~2" -"%GLSLC%" %O% %F% -DKH=3u -DKW=3u -DSH=1u -DSW=1u "%CT%" -o "%DIR%\conv_k33s1%S%.spv" || exit /b 1 -"%GLSLC%" %O% %F% -DKH=3u -DKW=3u -DSH=2u -DSW=2u "%CT%" -o "%DIR%\conv_k33s2%S%.spv" || exit /b 1 -"%GLSLC%" %O% %F% -DKH=1u -DKW=1u -DSH=1u -DSW=1u "%CT%" -o "%DIR%\conv_k11s1%S%.spv" || exit /b 1 -"%GLSLC%" %O% %F% "%CT%" -o "%DIR%\conv_gen%S%.spv" || exit /b 1 exit /b 0 diff --git a/src/Sdcb.SimdGcdrNet/Engine/CompiledModel.cs b/src/Sdcb.SimdGcdrNet/Engine/CompiledModel.cs deleted file mode 100644 index 59f24c9..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/CompiledModel.cs +++ /dev/null @@ -1,30 +0,0 @@ -using Sdcb.SimdGcdrNet.Onnx; - -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// An specialized to a concrete input shape: all -/// dynamic-shape plumbing folded, ops lowered to the exec set, buffers -/// planned. Immutable; share across sessions freely. -/// -public sealed class CompiledModel -{ - internal ExecPlan Plan { get; } - - private CompiledModel(ExecPlan plan) => Plan = plan; - - /// Compiles for a fixed input shape. - /// The ONNX model to compile. - /// Concrete input shape, e.g. [1, 3, H, W]. - public static CompiledModel Compile(OnnxModel model, ReadOnlySpan inputShape) => - new(GraphCompiler.Compile(model, inputShape)); - - /// Creates a new CPU with its own arena. - public InferenceSession CreateSession() => new(Plan); - - /// Number of executable nodes after folding and fusion. - public int NodeCount => Plan.Nodes.Length; - - /// Size of the activation arena, in floats, a session needs. - public int ArenaFloats => Plan.ArenaFloats; -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/CpuKernels.cs b/src/Sdcb.SimdGcdrNet/Engine/CpuKernels.cs deleted file mode 100644 index 7a6e336..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/CpuKernels.cs +++ /dev/null @@ -1,76 +0,0 @@ -using Sdcb.SimdGcdrNet.Kernels; - -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// CPU-private packed weights (conv panels + Winograd transforms, Gemm panels, -/// UpHead lanes) stored in the plan's op params. Per-size plans of one model -/// carry identical weights, so a shared across sizes packs -/// each layer once: entries are keyed by node identity and only reused when the -/// weights (and bias) are element-for-element equal. -/// -internal static class CpuKernels -{ - internal sealed class Cache - { - private readonly Dictionary<(ExecOp op, string name, int src), (float[] w, float[]? b, object kp)> _d = new(); - - internal object GetOrPack(ExecNode n, float[] w, float[]? b, Func pack) - { - var key = (n.Op, n.Name, n.SrcNode); - lock (_d) - if (_d.TryGetValue(key, out var e) && Same(e.w, w) && Same(e.b, b)) return e.kp; - object kp = pack(); - lock (_d) _d[key] = (w, b, kp); - return kp; - } - - private static bool Same(float[]? a, float[]? b) => - ReferenceEquals(a, b) || (a is not null && b is not null && a.AsSpan().SequenceEqual(b)); - } - - /// Fills missing kernel plans of an NHWC plan (idempotent; plans may be shared). - internal static void Pack(ExecPlan plan, Cache? cache) - { - if (!plan.Nhwc) return; - CValue[] V = plan.Values; - foreach (ExecNode n in plan.Nodes) - switch (n.P) - { - case GemmP gp when gp.KernelPlan is null && V[n.In[1]].Floats is { } bw: - { - int[] bs = V[n.In[1]].Shape; - int kk = gp.TransB ? bs[1] : bs[0], nn = gp.TransB ? bs[0] : bs[1]; - float[]? gb = gp.BiasValue >= 0 ? V[gp.BiasValue].Floats : null; - object kp = Get(cache, n, bw, gb, () => Nhwc.PackGemm(bw, kk, nn, gp.TransB, gb)); - Interlocked.CompareExchange(ref gp.KernelPlan, kp, null); - break; - } - case ConvP cp when cp.KernelPlan is null: - { - CValue w = V[n.In[1]]; - float[]? b = cp.BiasValue >= 0 ? V[cp.BiasValue].Floats : null; - int ic = V[n.In[0]].Shape[1]; - object kp = Get(cache, n, w.Floats!, b, () => - { - NhwcConvPlan p = Nhwc.Pack(w.Floats!, w.Shape, b, cp.Group, ic); - if (Nhwc.WinogradEligible(cp, w.Shape, ic)) p.WinoU = Nhwc.PackWinograd(w.Floats!, w.Shape[0], ic); - return p; - }); - Interlocked.CompareExchange(ref cp.KernelPlan, kp, null); - break; - } - case UpHeadP up when up.KernelPlan is null: - { - CValue w = V[n.In[2]]; - float[]? b = n.In.Length > 3 ? V[n.In[3]].Floats : null; - object kp = Get(cache, n, w.Floats!, b, () => Nhwc.PackUpHead(w.Floats!, w.Shape[0], w.Shape[1], b)); - Interlocked.CompareExchange(ref up.KernelPlan, kp, null); - break; - } - } - } - - private static object Get(Cache? cache, ExecNode n, float[] w, float[]? b, Func pack) => - cache is null ? pack() : cache.GetOrPack(n, w, b, pack); -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/ExecPlan.cs b/src/Sdcb.SimdGcdrNet/Engine/ExecPlan.cs deleted file mode 100644 index dca1a69..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/ExecPlan.cs +++ /dev/null @@ -1,208 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Engine; - -/// Executable op set — the lowered form the session runs. -internal enum ExecOp -{ - Conv, // NCHW conv, optional bias + fused activation - MaxPool, // NCHW pooling - Elementwise, // Add/Sub/Mul/Div with broadcast (incl. scalar const) - Relu, - Sigmoid, - Erf, // elementwise erf (normally fused into Gelu) - Gelu, // exact-erf GELU, elementwise - LayerNorm, // normalize over last dim - Gemm, // [.., M, K] x [K, N] + bias[N] - ResizeBilinear, // NCHW, half_pixel, per-axis scales - Pad, // constant / reflect on trailing dims - Transpose, // arbitrary permute (physical perm in layout-planned plans) - Slice, // strided subrange copy - Concat, // axis join - ChannelShift, // fused shift-MLP: per channel group, shift along H or W (zero-fill) - Copy, // plain copy (for Squeeze/Unsqueeze/identity leftovers) - UpHead, // fused NHWC tail: act(W·(resize(x) + [relu](skip)) + b), pointwise W with few outputs -} - -internal enum ActivationKind { None, Relu, Sigmoid, Gelu } - -internal enum PadMode { Constant, Reflect } - -internal enum EwOp { Add, Sub, Mul, Div } - -// ---- per-op parameter records (resolved at compile time) ---- - -internal sealed class ConvP -{ - public required int[] Pads; // [top, left, bottom, right] spatial only - public required int[] Strides; // [sh, sw] - public required int[] Dilations; // [dh, dw] - public required int[] Kernel; // [kh, kw] - public int Group = 1; - public ActivationKind Act; - public PadMode PadMode = PadMode.Constant; - public float PadValue; // constant-mode fill - public int BiasValue = -1; // value id of per-channel bias, -1 none - public int AddValue = -1; // value id of same-shape residual addend, -1 none - public object? KernelPlan; // packed weights / kernel choice, filled at plan build - // NHWC plans: fused 2x2/2 max pool epilogue — Out[0] is the pooled tensor - public PoolP? Pool; -} - -internal sealed class PoolP -{ - public required int[] Kernel; - public required int[] Strides; - public required int[] Pads; - public bool FuseRelu; -} - -internal sealed class EwP -{ - public required EwOp Op; - public ActivationKind Act; -} - -internal sealed class LnP -{ - public float Eps; -} - -internal sealed class GemmP -{ - public bool TransB; // B stored [N,K] when true (onnx transB=1) - public int BiasValue = -1; - public ActivationKind Act; - public object? KernelPlan; // backend-private packed B, filled at session build -} - -internal sealed class ResizeP -{ - public required float[] Scales; // per-dim, only [.., sh, sw] used - public bool FuseRelu; // relu on the resized value (before the optional add) - // optional same-shape addend in In[1] (NHWC plans): out = [relu](resize(x)) + [relu](In[1]) - public bool AddRelu; // relu on the addend -} - -/// -/// Fused NHWC network tail (In = [x, skip, w, bias?]): -/// out[o] = act(bias[o] + Σ_c w[o,c]·s[c]) with s = [relu](resize(x)) + [relu](skip). -/// Replaces Resize → (Relu) → Add → 1x1 Conv → (act) so the full-resolution -/// intermediates are never stored. -/// -internal sealed class UpHeadP -{ - public required float[] Scales; - public bool ResizeRelu; - public bool SkipRelu; - public ActivationKind Act; - public object? KernelPlan; // backend-private packed weights - // NHWC plans: In[1] is the input of this (still scheduled) conv and the skip is - // recomputed, so the full-resolution skip tensor need not stay live - public ConvP? SkipConv; - public int SkipW = -1, SkipB = -1; // the skip conv's weight / bias value ids -} - -internal sealed class PadP -{ - public required int[] Pads; // [2*rank] begin..end - public PadMode Mode = PadMode.Constant; - public float Value; -} - -internal sealed class TransposeP -{ - public required int[] Perm; -} - -internal sealed class SliceP -{ - public required int[] Starts; - public required int[] Ends; - public required int[] Axes; - public required int[] Steps; -} - -internal sealed class ConcatP -{ - public int Axis; -} - -internal sealed class ShiftP -{ - public int Axis; // logical spatial axis index (2 = H, 3 = W) - public required int[] GroupEnds; // cumulative channel-group end indices (len 5) - public required int[] Offsets; // per-group shift (len 5) -} - -/// A resolved tensor value: compile-time constant or activation. -internal sealed class CValue -{ - public int[] Shape = []; - public int DType; // onnx dtype: 1 float, 7 int64 - public long[]? Ints; // compile-time int content - public float[]? Floats; // compile-time float content (weights or small consts) - public int Buf = -1; // activation buffer id; >= 0 → runtime tensor - public string Name = ""; - /// Physical dim order: physical dim i holds logical dim Layout[i]. null = identity. - public int[]? Layout; - /// Zero-copy view source (reshape/squeeze/unsqueeze/identity); -1 none. - public int AliasOf = -1; - public bool IsInt => Ints != null; - public bool IsConst => Ints != null || Floats != null; - public bool IsActivation => Buf >= 0; - public long ElementCount => Shape.Aggregate(1L, (a, b) => a * b); - /// Shape in storage order — what kernels address by. - public int[] PhysShape => Layout is null ? Shape : Engine.Layout.Phys(Shape, Layout); -} - -/// A buffer reference into either the activation arena or a const array. -internal readonly unsafe struct FBuf -{ - public readonly float[]? Data; - public readonly float* Ptr; - public readonly int Offset; - public readonly int Length; - public FBuf(float[] data, int offset, int length) { Data = data; Offset = offset; Length = length; } - public FBuf(float* ptr, int length) { Ptr = ptr; Length = length; } - public Span Span => Data != null ? Data.AsSpan(Offset, Length) : new Span(Ptr, Length); -} - -/// One executable step: op + value ids. -internal sealed class ExecNode -{ - public required ExecOp Op; - public required int[] In; // value ids - public required int[] Out; - public object? P; - public int SrcNode = -1; // onnx node index (diagnostics) - public string Name = ""; -} - -/// Compiled graph ready for a session: exec nodes + buffer plan. -internal sealed class ExecPlan -{ - public required ExecNode[] Nodes; - public required CValue[] Values; - public required int[] BufSize; // floats per activation buffer - public required int[] BufOffset; // arena offset per buffer - public required int ArenaFloats; - public required int InputValue; - public required int OutputValue; - /// Rank-4 activations stored channels-last (layout pass ran). - public bool Nhwc; - /// Input/output stay in plan layout (no boundary relayout); else logical NCHW. - public bool PhysicalIo; -} - -/// Plan compilation options. -/// run the layout pass: rank-4 activations stored channels-last. -/// keep input/output in plan layout (no boundary relayout); else logical NCHW. -/// memory fusions (conv+pool, UpHead skip recompute; NHWC only). -/// plan for the Vulkan backend (restricts fusions to what its shaders implement). -internal sealed record CompileOptions(bool Nhwc, bool PhysicalIo = false, bool MemFuse = true, bool Gpu = false) -{ - /// GCDRNET_NHWC=0 forces the legacy all-NCHW plan (A/B oracle). - public static bool DefaultNhwc { get; } = Environment.GetEnvironmentVariable("GCDRNET_NHWC") != "0"; - public static CompileOptions Default => new(DefaultNhwc); - /// GCDRNET_NOFUSE_MEM=1 disables the CPU memory fusions (conv+pool, skip recompute). - public static bool MemFusions { get; } = Environment.GetEnvironmentVariable("GCDRNET_NOFUSE_MEM") != "1"; -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/GraphCompiler.cs b/src/Sdcb.SimdGcdrNet/Engine/GraphCompiler.cs deleted file mode 100644 index 662de6a..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/GraphCompiler.cs +++ /dev/null @@ -1,1529 +0,0 @@ -using Sdcb.SimdGcdrNet.Onnx; - -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// Compiles an into an for a -/// concrete input shape. All ONNX dynamic-shape plumbing (Shape/Squeeze/Div/ -/// Mod/Concat/Slice on int64 scalars) is evaluated at compile time; remaining -/// float ops are emitted with fully resolved shapes. -/// -internal static class GraphCompiler -{ - public static ExecPlan Compile(OnnxModel model, ReadOnlySpan inputShape) => - Compile(model, inputShape, CompileOptions.Default); - - public static ExecPlan Compile(OnnxModel model, ReadOnlySpan inputShape, CompileOptions opt) - { - var c = new Compiler(model); - return c.Run(inputShape, opt); - } - - /// Vulkan small-conv shaders (uphead skip recompute, convpool): dense stride-1 - /// conv, ic < 8, oc a multiple of 8 (<= 128), weights fit the 16 KB staging. - internal static bool GpuSmallConv(ConvP cp, int[] wShape, int ic) => - cp.Group == 1 && ic < 8 && wShape.Length == 4 && wShape[0] % 8 == 0 && wShape[0] <= 128 - && (long)wShape[0] * ic * wShape[2] * wShape[3] <= 4096 - && cp.Strides.All(s => s == 1) && cp.Dilations.All(d => d == 1) - && cp.Act is ActivationKind.None or ActivationKind.Relu; - - private sealed class Compiler - { - private readonly OnnxModel _m; - private readonly List _values = []; - private readonly Dictionary _byName = new(StringComparer.Ordinal); - private readonly List _nodes = []; - private readonly List _bufSize = []; - private int _inputValue = -1; - - public Compiler(OnnxModel m) => _m = m; - - private int AddValue(CValue v) - { - int id = _values.Count; - _values.Add(v); - if (v.Name.Length != 0) _byName[v.Name] = id; - return id; - } - - private int NewBuf(int size) - { - int id = _bufSize.Count; - _bufSize.Add(size); - return id; - } - - private CValue V(int id) => _values[id]; - - private int Input(int i, OnnxNodeData n) - { - string name = n.Inputs[i]; - if (name.Length == 0) return -1; - if (!_byName.TryGetValue(name, out int id)) - throw new InvalidDataException($"Node '{n.Name}' ({n.OpType}): input '{name}' not produced."); - return id; - } - - private int[] Inputs(OnnxNodeData n, int skip = 0) - { - int[] ids = new int[n.Inputs.Count - skip]; - for (int i = skip; i < n.Inputs.Count; i++) ids[i - skip] = Input(i, n); - return ids; - } - - private int _curNi; // onnx node index currently being emitted (for SrcNode) - - private int Emit(ExecOp op, int[] ins, CValue output, object? p, OnnxNodeData n) - { - int outId = AddValue(output); - _nodes.Add(new ExecNode { Op = op, In = ins, Out = [outId], P = p, SrcNode = _curNi, Name = n.Name }); - return outId; - } - - public ExecPlan Run(ReadOnlySpan inputShape, CompileOptions opt) - { - OnnxGraphData g = _m.Graph; - foreach (OnnxTensorData t in g.Initializers) - { - int[] dims = t.DimsArray(); - if (t.DataType == OnnxTensorData.TypeFloat) - AddValue(new CValue { Name = t.Name, Shape = dims, DType = 1, Floats = t.AsFloats() }); - else - AddValue(new CValue { Name = t.Name, Shape = dims, DType = t.DataType, Ints = t.AsInt64s() }); - } - if (g.Inputs.Count != 1) throw new NotSupportedException("Expected single graph input."); - { - string inName = g.Inputs[0].Name; - int elems = 1; - int[] shape = inputShape.ToArray(); - foreach (int d in shape) elems *= d; - int buf = NewBuf(elems); - _inputValue = AddValue(new CValue { Name = inName, Shape = shape, DType = 1, Buf = buf }); - } - - for (int ni = 0; ni < g.Nodes.Count; ni++) - { - OnnxNodeData n = g.Nodes[ni]; - _curNi = ni; - EmitNode(n); - } - - // graph outputs - if (g.Outputs.Count != 1) throw new NotSupportedException("Expected single graph output."); - int outputValue = _byName[g.Outputs[0].Name]; - - return FuseAndPlan(_nodes, _values, _bufSize, _inputValue, outputValue, opt); - } - - // ---------- fusion + buffer planning ---------- - private static ExecPlan FuseAndPlan(List nodes, List values, - List bufSize, int inputValue, int outputValue, CompileOptions opt) - { - // fusion passes operate on the logical graph and are layout-blind; - // physical layout is decided afterwards by LayoutPlanner - if (Environment.GetEnvironmentVariable("GCDRNET_NOFUSE") != "1") - { - Fuse(nodes, values); - // NHWC-only kernels (CPU channels-last + Vulkan); the NCHW oracle plan keeps the plain ops - if (opt.Nhwc && Environment.GetEnvironmentVariable("GCDRNET_NOFUSE_UP") != "1") - FuseUpsample(nodes, values, outputValue); - } - if (opt.Nhwc) - LayoutPlanner.Run(nodes, values, bufSize, inputValue, ref outputValue, opt.PhysicalIo); - if (opt.Nhwc && opt.MemFuse && CompileOptions.MemFusions - && Environment.GetEnvironmentVariable("GCDRNET_NOFUSE") != "1") - FuseMemory(nodes, values, outputValue, opt.Gpu); - - PlanArena(nodes, values, bufSize, inputValue, outputValue, - out int[] offsets, out int arenaFloats); - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "2") - { - string Vs(int v) => v < 0 ? "-" : $"v{v}/b{values[v].Buf}[{string.Join('x', values[v].Shape)}]{(values[v].Layout is { } l && !Layout.IsIdentity(l) ? Layout.Str(l) : "")}"; - for (int i = 0; i < nodes.Count; i++) - { - ExecNode m = nodes[i]; - string extra = m.P switch - { - TransposeP t => $" perm=[{string.Join(',', t.Perm)}]", - ConvP c => $" k={c.Kernel[0]}x{c.Kernel[1]} s={c.Strides[0]} g={c.Group} add={Vs(c.AddValue)} act={c.Act}{(c.Pool is { } pl ? $" +maxpool{(pl.FuseRelu ? "+relu" : "")}" : "")}", - UpHeadP u => $" rrelu={u.ResizeRelu} srelu={u.SkipRelu} act={u.Act}" + (u.SkipConv is { } sc ? $" skip=conv{sc.Kernel[0]}x{sc.Kernel[1]}(In[1]) sact={sc.Act}" : ""), - GemmP g => $" act={g.Act}", - ConcatP c => $" axis={c.Axis}", - ShiftP s => $" axis={s.Axis}", - EwP e => $" {e.Op} act={e.Act}", - _ => "", - }; - Console.Error.WriteLine($" #{i,-3} {m.Op,-14} {string.Join(' ', m.In.Select(Vs))} -> {string.Join(' ', m.Out.Select(Vs))}{extra}"); - } - } - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - { - var hist = nodes.GroupBy(m => m.Op).OrderByDescending(g => g.Count()); - Console.Error.WriteLine($"[plan] {nodes.Count} nodes, arena {arenaFloats * 4.0 / 1e6:F1} MB"); - foreach (var h in hist) Console.Error.WriteLine($" {h.Key,-14} x{h.Count()}"); - } - return new ExecPlan - { - Nodes = nodes.ToArray(), - Values = values.ToArray(), - BufSize = bufSize.ToArray(), - BufOffset = offsets, - ArenaFloats = arenaFloats, - InputValue = inputValue, - OutputValue = outputValue, - Nhwc = opt.Nhwc, - PhysicalIo = opt.Nhwc && opt.PhysicalIo, - }; - } - - /// Structural fusions over the exec node list (single consumer chains). - private static void Fuse(List nodes, List values) - { - // pass 1: Pad -> Conv (fold reflect/constant padding into the conv) - // pass 2: Conv -> Relu (activation epilogue) - // pass 3: Div(x,√2) Erf Add(x,1) Mul(x,0.5) Mul(x0) -> Gelu - // pass 4: Pad(+2) -> 5x Slice -> 10x Slice + 5x Concat -> Concat -> Slice - // (shift-MLP) -> ChannelShift - for (int i = 0; i < nodes.Count; i++) - { - ExecNode n = nodes[i]; - if (n.Op == ExecOp.Conv && n.P is ConvP cp && cp.Pads.All(p => p == 0)) - { - int src = n.In[0]; - ExecNode? pad = nodes.FirstOrDefault(m => m.Out.Length == 1 && m.Out[0] == src && m.Op == ExecOp.Pad); - if (pad != null && SoleUse(nodes, src, i) && pad.P is PadP pp && - PadsOnlySpatial(pp.Pads, values[src].Shape.Length)) - { - cp.Pads = [pp.Pads[2], pp.Pads[3], pp.Pads[2 + values[src].Shape.Length], pp.Pads[3 + values[src].Shape.Length]]; - cp.PadMode = pp.Mode; - cp.PadValue = pp.Value; - n.In[0] = pad.In[0]; - nodes.RemoveAt(nodes.IndexOf(pad)); - i--; - continue; - } - } - } - // Relu/Sigmoid/elementwise-tail fusions - for (int i = 0; i + 1 < nodes.Count; i++) - { - ExecNode a = nodes[i], b = nodes[i + 1]; - if (a.Op == ExecOp.Conv && b.Op == ExecOp.Relu && b.In[0] == a.Out[0] && - SoleUse(nodes, a.Out[0], i + 1)) - { - ((ConvP)a.P!).Act = ActivationKind.Relu; - // rewrite consumers of b.Out to use a.Out - Redirect(nodes, values, b.Out[0], a.Out[0]); - nodes.RemoveAt(i + 1); - i--; - } - else if (a.Op == ExecOp.MaxPool && b.Op == ExecOp.Relu && b.In[0] == a.Out[0] && - SoleUse(nodes, a.Out[0], i + 1)) - { - ((PoolP)a.P!).FuseRelu = true; - Redirect(nodes, values, b.Out[0], a.Out[0]); - nodes.RemoveAt(i + 1); - i--; - } - else if (a.Op == ExecOp.ResizeBilinear && b.Op == ExecOp.Relu && b.In[0] == a.Out[0] && - SoleUse(nodes, a.Out[0], i + 1)) - { - ((ResizeP)a.P!).FuseRelu = true; - Redirect(nodes, values, b.Out[0], a.Out[0]); - nodes.RemoveAt(i + 1); - i--; - } - else if (a.Op == ExecOp.Elementwise && b.Op == ExecOp.Relu && b.In[0] == a.Out[0] && - SoleUse(nodes, a.Out[0], i + 1)) - { - ((EwP)a.P!).Act = ActivationKind.Relu; - Redirect(nodes, values, b.Out[0], a.Out[0]); - nodes.RemoveAt(i + 1); - i--; - } - } - // Conv -> Add residual (skip connections): fold same-shape addend into conv epilogue - for (int i = 0; i + 1 < nodes.Count; i++) - { - ExecNode a = nodes[i], b = nodes[i + 1]; - if (a.Op != ExecOp.Conv || b.Op != ExecOp.Elementwise || - ((EwP)b.P!).Op != EwOp.Add || - ((EwP)b.P!).Act is not (ActivationKind.None or ActivationKind.Relu)) - continue; - var cp = (ConvP)a.P!; - if (cp.Act != ActivationKind.None || cp.Group > 1) continue; // depthwise shader has no residual path - int outV = a.Out[0]; - int other = b.In[0] == outV ? b.In[1] : b.In[1] == outV ? b.In[0] : -1; - if (other < 0 || !SoleUse(nodes, outV, i + 1)) continue; - if (!values[other].Shape.SequenceEqual(values[outV].Shape)) continue; - cp.AddValue = other; - cp.Act = ((EwP)b.P!).Act; // epilogue: +residual then activation (matches conv shader) - Redirect(nodes, values, b.Out[0], outV); - nodes.RemoveAt(i + 1); - i--; - } - FuseGelu(nodes, values); - FuseShift(nodes, values); - CommuteActBeforeTranspose(nodes, values); - FuseLateAct(nodes, values); - } - - // act(T(x)) == T(act(x)) for elementwise unary act: move a sole-consumer - // act node in front of its transpose producer so producer fusions (e.g. - // Gemm->Gelu) can see it. - private static void CommuteActBeforeTranspose(List nodes, List values) - { - for (int g = 0; g < nodes.Count; g++) - { - ExecNode act = nodes[g]; - if (act.Op is not (ExecOp.Relu or ExecOp.Gelu or ExecOp.Sigmoid or ExecOp.Erf) - || act.In.Length != 1) continue; - int t = nodes.FindIndex(m => m.Out.Length == 1 && m.Out[0] == act.In[0]); - if (t < 0 || t >= g || nodes[t].Op != ExecOp.Transpose) continue; - ExecNode tr = nodes[t]; - if (tr.In.Length != 1 || !SoleUse(nodes, tr.Out[0], g)) continue; - // rewire as x --act--> y --T--> z: swap the two Out ids so - // downstream consumers of z read the transposed value as before. - // y now holds act(x) in x's layout — fix its registered shape. - int y = tr.Out[0], z = act.Out[0]; - values[y].Shape = values[tr.In[0]].Shape; - act.In[0] = tr.In[0]; act.Out[0] = y; - tr.In[0] = y; tr.Out[0] = z; - nodes.RemoveAt(g); - nodes.Insert(t, act); - } - } - - // Sole-consumer activation fusion, buffer-aware (consumers reached through - // reshape/squeeze Buf-aliases count as uses of the producer's buffer) and - // adjacency-independent. Retargets the act's output buffer onto the - // producer's so downstream aliases keep their shape metadata. - private static void FuseLateAct(List nodes, List values) - { - for (int i = 0; i < nodes.Count; i++) - { - ExecNode a = nodes[i]; - bool can = a.P switch - { - ConvP c => c.Act == ActivationKind.None, - PoolP p => !p.FuseRelu, - ResizeP r => !r.FuseRelu, - EwP e => e.Act == ActivationKind.None, - GemmP g => g.Act == ActivationKind.None, - _ => false, - }; - if (!can || a.Out.Length != 1) continue; - int ob = values[a.Out[0]].Buf; - if (ob < 0) continue; - - int u = -1, uses = 0; - for (int j = 0; j < nodes.Count && uses <= 1; j++) - if (j != i && nodes[j].In.Any(iv => values[iv].Buf == ob)) { u = j; uses++; } - if (uses != 1) continue; - ExecNode b = nodes[u]; - if (b.In.Length != 1 || values[b.In[0]].Buf != ob) continue; - ActivationKind got = b.Op switch - { - ExecOp.Relu => ActivationKind.Relu, - ExecOp.Gelu => ActivationKind.Gelu, - ExecOp.Sigmoid => ActivationKind.Sigmoid, - _ => ActivationKind.None, - }; - bool ok = a.P switch - { - ConvP => got is ActivationKind.Relu or ActivationKind.Gelu, - PoolP or ResizeP or EwP => got == ActivationKind.Relu, - GemmP => got is ActivationKind.Relu or ActivationKind.Gelu, - _ => false, - }; - if (!ok) continue; - - switch (a.P) - { - case ConvP c: c.Act = got; break; - case PoolP p: p.FuseRelu = true; break; - case ResizeP r: r.FuseRelu = true; break; - case EwP e: e.Act = ActivationKind.Relu; break; - case GemmP g: g.Act = got; break; - } - int ab = values[b.Out[0]].Buf; - if (ab >= 0 && ab != ob) - { - foreach (CValue v in values) - if (v.Buf == ab) v.Buf = ob; - // b.Out is now an unproduced view of a.Out: record it so the - // layout pass can follow the alias chain - values[b.Out[0]].AliasOf = a.Out[0]; - } - else - { - Redirect(nodes, values, b.Out[0], a.Out[0]); - } - nodes.RemoveAt(u); - if (u < i) i--; - } - } - - /// - /// NHWC memory fusions (after layout planning) that keep full-resolution - /// tensors from ever being stored: - /// - /// UpHead skip recompute: when the skip is a cheap direct conv of a smaller - /// input (stem conv: 3/6 → 16/32 channels), UpHead reads the conv's input and - /// recomputes the skip rows, so the skip need not stay live across the network. - /// Conv → MaxPool(2x2/2): the conv output's only reader is the pool, so the - /// conv emits pooled rows directly (). - /// - /// Vulkan plans () only get what its shaders implement: small-ic - /// (stem) convs with staged weights, both for the recompute and for conv+pool. - /// - private static void FuseMemory(List nodes, List values, int outputValue, bool gpu) - { - int Producer(int v) => nodes.FindIndex(m => m.Out.Length == 1 && m.Out[0] == v); - int Uses(int v) - { - int b = values[v].Buf; - if (b < 0 || v == outputValue || values[outputValue].Buf == b) return int.MaxValue; - int n = 0; - foreach (ExecNode m in nodes) - { - foreach (int iv in m.In) if (values[iv].Buf == b) n++; - int a1 = m.P switch { ConvP c => c.AddValue, _ => -1 }; - int a2 = m.P switch { ConvP c => c.BiasValue, GemmP g => g.BiasValue, _ => -1 }; - if (a1 >= 0 && values[a1].Buf == b) n++; - if (a2 >= 0 && values[a2].Buf == b) n++; - } - return n; - } - bool Nhwc4(int v) => values[v].Shape.Length == 4 && values[v].Layout is [0, 2, 3, 1]; - bool SameHw(int a, int b) => values[a].Shape[2] == values[b].Shape[2] && values[a].Shape[3] == values[b].Shape[3]; - - // 1) UpHead skip recompute - foreach (ExecNode u in nodes) - { - if (u.P is not UpHeadP up || up.SkipConv is not null) continue; - int s = u.In[1], ci = Producer(s); - if (ci < 0 || nodes[ci].P is not ConvP cp || cp.Group != 1 || cp.AddValue >= 0 || cp.Pool is not null - || cp.Strides.Any(x => x != 1) || cp.Dilations.Any(x => x != 1)) continue; - int xin = nodes[ci].In[0]; - CValue w = values[nodes[ci].In[1]]; - if (w.Floats is null || w.Shape.Length != 4 || !Nhwc4(s) || !Nhwc4(xin)) continue; - int ic = values[xin].Shape[1], c = values[s].Shape[1]; - // cheap to recompute, much smaller to keep; the conv itself stays scheduled - // for its other reader(s) (and keeps the shared ConvP packed) - if (ic * 2 > c || Kernels.Nhwc.WinogradEligible(cp, w.Shape, ic) || Uses(s) < 2) continue; - if (cp.BiasValue >= 0 && values[cp.BiasValue].Floats is null) continue; - if (gpu && (!GpuSmallConv(cp, w.Shape, ic) || c > 32 || !SameHw(xin, s))) continue; - u.In[1] = xin; - up.SkipConv = cp; - up.SkipW = nodes[ci].In[1]; - up.SkipB = cp.BiasValue; - } - - // 2) Conv -> MaxPool(2x2, stride 2, no pad, exact halving) - for (int pi = 0; pi < nodes.Count; pi++) - { - ExecNode m = nodes[pi]; - if (m.Op != ExecOp.MaxPool || m.P is not PoolP pp || pp.Kernel is not [2, 2] || pp.Strides is not [2, 2] - || pp.Pads.Any(x => x != 0) || m.In.Length != 1) continue; - int v = m.In[0], ci = Producer(v); - if (ci < 0 || ci > pi || nodes[ci].P is not ConvP cp || cp.Group != 1 || cp.AddValue >= 0 || cp.Pool is not null - || Uses(v) != 1 || !Nhwc4(v) || !Nhwc4(m.Out[0])) continue; - int[] cs = values[v].Shape, ps = values[m.Out[0]].Shape; - if (cs[2] != ps[2] * 2 || cs[3] != ps[3] * 2) continue; - if (gpu && (!GpuSmallConv(cp, values[nodes[ci].In[1]].Shape, values[nodes[ci].In[0]].Shape[1]) || !SameHw(nodes[ci].In[0], v))) continue; - cp.Pool = pp; - nodes[ci].Out = [m.Out[0]]; - nodes.RemoveAt(pi); - pi--; - } - } - - /// - /// Upsample fusions (NHWC plans): - /// - /// Resize → (Relu) → Add(·, [Relu](skip)) → 1x1 Conv (few outputs) → (Sigmoid|Relu) - /// becomes one — the network tail, where every - /// intermediate is a full-resolution tensor that would otherwise round-trip memory. - /// Resize → Add(·, [Relu](skip)) becomes a Resize with an addend (In[1]). - /// - /// Fused nodes take the position of the last node they replace, so every input is - /// produced before them. Uses are counted by buffer (aliases and conv/gemm param - /// value ids included), so an intermediate is only dropped when nothing else reads it. - /// - private static void FuseUpsample(List nodes, List values, int outputValue) - { - int Producer(int v) => nodes.FindIndex(m => m.Out.Length == 1 && m.Out[0] == v); - int Uses(int v) - { - int b = values[v].Buf; - if (b < 0 || v == outputValue || values[outputValue].Buf == b) return int.MaxValue; - int n = 0; - foreach (ExecNode m in nodes) - { - foreach (int iv in m.In) if (values[iv].Buf == b) n++; - int a1 = m.P switch { ConvP c => c.AddValue, _ => -1 }; - int a2 = m.P switch { ConvP c => c.BiasValue, GemmP g => g.BiasValue, _ => -1 }; - if (a1 >= 0 && values[a1].Buf == b) n++; - if (a2 >= 0 && values[a2].Buf == b) n++; - } - return n; - } - bool Same(int a, int b) => values[a].Shape.SequenceEqual(values[b].Shape); - - // resolve add(resize(x), other): (resize node, skip value, skip relu node or null) - (ExecNode rs, int skip, ExecNode? relu)? MatchAdd(ExecNode add) - { - if (add.Op != ExecOp.Elementwise || add.In.Length != 2 || add.P is not EwP { Op: EwOp.Add, Act: ActivationKind.None }) - return null; - for (int k = 0; k < 2; k++) - { - int u = add.In[k], o = add.In[1 - k]; - int ri = Producer(u); - if (ri < 0 || nodes[ri].Op != ExecOp.ResizeBilinear || nodes[ri].In.Length != 1) continue; - if (Uses(u) != 1 || !Same(u, add.Out[0]) || !Same(o, add.Out[0]) || values[o].Buf < 0) continue; - int pi = Producer(o); - if (pi >= 0 && nodes[pi].Op == ExecOp.Relu && nodes[pi].In.Length == 1 && Uses(o) == 1 - && values[nodes[pi].In[0]].Buf >= 0 && Same(nodes[pi].In[0], o)) - return (nodes[ri], nodes[pi].In[0], nodes[pi]); - return (nodes[ri], o, null); - } - return null; - } - - // 1) tails: ... -> Add -> Conv1x1 -> (act) - for (int ci = 0; ci < nodes.Count; ci++) - { - ExecNode c = nodes[ci]; - if (c.Op != ExecOp.Conv || c.P is not ConvP cp || cp.Group != 1 || cp.Act != ActivationKind.None - || cp.AddValue >= 0 || cp.Kernel[0] != 1 || cp.Kernel[1] != 1 || cp.Strides.Any(s => s != 1) - || cp.Pads.Any(p => p != 0) || cp.Dilations.Any(d => d != 1)) continue; - CValue w = values[c.In[1]]; - if (w.Floats is null || w.Shape.Length != 4 || w.Shape[0] > 8 || w.Shape[1] > 128 || w.Shape[1] % 4 != 0) continue; // kernel limits (oc lanes, staged weights, vec4 channels) - if (cp.BiasValue >= 0 && values[cp.BiasValue].Floats is null) continue; - int s = c.In[0]; - int ai = Producer(s); - if (ai < 0 || Uses(s) != 1 || values[s].Shape.Length != 4) continue; - ExecNode addNode = nodes[ai]; - if (MatchAdd(addNode) is not { } mt) continue; - var (rs, skip, relu) = mt; - - ExecNode? act = null; - int y = c.Out[0]; - if (Uses(y) == 1) - { - act = nodes.FirstOrDefault(m => m.In.Length == 1 && m.In[0] == y - && m.Op is ExecOp.Sigmoid or ExecOp.Relu && values[m.Out[0]].Buf >= 0); - } - var head = new ExecNode - { - Op = ExecOp.UpHead, - In = cp.BiasValue >= 0 ? [rs.In[0], skip, c.In[1], cp.BiasValue] : [rs.In[0], skip, c.In[1]], - Out = [act?.Out[0] ?? y], - P = new UpHeadP - { - Scales = ((ResizeP)rs.P!).Scales, - ResizeRelu = ((ResizeP)rs.P!).FuseRelu, - SkipRelu = relu is not null, - Act = act is null ? ActivationKind.None : act.Op == ExecOp.Sigmoid ? ActivationKind.Sigmoid : ActivationKind.Relu, - }, - SrcNode = c.SrcNode, - Name = c.Name + ".uphead", - }; - nodes[ci] = head; - nodes.Remove(rs); nodes.Remove(addNode); - if (relu is not null) nodes.Remove(relu); - if (act is not null) nodes.Remove(act); - ci = -1; // indices shifted; rescan (few nodes, cheap) - } - - // 2) Resize -> Add - for (int ai = 0; ai < nodes.Count; ai++) - { - ExecNode add = nodes[ai]; - if (MatchAdd(add) is not { } mt) continue; - var (rs, skip, relu) = mt; - int ri = nodes.IndexOf(rs); - var rp = (ResizeP)rs.P!; - rs.In = [rs.In[0], skip]; - rp.AddRelu = relu is not null; - rs.Out = [add.Out[0]]; - nodes[ai] = rs; // producer precedes consumer: ri < ai - nodes.RemoveAt(ri); - if (relu is not null) nodes.Remove(relu); - ai = -1; - } - } - - private static void Redirect(List nodes, List values, int oldValue, int newValue) - { - foreach (ExecNode m in nodes) - for (int i = 0; i < m.In.Length; i++) - if (m.In[i] == oldValue) m.In[i] = newValue; - if (values[oldValue].Name.Length != 0 && values[newValue].Name.Length == 0) - values[newValue].Name = values[oldValue].Name; - } - - /// True when value `v`'s only consumer is node index `useIdx`, once. - private static bool SoleUse(List nodes, int v, int useIdx) - { - int count = 0; - for (int i = 0; i < nodes.Count; i++) - foreach (int inp in nodes[i].In) - if (inp == v) - { - if (i != useIdx) return false; - if (++count > 1) return false; - } - return count == 1; - } - - private static bool PadsOnlySpatial(int[] pads, int rank) - { - if (pads.Length != 2 * rank) return false; - for (int i = 0; i < 2; i++) if (pads[i] != 0 || pads[i + rank] != 0) return false; - return true; - } - - private static void FuseGelu(List nodes, List values) - { - // Div(X, sqrt2) -> Erf -> Add(+1) -> Mul(*0.5) -> Mul(*X) - for (int i = 0; i + 4 < nodes.Count; i++) - { - ExecNode d = nodes[i], erf = nodes[i + 1], a1 = nodes[i + 2], mh = nodes[i + 3], mx = nodes[i + 4]; - if (d.Op != ExecOp.Elementwise || ((EwP)d.P!).Op != EwOp.Div || - erf.Op != ExecOp.Erf || - a1.Op != ExecOp.Elementwise || ((EwP)a1.P!).Op != EwOp.Add || - mh.Op != ExecOp.Elementwise || ((EwP)mh.P!).Op != EwOp.Mul || - mx.Op != ExecOp.Elementwise || ((EwP)mx.P!).Op != EwOp.Mul) - continue; - if (erf.In[0] != d.Out[0] || a1.In[0] != erf.Out[0] && a1.In[1] != erf.Out[0] || - (mh.In[0] != a1.Out[0] && mh.In[1] != a1.Out[0]) || - (mx.In[0] != mh.Out[0] && mx.In[1] != mh.Out[0])) - continue; - if (!IsScalarConst(values, d.In[1], MathF.Sqrt(2f))) continue; - int a1Other = a1.In[0] == erf.Out[0] ? a1.In[1] : a1.In[0]; - int mhOther = mh.In[0] == a1.Out[0] ? mh.In[1] : mh.In[0]; - int mxOther = mx.In[0] == mh.Out[0] ? mx.In[1] : mx.In[0]; - if (!IsScalarConst(values, a1Other, 1f) || !IsScalarConst(values, mhOther, 0.5f)) continue; - if (mxOther != d.In[0]) continue; - // emit single Gelu - values[mx.Out[0]].Shape = values[d.In[0]].Shape; - nodes[i] = new ExecNode { Op = ExecOp.Gelu, In = [d.In[0]], Out = mx.Out, P = null, SrcNode = d.SrcNode, Name = d.Name }; - nodes.RemoveRange(i + 1, 4); - } - } - - private static bool IsScalarConst(List values, int v, float expected) - { - CValue cv = values[v]; - return cv.Floats is { Length: 1 } f && MathF.Abs(f[0] - expected) < 1e-6f; - } - - /// - /// Recognizes the shift-MLP subgraph: Pad(const ±p on H or W) -> channel - /// Split (5 groups via Slice) -> per-group roll (Slice+Slice+Concat on the - /// spatial axis) -> Concat(axis=1) -> Slice(unpad). Replaced by a single - /// ChannelShift node writing the shifted [1,C,h,w] tensor. - /// - private static void FuseShift(List nodes, List values) - { - for (int i = 0; i < nodes.Count; i++) - { - ExecNode pad = nodes[i]; - if (pad.Op != ExecOp.Pad || pad.P is not PadP pp || pp.Mode != PadMode.Constant) - continue; - int inId = pad.In[0], padOut = pad.Out[0]; - int rank = values[padOut].Shape.Length; - if (rank != 4) continue; - // spatial pad amount (must be uniform ±p on exactly one spatial axis pair? ours: both H,W ±2) - int padH = pp.Pads[2], padW = pp.Pads[3]; - if (pp.Pads[2 + rank] != padH || pp.Pads[3 + rank] != padW) continue; - - // find all consumer slices of padOut (channel split pieces) - var pieces = nodes.Where(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == padOut).ToList(); - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($"[shift?] pad {pad.Name} pieces={pieces.Count}"); - if (pieces.Count != 5) continue; - int[] padShape = values[padOut].Shape; - // SliceP.Starts/Ends are per-Axes-position arrays (len == Axes.Length). - // For a slice emitted from a Split over axis C, every axes entry is - // identity except the one at axis 1, which carries the group range. - bool RangeAt(SliceP s, int ax, int[] inShape, out int st, out int en) - { - st = 0; en = inShape[ax]; - for (int k = 0; k < s.Axes.Length; k++) - if (s.Axes[k] == ax) { st = s.Starts[k]; en = s.Ends[k]; break; } - return s.Axes.Where((a, k) => a != ax).All(a => - s.Starts[Array.IndexOf(s.Axes, a)] == 0 && - s.Ends[Array.IndexOf(s.Axes, a)] >= inShape[a]); - } - var pieceInfo = pieces - .Select(m => (node: m, s: (SliceP)m.P!)) - .Where(t => RangeAt(t.s, 1, padShape, out _, out _)) - .OrderBy(t => { RangeAt(t.s, 1, padShape, out int st, out _); return st; }) - .ToArray(); - if (pieceInfo.Length != 5) - { - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($" pieceInfo={pieceInfo.Length} (OnlyAxis fail)"); - continue; - } - int[] groupEnds = pieceInfo.Select(t => - { - RangeAt(t.s, 1, padShape, out _, out int en); return en; - }).ToArray(); - - // for each piece find its roll: two Slice consumers + one Concat - int[] offsets = new int[5]; - int shiftAxis = -1; - int[] pieceOutIds = pieceInfo.Select(t => t.node.Out[0]).ToArray(); - var smallConcatOuts = new List(5); - bool fail = false; - var toRemove = new List(pieces); - for (int gI = 0; gI < 5; gI++) - { - int pieceId = pieceOutIds[gI]; - var rollSlices = nodes.Where(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == pieceId).ToList(); - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - { - var allCons = nodes.Where(m => m.In.Contains(pieceId)) - .Select(m => m.Op.ToString()); - Console.Error.WriteLine($" g{gI} piece {pieceId} consumers=[{string.Join(",", allCons)}] rollSlices={rollSlices.Count}"); - } - if (rollSlices.Count != 2) { fail = true; break; } - var concat = nodes.FirstOrDefault(m => m.Op == ExecOp.Concat && m.In.Length == 2 && - m.In.Contains(rollSlices[0].Out[0]) && m.In.Contains(rollSlices[1].Out[0])); - if (concat is null) { fail = true; break; } - var sp = (SliceP)rollSlices[0].P!; var sp2 = (SliceP)rollSlices[1].P!; - // both roll slices must slice a single spatial axis - int SliceAxis(SliceP s, int[] pieceShape) - { - int found = -1; - for (int k = 0; k < s.Axes.Length; k++) - { - int d = s.Axes[k]; - bool sliced = s.Starts[k] != 0 || s.Ends[k] < pieceShape[d]; - if (sliced && found != -1) return -1; - if (sliced) found = d; - } - return found; - } - int[] pieceShape = values[pieceId].Shape; - int ax = SliceAxis(sp, pieceShape); - int ax2 = SliceAxis(sp2, pieceShape); - // allow one side to be unsliced (identity roll for the 0-offset group) - if (ax < 0) ax = ax2; - if (ax2 >= 0 && ax2 != ax) ax = -1; - if (ax < 0 || (ax != 2 && ax != 3)) - { - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($" ax={ax} ax2={ax2} s0.Axes=[{string.Join(",", sp.Axes)}] s1.Axes=[{string.Join(",", sp2.Axes)}]"); - fail = true; break; - } - if (shiftAxis == -1) shiftAxis = ax; - else if (shiftAxis != ax) { fail = true; break; } - int L = pieceShape[ax]; - // concat order = [B, A] where A = piece[0..cut] is the "head" - var sA = rollSlices[0].Out[0] == concat.In[0] ? sp2 : sp; - var sB = rollSlices[0].Out[0] == concat.In[0] ? sp : sp2; - RangeAt(sA, ax, pieceShape, out int saSt, out int saEn); - RangeAt(sB, ax, pieceShape, out int sbSt, out int sbEn); - // sanity: A is the head [0..cut], B is the tail [cut..L] - if (saSt != 0 || sbSt != saEn || sbEn < L) { fail = true; break; } - int cut = saEn - saSt; - // effective shift on UNPADDED axis: out[y] = orig[y + s], zero-fill. - // rolled[i] = padded[(i+cut) mod L]; sliced[y] = rolled[y+padAmt] - // = padded[(y+padAmt+cut) mod L]; orig coord = that - padAmt. - int dim = values[pad.In[0]].Shape[shiftAxis]; - int padAmt = shiftAxis == 2 ? padH : padW; - offsets[gI] = ((padAmt + cut) % L) - padAmt; - toRemove.AddRange(rollSlices); - toRemove.Add(concat); - smallConcatOuts.Add(concat.Out[0]); - } - ExecNode? bigConcat = fail ? null : nodes.FirstOrDefault(m => - m.Op == ExecOp.Concat && m.In.Length == 5 && - smallConcatOuts.All(m.In.Contains)); - if (fail || bigConcat is null) - { - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - { - Console.Error.WriteLine($" fail={fail} bigConcat={(bigConcat is null ? "null" : "ok")} (gI loop)"); - foreach (var cc in nodes.Where(m => m.Op == ExecOp.Concat)) - Console.Error.WriteLine($" concat in[{string.Join("+", cc.In.Select(v => v.ToString()))}] outs=[{string.Join("+", cc.Out.Select(v => v.ToString()))}]"); - } - continue; - } - // unpad slice consuming bigConcat output - var unpad = nodes.FirstOrDefault(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == bigConcat.Out[0]); - if (unpad is null) - { - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($" unpad missing for concat out {bigConcat.Out[0]}"); - continue; - } - toRemove.Add(bigConcat); - toRemove.Add(unpad); - toRemove.Add(pad); - - var shift = new ExecNode - { - Op = ExecOp.ChannelShift, - In = [inId], - Out = unpad.Out, - P = new ShiftP { Axis = shiftAxis, GroupEnds = groupEnds, Offsets = offsets }, - SrcNode = pad.SrcNode, - Name = pad.Name + ".shift", - }; - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($"[shift!] {pad.Name} axis={shiftAxis} ends=[{string.Join(",", groupEnds)}] offs=[{string.Join(",", offsets)}]"); - foreach (ExecNode m in toRemove) nodes.Remove(m); - nodes.Insert(Math.Min(i, nodes.Count), shift); - } - } - - /// Lifetime-based arena packing (def-order, free-list reuse). - private static void PlanArena(List nodes, List values, - List bufSize, int inputValue, int outputValue, - out int[] offsets, out int arenaFloats) - { - int bufCount = bufSize.Count; - const int Align = 16; // floats (64B) - int AlignUp(int v) => (v + Align - 1) & ~(Align - 1); - - int[] def = Enumerable.Repeat(int.MaxValue, bufCount).ToArray(); - int[] use = Enumerable.Repeat(-1, bufCount).ToArray(); - for (int ni = 0; ni < nodes.Count; ni++) - { - ExecNode n = nodes[ni]; - foreach (int iv in n.In) - { - int b = values[iv].Buf; - if (b >= 0) { use[b] = Math.Max(use[b], ni); def[b] = Math.Min(def[b], ni); } - } - // value ids carried in params (conv bias/residual, gemm bias) count as uses - int aux = n.P switch - { - ConvP c => Math.Max(c.BiasValue, c.AddValue), - GemmP g => g.BiasValue, - _ => -1, - }; - if (aux >= 0) - { - // note: for ConvP, BiasValue and AddValue may differ — scan both - if (n.P is ConvP cp2) - foreach (int iv in new[] { cp2.BiasValue, cp2.AddValue }) - { - if (iv < 0) continue; - int b = values[iv].Buf; - if (b >= 0) { use[b] = Math.Max(use[b], ni); def[b] = Math.Min(def[b], ni); } - } - else - { - int b = values[aux].Buf; - if (b >= 0) { use[b] = Math.Max(use[b], ni); def[b] = Math.Min(def[b], ni); } - } - } - foreach (int ov in n.Out) - { - int b = values[ov].Buf; - if (b >= 0) def[b] = Math.Min(def[b], ni); - } - } - int inputBuf = values[inputValue].Buf; - int outputBuf = values[outputValue].Buf; - def[inputBuf] = -1; - // keep input + output live for the whole run: callers may read - // Input (e.g. two-stage pipelines reusing it) after Run(). - use[inputBuf] = nodes.Count; - use[outputBuf] = nodes.Count; - - var order = Enumerable.Range(0, bufCount) - .Where(b => def[b] != int.MaxValue) - .OrderBy(b => def[b]).ThenByDescending(b => bufSize[b]).ToList(); - var active = new List(); // placed buf ids still live - var free = new List<(int off, int size)>(); - offsets = new int[bufCount]; - Array.Fill(offsets, -1); - int bump = 0; - foreach (int b in order) - { - // release dead bufs before allocating this one - for (int i = active.Count - 1; i >= 0; i--) - { - int ob = active[i]; - if (use[ob] < def[b]) - { - active.RemoveAt(i); - free.Add((offsets[ob], AlignUp(bufSize[ob]))); - } - } - int need = AlignUp(bufSize[b]); - int bestIdx = -1; - for (int fi = 0; fi < free.Count; fi++) - if (free[fi].size >= need && (bestIdx < 0 || free[fi].size < free[bestIdx].size)) - bestIdx = fi; - if (bestIdx >= 0) - { - var chunk = free[bestIdx]; - offsets[b] = chunk.off; - free.RemoveAt(bestIdx); - if (chunk.size > need) free.Add((chunk.off + need, chunk.size - need)); - } - else - { - offsets[b] = bump; - bump += need; - } - active.Add(b); - } - arenaFloats = bump; - - // greedy by size (largest first, lowest-waste gap among lifetime-overlapping - // placements): usually much tighter than def-order free lists; keep the better one - { - // the input (live for the whole run) goes first, at offset 0: pipelines can - // then stack another plan's arena right after it - var bySize = order.OrderByDescending(b => b == inputBuf).ThenByDescending(b => bufSize[b]).ThenBy(b => def[b]).ToList(); - var placed = new List(bySize.Count); - int[] off2 = new int[bufCount]; - Array.Fill(off2, -1); - int top = 0; - var conflicts = new List<(int off, int end)>(); - foreach (int b in bySize) - { - int need = AlignUp(bufSize[b]); - conflicts.Clear(); - foreach (int o in placed) - if (!(use[o] < def[b] || use[b] < def[o])) - conflicts.Add((off2[o], off2[o] + AlignUp(bufSize[o]))); - conflicts.Sort(); - int best = -1, bestWaste = int.MaxValue, cur = 0; - foreach (var (o, e) in conflicts) - { - if (o - cur >= need && o - cur - need < bestWaste) { best = cur; bestWaste = o - cur - need; } - cur = Math.Max(cur, e); - } - off2[b] = best >= 0 ? best : cur; - top = Math.Max(top, off2[b] + need); - placed.Add(b); - } - if (top <= bump) - { - offsets = off2; - arenaFloats = top; - } - } - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "3") - { - long peak = 0; int at = -1; - for (int ni = -1; ni <= nodes.Count; ni++) - { - long s = 0; - for (int b = 0; b < bufCount; b++) if (def[b] != int.MaxValue && def[b] <= ni && use[b] >= ni) s += AlignUp(bufSize[b]); - if (s > peak) { peak = s; at = ni; } - } - Console.Error.WriteLine($"[arena] {arenaFloats * 4.0 / 1e6:F1} MB (freelist {bump * 4.0 / 1e6:F1}), live-peak {peak * 4.0 / 1e6:F1} MB at #{at}, input {bufSize[inputBuf] * 4.0 / 1e6:F1} MB off {offsets[inputBuf]}"); - for (int b = 0; b < bufCount; b++) - if (def[b] != int.MaxValue && def[b] <= at && use[b] >= at) - Console.Error.WriteLine($" b{b,-4} {bufSize[b] * 4.0 / 1e6,8:F1} MB #{def[b]}..#{use[b]} {(def[b] >= 0 && def[b] < nodes.Count ? nodes[def[b]].Op : "input")}"); - } - } - - private static readonly HashSet IntOps = new(StringComparer.Ordinal) - { "Squeeze", "Unsqueeze", "Div", "Mod", "Add", "Sub", "Mul", "Reshape", - "Concat", "Slice", "Gather", "Cast", "Floor", "Ceil", "Neg", "Abs", - "Min", "Max", "Equal", "Less", "Where", "ConstantOfShape", "Expand", - "Identity", "Tile" }; - - private void EmitNode(OnnxNodeData n) - { - string op = n.OpType; - // Shape → int tensor of input dims - if (op == "Shape") - { - CValue x = V(Input(0, n)); - if (x.Shape is null) throw new InvalidDataException("Shape input has no known shape."); - int start = (int)n.IntAttr("start", 0); - int end = (int)n.IntAttr("end", x.Shape.Length); - int rank = x.Shape.Length; - if (start < 0) start += rank; - if (end < 0) end += rank; - start = Math.Clamp(start, 0, rank); - end = Math.Clamp(end, 0, rank); - long[] dims = x.Shape[start..end].Select(d => (long)d).ToArray(); - AddValue(new CValue { Name = n.Outputs[0], Shape = [dims.Length], DType = 7, Ints = dims }); - return; - } - if (op == "Size") - { - CValue x = V(Input(0, n)); - AddValue(new CValue { Name = n.Outputs[0], Shape = [], DType = 7, Ints = [x.ElementCount] }); - return; - } - if (op == "Constant") - { - OnnxAttributeData? a = n.Attr("value"); - if (a?.Tensor is OnnxTensorData t) - { - if (t.DataType == OnnxTensorData.TypeFloat) - AddValue(new CValue { Name = n.Outputs[0], Shape = t.DimsArray(), DType = 1, Floats = t.AsFloats() }); - else - AddValue(new CValue { Name = n.Outputs[0], Shape = t.DimsArray(), DType = t.DataType, Ints = t.AsInt64s() }); - return; - } - throw new NotSupportedException("Constant without tensor value."); - } - - // int-only evaluation: if every present input is a const int tensor - // and the op is elementwise/shape math, evaluate now. - bool allInt = n.Inputs.All(s => s.Length == 0 || (V(_byName[s]).IsInt)); - if (allInt && IntOps.Contains(op)) - { - EvalInt(n); - return; - } - // Cast on int → still int - if (op == "Cast" && V(Input(0, n)).IsInt) - { - int to = (int)n.IntAttr("to"); - CValue x = V(Input(0, n)); - if (to == 1) - AddValue(new CValue { Name = n.Outputs[0], Shape = x.Shape, DType = 1, Floats = x.Ints!.Select(v => (float)v).ToArray() }); - else - AddValue(new CValue { Name = n.Outputs[0], Shape = x.Shape, DType = 7, Ints = x.Ints }); - return; - } - - // ---- float ops ---- - switch (op) - { - case "Pad": EmitPad(n); break; - case "Conv": EmitConv(n); break; - case "MaxPool": EmitPool(n); break; - case "Relu": EmitUnary(n, ExecOp.Relu); break; - case "Sigmoid": EmitUnary(n, ExecOp.Sigmoid); break; - case "Erf": EmitUnary(n, ExecOp.Erf); break; - case "LayerNormalization": EmitLayerNorm(n); break; - case "Gemm": EmitGemm(n); break; - case "Resize": EmitResize(n); break; - case "Transpose": EmitTranspose(n); break; - case "Slice": EmitSlice(n); break; - case "Concat": EmitConcat(n); break; - case "Split": EmitSplit(n); break; - case "Reshape": EmitReshape(n); break; - case "Squeeze": EmitSqueeze(n); break; - case "Unsqueeze": EmitUnsqueeze(n); break; - case "Add" or "Sub" or "Mul" or "Div": EmitElementwise(n, op); break; - case "Identity": Alias(n); break; - default: - throw new NotSupportedException($"Op '{op}' not supported (node '{n.Name}')."); - } - } - - // ---------- int evaluation ---------- - private void EvalInt(OnnxNodeData n) - { - string op = n.OpType; - CValue In(int i) => V(Input(i, n)); - CValue Out(long[] data, int[] shape) - { - var v = new CValue { Name = n.Outputs[0], Shape = shape, DType = 7, Ints = data }; - AddValue(v); - return v; - } - switch (op) - { - case "Squeeze": - { - CValue x = In(0); - long[]? axes = n.Inputs.Count > 1 && n.Inputs[1].Length > 0 ? In(1).Ints : null; - int[] shape; - if (axes is null) - shape = x.Shape.Where(d => d != 1).ToArray(); - else - { - var dims = x.Shape.ToList(); - foreach (long a in axes.OrderByDescending(v => v)) - { - int ax = (int)(a < 0 ? a + dims.Count : a); - if (dims[ax] != 1) throw new InvalidDataException("Squeeze axis not 1."); - dims.RemoveAt(ax); - } - shape = dims.ToArray(); - } - Out(x.Ints!, shape); - return; - } - case "Unsqueeze": - { - CValue x = In(0); - long[] axes = In(1).Ints!; - var dims = x.Shape.ToList(); - foreach (long a in axes.OrderBy(v => v)) - { - int ax = (int)(a < 0 ? a + dims.Count + 1 : a); - dims.Insert(ax, 1); - } - Out(x.Ints!, dims.ToArray()); - return; - } - case "Reshape": - { - CValue x = In(0); - long[] spec = In(1).Ints!; - bool allowZero = n.IntAttr("allowzero") == 1; - Out(x.Ints!, ResolveReshape(x.Shape, spec, allowZero, x.Ints!.Length)); - return; - } - case "Concat": - { - int axis = (int)n.IntAttr("axis"); - var inputs = Inputs(n).Select(V).ToArray(); - int rank = inputs[0].Shape.Length; - if (axis < 0) axis += rank; - int[] shape = (int[])inputs[0].Shape.Clone(); - shape[axis] = inputs.Sum(v => v.Shape[axis]); - long[] data = IntConcat(inputs.Select(v => v.Ints!).ToArray(), - inputs.Select(v => v.Shape).ToArray(), axis, shape); - Out(data, shape); - return; - } - case "Slice": - { - CValue x = In(0); - long[] starts = In(1).Ints!; - long[] ends = n.Inputs.Count > 2 && n.Inputs[2].Length > 0 ? In(2).Ints! : x.Shape.Select(d => (long)d).ToArray(); - long[] axes = n.Inputs.Count > 3 && n.Inputs[3].Length > 0 ? In(3).Ints! : Enumerable.Range(0, starts.Length).Select(i => (long)i).ToArray(); - long[] steps = n.Inputs.Count > 4 && n.Inputs[4].Length > 0 ? In(4).Ints! : starts.Select(_ => 1L).ToArray(); - SliceBounds(x.Shape, starts, ends, axes, steps, - out int[] outShape, out int[] st, out int[] en, out int[] stepI); - long[] data = IntSlice(x.Ints!, x.Shape, - axes.Select(a => (int)(a < 0 ? a + x.Shape.Length : a)).ToArray(), st, stepI, outShape); - Out(data, outShape); - return; - } - case "Gather": - { - CValue x = In(0); CValue idx = In(1); - int axis = (int)n.IntAttr("axis"); - if (axis < 0) axis += x.Shape.Length; - if (x.Shape.Length != 1 || axis != 0) throw new NotSupportedException("Gather eval supports 1-D axis 0."); - Out(idx.Ints!.Select(i => x.Ints![checked((int)i)]).ToArray(), idx.Shape); - return; - } - case "Identity": - { - CValue x = In(0); - Out(x.Ints!, x.Shape); - return; - } - case "Add" or "Sub" or "Mul" or "Div" or "Mod" or "Min" or "Max": - { - long[] a = In(0).Ints!, b = In(1).Ints!; - long[] r = new long[Math.Max(a.Length, b.Length)]; - for (int i = 0; i < r.Length; i++) - { - long av = a.Length == 1 ? a[0] : a[i], bv = b.Length == 1 ? b[0] : b[i]; - r[i] = op switch - { - "Add" => av + bv, "Sub" => av - bv, "Mul" => av * bv, - "Div" => av / bv, "Mod" => ((av % bv) + bv) % bv, - "Min" => Math.Min(av, bv), _ => Math.Max(av, bv), - }; - } - int[] shape = In(0).Shape.Length >= In(1).Shape.Length ? In(0).Shape : In(1).Shape; - Out(r, shape); - return; - } - case "Neg" or "Abs" or "Floor" or "Ceil": - { - long[] a = In(0).Ints!; - // Floor/Ceil on ints are identity - long[] r = a.Select(v => op == "Neg" ? -v : op == "Abs" ? Math.Abs(v) : v).ToArray(); - Out(r, In(0).Shape); - return; - } - default: - throw new NotSupportedException($"Int eval for '{op}' not implemented (node '{n.Name}')."); - } - } - - // ---------- float ops ---------- - private CValue ActivationValue(string name, int[] shape) - { - int elems = 1; - foreach (int d in shape) elems = checked(elems * d); - return new CValue { Name = name, Shape = shape, DType = 1, Buf = NewBuf(elems) }; - } - - private void Alias(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - AddValue(new CValue { Name = n.Outputs[0], Shape = x.Shape, DType = 1, Buf = x.Buf, Floats = x.Floats, Ints = x.Ints, AliasOf = x.Buf >= 0 ? Input(0, n) : -1 }); - } - - private void EmitPad(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - long[] pads = V(Input(1, n)).Ints!; - float val = n.Inputs.Count > 2 && n.Inputs[2].Length > 0 ? V(Input(2, n)).Floats![0] : 0f; - PadMode mode = n.StringAttr("mode", "constant") switch - { - "reflect" => PadMode.Reflect, - "constant" => PadMode.Constant, - var s => throw new NotSupportedException($"Pad mode '{s}'"), - }; - int[] shape = x.Shape.ToArray(); - for (int i = 0; i < shape.Length; i++) - shape[i] += checked((int)(pads[i] + pads[i + shape.Length])); - Emit(ExecOp.Pad, [Input(0, n)], ActivationValue(n.Outputs[0], shape), - new PadP { Pads = pads.Select(p => (int)p).ToArray(), Mode = mode, Value = val }, n); - } - - private void EmitConv(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - CValue w = V(Input(1, n)); - int[] wd = w.Shape; - int[] pads = n.IntsAttr("pads").Select(v => (int)v).ToArray(); - if (pads.Length == 0) pads = new int[2 * (wd.Length - 2)]; - int[] strides = n.IntsAttr("strides").Select(v => (int)v).ToArray(); - if (strides.Length == 0) strides = Enumerable.Repeat(1, wd.Length - 2).ToArray(); - int[] dil = n.IntsAttr("dilations").Select(v => (int)v).ToArray(); - if (dil.Length == 0) dil = Enumerable.Repeat(1, wd.Length - 2).ToArray(); - int group = (int)n.IntAttr("group", 1); - int[] outShape = new int[wd.Length]; - outShape[0] = x.Shape[0]; - outShape[1] = wd[0]; - for (int d = 0; d < wd.Length - 2; d++) - { - int eff = dil[d] * (wd[2 + d] - 1) + 1; - outShape[2 + d] = (x.Shape[2 + d] + pads[d] + pads[d + wd.Length - 2] - eff) / strides[d] + 1; - } - int biasV = n.Inputs.Count > 2 && n.Inputs[2].Length > 0 ? Input(2, n) : -1; - Emit(ExecOp.Conv, [Input(0, n), Input(1, n)], ActivationValue(n.Outputs[0], outShape), - new ConvP - { - Pads = pads, Strides = strides, Dilations = dil, - Kernel = wd[2..], Group = group, BiasValue = biasV, - }, n); - } - - private void EmitPool(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - int[] kernel = n.IntsAttr("kernel_shape").Select(v => (int)v).ToArray(); - int[] strides = n.IntsAttr("strides").Select(v => (int)v).ToArray(); - if (strides.Length == 0) strides = (int[])kernel.Clone(); - int[] pads = n.IntsAttr("pads").Select(v => (int)v).ToArray(); - if (pads.Length == 0) pads = new int[2 * kernel.Length]; - int[] outShape = (int[])x.Shape.Clone(); - for (int d = 0; d < kernel.Length; d++) - outShape[2 + d] = (x.Shape[2 + d] + pads[d] + pads[d + kernel.Length] - kernel[d]) / strides[d] + 1; - Emit(ExecOp.MaxPool, [Input(0, n)], ActivationValue(n.Outputs[0], outShape), - new PoolP { Kernel = kernel, Strides = strides, Pads = pads }, n); - } - - private void EmitUnary(OnnxNodeData n, ExecOp op) - { - CValue x = V(Input(0, n)); - Emit(op, [Input(0, n)], ActivationValue(n.Outputs[0], x.Shape), null, n); - } - - private void EmitLayerNorm(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - float eps = n.FloatAttr("epsilon", 1e-5f); - int axis = (int)n.IntAttr("axis", -1); - if (axis < 0) axis += x.Shape.Length; - if (axis != x.Shape.Length - 1) - throw new NotSupportedException("LayerNorm only over last dim."); - Emit(ExecOp.LayerNorm, [Input(0, n), Input(1, n), Input(2, n)], - ActivationValue(n.Outputs[0], x.Shape), new LnP { Eps = eps }, n); - } - - private void EmitGemm(OnnxNodeData n) - { - CValue a = V(Input(0, n)); - CValue b = V(Input(1, n)); - bool transB = n.IntAttr("transB") == 1; - float alpha = n.FloatAttr("alpha", 1f); - if (alpha != 1f) throw new NotSupportedException("Gemm alpha != 1."); - int k = transB ? b.Shape[1] : b.Shape[0]; - int ncol = transB ? b.Shape[0] : b.Shape[1]; - if (a.Shape[^1] != k) throw new InvalidDataException("Gemm K mismatch."); - int[] outShape = (int[])a.Shape.Clone(); - outShape[^1] = ncol; - int biasV = n.Inputs.Count > 2 && n.Inputs[2].Length > 0 ? Input(2, n) : -1; - Emit(ExecOp.Gemm, [Input(0, n), Input(1, n)], ActivationValue(n.Outputs[0], outShape), - new GemmP { TransB = transB, BiasValue = biasV }, n); - } - - private void EmitResize(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - float[] scales; - int[]? sizesI = null; - if (n.Inputs.Count > 3 && n.Inputs[3].Length > 0) - { - // sizes input (int tensor): authoritative output shape - long[] sizes = V(Input(3, n)).Ints!; - sizesI = sizes.Select(s => (int)s).ToArray(); - scales = sizes.Select((s, i) => (float)s / x.Shape[i]).ToArray(); - } - else - { - scales = V(Input(2, n)).Floats!; - } - if (n.StringAttr("mode") != "linear") - throw new NotSupportedException("Resize mode != linear."); - int[] outShape = sizesI ?? (int[])x.Shape.Clone(); - if (sizesI == null) - for (int i = 0; i < outShape.Length; i++) - outShape[i] = (int)MathF.Floor(x.Shape[i] * scales[i]); - Emit(ExecOp.ResizeBilinear, [Input(0, n)], ActivationValue(n.Outputs[0], outShape), - new ResizeP { Scales = scales }, n); - } - - private void EmitTranspose(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - int[] perm = n.IntsAttr("perm").Select(v => (int)v).ToArray(); - if (perm.Length == 0) perm = Enumerable.Range(0, x.Shape.Length).Reverse().ToArray(); - int[] outShape = perm.Select(p => x.Shape[p]).ToArray(); - Emit(ExecOp.Transpose, [Input(0, n)], ActivationValue(n.Outputs[0], outShape), - new TransposeP { Perm = perm }, n); - } - - private void EmitSlice(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - long[] starts = V(Input(1, n)).Ints!; - long[] ends = V(Input(2, n)).Ints!; - long[] axes = n.Inputs.Count > 3 && n.Inputs[3].Length > 0 - ? V(Input(3, n)).Ints! : Enumerable.Range(0, starts.Length).Select(i => (long)i).ToArray(); - long[] steps = n.Inputs.Count > 4 && n.Inputs[4].Length > 0 - ? V(Input(4, n)).Ints! : starts.Select(_ => 1L).ToArray(); - SliceBounds(x.Shape, starts, ends, axes, steps, - out int[] outShape, out int[] st, out int[] en, out int[] stepI); - Emit(ExecOp.Slice, [Input(0, n)], ActivationValue(n.Outputs[0], outShape), - new SliceP { Starts = st, Ends = en, - Axes = axes.Select(a => (int)a).ToArray(), Steps = stepI }, n); - } - - private void EmitConcat(OnnxNodeData n) - { - int axis = (int)n.IntAttr("axis"); - int[] ins = Inputs(n); - int[] shape = (int[])V(ins[0]).Shape.Clone(); - int rank = shape.Length; - if (axis < 0) axis += rank; - shape[axis] = ins.Sum(id => V(id).Shape[axis]); - Emit(ExecOp.Concat, ins, ActivationValue(n.Outputs[0], shape), - new ConcatP { Axis = axis }, n); - } - - private void EmitSplit(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - int axis = (int)n.IntAttr("axis"); - if (axis < 0) axis += x.Shape.Length; - int[] sizes; - if (n.Inputs.Count > 1 && n.Inputs[1].Length > 0) - sizes = V(Input(1, n)).Ints!.Select(v => (int)v).ToArray(); - else - { - int outputs = (int)n.IntAttr("num_outputs", n.Outputs.Count); - int each = x.Shape[axis] / outputs; - sizes = Enumerable.Repeat(each, outputs).ToArray(); - sizes[^1] += x.Shape[axis] - each * outputs; - } - // Split is implemented as a per-output Slice at exec time. - int[] starts = new int[x.Shape.Length]; - for (int i = 0; i < sizes.Length; i++) - { - int[] shape = (int[])x.Shape.Clone(); - shape[axis] = sizes[i]; - var slice = new SliceP - { - Starts = (int[])starts.Clone(), - Ends = (int[])x.Shape.Clone(), - Axes = Enumerable.Range(0, x.Shape.Length).ToArray(), - Steps = Enumerable.Repeat(1, x.Shape.Length).ToArray(), - }; - slice.Ends[axis] = starts[axis] + sizes[i]; - _nodes.Add(new ExecNode { Op = ExecOp.Slice, In = [Input(0, n)], Out = [AddValue(ActivationValue(n.Outputs[i], shape))], P = slice, SrcNode = _nodes.Count, Name = n.Name }); - starts[axis] += sizes[i]; - } - } - - private void EmitReshape(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - long[] spec = V(Input(1, n)).Ints!; - bool allowZero = n.IntAttr("allowzero") == 1; - int[] shape = ResolveReshape(x.Shape, spec, allowZero, checked((int)x.ElementCount)); - // zero-copy alias (consts keep their payload so consumers see a materialized tensor) - AddValue(new CValue { Name = n.Outputs[0], Shape = shape, DType = 1, Buf = x.Buf, Floats = x.Floats, AliasOf = x.Buf >= 0 ? Input(0, n) : -1 }); - } - - private void EmitSqueeze(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - long[]? axes = n.Inputs.Count > 1 && n.Inputs[1].Length > 0 ? V(Input(1, n)).Ints : null; - int[] shape; - if (axes is null) shape = x.Shape.Where(d => d != 1).ToArray(); - else - { - var dims = x.Shape.ToList(); - foreach (long a in axes.OrderByDescending(v => v)) dims.RemoveAt((int)(a < 0 ? a + dims.Count : a)); - shape = dims.ToArray(); - } - AddValue(new CValue { Name = n.Outputs[0], Shape = shape, DType = 1, Buf = x.Buf, Floats = x.Floats, AliasOf = x.Buf >= 0 ? Input(0, n) : -1 }); - } - - private void EmitUnsqueeze(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - long[] axes = V(Input(1, n)).Ints!; - var dims = x.Shape.ToList(); - foreach (long a in axes.OrderBy(v => v)) dims.Insert((int)(a < 0 ? a + dims.Count + 1 : a), 1); - AddValue(new CValue { Name = n.Outputs[0], Shape = dims.ToArray(), DType = 1, Buf = x.Buf, Floats = x.Floats, AliasOf = x.Buf >= 0 ? Input(0, n) : -1 }); - } - - private void EmitElementwise(OnnxNodeData n, string op) - { - int a = Input(0, n), b = Input(1, n); - CValue va = V(a), vb = V(b); - int[] shape = BroadcastShape(va.Shape, vb.Shape); - var eop = op switch { "Add" => EwOp.Add, "Sub" => EwOp.Sub, "Mul" => EwOp.Mul, "Div" => EwOp.Div, _ => throw new() }; - Emit(ExecOp.Elementwise, [a, b], ActivationValue(n.Outputs[0], shape), new EwP { Op = eop }, n); - } - - // ---------- helpers ---------- - private static int[] BroadcastShape(int[] a, int[] b) - { - int rank = Math.Max(a.Length, b.Length); - int[] r = new int[rank]; - for (int i = 0; i < rank; i++) - { - int ad = i - (rank - a.Length) >= 0 ? a[i - (rank - a.Length)] : 1; - int bd = i - (rank - b.Length) >= 0 ? b[i - (rank - b.Length)] : 1; - if (ad != bd && ad != 1 && bd != 1) - throw new InvalidDataException($"Broadcast mismatch {ad} vs {bd}."); - r[i] = Math.Max(ad, bd); - } - return r; - } - - internal static int[] ResolveReshape(int[] inShape, long[] spec, bool allowZero, int inElems) - { - int[] shape = new int[spec.Length]; - int neg = -1; long prod = 1; - for (int i = 0; i < spec.Length; i++) - { - long s = spec[i]; - if (s == -1) { neg = i; shape[i] = 1; } - else if (s == 0 && !allowZero) { shape[i] = inShape[i]; prod *= shape[i]; } - else { shape[i] = checked((int)s); prod *= shape[i]; } - } - if (neg >= 0) shape[neg] = checked((int)(inElems / prod)); - return shape; - } - - internal static void SliceBounds(int[] inShape, long[] starts, long[] ends, - long[] axes, long[] steps, out int[] outShape, out int[] stOut, out int[] enOut, out int[] stepOut) - { - int rank = inShape.Length; - stOut = new int[starts.Length]; - enOut = new int[starts.Length]; - stepOut = new int[starts.Length]; - outShape = (int[])inShape.Clone(); - for (int i = 0; i < starts.Length; i++) - { - int ax = (int)(axes[i] < 0 ? axes[i] + rank : axes[i]); - int dim = inShape[ax]; - int step = (int)steps[i]; - long s = starts[i] < 0 ? starts[i] + dim : starts[i]; - long e = ends[i] < 0 ? ends[i] + dim : ends[i]; - if (step > 0) - { - s = Math.Clamp(s, 0, dim); - e = Math.Clamp(e, 0, dim); - } - else - { - s = Math.Clamp(s, -1, dim - 1); - e = Math.Clamp(e, -1, dim - 1); - } - int len = step > 0 ? Math.Max(0, (int)(e - s + step - 1) / step) - : Math.Max(0, (int)(s - e - step - 1) / -step); - stOut[i] = (int)s; - enOut[i] = (int)e; - stepOut[i] = step; - outShape[ax] = len; - } - } - - private static long[] IntConcat(long[][] inputs, int[][] shapes, int axis, int[] outShape) - { - // small tensors only — generic n-d concat - int outElems = outShape.Aggregate(1, (a, b) => a * b); - long[] dst = new long[outElems]; - int[] strides = Strides(outShape); - int[] coord = new int[outShape.Length]; - for (int i = 0; i < inputs.Length; i++) - { - int[] inStrides = Strides(shapes[i]); - int inElems = shapes[i].Aggregate(1, (a, b) => a * b); - for (int e = 0; e < inElems; e++) - { - int rem = e, srcOff = 0; - for (int d = 0; d < shapes[i].Length; d++) - { - coord[d] = rem / inStrides[d]; - srcOff += coord[d] * inStrides[d]; - rem %= inStrides[d]; - } - int dstOff = 0; - for (int d = 0; d < outShape.Length; d++) - dstOff += (coord[d] + (d == axis ? shapes.Take(i).Sum(s => s[axis]) : 0)) * strides[d]; - dst[dstOff] = inputs[i][srcOff]; - } - } - return dst; - } - - private static long[] IntSlice(long[] src, int[] inShape, int[] axes, int[] starts, int[] steps, int[] outShape) - { - int rank = inShape.Length; - int[] fullStart = new int[rank], fullStep = Enumerable.Repeat(1, rank).ToArray(); - for (int i = 0; i < axes.Length; i++) { fullStart[axes[i]] = starts[i]; fullStep[axes[i]] = steps[i]; } - int outElems = outShape.Aggregate(1, (a, b) => a * b); - long[] dst = new long[outElems]; - int[] inStrides = Strides(inShape), outStrides = Strides(outShape); - for (int e = 0; e < outElems; e++) - { - int rem = e, srcOff = 0; - for (int d = 0; d < rank; d++) - { - int cc = rem / outStrides[d]; - rem %= outStrides[d]; - srcOff += (fullStart[d] + cc * fullStep[d]) * inStrides[d]; - } - dst[e] = src[srcOff]; - } - return dst; - } - - internal static int[] Strides(int[] shape) - { - int[] s = new int[shape.Length]; - int acc = 1; - for (int i = shape.Length - 1; i >= 0; i--) { s[i] = acc; acc *= shape[i]; } - return s; - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/Layout.cs b/src/Sdcb.SimdGcdrNet/Engine/Layout.cs deleted file mode 100644 index 755fef0..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/Layout.cs +++ /dev/null @@ -1,111 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// Physical-layout algebra. A layout L lists, for each physical -/// (storage) dim i, the logical dim it holds: storage is row-major over -/// [shape[L[0]], shape[L[1]], ...]. Logical shapes never change — -/// fusion passes and perm reasoning stay in logical space; kernels address by -/// . Size-1 dims carry no stride, so layouts that order the -/// non-1 dims identically are interchangeable (). -/// -internal static class Layout -{ - /// Channels-last for rank-4 [n,c,h,w]: storage [n,h,w,c]. - public static readonly int[] Nhwc = [0, 2, 3, 1]; - - public static int[] Identity(int rank) - { - int[] l = new int[rank]; - for (int i = 0; i < rank; i++) l[i] = i; - return l; - } - - public static bool IsIdentity(int[] l) - { - for (int i = 0; i < l.Length; i++) if (l[i] != i) return false; - return true; - } - - public static int[] Phys(int[] shape, int[] l) - { - int[] p = new int[l.Length]; - for (int i = 0; i < l.Length; i++) p[i] = shape[l[i]]; - return p; - } - - public static int[] Inverse(int[] p) - { - int[] inv = new int[p.Length]; - for (int i = 0; i < p.Length; i++) inv[p[i]] = i; - return inv; - } - - /// - /// Layout of y = Transpose(x, perm) as a view of x's storage. y logical k - /// is x logical perm[k], so x logical d is y logical perm⁻¹[d]: - /// L'[i] = perm⁻¹[L[i]]. Always representable — every Transpose dissolves. - /// - public static int[] Transposed(int[] l, int[] perm) - { - int[] inv = Inverse(perm); - int[] r = new int[l.Length]; - for (int i = 0; i < l.Length; i++) r[i] = inv[l[i]]; - return r; - } - - /// Same storage order for (size-1 dims ignored). - public static bool Equivalent(int[] shape, int[] a, int[] b) => - a.Where(d => shape[d] != 1).SequenceEqual(b.Where(d => shape[d] != 1)); - - /// Physical perm relaying storage from layout to - /// : out phys dim i = in phys dim q[i]. - public static int[] PhysPerm(int[] from, int[] to) => - to.Select(d => Array.IndexOf(from, d)).ToArray(); - - /// - /// Layout of a zero-copy reshape → - /// of storage laid out as , or null when the reshape - /// needs data movement. The logical reshape factors into groups (a run of - /// source dims ↔ a run of target dims with equal product); it is a view iff - /// each group's non-1 source dims sit consecutively, in logical order, in - /// storage. Target dims are then ordered by their group's storage position. - /// - public static int[]? Reshaped(int[] src, int[] l, int[] dst) - { - int[] sd = Enumerable.Range(0, src.Length).Where(d => src[d] != 1).ToArray(); - int[] td = Enumerable.Range(0, dst.Length).Where(d => dst[d] != 1).ToArray(); - // storage rank of each non-1 source dim - int[] rank = new int[src.Length]; - int r = 0; - foreach (int d in l) if (src[d] != 1) rank[d] = r++; - - double[] key = new double[dst.Length]; - int i = 0, j = 0; - while (i < sd.Length || j < td.Length) - { - if (i >= sd.Length || j >= td.Length) return null; // element count mismatch - var gs = new List { sd[i] }; long ps = src[sd[i++]]; - var gt = new List { td[j] }; long pt = dst[td[j++]]; - while (ps != pt) - { - if (ps < pt) { if (i >= sd.Length) return null; gs.Add(sd[i]); ps *= src[sd[i++]]; } - else { if (j >= td.Length) return null; gt.Add(td[j]); pt *= dst[td[j++]]; } - } - for (int k = 1; k < gs.Count; k++) - if (rank[gs[k]] != rank[gs[k - 1]] + 1) return null; - for (int k = 0; k < gt.Count; k++) - key[gt[k]] = rank[gs[0]] + k / (double)(gt.Count + 1); - } - // size-1 target dims ride right behind their logical predecessor (or lead), - // so an identity source yields an identity target - double prev = -1; - for (int e = 0; e < dst.Length; e++) - { - if (dst[e] != 1) prev = key[e]; - else key[e] = prev + 1e-3 * (e + 1) / (dst.Length + 1); - } - return Enumerable.Range(0, dst.Length).OrderBy(e => key[e]).ThenBy(e => e).ToArray(); - } - - public static string Str(int[]? l) => l is null ? "-" : "[" + string.Join(',', l) + "]"; -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/LayoutPlanner.cs b/src/Sdcb.SimdGcdrNet/Engine/LayoutPlanner.cs deleted file mode 100644 index 87c0194..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/LayoutPlanner.cs +++ /dev/null @@ -1,243 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// Physical layout pass (runs after fusion, before arena planning). Every -/// activation value gets a ; logical shapes are -/// untouched, so fusions and golden shapes stay in logical NCHW terms. -/// -/// Rules (global channels-last; no runtime segment decisions — this model is a -/// homogeneous conv + token-MLP graph): -/// -/// Conv / MaxPool / Resize / Pad / ChannelShift consume and produce -/// rank-4 NHWC (). -/// LayerNorm / Gemm need their last logical dim innermost and keep the -/// input layout; elementwise / unary ops follow their full-shape input. -/// Every graph Transpose dissolves into a zero-copy view -/// (); reshape/squeeze aliases stay views when -/// can express them. Layout bookkeeping follows -/// the Buf alias chain (), not just ExecNodes — -/// aliases have no producer node. -/// A consumer whose requirement is not met gets an explicit relayout -/// (a physical Transpose node). On this model the only ones are the I/O -/// boundary relayouts when physicalIo is off. -/// -/// Op params that name axes are rewritten into physical terms (Slice axes, -/// Concat axis, relayout perms), so generic kernels address purely by -/// . -/// -internal sealed class LayoutPlanner -{ - private readonly List _nodes; - private readonly List _values; - private readonly List _bufSize; - private readonly Dictionary _perm = []; // dissolved transpose → logical perm - private readonly Dictionary<(int v, string l), int> _relayout = []; - private int _at; // index of the node being planned - private int _dissolved; - - private LayoutPlanner(List nodes, List values, List bufSize) - { - _nodes = nodes; _values = values; _bufSize = bufSize; - } - - public static void Run(List nodes, List values, List bufSize, - int inputValue, ref int outputValue, bool physicalIo) - { - var lp = new LayoutPlanner(nodes, values, bufSize); - CValue inV = values[inputValue]; - inV.Layout = physicalIo && inV.Shape.Length == 4 ? Layout.Nhwc : Layout.Identity(inV.Shape.Length); - - for (lp._at = 0; lp._at < nodes.Count; lp._at++) - lp.Plan(nodes[lp._at]); - - lp.L(outputValue); - if (!physicalIo) - { - CValue ov = values[outputValue]; - int[] id = Layout.Identity(ov.Shape.Length); - if (!Layout.Equivalent(ov.Shape, ov.Layout!, id)) - { - lp._at = nodes.Count; - outputValue = lp.Relayout(outputValue, id); - } - } - // dead aliases (never consumed): identity layout so dumps never see null. - // Resolving them could insert relayout nodes for values nobody reads. - for (int v = 0; v < values.Count; v++) - if (values[v].Buf >= 0 && values[v].Layout is null) - values[v].Layout = Layout.Identity(values[v].Shape.Length); - - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($"[layout] dissolved {lp._dissolved} transposes, {lp._relayout.Count} relayouts"); - } - - /// Resolves (and caches) the layout of value , following aliases. - private int[] L(int v) - { - CValue cv = _values[v]; - if (cv.Layout is not null) return cv.Layout; - if (cv.Buf < 0) return cv.Layout = Layout.Identity(cv.Shape.Length); // const - if (cv.AliasOf < 0) - throw new InvalidOperationException( - $"layout: value '{cv.Name}' (v{v}) consumed before its producer was planned."); - int s = cv.AliasOf; - int[] ls = L(s); - if (_perm.TryGetValue(v, out int[]? perm)) - { - cv.Buf = _values[s].Buf; - return cv.Layout = Layout.Transposed(ls, perm); - } - int[]? r = Layout.Reshaped(_values[s].Shape, ls, cv.Shape); - if (r is null) - { - // the reshape needs data movement: materialize the source in - // logical order, then the reshape is an identity-layout view of that - int[] id = Layout.Identity(_values[s].Shape.Length); - s = Relayout(s, id); - cv.AliasOf = s; - r = Layout.Reshaped(_values[s].Shape, id, cv.Shape) - ?? throw new InvalidOperationException("identity reshape must be a view"); - } - cv.Buf = _values[s].Buf; - return cv.Layout = r; - } - - /// Inserts (before the current node) a physical copy of in layout . - private int Relayout(int v, int[] target) - { - var key = (v, Layout.Str(target)); - if (_relayout.TryGetValue(key, out int done)) return done; - CValue src = _values[v]; - int[] from = L(v); - int buf = _bufSize.Count; - _bufSize.Add(checked((int)src.ElementCount)); - int id = _values.Count; - _values.Add(new CValue { Name = src.Name + "@relayout", Shape = src.Shape, DType = 1, Buf = buf, Layout = target }); - _nodes.Insert(_at++, new ExecNode - { - Op = ExecOp.Transpose, In = [v], Out = [id], - P = new TransposeP { Perm = Layout.PhysPerm(from, target) }, - Name = $"relayout{Layout.Str(from)}->{Layout.Str(target)}", - }); - _relayout[key] = id; - return id; - } - - private int Need(int v, int[] target) => - Layout.Equivalent(_values[v].Shape, L(v), target) ? v : Relayout(v, target); - - private void Need(ExecNode n, int k, int[] target) => n.In[k] = Need(n.In[k], target); - - private static bool LastInnermost(int[] shape, int[] l) - { - int r = shape.Length - 1; - if (r < 0 || shape[r] == 1) return true; - for (int i = l.Length - 1; i >= 0; i--) - if (shape[l[i]] != 1) return l[i] == r; - return true; - } - - private void SetOut(ExecNode n, int[] l) => _values[n.Out[0]].Layout = l; - - private void Plan(ExecNode n) - { - switch (n.Op) - { - case ExecOp.Transpose: - { - // dissolve: y = T(x) becomes a view of x's storage - int x = n.In[0], o = n.Out[0]; - L(x); - _values[o].AliasOf = x; - _values[o].Layout = null; - _perm[o] = ((TransposeP)n.P!).Perm; - L(o); - _nodes.RemoveAt(_at--); - _dissolved++; - break; - } - case ExecOp.Conv or ExecOp.MaxPool or ExecOp.ResizeBilinear or ExecOp.Pad or ExecOp.ChannelShift: - { - if (_values[n.In[0]].Shape.Length != 4 || _values[n.Out[0]].Shape.Length != 4) - throw new NotSupportedException($"layout: {n.Op} '{n.Name}' is not rank-4."); - if (n.P is PadP pp) - { - int r = 4; - if (pp.Pads[0] != 0 || pp.Pads[1] != 0 || pp.Pads[r] != 0 || pp.Pads[r + 1] != 0) - throw new NotSupportedException($"layout: Pad '{n.Name}' pads N/C."); - } - if (n.P is ConvP cp) - { - if (cp.Dilations.Any(d => d != 1)) - throw new NotSupportedException($"layout: dilated Conv '{n.Name}'."); - if (cp.AddValue >= 0) cp.AddValue = Need(cp.AddValue, Layout.Nhwc); - } - Need(n, 0, Layout.Nhwc); - if (n.Op == ExecOp.ResizeBilinear) - for (int k = 1; k < n.In.Length; k++) Need(n, k, Layout.Nhwc); // fused addend - SetOut(n, Layout.Nhwc); - break; - } - case ExecOp.UpHead: - { - // In = [x, skip, w, bias?]: activations channels-last, weights stay logical - if (_values[n.In[0]].Shape.Length != 4 || _values[n.In[1]].Shape.Length != 4) - throw new NotSupportedException($"layout: UpHead '{n.Name}' is not rank-4."); - Need(n, 0, Layout.Nhwc); - Need(n, 1, Layout.Nhwc); - SetOut(n, Layout.Nhwc); - break; - } - case ExecOp.LayerNorm or ExecOp.Gemm: - { - int x = n.In[0]; - if (!LastInnermost(_values[x].Shape, L(x))) - Need(n, 0, Layout.Identity(_values[x].Shape.Length)); - SetOut(n, L(n.In[0])); - break; - } - case ExecOp.Elementwise: - { - int[] os = _values[n.Out[0]].Shape; - int rank = os.Length; - int ri = Array.FindIndex(n.In, v => _values[v].IsActivation && _values[v].Shape.SequenceEqual(os)); - int[] lo = ri >= 0 ? L(n.In[ri]) : Layout.Identity(rank); - for (int k = 0; k < n.In.Length; k++) - { - CValue cv = _values[n.In[k]]; - if (cv.ElementCount == 1) continue; - // operand storage must match lo restricted to its own dims - int off = rank - cv.Shape.Length; - int[] tv = lo.Where(d => d >= off).Select(d => d - off).ToArray(); - Need(n, k, tv); - } - SetOut(n, lo); - break; - } - case ExecOp.Relu or ExecOp.Sigmoid or ExecOp.Erf or ExecOp.Gelu or ExecOp.Copy: - SetOut(n, L(n.In[0])); - break; - case ExecOp.Slice: - { - int[] lx = L(n.In[0]); - int rank = lx.Length; - var sp = (SliceP)n.P!; - for (int i = 0; i < sp.Axes.Length; i++) - sp.Axes[i] = Array.IndexOf(lx, sp.Axes[i] < 0 ? sp.Axes[i] + rank : sp.Axes[i]); - SetOut(n, lx); - break; - } - case ExecOp.Concat: - { - int[] lo = L(n.In[0]); - for (int k = 1; k < n.In.Length; k++) Need(n, k, lo); - var cp = (ConcatP)n.P!; - cp.Axis = Array.IndexOf(lo, cp.Axis); - SetOut(n, lo); - break; - } - default: - throw new NotSupportedException($"layout: no rule for {n.Op} '{n.Name}'."); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/Session.cs b/src/Sdcb.SimdGcdrNet/Engine/Session.cs deleted file mode 100644 index 2c06d33..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/Session.cs +++ /dev/null @@ -1,327 +0,0 @@ -using System.Runtime.InteropServices; -using Sdcb.SimdGcdrNet.Kernels; -using Sdcb.SimdGcdrNet.Onnx; - -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// A reusable inference session over a compiled . -/// Not thread-safe; create one per thread or serialize externally. -/// -public sealed unsafe class InferenceSession : IDisposable -{ - private readonly ExecPlan _plan; - // native so Dispose frees it deterministically: multi-GB LOH arenas of evicted - // sessions otherwise pile up until a (rare, on big-RAM hosts) gen2 GC - private nint _arena; - private readonly long _arenaBytes; - private readonly bool _ownsArena; - private readonly int[] _offsets; - private readonly long[] _nodeTicks; - private readonly int _inputValue, _outputValue; - - /// Creates a session over that owns a fresh native arena. - public InferenceSession(CompiledModel model) : this(model.Plan) { } - - internal InferenceSession(ExecPlan plan) : this(plan, 0) { } - - /// - /// Session over . With = 0 the session owns a - /// fresh zeroed native arena; otherwise it borrows the caller's arena (at least - /// ArenaFloats floats; contents need not be zero — every buffer is written before it is - /// read), which outlives the session and is never freed by it. - /// - internal InferenceSession(ExecPlan plan, nint arena) - { - _plan = plan; - _arenaBytes = (long)plan.ArenaFloats * sizeof(float); - if (arena == 0) - { - // calloc: large blocks come straight from fresh OS pages (already zero, faulted in lazily - // by the parallel kernels) instead of a single-threaded multi-GB memset up front - _arena = (nint)NativeMemory.AllocZeroed((nuint)Math.Max(_arenaBytes, 64)); - _ownsArena = true; - if (_arenaBytes > 0) GC.AddMemoryPressure(_arenaBytes); - } - else _arena = arena; - _offsets = plan.BufOffset; - _inputValue = plan.InputValue; - _outputValue = plan.OutputValue; - _nodeTicks = new long[plan.Nodes.Length]; - CpuKernels.Pack(plan, null); - } - /// Input tensor shape (e.g. [1,3,H,W]). - public int[] InputShape => _plan.Values[_inputValue].Shape; - /// Output tensor shape (e.g. [1,3,H,W]). - public int[] OutputShape => _plan.Values[_outputValue].Shape; - - /// The writable input window inside the arena (NCHW float). - public Span Input => GetSpan(_inputValue); - /// The output tensor after . - public ReadOnlySpan Output => GetSpan(_outputValue); - - private Span GetSpan(int valueId) - { - CValue v = _plan.Values[valueId]; - if (v.Buf >= 0) - return new Span(Arena + _offsets[v.Buf], checked((int)v.ElementCount)); - if (v.Floats != null) return v.Floats; - throw new InvalidOperationException($"Value '{v.Name}' has no float storage."); - } - - private ReadOnlySpan GetIn(int valueId) - { - if (valueId < 0) return ReadOnlySpan.Empty; - return GetSpan(valueId); - } - - /// Runs the whole graph. Input must already be written to . - public void Run() - { - ExecNode[] nodes = _plan.Nodes; - CValue[] values = _plan.Values; - for (int i = 0; i < nodes.Length; i++) - { - ExecNode n = nodes[i]; - long t0 = s_profile ? System.Diagnostics.Stopwatch.GetTimestamp() : 0; - try { Exec(n); } - catch (Exception ex) - { - string In(int v) => v < 0 ? "-" : $"{values[v].Name}:{string.Join('x', values[v].Shape)}"; - throw new InvalidOperationException( - $"node[{i}] {n.Op} '{n.Name}' in=[{string.Join(", ", n.In.Select(In))}] out=[{string.Join(", ", n.Out.Select(In))}]", ex); - } - if (s_profile) _nodeTicks[i] += System.Diagnostics.Stopwatch.GetTimestamp() - t0; - } - } - - private static readonly bool s_profile = Environment.GetEnvironmentVariable("GCDRNET_PROF") == "1"; - - /// Per-op-type profile (ticks), populated when GCDRNET_PROF=1. - internal Dictionary Profile() - { - var d = new Dictionary(); - for (int i = 0; i < _plan.Nodes.Length; i++) - { - ExecOp op = _plan.Nodes[i].Op; - d.TryGetValue(op, out var cur); - d[op] = cur + TimeSpan.FromSeconds((double)_nodeTicks[i] / System.Diagnostics.Stopwatch.Frequency); - } - return d; - } - - /// Per-node profile (node index, op, name, ticks→time), populated when GCDRNET_PROF=1. - internal IEnumerable<(int i, ExecNode n, TimeSpan t)> NodeProfile() => - _plan.Nodes.Select((n, i) => (i, n, TimeSpan.FromSeconds((double)_nodeTicks[i] / System.Diagnostics.Stopwatch.Frequency))); - - /// Clears the GCDRNET_PROF counters (e.g. after a warmup run). - internal void ResetProfile() => Array.Clear(_nodeTicks); - - /// Debug/test: run a single node by index. Arena must be pre-initialized. - internal void ExecNode(int i) => Exec(_plan.Nodes[i]); - - internal int[] ShapeOf(int valueId) => _plan.Values[valueId].Shape; - - /// Debug/test: read an activation value's arena region. - internal ReadOnlySpan ValueSpan(int valueId) => GetSpan(valueId); - - private void Exec(ExecNode n) - { - CValue[] values = _plan.Values; - if (_plan.Nhwc && ExecNhwc(n)) return; - switch (n.Op) - { - case ExecOp.Conv: - { - var p = (ConvP)n.P!; - CValue x = values[n.In[0]], w = values[n.In[1]], o = values[n.Out[0]]; - ReadOnlySpan bias = p.BiasValue >= 0 ? GetIn(p.BiasValue) : ReadOnlySpan.Empty; - ReadOnlySpan add = p.AddValue >= 0 ? GetIn(p.AddValue) : ReadOnlySpan.Empty; - Ops.Conv(GetIn(n.In[0]), x.Shape, GetIn(n.In[1]), w.Shape, bias, - GetSpan(n.Out[0]), o.Shape, p, add); - break; - } - case ExecOp.MaxPool: - { - var p = (PoolP)n.P!; - Ops.MaxPool(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p); - break; - } - case ExecOp.Elementwise: - { - var p = (EwP)n.P!; - Ops.Elementwise(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetIn(n.In[1]), values[n.In[1]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p.Op, p.Act); - break; - } - case ExecOp.Relu: Ops.Relu(GetIn(n.In[0]), GetSpan(n.Out[0])); break; - case ExecOp.Sigmoid: Ops.Sigmoid(GetIn(n.In[0]), GetSpan(n.Out[0])); break; - case ExecOp.Erf: Ops.Erf(GetIn(n.In[0]), GetSpan(n.Out[0])); break; - case ExecOp.Gelu: Ops.Gelu(GetIn(n.In[0]), GetSpan(n.Out[0])); break; - case ExecOp.LayerNorm: - { - var p = (LnP)n.P!; - Ops.LayerNorm(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetIn(n.In[1]), GetIn(n.In[2]), GetSpan(n.Out[0]), p.Eps); - break; - } - case ExecOp.Gemm: - { - var p = (GemmP)n.P!; - ReadOnlySpan bias = p.BiasValue >= 0 ? GetIn(p.BiasValue) : ReadOnlySpan.Empty; - Ops.Gemm(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetIn(n.In[1]), values[n.In[1]].PhysShape, bias, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p); - break; - } - case ExecOp.ResizeBilinear: - { - var p = (ResizeP)n.P!; - Ops.ResizeBilinear(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p.Scales, - p.FuseRelu ? ActivationKind.Relu : ActivationKind.None); - break; - } - case ExecOp.Pad: - { - var p = (PadP)n.P!; - Ops.Pad(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p); - break; - } - case ExecOp.Transpose: - { - var p = (TransposeP)n.P!; - Ops.Transpose(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p.Perm); - break; - } - case ExecOp.Slice: - { - var p = (SliceP)n.P!; - Ops.Slice(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p); - break; - } - case ExecOp.Concat: - { - var p = (ConcatP)n.P!; - var ins = new FBuf[n.In.Length]; - var shapes = new int[n.In.Length][]; - for (int k = 0; k < n.In.Length; k++) - { - CValue cv = values[n.In[k]]; - ins[k] = cv.Buf >= 0 - ? new FBuf(Arena + _offsets[cv.Buf], checked((int)cv.ElementCount)) - : new FBuf(cv.Floats!, 0, checked((int)cv.ElementCount)); - shapes[k] = cv.PhysShape; - } - Ops.Concat(ins, shapes, GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p.Axis); - break; - } - case ExecOp.ChannelShift: - { - var p = (ShiftP)n.P!; - Ops.ChannelShift(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p); - break; - } - case ExecOp.Copy: - GetIn(n.In[0]).CopyTo(GetSpan(n.Out[0])); - break; - default: - throw new NotSupportedException($"ExecOp {n.Op}"); - } - } - - /// Channels-last kernels for the layout-sensitive ops; false = generic (storage-shape) path. - private bool ExecNhwc(ExecNode n) - { - CValue[] values = _plan.Values; - switch (n.Op) - { - case ExecOp.Conv: - { - var p = (ConvP)n.P!; - int[] i = values[n.In[0]].PhysShape, o = values[n.Out[0]].PhysShape; - if (p.Pool is not null) - { - Nhwc.ConvPool(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), o[1], o[2], o[3], - p, (NhwcConvPlan)p.KernelPlan!); - return true; - } - ReadOnlySpan add = p.AddValue >= 0 ? GetIn(p.AddValue) : ReadOnlySpan.Empty; - Nhwc.Conv(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), o[1], o[2], o[3], - p, (NhwcConvPlan)p.KernelPlan!, add); - return true; - } - case ExecOp.MaxPool: - { - int[] i = values[n.In[0]].PhysShape, o = values[n.Out[0]].PhysShape; - Nhwc.MaxPool(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), o[1], o[2], (PoolP)n.P!); - return true; - } - case ExecOp.ResizeBilinear: - { - var p = (ResizeP)n.P!; - int[] i = values[n.In[0]].PhysShape, o = values[n.Out[0]].PhysShape; - Nhwc.ResizeBilinear(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), o[1], o[2], - p.Scales[2], p.Scales[3], p.FuseRelu ? ActivationKind.Relu : ActivationKind.None, - n.In.Length > 1 ? GetIn(n.In[1]) : default, p.AddRelu); - return true; - } - case ExecOp.UpHead: - { - var p = (UpHeadP)n.P!; - int[] i = values[n.In[0]].PhysShape, o = values[n.Out[0]].PhysShape, k = values[n.In[1]].PhysShape; - Nhwc.UpHead(GetIn(n.In[0]), i[1], i[2], i[3], GetIn(n.In[1]), GetSpan(n.Out[0]), o[1], o[2], - p.Scales[2], p.Scales[3], p, (UpHeadPlan)p.KernelPlan!, k[1], k[2], k[3]); - return true; - } - case ExecOp.Pad: - { - int[] i = values[n.In[0]].PhysShape, o = values[n.Out[0]].PhysShape; - Nhwc.Pad(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), o[1], o[2], (PadP)n.P!); - return true; - } - case ExecOp.Gemm when ((GemmP)n.P!).KernelPlan is NhwcConvPlan gk: - { - var p = (GemmP)n.P!; - int[] a = values[n.In[0]].PhysShape; - int k = a[^1], m = (int)(values[n.In[0]].ElementCount / k); - Nhwc.Gemm(GetIn(n.In[0]), m, k, GetSpan(n.Out[0]), values[n.Out[0]].Shape[^1], gk, p.Act); - return true; - } - case ExecOp.ChannelShift: - { - int[] i = values[n.In[0]].PhysShape; - Nhwc.ChannelShift(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), (ShiftP)n.P!); - return true; - } - default: - return false; - } - } - - private float* Arena => _arena != 0 ? (float*)_arena : throw new ObjectDisposedException(nameof(InferenceSession)); - - /// Frees the native arena (if owned by this session). - public void Dispose() - { - Free(); - GC.SuppressFinalize(this); - } - - /// Frees the native arena if was not called. - ~InferenceSession() => Free(); - - private void Free() - { - nint p = Interlocked.Exchange(ref _arena, 0); - if (p == 0 || !_ownsArena) return; - NativeMemory.Free((void*)p); - if (_arenaBytes > 0) GC.RemoveMemoryPressure(_arenaBytes); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/ChannelShiftOp.cs b/src/Sdcb.SimdGcdrNet/Extensions/ChannelShiftOp.cs new file mode 100644 index 0000000..b7a9f64 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/ChannelShiftOp.cs @@ -0,0 +1,189 @@ +using Sdcb.SimdOnnx.Backends.Metal; +using Sdcb.SimdOnnx.Backends.Vulkan; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; + +namespace Sdcb.SimdGcdrNet.Extensions; + +internal sealed class ShiftP +{ + public int Axis; // logical spatial axis index (2 = H, 3 = W) + public required int[] GroupEnds; // cumulative channel-group end indices (len 5) + public required int[] Offsets; // per-group shift (len 5) +} + +/// Fused shift-MLP: per channel group, shift along H or W (zero-fill). In = [x], P = . +internal sealed class ChannelShiftOp : ExtOp +{ + public static readonly ChannelShiftOp Instance = new(); + public override string Name => "ChannelShift"; + + public override void RunCpu(InferenceSession s, ExecNode n) + { + var p = (ShiftP)n.P!; + CValue[] v = s.Plan.Values; + if (s.Plan.Nhwc) + { + int[] i = v[n.In[0]].PhysShape; + GcdrKernels.ChannelShift(s.GetIn(n.In[0]), i[1], i[2], i[3], s.GetSpan(n.Out[0]), p); + } + else + GcdrKernels.ChannelShiftNchw(s.GetIn(n.In[0]), v[n.In[0]].PhysShape, s.GetSpan(n.Out[0]), v[n.Out[0]].PhysShape, p); + } + + private static (uint[] pc, bool v4) Pc(int[] si, ShiftP p, bool vec4) + { + var ge = p.GroupEnds; var of = p.Offsets; + if (ge.Length > 5) throw new NotSupportedException("GPU shift: > 5 groups"); + var pc = new List { (uint)si[3], (uint)si[1], (uint)si[2], (uint)p.Axis }; + for (int i = 0; i < 5; i++) pc.Add((uint)(i < ge.Length ? ge[i] : si[3])); + for (int i = 0; i < 5; i++) pc.Add((uint)(i < of.Length ? of[i] : 0)); + return (pc.ToArray(), vec4 && si[3] % 4 == 0 && ge.All(e => e % 4 == 0)); + } + + public override IEnumerable EmitVulkan(VkGraphModel g, ExecNode n) + { + g.RequireNhwc(n.In[0]); g.RequireNhwc(n.Out[0]); + int[] si = g.Plan.Values[n.In[0]].PhysShape; // [1,h,w,c] + var (pc, v4) = Pc(si, (ShiftP)n.P!, g.Device.Vec4Eltwise); + uint count = (uint)g.Plan.Values[n.Out[0]].ElementCount / (v4 ? 4u : 1u); + yield return g.Dispatch(n, v4 ? "shift4" : "shift", [g.Bind(n.In[0]), g.Bind(n.Out[0])], + VkGraphModel.PcOf(pc), VkGraphModel.G(count), owner: GcdrExtensions.Owner); + } + + public override IEnumerable EmitMetal(MtlGraphModel g, ExecNode n) + { + g.RequireNhwc(n.In[0]); g.RequireNhwc(n.Out[0]); + int[] si = g.Plan.Values[n.In[0]].PhysShape; + var (pc, v4) = Pc(si, (ShiftP)n.P!, true); + uint count = (uint)g.Plan.Values[n.Out[0]].ElementCount / (v4 ? 4u : 1u); + yield return g.Dispatch(n, v4 ? "shift4" : "shift", [g.Bind(n.In[0]), g.Bind(n.Out[0])], + MtlGraphModel.PcOf(pc), MtlGraphModel.G(count), owner: GcdrExtensions.Owner); + } +} + +/// +/// Recognizes the shift-MLP subgraph: Pad(const ±p on H or W) -> channel Split (5 groups via +/// Slice) -> per-group roll (Slice+Slice+Concat on the spatial axis) -> Concat(axis=1) -> +/// Slice(unpad). Replaced by a single node writing the shifted [1,C,h,w]. +/// +internal sealed class FuseShiftPass : GraphPass +{ + public override PassStage Stage => PassStage.Logical; + + public override void Run(PassContext ctx) + { + List nodes = ctx.Nodes; + List values = ctx.Values; + for (int i = 0; i < nodes.Count; i++) + { + ExecNode pad = nodes[i]; + if (pad.Op != ExecOp.Pad || pad.P is not PadP pp || pp.Mode != PadMode.Constant) + continue; + int inId = pad.In[0], padOut = pad.Out[0]; + int rank = values[padOut].Shape.Length; + if (rank != 4) continue; + int padH = pp.Pads[2], padW = pp.Pads[3]; + if (pp.Pads[2 + rank] != padH || pp.Pads[3 + rank] != padW) continue; + + // channel split pieces of the padded tensor + var pieces = nodes.Where(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == padOut).ToList(); + if (pieces.Count != 5) continue; + int[] padShape = values[padOut].Shape; + // SliceP.Starts/Ends are per-Axes-position arrays. For a slice emitted from a Split + // over axis C, every axes entry is identity except axis 1 (the group range). + bool RangeAt(SliceP s, int ax, int[] inShape, out int st, out int en) + { + st = 0; en = inShape[ax]; + for (int k = 0; k < s.Axes.Length; k++) + if (s.Axes[k] == ax) { st = s.Starts[k]; en = s.Ends[k]; break; } + return s.Axes.Where(a => a != ax).All(a => + s.Starts[Array.IndexOf(s.Axes, a)] == 0 && + s.Ends[Array.IndexOf(s.Axes, a)] >= inShape[a]); + } + var pieceInfo = pieces + .Select(m => (node: m, s: (SliceP)m.P!)) + .Where(t => RangeAt(t.s, 1, padShape, out _, out _)) + .OrderBy(t => { RangeAt(t.s, 1, padShape, out int st, out _); return st; }) + .ToArray(); + if (pieceInfo.Length != 5) continue; + int[] groupEnds = pieceInfo.Select(t => { RangeAt(t.s, 1, padShape, out _, out int en); return en; }).ToArray(); + + // each piece's roll: two Slice consumers + one Concat + int[] offsets = new int[5]; + int shiftAxis = -1; + int[] pieceOutIds = pieceInfo.Select(t => t.node.Out[0]).ToArray(); + var smallConcatOuts = new List(5); + bool fail = false; + var toRemove = new List(pieces); + for (int gI = 0; gI < 5; gI++) + { + int pieceId = pieceOutIds[gI]; + var rollSlices = nodes.Where(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == pieceId).ToList(); + if (rollSlices.Count != 2) { fail = true; break; } + var concat = nodes.FirstOrDefault(m => m.Op == ExecOp.Concat && m.In.Length == 2 && + m.In.Contains(rollSlices[0].Out[0]) && m.In.Contains(rollSlices[1].Out[0])); + if (concat is null) { fail = true; break; } + var sp = (SliceP)rollSlices[0].P!; var sp2 = (SliceP)rollSlices[1].P!; + // both roll slices must slice a single spatial axis + int SliceAxis(SliceP s, int[] pieceShape) + { + int found = -1; + for (int k = 0; k < s.Axes.Length; k++) + { + int d = s.Axes[k]; + bool sliced = s.Starts[k] != 0 || s.Ends[k] < pieceShape[d]; + if (sliced && found != -1) return -1; + if (sliced) found = d; + } + return found; + } + int[] pieceShape = values[pieceId].Shape; + int ax = SliceAxis(sp, pieceShape); + int ax2 = SliceAxis(sp2, pieceShape); + // one side may be unsliced (identity roll for the 0-offset group) + if (ax < 0) ax = ax2; + if (ax2 >= 0 && ax2 != ax) ax = -1; + if (ax < 0 || (ax != 2 && ax != 3)) { fail = true; break; } + if (shiftAxis == -1) shiftAxis = ax; + else if (shiftAxis != ax) { fail = true; break; } + int L = pieceShape[ax]; + // concat order = [B, A] where A = piece[0..cut] is the "head" + var sA = rollSlices[0].Out[0] == concat.In[0] ? sp2 : sp; + var sB = rollSlices[0].Out[0] == concat.In[0] ? sp : sp2; + RangeAt(sA, ax, pieceShape, out int saSt, out int saEn); + RangeAt(sB, ax, pieceShape, out int sbSt, out int sbEn); + if (saSt != 0 || sbSt != saEn || sbEn < L) { fail = true; break; } + int cut = saEn - saSt; + // effective shift on the unpadded axis: out[y] = orig[y + s], zero-fill. + // rolled[i] = padded[(i+cut) mod L]; sliced[y] = rolled[y+padAmt] + int padAmt = shiftAxis == 2 ? padH : padW; + offsets[gI] = ((padAmt + cut) % L) - padAmt; + toRemove.AddRange(rollSlices); + toRemove.Add(concat); + smallConcatOuts.Add(concat.Out[0]); + } + ExecNode? bigConcat = fail ? null : nodes.FirstOrDefault(m => + m.Op == ExecOp.Concat && m.In.Length == 5 && smallConcatOuts.All(m.In.Contains)); + if (fail || bigConcat is null) continue; + var unpad = nodes.FirstOrDefault(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == bigConcat.Out[0]); + if (unpad is null) continue; + toRemove.Add(bigConcat); + toRemove.Add(unpad); + toRemove.Add(pad); + + var shift = new ExecNode + { + Op = ExecOp.Ext, + Ext = ChannelShiftOp.Instance, + In = [inId], + Out = unpad.Out, + P = new ShiftP { Axis = shiftAxis, GroupEnds = groupEnds, Offsets = offsets }, + SrcNode = pad.SrcNode, + Name = pad.Name + ".shift", + }; + foreach (ExecNode m in toRemove) nodes.Remove(m); + nodes.Insert(Math.Min(i, nodes.Count), shift); + } + } +} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/GcdrExtensions.cs b/src/Sdcb.SimdGcdrNet/Extensions/GcdrExtensions.cs new file mode 100644 index 0000000..bc7ce33 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/GcdrExtensions.cs @@ -0,0 +1,16 @@ +using System.Reflection; +using Sdcb.SimdOnnx.Engine; + +namespace Sdcb.SimdGcdrNet.Extensions; + +/// The GCDRNet-specific fusions (shift-MLP, upsample tail) registered with the engine. +internal static class GcdrExtensions +{ + /// Assembly embedding the extension shaders. + public static Assembly Owner { get; } = typeof(GcdrExtensions).Assembly; + + public static EngineExtensions Instance { get; } = new EngineExtensions() + .Add(new FuseShiftPass()) + .Add(new FuseUpHeadPass()) + .Add(new UpHeadSkipRecomputePass()); +} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Avx512.cs b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Avx512.cs new file mode 100644 index 0000000..df81c46 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Avx512.cs @@ -0,0 +1,88 @@ +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; +using static Sdcb.SimdOnnx.Kernels.Nhwc; + +using F32x8 = System.Runtime.Intrinsics.Vector256; +using F32x16 = System.Runtime.Intrinsics.Vector512; +using V128 = System.Runtime.Intrinsics.Vector128; +using V4 = System.Runtime.Intrinsics.Vector128; +using V256 = System.Runtime.Intrinsics.Vector256; + +namespace Sdcb.SimdGcdrNet.Extensions; + +internal static unsafe partial class GcdrKernels +{ + /// + /// for the AVX-512 tier: 16-lane resize, and the c-long oc dot runs as four + /// interleaved per-pixel FMA chains (one chain per pixel was FMA-latency bound). Each pixel keeps + /// its bias-initialized, channel-ascending chain, so outputs match the AVX2 path bit for bit. + /// + internal static void UpHead512(float* sp, int iw, int c, float* kpp, bool skipConv, DenseArgs ca, float* dp, int oh, int ow, + int[] y0, int[] y1, int[] x0, int[] x1, float[] wy, float[] wx, bool relu, bool skipRelu, + float* wp, float* bp, int oc, ActivationKind act) + { + bool v512 = c % 16 == 0; + For(oh, (long)oh * ow * c * (oc + 4), y => + { + float* s = stackalloc float[4 * c + 8]; + float* acc = stackalloc float[4 * 8]; + float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c; + var vY = new System.Numerics.Vector(wy[y]); + var vY16 = Vector512.Create(wy[y]); + float wY = wy[y]; + float* orow = dp + (long)y * ow * oc; + float* krow = kpp + (long)y * ow * c; + float[]? band = skipConv ? BandBuf((long)ow * c) : null; + fixed (float* sb = band) + { + if (skipConv) + { + ConvRow(ca, y, sb, null); // the skip row, recomputed by the same conv kernel + krow = sb; + } + for (int x = 0; x < ow; x += 4) + { + int n = Math.Min(4, ow - x); + for (int i = 0; i < n; i++) + { + int xx = x + i; + float* a = r0 + (long)x0[xx] * c, b = r0 + (long)x1[xx] * c, e = r1 + (long)x0[xx] * c, f = r1 + (long)x1[xx] * c; + if (v512) ResizePixel512(a, b, e, f, Vector512.Create(wx[xx]), vY16, c, relu, krow + (long)xx * c, skipRelu, s + i * c); + else ResizePixel(a, b, e, f, wx[xx], wY, new System.Numerics.Vector(wx[xx]), vY, c, relu, krow + (long)xx * c, skipRelu, s + i * c); + } + UpHeadDot4(s, c, wp, bp, acc, n); + for (int i = 0; i < n; i++) + { + float* o = orow + (long)(x + i) * oc; + for (int l = 0; l < oc; l++) o[l] = acc[i * 8 + l]; + } + } + if (act != ActivationKind.None) ActRow(orow, ow * oc, act, true); + } + }); + } + + /// acc[i*8 + l] = bias[l] + Σ_ch s_i[ch]·w[ch][l] for 4 pixels (s_i = s + i*c), chains interleaved. + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + internal static void UpHeadDot4(float* s, int c, float* wp, float* bp, float* acc, int n) + { + float* s0 = s, s1 = n > 1 ? s + c : s, s2 = n > 2 ? s + 2 * c : s, s3 = n > 3 ? s + 3 * c : s; + F32x8 a0 = Avx.LoadVector256(bp), a1 = a0, a2 = a0, a3 = a0; + float* w = wp; + for (int ch = 0; ch < c; ch++, w += 8) + { + F32x8 wv = Avx.LoadVector256(w); + a0 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s0 + ch), wv, a0); + a1 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s1 + ch), wv, a1); + a2 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s2 + ch), wv, a2); + a3 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s3 + ch), wv, a3); + } + Avx.Store(acc, a0); Avx.Store(acc + 8, a1); Avx.Store(acc + 16, a2); Avx.Store(acc + 24, a3); + } +} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Neon.cs b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Neon.cs new file mode 100644 index 0000000..9e9c556 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Neon.cs @@ -0,0 +1,59 @@ +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; +using static Sdcb.SimdOnnx.Kernels.Nhwc; + +using F32x8 = System.Runtime.Intrinsics.Vector256; +using F32x16 = System.Runtime.Intrinsics.Vector512; +using V128 = System.Runtime.Intrinsics.Vector128; +using V4 = System.Runtime.Intrinsics.Vector128; +using V256 = System.Runtime.Intrinsics.Vector256; + +namespace Sdcb.SimdGcdrNet.Extensions; + +internal static unsafe partial class GcdrKernels +{ + /// + /// One output row for oc <= 4, c % 4 == 0: per 4 channels the bilinear + /// lerps (fma), [relu], + [relu](skip), then fmla by element into one 4-lane accumulator against + /// the packed [c][8] weights — the resized pixel never leaves registers. + /// + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + internal static void UpHeadRowNeon(float* r0, float* r1, int* x0, int* x1, float* wx, float wy, float* krow, + int c, int ow, float* wp, float* bp, float* orow, int oc, bool relu, bool skipRelu, ActivationKind act) + { + V4 vY = Vector128.Create(wy), bias = Ld4(bp); + for (int x = 0; x < ow; x++) + { + float* a = r0 + (long)x0[x] * c, b = r0 + (long)x1[x] * c, e = r1 + (long)x0[x] * c, f = r1 + (long)x1[x] * c; + float* k = krow + (long)x * c; + V4 vX = Vector128.Create(wx[x]), acc = bias; + float* w = wp; + for (int ch = 0; ch < c; ch += 4, w += 32) + { + V4 va = Ld4(a + ch), ve = Ld4(e + ch); + V4 top = Fma4(va, AdvSimd.Subtract(Ld4(b + ch), va), vX); + V4 bot = Fma4(ve, AdvSimd.Subtract(Ld4(f + ch), ve), vX); + V4 v = Fma4(top, AdvSimd.Subtract(bot, top), vY); + if (relu) v = AdvSimd.Max(v, V4.Zero); + V4 s = Ld4(k + ch); + v = AdvSimd.Add(v, skipRelu ? AdvSimd.Max(s, V4.Zero) : s); + acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w), v, 0); + acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 8), v, 1); + acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 16), v, 2); + acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 24), v, 3); + } + acc = Act4(acc, act); + float* o = orow + (long)x * oc; + o[0] = acc.ToScalar(); + if (oc > 1) o[1] = acc.GetElement(1); + if (oc > 2) o[2] = acc.GetElement(2); + if (oc > 3) o[3] = acc.GetElement(3); + } + } +} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.PackedSimd.cs b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.PackedSimd.cs new file mode 100644 index 0000000..35f5daf --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.PackedSimd.cs @@ -0,0 +1,59 @@ +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; +using static Sdcb.SimdOnnx.Kernels.Nhwc; + +using F32x8 = System.Runtime.Intrinsics.Vector256; +using F32x16 = System.Runtime.Intrinsics.Vector512; +using V128 = System.Runtime.Intrinsics.Vector128; +using V4 = System.Runtime.Intrinsics.Vector128; +using V256 = System.Runtime.Intrinsics.Vector256; + +namespace Sdcb.SimdGcdrNet.Extensions; + +internal static unsafe partial class GcdrKernels +{ + /// + /// One output row for oc <= 4, c % 4 == 0 (see ): + /// per 4 channels the bilinear lerps, [relu], + [relu](skip), then each channel is splatted back + /// from a 16-byte spill (WASM SIMD has no by-element multiply) into one 4-lane accumulator. + /// Same per-element ops as the unfused Vector<float> path. + /// + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + internal static void UpHeadRowPk(float* r0, float* r1, int* x0, int* x1, float* wx, float wy, float* krow, + int c, int ow, float* wp, float* bp, float* orow, int oc, bool relu, bool skipRelu, ActivationKind act) + { + V4 vY = PackedSimd.Splat(wy), bias = LdP(bp); + float* s = stackalloc float[4]; + for (int x = 0; x < ow; x++) + { + float* a = r0 + (long)x0[x] * c, b = r0 + (long)x1[x] * c, e = r1 + (long)x0[x] * c, f = r1 + (long)x1[x] * c; + float* k = krow + (long)x * c; + V4 vX = PackedSimd.Splat(wx[x]), acc = bias; + float* w = wp; + for (int ch = 0; ch < c; ch += 4, w += 32) + { + V4 va = LdP(a + ch), ve = LdP(e + ch); + V4 top = MadP(va, PackedSimd.Subtract(LdP(b + ch), va), vX); + V4 bot = MadP(ve, PackedSimd.Subtract(LdP(f + ch), ve), vX); + V4 v = MadP(top, PackedSimd.Subtract(bot, top), vY); + if (relu) v = PackedSimd.PseudoMax(V4.Zero, v); + V4 sk = LdP(k + ch); + v = PackedSimd.Add(v, skipRelu ? PackedSimd.PseudoMax(V4.Zero, sk) : sk); + PackedSimd.Store(s, v); + acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s), LdP(w)); + acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 1), LdP(w + 8)); + acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 2), LdP(w + 16)); + acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 3), LdP(w + 24)); + } + PackedSimd.Store(s, ActP(acc, act)); + float* o = orow + (long)x * oc; + for (int l = 0; l < oc; l++) o[l] = s[l]; + } + } +} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.cs b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.cs new file mode 100644 index 0000000..eb01525 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.cs @@ -0,0 +1,312 @@ +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; +using static Sdcb.SimdOnnx.Kernels.Nhwc; + +using F32x8 = System.Runtime.Intrinsics.Vector256; +using F32x16 = System.Runtime.Intrinsics.Vector512; +using V128 = System.Runtime.Intrinsics.Vector128; +using V4 = System.Runtime.Intrinsics.Vector128; +using V256 = System.Runtime.Intrinsics.Vector256; + +namespace Sdcb.SimdGcdrNet.Extensions; + +/// Packed UpHead weights: [c][8] oc lanes (zero-padded) + bias[8]. +internal sealed class UpHeadPlan +{ + public required float[] W; + public required float[] Bias; + public int Oc, C; +} + + +/// +/// CPU kernels of the GCDR extension ops (, ), +/// built on the engine's NHWC helpers (). +/// +internal static unsafe partial class GcdrKernels +{ + private static readonly ParallelOptions s_po = new() { MaxDegreeOfParallelism = Avx2Kernels.Threads }; + + /// Packs UpHead weights [oc, c, 1, 1] as [c][8] (oc lanes, zero-padded) + bias[8]. + public static UpHeadPlan PackUpHead(ReadOnlySpan w, int oc, int c, ReadOnlySpan bias) + { + if (oc > 8) throw new NotSupportedException($"UpHead supports oc <= 8 (got {oc})."); + float[] pw = new float[c * 8], pb = new float[8]; + for (int o = 0; o < oc; o++) + for (int ch = 0; ch < c; ch++) pw[ch * 8 + o] = w[o * c + ch]; + if (!bias.IsEmpty) bias.CopyTo(pb); + return new UpHeadPlan { W = pw, Bias = pb, Oc = oc, C = c }; + } + + /// + /// out[y,x,o] = act(bias[o] + Σ_c w[o,c]·s[c]), s = [relu](resize(src)) + [relu](skip): the + /// full-resolution resize/add tensors live only in a per-pixel stack buffer. Per-element + /// math follows the unfused nodes (bias-initialized, channel-ascending accumulation). + /// + public static void UpHead(ReadOnlySpan src, int ih, int iw, int c, ReadOnlySpan skip, + Span dst, int oh, int ow, float sy, float sx, UpHeadP p, UpHeadPlan kp, + int skipH = 0, int skipW = 0, int skipC = 0) + { + // SkipConv: `skip` is the conv's input [skipH, skipW, skipC]; each task recomputes + // its skip row (same direct-conv kernel → same values) instead of reading a stored one + ConvP? sc = p.SkipConv; + var skp = sc?.KernelPlan as NhwcConvPlan; + if (kp.C != c || dst.Length != (long)oh * ow * kp.Oc + || (sc is null ? skip.Length != (long)oh * ow * c + : skp is null || skp.Depthwise || skp.WinoU is not null || sc.AddValue >= 0 || skip.Length != (long)skipH * skipW * skipC)) + throw new NotSupportedException("UpHead shape mismatch."); + int[] y0 = new int[oh], y1 = new int[oh], x0 = new int[ow], x1 = new int[ow]; + float[] wy = new float[oh], wx = new float[ow]; + ResizeAxis(oh, ih, sy, y0, y1, wy); + ResizeAxis(ow, iw, sx, x0, x1, wx); + bool relu = p.ResizeRelu, skipRelu = p.SkipRelu, avx = UseAvx; + int oc = kp.Oc; + bool neonUp = UseNeon && c % 4 == 0 && oc <= 4; + bool pkUp = UsePacked && c % 4 == 0 && oc <= 4; + ActivationKind act = p.Act; + int[] rm = [], cm = []; + float[] padVec = []; + if (sc is not null) + { + rm = Map(oh, sc.Kernel[0], sc.Strides[0], sc.Pads[0], skipH, sc.PadMode); + cm = Map(ow, sc.Kernel[1], sc.Strides[1], sc.Pads[1], skipW, sc.PadMode); + padVec = new float[skipC * sc.Kernel[1] + Ob]; // one constant-pad kernel row (Dense512Row) + Array.Fill(padVec, sc.PadValue); + } + fixed (float* sp0 = src, dp0 = dst, kp0 = skip, wp0 = kp.W, bp0 = kp.Bias) + fixed (float* cw = skp?.W, cb = skp?.Bias, pv = padVec) + fixed (int* rmp = rm, cmp = cm) + { + float* sp = sp0, dp = dp0, kpp = kp0, wp = wp0, bp = bp0; + var ca = sc is null ? default + : new DenseArgs(kp0, cw, cb, pv, rmp, cmp, skipW, skipC, ow, c, sc.Kernel[0], sc.Kernel[1], skp!.Blocks, sc.Act, avx); + if (UseAvx512) + { + UpHead512(sp, iw, c, kpp, sc is not null, ca, dp, oh, ow, y0, y1, x0, x1, wy, wx, relu, skipRelu, wp, bp, oc, act); + return; + } + For(oh, (long)oh * ow * c * (oc + 4), y => + { + float* s = stackalloc float[c + 8]; + float* acc = stackalloc float[8]; + float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c; + var vY = new Vector(wy[y]); + float wY = wy[y]; + float* orow = dp + (long)y * ow * oc; + float[]? band = sc is null ? null : BandBuf((long)ow * c); + fixed (float* sb = band) + { + float* krow = kpp + (long)y * ow * c; + if (sc is not null) + { + ConvRow(ca, y, sb, null); + krow = sb; + } + if (neonUp) + { + fixed (int* xa = x0, xb = x1) + fixed (float* wxp = wx) + UpHeadRowNeon(r0, r1, xa, xb, wxp, wY, krow, c, ow, wp, bp, orow, oc, relu, skipRelu, act); + return; + } + if (pkUp) + { + fixed (int* xa = x0, xb = x1) + fixed (float* wxp = wx) + UpHeadRowPk(r0, r1, xa, xb, wxp, wY, krow, c, ow, wp, bp, orow, oc, relu, skipRelu, act); + return; + } + for (int x = 0; x < ow; x++) + { + ResizePixel(r0 + (long)x0[x] * c, r0 + (long)x1[x] * c, r1 + (long)x0[x] * c, r1 + (long)x1[x] * c, + wx[x], wY, new Vector(wx[x]), vY, c, relu, krow + (long)x * c, skipRelu, s); + float* o = orow + (long)x * oc; + if (avx) + { + F32x8 a = Avx.LoadVector256(bp); + float* w = wp; + for (int ch = 0; ch < c; ch++, w += 8) + a = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s + ch), Avx.LoadVector256(w), a); + Avx.Store(acc, a); + } + else if (UseNeon) + { + V128 a = AdvSimd.LoadVector128(bp), b = AdvSimd.LoadVector128(bp + 4); + float* w = wp; + for (int ch = 0; ch < c; ch++, w += 8) + { + V128 v = AdvSimd.LoadAndReplicateToVector128(s + ch); + a = AdvSimd.FusedMultiplyAdd(a, AdvSimd.LoadVector128(w), v); + b = AdvSimd.FusedMultiplyAdd(b, AdvSimd.LoadVector128(w + 4), v); + } + AdvSimd.Store(acc, a); AdvSimd.Store(acc + 4, b); + } + else + { + for (int l = 0; l < 8; l++) acc[l] = bp[l]; + float* w = wp; + for (int ch = 0; ch < c; ch++, w += 8) + { + float v = s[ch]; + for (int l = 0; l < oc; l++) acc[l] += v * w[l]; + } + } + for (int l = 0; l < oc; l++) o[l] = acc[l]; + } + if (act != ActivationKind.None) ActRow(orow, ow * oc, act, avx); + } + }); + } + } + + + // out[y,x,c] = in[y+off_g, x, c] (logical axis 2) or in[y, x+off_g, c] (axis 3), zero-fill + public static void ChannelShift(ReadOnlySpan src, int h, int w, int c, + Span dst, ShiftP p) + { + int[] ends = p.GroupEnds, offs = p.Offsets; + bool alongH = p.Axis == 2; + fixed (float* sp0 = src, dp0 = dst) + { + float* sp = sp0, dp = dp0; + For(h, (long)h * w * c, y => + { + for (int x = 0; x < w; x++) + { + float* o = dp + ((long)y * w + x) * c; + int g0 = 0; + for (int g = 0; g < offs.Length; g++) + { + int g1 = ends[g]; + int sy = alongH ? y + offs[g] : y, sx = alongH ? x : x + offs[g]; + if ((uint)sy < (uint)h && (uint)sx < (uint)w) + Copy(sp + ((long)sy * w + sx) * c + g0, o + g0, g1 - g0); + else + Fill(o + g0, g1 - g0, 0f); + g0 = g1; + } + if (g0 < c) Copy(sp + ((long)y * w + x) * c + g0, o + g0, c - g0); + } + }); + } + } + // out[c,y,x] = in[c, y+dy_g] (axis=2) or in[c,y,x+dx_g] (axis=3), zero-fill + public static void ChannelShiftNchw(ReadOnlySpan src, int[] inShape, Span dst, + int[] outShape, ShiftP p) + { + int c = inShape[1], h = inShape[2], w = inShape[3]; + if (Avx2.IsSupported && Fma.IsSupported && c * h * w >= 8192) + { + ChannelShiftNchwAvx2(src, c, h, w, dst, p.Axis, p.Offsets, p.GroupEnds); + return; + } + int g0 = 0; + for (int g = 0; g < p.Offsets.Length; g++) + { + int g1 = p.GroupEnds[g]; + int off = p.Offsets[g]; + for (int ch = g0; ch < g1; ch++) + { + ReadOnlySpan s = src.Slice(ch * h * w, h * w); + Span d = dst.Slice(ch * h * w, h * w); + if (p.Axis == 2) + { + for (int y = 0; y < h; y++) + { + int sy = y + off; + if ((uint)sy < (uint)h) s.Slice(sy * w, w).CopyTo(d.Slice(y * w, w)); + else d.Slice(y * w, w).Clear(); + } + } + else if (Math.Abs(off) >= w) + { + d.Clear(); // shifted fully out of range -> all zeros + } + else + { + // out[c,y,x] = in[c,y,x+off], zero-fill — same convention as axis 2 + if (off >= 0) + for (int y = 0; y < h; y++) + { + ReadOnlySpan sr = s.Slice(y * w, w); + Span dr = d.Slice(y * w, w); + sr.Slice(off).CopyTo(dr.Slice(0, w - off)); + dr.Slice(w - off).Clear(); + } + else + for (int y = 0; y < h; y++) + { + ReadOnlySpan sr = s.Slice(y * w, w); + Span dr = d.Slice(y * w, w); + sr.Slice(0, w + off).CopyTo(dr.Slice(-off)); + dr.Slice(0, -off).Clear(); + } + } + } + g0 = g1; + } + } + private static void ChannelShiftNchwAvx2(ReadOnlySpan src, int c, int h, int w, + Span dst, int axis, int[] offsets, int[] groupEnds) + { + fixed (float* sp0 = src, dp0 = dst) + fixed (int* offp = offsets, endp = groupEnds) + { + float* sp = sp0, dp = dp0; + int* offa = offp, enda = endp; + int groups = offsets.Length; + Parallel.For(0, c * h, s_po, t => + { + int ch = t / h, y = t % h; + int off = 0; + for (int g = 0, g0 = 0; g < groups; g++) + { + int g1 = enda[g]; + if (ch >= g0 && ch < g1) { off = offa[g]; break; } + g0 = g1; + } + float* sr = sp + (ch * h + y) * w; + float* dr = dp + (ch * h + y) * w; + if (axis == 2) + { + int sy = y + off; + if ((uint)sy < (uint)h) + Buffer.MemoryCopy(sp + (ch * h + sy) * w, dr, w * 4L, w * 4L); + else + { + F32x8 z = F32x8.Zero; + int x = 0; + for (; x + 8 <= w; x += 8) Avx.Store(dr + x, z); + for (; x < w; x++) dr[x] = 0f; + } + } + else if (Math.Abs(off) >= w) + { + int x = 0; + for (; x + 8 <= w; x += 8) Avx.Store(dr + x, F32x8.Zero); + for (; x < w; x++) dr[x] = 0f; + } + else if (off >= 0) + { + // dr[x] = sr[x+off]; tail zero + Buffer.MemoryCopy(sr + off, dr, (w - off) * 4L, (w - off) * 4L); + int x = w - off; + for (; x < w; x++) dr[x] = 0f; + } + else + { + // dr[x] = sr[x+off] with off<0: head zero, then copy + int x = 0; + for (; x + 8 <= -off; x += 8) Avx.Store(dr + x, F32x8.Zero); + for (; x < -off; x++) dr[x] = 0f; + Buffer.MemoryCopy(sr, dr - off, (w + off) * 4L, (w + off) * 4L); + } + }); + } + }} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/UpHeadOp.cs b/src/Sdcb.SimdGcdrNet/Extensions/UpHeadOp.cs new file mode 100644 index 0000000..d23536a --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/UpHeadOp.cs @@ -0,0 +1,279 @@ +using Sdcb.SimdOnnx.Backends.Metal; +using Sdcb.SimdOnnx.Backends.Vulkan; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; + +namespace Sdcb.SimdGcdrNet.Extensions; + +/// +/// Fused NHWC network tail (In = [x, skip, w, bias?]): +/// out[o] = act(bias[o] + Σ_c w[o,c]·s[c]) with s = [relu](resize(x)) + [relu](skip). +/// Replaces Resize → (Relu) → Add → 1x1 Conv → (act) so the full-resolution +/// intermediates are never stored. +/// +internal sealed class UpHeadP +{ + public required float[] Scales; + public bool ResizeRelu; + public bool SkipRelu; + public ActivationKind Act; + public object? KernelPlan; // backend-private packed weights + // In[1] is the input of this (still scheduled) conv and the skip is recomputed, + // so the full-resolution skip tensor need not stay live + public ConvP? SkipConv; + public int SkipW = -1, SkipB = -1; // the skip conv's weight / bias value ids + + public override string ToString() => $"rrelu={ResizeRelu} srelu={SkipRelu} act={Act}" + + (SkipConv is { } sc ? $" skip=conv{sc.Kernel[0]}x{sc.Kernel[1]}(In[1]) sact={sc.Act}" : ""); +} + +internal sealed class UpHeadOp : ExtOp +{ + public static readonly UpHeadOp Instance = new(); + public override string Name => "UpHead"; + + public override IEnumerable ParamRefs(ExecNode n) + { + var p = (UpHeadP)n.P!; + if (p.SkipW >= 0) yield return p.SkipW; + if (p.SkipB >= 0) yield return p.SkipB; + } + + public override void PlanLayout(LayoutPlanner lp, ExecNode n) + { + // activations channels-last, weights stay logical + if (lp.Values[n.In[0]].Shape.Length != 4 || lp.Values[n.In[1]].Shape.Length != 4) + throw new NotSupportedException($"layout: UpHead '{n.Name}' is not rank-4."); + lp.Need(n, 0, Layout.Nhwc); + lp.Need(n, 1, Layout.Nhwc); + lp.Values[n.Out[0]].Layout = Layout.Nhwc; + } + + public override void PackCpu(ExecPlan plan, ExecNode n, CpuKernels.Cache? cache) + { + var up = (UpHeadP)n.P!; + if (up.KernelPlan is not null) return; + CValue w = plan.Values[n.In[2]]; + float[]? b = n.In.Length > 3 ? plan.Values[n.In[3]].Floats : null; + object kp = CpuKernels.Get(cache, n, w.Floats!, b, () => GcdrKernels.PackUpHead(w.Floats!, w.Shape[0], w.Shape[1], b)); + Interlocked.CompareExchange(ref up.KernelPlan, kp, null); + } + + public override void RunCpu(InferenceSession s, ExecNode n) + { + var p = (UpHeadP)n.P!; + CValue[] v = s.Plan.Values; + int[] i = v[n.In[0]].PhysShape, o = v[n.Out[0]].PhysShape, k = v[n.In[1]].PhysShape; + GcdrKernels.UpHead(s.GetIn(n.In[0]), i[1], i[2], i[3], s.GetIn(n.In[1]), s.GetSpan(n.Out[0]), o[1], o[2], + p.Scales[2], p.Scales[3], p, (UpHeadPlan)p.KernelPlan!, k[1], k[2], k[3]); + } + + public override IEnumerable<(int value, ConstKind kind)> GpuConsts(ExecNode n) + { + var up = (UpHeadP)n.P!; + yield return (n.In[2], ConstKind.Raw); + if (n.In.Length > 3) yield return (n.In[3], ConstKind.F32); + if (up.SkipConv is not null) + { + yield return (up.SkipW, ConstKind.ConvW); + if (up.SkipB >= 0) yield return (up.SkipB, ConstKind.F32); + } + } + + private static (uint c, uint oc, uint flags) Common(CValue[] V, ExecNode n, string backend) + { + var p = (UpHeadP)n.P!; + int[] si = V[n.In[0]].PhysShape, so = V[n.Out[0]].PhysShape; + uint c = (uint)si[3], oc = (uint)so[3]; + if (oc > 8 || c > 128 || c % 4 != 0) throw new NotSupportedException($"{backend} UpHead: c={c} oc={oc}"); + uint act = p.Act switch + { + ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, ActivationKind.None => 0u, + _ => throw new NotSupportedException($"{backend} UpHead: {p.Act}"), + }; + uint flags = (p.ResizeRelu ? 1u : 0u) | (p.SkipRelu ? 2u : 0u) | (n.In.Length > 3 ? 4u : 0u) | (act << 3); + return (c, oc, flags); + } + + private static byte[] SkipPc(CValue[] V, ExecNode n, uint c, string backend) + { + var p = (UpHeadP)n.P!; + var sc = p.SkipConv!; + // skip recomputed from the stem conv's input (GraphCompiler.GpuSmallConv, c <= 32) + int[] xi = V[n.In[1]].PhysShape, cw = V[p.SkipW].Shape, so = V[n.Out[0]].PhysShape; + if (!GraphCompiler.GpuSmallConv(sc, cw, xi[3]) || cw[0] != c || c > 32 || xi[1] != so[1] || xi[2] != so[2]) + throw new NotSupportedException($"{backend} UpHead: unsupported skip conv"); + uint cf = (sc.BiasValue >= 0 ? 1u : 0u) | (sc.PadMode == PadMode.Reflect ? 2u : 0u) + | (sc.Act == ActivationKind.Relu ? 4u : 0u); + return VkGraphModel.PcOfF(VkGraphModel.U(xi[3], sc.Kernel[0], sc.Kernel[1], sc.Pads[0], sc.Pads[1], (int)cf), + sc.PadMode == PadMode.Constant ? sc.PadValue : 0f); + } + + public override IEnumerable EmitVulkan(VkGraphModel g, ExecNode n) + { + var p = (UpHeadP)n.P!; + CValue[] V = g.Plan.Values; + g.RequireNhwc(n.In[0]); g.RequireNhwc(n.In[1]); g.RequireNhwc(n.Out[0]); + int[] si = V[n.In[0]].PhysShape, so = V[n.Out[0]].PhysShape; + var (c, oc, flags) = Common(V, n, "Vulkan"); + byte[] head = VkGraphModel.PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], p.Scales[2], p.Scales[3]); + VkBind bias = n.In.Length > 3 ? g.Bind(n.In[3], ConstKind.F32) : g.BindOrDummy(-1, ConstKind.F32); + if (p.SkipConv is not null) + { + uint np = g.Device.ConvDot && g.F16 ? 2u : 1u; + yield return g.Dispatch(n, np > 1 ? $"uphead_sc{np}" : "uphead_sc", + [g.Bind(n.In[0]), g.Bind(n.In[1]), g.Bind(n.In[2]), bias, g.Bind(n.Out[0]), + g.Bind(p.SkipW, ConstKind.ConvW), g.BindOrDummy(p.SkipB, ConstKind.F32)], + [.. head, .. SkipPc(V, n, c, "Vulkan")], + VkGraphModel.G((uint)so[1] * (((uint)so[2] + np - 1) / np)), owner: GcdrExtensions.Owner); + yield break; + } + yield return g.Dispatch(n, "uphead", + [g.Bind(n.In[0]), g.Bind(n.In[1]), g.Bind(n.In[2]), bias, g.Bind(n.Out[0])], + head, VkGraphModel.G((uint)so[1] * (uint)so[2]), owner: GcdrExtensions.Owner); + } + + public override IEnumerable EmitMetal(MtlGraphModel g, ExecNode n) + { + var p = (UpHeadP)n.P!; + CValue[] V = g.Plan.Values; + g.RequireNhwc(n.In[0]); g.RequireNhwc(n.In[1]); g.RequireNhwc(n.Out[0]); + int[] si = V[n.In[0]].PhysShape, so = V[n.Out[0]].PhysShape; + var (c, oc, flags) = Common(V, n, "Metal"); + byte[] head = MtlGraphModel.PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], p.Scales[2], p.Scales[3]); + MtlBind bias = n.In.Length > 3 ? g.Bind(n.In[3], ConstKind.F32) : g.BindOrDummy(-1, ConstKind.F32); + if (p.SkipConv is not null) + { + yield return g.Dispatch(n, "uphead_sc", + [g.Bind(n.In[0]), g.Bind(n.In[1]), g.Bind(n.In[2]), bias, g.Bind(n.Out[0]), + g.Bind(p.SkipW, ConstKind.ConvW), g.BindOrDummy(p.SkipB, ConstKind.F32)], + [.. head, .. SkipPc(V, n, c, "Metal")], + MtlGraphModel.G((uint)so[1] * (uint)so[2]), owner: GcdrExtensions.Owner); + yield break; + } + yield return g.Dispatch(n, "uphead", + [g.Bind(n.In[0]), g.Bind(n.In[1]), g.Bind(n.In[2]), bias, g.Bind(n.Out[0])], + head, MtlGraphModel.G((uint)so[1] * (uint)so[2]), owner: GcdrExtensions.Owner); + } +} + +/// +/// Resize → (Relu) → Add(·, [Relu](skip)) → 1x1 Conv (few outputs) → (Sigmoid|Relu) becomes one +/// node in NHWC plans — the network tail, where every intermediate is a +/// full-resolution tensor that would otherwise round-trip memory. The fused node takes the +/// position of the last node it replaces, so every input is produced before it. +/// +internal sealed class FuseUpHeadPass : GraphPass +{ + public override PassStage Stage => PassStage.Logical; + + public override void Run(PassContext ctx) + { + if (!ctx.Options.Nhwc || Env.Flag("NOFUSE_UP") == "1") return; + List nodes = ctx.Nodes; + List values = ctx.Values; + + // resolve add(resize(x), other): (resize node, skip value, skip relu node or null) + (ExecNode rs, int skip, ExecNode? relu)? MatchAdd(ExecNode add) + { + if (add.Op != ExecOp.Elementwise || add.In.Length != 2 || add.P is not EwP { Op: EwOp.Add, Act: ActivationKind.None }) + return null; + for (int k = 0; k < 2; k++) + { + int u = add.In[k], o = add.In[1 - k]; + int ri = ctx.Producer(u); + if (ri < 0 || nodes[ri].Op != ExecOp.ResizeBilinear || nodes[ri].In.Length != 1) continue; + if (ctx.Uses(u) != 1 || !ctx.SameShape(u, add.Out[0]) || !ctx.SameShape(o, add.Out[0]) || values[o].Buf < 0) continue; + int pi = ctx.Producer(o); + if (pi >= 0 && nodes[pi].Op == ExecOp.Relu && nodes[pi].In.Length == 1 && ctx.Uses(o) == 1 + && values[nodes[pi].In[0]].Buf >= 0 && ctx.SameShape(nodes[pi].In[0], o)) + return (nodes[ri], nodes[pi].In[0], nodes[pi]); + return (nodes[ri], o, null); + } + return null; + } + + for (int ci = 0; ci < nodes.Count; ci++) + { + ExecNode c = nodes[ci]; + if (c.Op != ExecOp.Conv || c.P is not ConvP cp || cp.Group != 1 || cp.Act != ActivationKind.None + || cp.AddValue >= 0 || cp.Kernel[0] != 1 || cp.Kernel[1] != 1 || cp.Strides.Any(s => s != 1) + || cp.Pads.Any(p => p != 0) || cp.Dilations.Any(d => d != 1)) continue; + CValue w = values[c.In[1]]; + // kernel limits (oc lanes, staged weights, vec4 channels) + if (w.Floats is null || w.Shape.Length != 4 || w.Shape[0] > 8 || w.Shape[1] > 128 || w.Shape[1] % 4 != 0) continue; + if (cp.BiasValue >= 0 && values[cp.BiasValue].Floats is null) continue; + int s = c.In[0]; + int ai = ctx.Producer(s); + if (ai < 0 || ctx.Uses(s) != 1 || values[s].Shape.Length != 4) continue; + ExecNode addNode = nodes[ai]; + if (MatchAdd(addNode) is not { } mt) continue; + var (rs, skip, relu) = mt; + + ExecNode? act = null; + int y = c.Out[0]; + if (ctx.Uses(y) == 1) + act = nodes.FirstOrDefault(m => m.In.Length == 1 && m.In[0] == y + && m.Op is ExecOp.Sigmoid or ExecOp.Relu && values[m.Out[0]].Buf >= 0); + var head = new ExecNode + { + Op = ExecOp.Ext, + Ext = UpHeadOp.Instance, + In = cp.BiasValue >= 0 ? [rs.In[0], skip, c.In[1], cp.BiasValue] : [rs.In[0], skip, c.In[1]], + Out = [act?.Out[0] ?? y], + P = new UpHeadP + { + Scales = ((ResizeP)rs.P!).Scales, + ResizeRelu = ((ResizeP)rs.P!).FuseRelu, + SkipRelu = relu is not null, + Act = act is null ? ActivationKind.None : act.Op == ExecOp.Sigmoid ? ActivationKind.Sigmoid : ActivationKind.Relu, + }, + SrcNode = c.SrcNode, + Name = c.Name + ".uphead", + }; + nodes[ci] = head; + nodes.Remove(rs); nodes.Remove(addNode); + if (relu is not null) nodes.Remove(relu); + if (act is not null) nodes.Remove(act); + ci = -1; // indices shifted; rescan (few nodes, cheap) + } + } +} + +/// +/// UpHead skip recompute (after layout planning): when the skip is a cheap direct conv of a +/// smaller input (stem conv: 3/6 → 16/32 channels), UpHead reads the conv's input and recomputes +/// the skip rows, so the skip need not stay live across the network. GPU plans only get it for the +/// small-ic convs their shaders stage (). +/// +internal sealed class UpHeadSkipRecomputePass : GraphPass +{ + public override PassStage Stage => PassStage.Physical; + + public override void Run(PassContext ctx) + { + List nodes = ctx.Nodes; + List values = ctx.Values; + foreach (ExecNode u in nodes) + { + if (u.P is not UpHeadP up || up.SkipConv is not null) continue; + int s = u.In[1], ci = ctx.Producer(s); + if (ci < 0 || nodes[ci].P is not ConvP cp || cp.Group != 1 || cp.AddValue >= 0 || cp.Pool is not null + || cp.Strides.Any(x => x != 1) || cp.Dilations.Any(x => x != 1)) continue; + int xin = nodes[ci].In[0]; + CValue w = values[nodes[ci].In[1]]; + if (w.Floats is null || w.Shape.Length != 4 || !ctx.IsNhwc4(s) || !ctx.IsNhwc4(xin)) continue; + int ic = values[xin].Shape[1], c = values[s].Shape[1]; + // cheap to recompute, much smaller to keep; the conv itself stays scheduled + // for its other reader(s) (and keeps the shared ConvP packed) + if (ic * 2 > c || Nhwc.WinogradEligible(cp, w.Shape, ic) || ctx.Uses(s) < 2) continue; + if (cp.BiasValue >= 0 && values[cp.BiasValue].Floats is null) continue; + bool sameHw = values[xin].Shape[2] == values[s].Shape[2] && values[xin].Shape[3] == values[s].Shape[3]; + if (ctx.Options.Gpu && (!GraphCompiler.GpuSmallConv(cp, w.Shape, ic) || c > 32 || !sameHw)) continue; + u.In[1] = xin; + up.SkipConv = cp; + up.SkipW = nodes[ci].In[1]; + up.SkipB = cp.BiasValue; + } + } +} diff --git a/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs b/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs index a4c12ae..4028547 100644 --- a/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs +++ b/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs @@ -1,8 +1,12 @@ using System.Collections.Concurrent; using System.Runtime.InteropServices; -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdGcdrNet.Extensions; using Sdcb.SimdGcdrNet.Kernels; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Onnx; +using VkBackend = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend; +using VkDeviceLostException = Sdcb.SimdOnnx.Backends.Vulkan.VkDeviceLostException; +using MetalBackend = Sdcb.SimdOnnx.Backends.Metal.MetalBackend; namespace Sdcb.SimdGcdrNet; @@ -51,7 +55,7 @@ public sealed class GcdrEnhancer : IDisposable // change): one pool serves every size; idle workers beyond the first are kept // only while their device memory fits the budget // (3 GB; integrated GPUs: at most a quarter of the shared device-local heap) - private static long VkIdleBudget => (long)Backends.Vulkan.VkGcdrPipeline.BudgetFor(Backends.Vulkan.VkBackend.Default.Dev); + private static long VkIdleBudget => (long)Backends.Vulkan.VkGcdrPipeline.BudgetFor(VkBackend.Default.Dev); private SizePool? _vkPool; private SizePool? _mtlPool; private long _poolsCreated, _workersCreated, _poolsEvicted; @@ -95,7 +99,7 @@ public static GcdrEnhancer FromStreams(Stream gcnetOnnx, Stream drnetOnnx, // pipeline sessions keep I/O in plan layout: the glue kernels below read/write // channels-last directly, so the plan has no boundary relayouts - private static readonly CompileOptions CpuPlan = new(CompileOptions.DefaultNhwc, PhysicalIo: true); + private static readonly CompileOptions CpuPlan = new(CompileOptions.DefaultNhwc, PhysicalIo: true, Extensions: GcdrExtensions.Instance); private static bool CpuNhwc => CpuPlan.Nhwc; private static long Key(int h, int w) => (long)h * 65536 + w; @@ -143,7 +147,7 @@ private SizePool CpuPool() { if (_vkPool is { } p) return p; p = new SizePool(MaxPerSize, - () => { Interlocked.Increment(ref _workersCreated); return Backends.Vulkan.VkBackend.Default.NewWorker(_gcModel, _drModel); }, + () => { Interlocked.Increment(ref _workersCreated); return new Backends.Vulkan.VkGcdrPipeline(VkBackend.Default, _gcModel, _drModel); }, w => w.DeviceBytes, VkIdleBudget); if (Interlocked.CompareExchange(ref _vkPool, p, null) is { } won) { p.Dispose(); return won; } Interlocked.Increment(ref _poolsCreated); @@ -154,7 +158,7 @@ private SizePool CpuPool() { if (_mtlPool is { } p) return p; p = new SizePool(MaxPerSize, - () => { Interlocked.Increment(ref _workersCreated); return Backends.Metal.MetalBackend.Default.NewWorker(_gcModel, _drModel); }, + () => { Interlocked.Increment(ref _workersCreated); return new Backends.Metal.MtlGcdrPipeline(MetalBackend.Default, _gcModel, _drModel); }, w => w.DeviceBytes, (long)Backends.Metal.MtlGcdrPipeline.ArenaBudget); if (Interlocked.CompareExchange(ref _mtlPool, p, null) is { } won) { p.Dispose(); return won; } Interlocked.Increment(ref _poolsCreated); @@ -198,15 +202,23 @@ public void Enhance(ReadOnlySpan src, int width, int height, int srcStride || (_backend == GcdrBackend.Auto && VulkanOk())); if (useVk) { - var pool = VkPool(); - var pipe = pool.Rent(); - try + for (int attempt = 0; ; attempt++) { - pipe.Enhance(src, height, width, srcStride, dst, dstStride, format); - DebugAfterEnhance?.Invoke(pipe); + var pool = VkPool(); + var pipe = pool.Rent(); + try + { + pipe.Enhance(src, height, width, srcStride, dst, dstStride, format); + DebugAfterEnhance?.Invoke(pipe); + return; + } + catch (VkDeviceLostException) when (attempt == 0) + { + // the backend re-creates its device on next use; this pool's pipelines are dead + Interlocked.CompareExchange(ref _vkPool, null, pool); + } + finally { pool.Return(pipe); } } - finally { pool.Return(pipe); } - return; } int ph = (-height) & 31, pw = (-width) & 31; int h = height + ph, w = width + pw; @@ -261,12 +273,17 @@ public void Enhance(ReadOnlySpan src, int width, int height, int srcStride /// stage-2 glue that writes the dr input, and dr then reuses everything above. /// Returns the gc arena offset and the total size, in floats. /// - internal static (long gcOff, long total) StackArenas(ExecPlan gc, ExecPlan dr, long align = 16) + internal static (long gcOff, long total) StackArenas(ExecPlan gc, ExecPlan dr, long align = 16) => + StackArenas(gc.ArenaFloats, dr, dr.ArenaFloats, align); + + /// over each stage's full arena + /// (GPU graphs append scratch / tile-read slack past the plan's values). + internal static (long gcOff, long total) StackArenas(long gcArena, ExecPlan dr, long drArena, long align = 16) { CValue din = dr.Values[dr.InputValue]; long o = dr.BufOffset[din.Buf], end = (o + din.ElementCount + align - 1) / align * align; - long gcOff = o >= gc.ArenaFloats ? 0 : end; - return (gcOff, Math.Max(dr.ArenaFloats, gcOff + gc.ArenaFloats)); + long gcOff = o >= gcArena ? 0 : end; + return (gcOff, Math.Max(drArena, gcOff + gcArena)); } // browser/WASI: unlinked P/Invokes (vulkan-1, libobjc) trap the wasm module with @@ -275,7 +292,7 @@ internal static (long gcOff, long total) StackArenas(ExecPlan gc, ExecPlan dr, l private static bool VulkanOk() { - try { _ = Sdcb.SimdGcdrNet.Backends.Vulkan.VkBackend.Default.Dev.DeviceName; return true; } + try { _ = VkBackend.Default.Dev.DeviceName; return true; } catch { return false; } } @@ -283,11 +300,11 @@ private static bool MetalOk() { try { - var dev = Backends.Metal.MetalBackend.Default.Dev; + var dev = MetalBackend.Default.Dev; // Apple7+ (M1+) for simdgroup matrix ops; Pipe() forces the embedded // MSL library compile now rather than mid-Enhance on macOS 13. Intel Macs can // claim Apple7 yet build the simdgroup-MMA conv with 0 threads/group (Pipe throws). - return dev.SupportsApple7 && dev.Pipe("bgr2nhwc") is not null && dev.Pipe("conv_mm_32_0") is not null; + return dev.SupportsApple7 && dev.Pipe("bgr2nhwc", GcdrExtensions.Owner) is not null && dev.Pipe("conv_mm_32_0") is not null; } catch { return false; } } @@ -312,7 +329,7 @@ public void Dispose() /// /// Size-agnostic CPU worker: one native arena holding both stages (see - /// ), borrowed by per-call sessions. Within + /// ), borrowed by per-call sessions. Within /// it only grows (25% headroom on regrowth), so mixed sizes /// neither reallocate nor re-fault pages; an over-budget arena (one huge image) drops /// to exact size once a size that fits comes along. diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Avx2Kernels.cs b/src/Sdcb.SimdGcdrNet/Kernels/Avx2Kernels.cs deleted file mode 100644 index c8615d5..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Avx2Kernels.cs +++ /dev/null @@ -1,970 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; - -using F32x8 = System.Runtime.Intrinsics.Vector256; -using V256 = System.Runtime.Intrinsics.Vector256; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// AVX2+FMA kernels: multi-threaded, register-tiled NCHW conv (OC4 x YB2), -/// deinterleaved stride-2 conv, depthwise, 1x1, maxpool, GEMM, LayerNorm. -/// Inputs are pinned for the duration of each call so Parallel.For lambdas -/// can capture raw pointers (spans are not capturable). Edge pixels go -/// through a scalar spill path so every pad mode stays exact. -/// -internal static unsafe partial class Avx2Kernels -{ - internal static readonly int Threads = InitThreads(); - private static int InitThreads() - { - string? s = Environment.GetEnvironmentVariable("GCDRNET_THREADS"); - return int.TryParse(s, out int t) && t > 0 ? t : Environment.ProcessorCount; - } - private static readonly ParallelOptions PO = new() { MaxDegreeOfParallelism = Threads }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 Fma(F32x8 a, F32x8 b, F32x8 c) => V256.FusedMultiplyAdd(a, b, c); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 Ld(float* p) => Avx.LoadVector256(p); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void St(float* p, F32x8 v) => Avx.Store(p, v); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 Bc(float* p) => Avx2.BroadcastScalarToVector256(p); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int Reflect(int i, int dim) - { - if (dim <= 1) return 0; - while (i < 0 || i >= dim) i = i < 0 ? -i : 2 * dim - 2 - i; - return i; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void ApplyAct(ref F32x8 v, ActivationKind act) - { - if (act == ActivationKind.Relu) v = V256.Max(v, F32x8.Zero); - else if (act == ActivationKind.Sigmoid) v = SimdMath.SigmoidCore(v); - else if (act == ActivationKind.Gelu) v = SimdMath.GeluCore(v); - } - - // lane-wise fetch with pad resolution (edge chunks only) - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 Spill8(float* sp, int ih, int iw, int sy, int sx, - PadMode mode, float padVal, float* spill) - { - for (int l = 0; l < 8; l++) - { - int px = sx + l; - spill[l] = (uint)px < (uint)iw ? sp[sy * iw + px] - : mode == PadMode.Reflect ? sp[sy * iw + Reflect(px, iw)] - : padVal; - } - return Ld(spill); - } - - private static void StoreRow(float* dp, int oStride, - int xw, ActivationKind act, F32x8 a0, F32x8 a1, F32x8 a2, F32x8 a3) - { - ApplyAct(ref a0, act); ApplyAct(ref a1, act); - ApplyAct(ref a2, act); ApplyAct(ref a3, act); - if (xw == 8) - { - St(dp, a0); - St(dp + oStride, a1); - St(dp + 2 * oStride, a2); - St(dp + 3 * oStride, a3); - return; - } - float* tmp = stackalloc float[32]; - St(tmp, a0); St(tmp + 8, a1); St(tmp + 16, a2); St(tmp + 24, a3); - for (int l = 0; l < xw; l++) - for (int j = 0; j < 4; j++) - dp[j * oStride + l] = tmp[j * 8 + l]; - } - - // ======================== dense conv 3x3 s1 ======================== - public static void Conv3x3S1(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, Span dst, - int oc, int oh, int ow, int pt, int pl, - PadMode mode, float padVal, ActivationKind act) - { - int yb = (oh + 1) / 2, ob = oc / 4; - bool hasBias = !bias.IsEmpty; - fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst) - { - float* sp = sp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, ob * yb, PO, tile => - { - Conv3x3Tile(sp, ic, ih, iw, wp, hasBias ? bp : null, dp, oc, oh, ow, - pt, pl, mode, padVal, act, tile / yb * 4, tile % yb * 2); - }); - } - } - - private static void Conv3x3Tile(float* sp, int ic, int ih, int iw, - float* wp, float* bp, float* dp, - int oc, int oh, int ow, int pt, int pl, - PadMode mode, float padVal, ActivationKind act, int o0, int y0) - { - bool two = y0 + 1 < oh; - int y1 = two ? y0 + 1 : y0; - int wStride = ic * 9; - int ostride = oh * ow; - float* spill = stackalloc float[8]; - - for (int x0 = 0; x0 < ow; x0 += 8) - { - int xw = Math.Min(8, ow - x0); - F32x8 a0, a1, a2, a3, b0, b1, b2, b3; - if (bp is null) - { - a0 = a1 = a2 = a3 = b0 = b1 = b2 = b3 = F32x8.Zero; - } - else - { - a0 = b0 = Bc(bp + o0); - a1 = b1 = Bc(bp + o0 + 1); - a2 = b2 = Bc(bp + o0 + 2); - a3 = b3 = Bc(bp + o0 + 3); - } - for (int c = 0; c < ic; c++) - { - float* sbase = sp + c * ih * iw; - float* wb = wp + (o0 * ic + c) * 9; - for (int ky = 0; ky < 3; ky++) - { - int syA = y0 + ky - pt, syB = y1 + ky - pt; - int ryA = (uint)syA < (uint)ih ? syA - : mode == PadMode.Reflect ? Reflect(syA, ih) : -1; - int ryB = !two ? -1 - : (uint)syB < (uint)ih ? syB - : mode == PadMode.Reflect ? Reflect(syB, ih) : -1; - for (int kx = 0; kx < 3; kx++) - { - int sx = x0 + kx - pl; - int wi = ky * 3 + kx; - F32x8 wv0 = Bc(wb + wi); - F32x8 wv1 = Bc(wb + wi + wStride); - F32x8 wv2 = Bc(wb + wi + 2 * wStride); - F32x8 wv3 = Bc(wb + wi + 3 * wStride); - bool interior = (uint)sx < (uint)iw && sx + 8 <= iw; - if (ryA >= 0) - { - F32x8 v = interior - ? Ld(sbase + ryA * iw + sx) - : Spill8(sbase, ih, iw, ryA, sx, mode, padVal, spill); - a0 = Fma(v, wv0, a0); a1 = Fma(v, wv1, a1); a2 = Fma(v, wv2, a2); a3 = Fma(v, wv3, a3); - } - else if (padVal != 0f) - { - F32x8 pv = V256.Create(padVal); - a0 = Fma(pv, wv0, a0); a1 = Fma(pv, wv1, a1); - a2 = Fma(pv, wv2, a2); a3 = Fma(pv, wv3, a3); - } - if (ryB >= 0) - { - F32x8 v = interior - ? Ld(sbase + ryB * iw + sx) - : Spill8(sbase, ih, iw, ryB, sx, mode, padVal, spill); - b0 = Fma(v, wv0, b0); b1 = Fma(v, wv1, b1); b2 = Fma(v, wv2, b2); b3 = Fma(v, wv3, b3); - } - else if (ryB < 0 && padVal != 0f) - { - F32x8 pv = V256.Create(padVal); - b0 = Fma(pv, wv0, b0); b1 = Fma(pv, wv1, b1); - b2 = Fma(pv, wv2, b2); b3 = Fma(pv, wv3, b3); - } - } - } - } - float* db = dp + (o0 * oh + y0) * ow + x0; - StoreRow(db, ostride, xw, act, a0, a1, a2, a3); - if (two) StoreRow(db + ow, ostride, xw, act, b0, b1, b2, b3); - } - } - - // ======================== conv 1x1 ======================== - public static void Conv1x1(ReadOnlySpan src, int ic, int hw, - ReadOnlySpan w, ReadOnlySpan bias, Span dst, - int oc, ActivationKind act) - { - int stripes = Math.Clamp(hw / 32768, 1, 8); - int stride = hw / stripes; - bool hasBias = !bias.IsEmpty; - fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst) - { - float* sp = sp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, ((oc + 3) / 4) * stripes, PO, tile => - { - int o0 = tile / stripes * 4; - int hw0 = tile % stripes * stride; - int hw1 = tile % stripes == stripes - 1 ? hw : hw0 + stride; - Conv1x1Strip(sp, ic, wp, hasBias ? bp : null, dp, - o0, Math.Min(4, oc - o0), hw0, hw1, hw, act); - }); - } - } - - private static void Conv1x1Strip(float* sp, int ic, - float* wp, float* bp, float* dp, - int o0, int ocLim, int hw0, int hw1, int hw, ActivationKind act) - { - int i = hw0; - for (; i + 8 <= hw1; i += 8) - { - F32x8 a0 = V256.Create(bp is null ? 0f : bp[o0]); - F32x8 a1 = V256.Create(ocLim > 1 && bp is not null ? bp[o0 + 1] : 0f); - F32x8 a2 = V256.Create(ocLim > 2 && bp is not null ? bp[o0 + 2] : 0f); - F32x8 a3 = V256.Create(ocLim > 3 && bp is not null ? bp[o0 + 3] : 0f); - for (int c = 0; c < ic; c++) - { - F32x8 v = Ld(sp + c * hw + i); - a0 = Fma(v, Bc(wp + o0 * ic + c), a0); - if (ocLim > 1) a1 = Fma(v, Bc(wp + (o0 + 1) * ic + c), a1); - if (ocLim > 2) a2 = Fma(v, Bc(wp + (o0 + 2) * ic + c), a2); - if (ocLim > 3) a3 = Fma(v, Bc(wp + (o0 + 3) * ic + c), a3); - } - float* db = dp + o0 * hw + i; - ApplyAct(ref a0, act); - St(db, a0); - if (ocLim > 1) { ApplyAct(ref a1, act); St(db + hw, a1); } - if (ocLim > 2) { ApplyAct(ref a2, act); St(db + 2 * hw, a2); } - if (ocLim > 3) { ApplyAct(ref a3, act); St(db + 3 * hw, a3); } - } - for (; i < hw1; i++) - for (int j = 0; j < ocLim; j++) - { - float acc = bp is null ? 0f : bp[o0 + j]; - for (int c = 0; c < ic; c++) - acc += sp[c * hw + i] * wp[(o0 + j) * ic + c]; - float r = act == ActivationKind.Relu ? MathF.Max(acc, 0f) - : act == ActivationKind.Sigmoid ? 1f / (1f + MathF.Exp(-acc)) : acc; - dp[(o0 + j) * hw + i] = r; - } - } - - // ======================== conv 3x3 s2 (patch embed) ======================== - private static readonly Vector256 Evens = Vector256.Create(0, 2, 4, 6, 0, 0, 0, 0); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 LoadStride2(float* p) - { - F32x8 a = Ld(p); - F32x8 b = Ld(p + 8); - var ea = Avx2.PermuteVar8x32(a.AsInt32(), Evens); - var eb = Avx2.PermuteVar8x32(b.AsInt32(), Evens); - return V256.Create(ea.GetLower().AsSingle(), eb.GetLower().AsSingle()); - } - - public static void Conv3x3S2(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, Span dst, - int oc, int oh, int ow, int pt, int pl, - PadMode mode, float padVal, ActivationKind act) - { - int yb = (oh + 1) / 2, ob = oc / 4; - bool hasBias = !bias.IsEmpty; - fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst) - { - float* sp = sp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, ob * yb, PO, tile => - { - ConvS2Tile(sp, ic, ih, iw, wp, hasBias ? bp : null, dp, oc, oh, ow, - pt, pl, mode, padVal, act, tile / yb * 4, tile % yb * 2); - }); - } - } - - private static void ConvS2Tile(float* sp, int ic, int ih, int iw, - float* wp, float* bp, float* dp, - int oc, int oh, int ow, int pt, int pl, - PadMode mode, float padVal, ActivationKind act, int o0, int y0) - { - bool two = y0 + 1 < oh; - int y1 = two ? y0 + 1 : y0; - int wStride = ic * 9; - int ostride = oh * ow; - float* spill = stackalloc float[16]; - - for (int x0 = 0; x0 < ow; x0 += 8) - { - int xw = Math.Min(8, ow - x0); - F32x8 a0, a1, a2, a3, b0, b1, b2, b3; - if (bp is null) - { - a0 = a1 = a2 = a3 = b0 = b1 = b2 = b3 = F32x8.Zero; - } - else - { - a0 = b0 = Bc(bp + o0); - a1 = b1 = Bc(bp + o0 + 1); - a2 = b2 = Bc(bp + o0 + 2); - a3 = b3 = Bc(bp + o0 + 3); - } - for (int c = 0; c < ic; c++) - { - float* sbase = sp + c * ih * iw; - float* wb = wp + (o0 * ic + c) * 9; - for (int ky = 0; ky < 3; ky++) - { - int syA = y0 * 2 + ky - pt, syB = y1 * 2 + ky - pt; - int ryA = (uint)syA < (uint)ih ? syA - : mode == PadMode.Reflect ? Reflect(syA, ih) : -1; - int ryB = !two ? -1 - : (uint)syB < (uint)ih ? syB - : mode == PadMode.Reflect ? Reflect(syB, ih) : -1; - for (int kx = 0; kx < 3; kx++) - { - int sx = x0 * 2 + kx - pl; - int wi = ky * 3 + kx; - F32x8 wv0 = Bc(wb + wi); - F32x8 wv1 = Bc(wb + wi + wStride); - F32x8 wv2 = Bc(wb + wi + 2 * wStride); - F32x8 wv3 = Bc(wb + wi + 3 * wStride); - bool interior = sx >= 0 && sx + 15 < iw; - if (ryA >= 0) - { - F32x8 v; - if (interior) - { - v = LoadStride2(sbase + ryA * iw + sx); - } - else - { - for (int l = 0; l < 8; l++) - { - int px = sx + 2 * l; - spill[l] = x0 + l < ow && (uint)px < (uint)iw - ? sbase[ryA * iw + px] - : x0 + l < ow && mode == PadMode.Reflect - ? sbase[ryA * iw + Reflect(px, iw)] - : padVal; - } - v = Ld(spill); - } - a0 = Fma(v, wv0, a0); a1 = Fma(v, wv1, a1); a2 = Fma(v, wv2, a2); a3 = Fma(v, wv3, a3); - } - else if (padVal != 0f) - { - F32x8 pv = V256.Create(padVal); - a0 = Fma(pv, wv0, a0); a1 = Fma(pv, wv1, a1); - a2 = Fma(pv, wv2, a2); a3 = Fma(pv, wv3, a3); - } - if (ryB >= 0) - { - F32x8 v; - if (interior) - { - v = LoadStride2(sbase + ryB * iw + sx); - } - else - { - for (int l = 0; l < 8; l++) - { - int px = sx + 2 * l; - spill[l] = x0 + l < ow && (uint)px < (uint)iw - ? sbase[ryB * iw + px] - : x0 + l < ow && mode == PadMode.Reflect - ? sbase[ryB * iw + Reflect(px, iw)] - : padVal; - } - v = Ld(spill); - } - b0 = Fma(v, wv0, b0); b1 = Fma(v, wv1, b1); b2 = Fma(v, wv2, b2); b3 = Fma(v, wv3, b3); - } - else if (ryB < 0 && padVal != 0f) - { - F32x8 pv = V256.Create(padVal); - b0 = Fma(pv, wv0, b0); b1 = Fma(pv, wv1, b1); - b2 = Fma(pv, wv2, b2); b3 = Fma(pv, wv3, b3); - } - } - } - } - float* db = dp + (o0 * oh + y0) * ow + x0; - StoreRow(db, ostride, xw, act, a0, a1, a2, a3); - if (two) StoreRow(db + ow, ostride, xw, act, b0, b1, b2, b3); - } - } - - // ======================== depthwise conv ======================== - public static void Depthwise(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, Span dst, - int oh, int ow, int kh, int kw, int pt, int pl, - PadMode mode, float padVal, ActivationKind act) - { - bool hasBias = !bias.IsEmpty; - fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst) - { - float* sp = sp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, ic * oh, PO, t => - { - int c = t / oh, y = t % oh; - float* sbase = sp + c * ih * iw; - float* drow = dp + (c * oh + y) * ow; - float* wk = wp + c * kh * kw; - float b = bp is null ? 0f : bp[c]; - float* spill = stackalloc float[16]; - for (int x0 = 0; x0 < ow; x0 += 8) - { - int xw = Math.Min(8, ow - x0); - F32x8 acc = V256.Create(b); - for (int ky = 0; ky < kh; ky++) - { - int sy = y + ky - pt; - int ry = (uint)sy < (uint)ih ? sy - : mode == PadMode.Reflect ? Reflect(sy, ih) : -1; - for (int kx = 0; kx < kw; kx++) - { - int sx = x0 + kx - pl; - float wv = wk[ky * kw + kx]; - F32x8 v; - if (ry >= 0 && sx >= 0 && sx + 8 <= iw) - { - v = Ld(sbase + ry * iw + sx); - } - else - { - for (int l = 0; l < 8; l++) - { - int px = sx + l; - spill[l] = x0 + l < ow && ry >= 0 && (uint)px < (uint)iw - ? sbase[ry * iw + px] - : x0 + l < ow && ry >= 0 && mode == PadMode.Reflect - ? sbase[ry * iw + Reflect(px, iw)] - : padVal; - } - v = Ld(spill); - } - acc = Fma(v, V256.Create(wv), acc); - } - } - ApplyAct(ref acc, act); - if (xw == 8) - { - St(drow + x0, acc); - } - else - { - St(spill, acc); - for (int l = 0; l < xw; l++) drow[x0 + l] = spill[l]; - } - } - }); - } - } - - // ======================== maxpool 2x2 s2 ======================== - public static void MaxPool2x2(ReadOnlySpan src, int c, int ih, int iw, - Span dst, int oh, int ow, bool fuseRelu) - { - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - Parallel.For(0, c * oh, PO, t => - { - int ch = t / oh, y = t % oh; - float* sbase = sp + ch * ih * iw; - float* drow = dp + (ch * oh + y) * ow; - int r0 = Math.Min(y * 2, ih - 1) * iw, r1 = Math.Min(y * 2 + 1, ih - 1) * iw; - float* spill = stackalloc float[16]; - for (int x0 = 0; x0 < ow; x0 += 8) - { - int xw = Math.Min(8, ow - x0); - int sx = x0 * 2; - F32x8 m; - if (sx + 15 < iw) - { - m = V256.Max(LoadStride2(sbase + r0 + sx), LoadStride2(sbase + r0 + sx + 1)); - m = V256.Max(m, V256.Max(LoadStride2(sbase + r1 + sx), LoadStride2(sbase + r1 + sx + 1))); - } - else - { - for (int l = 0; l < 8; l++) - { - if (x0 + l >= ow) { spill[l] = float.NegativeInfinity; continue; } - int xx = Math.Min(sx + 2 * l, iw - 2); - float mx = MathF.Max(sbase[r0 + xx], sbase[r0 + xx + 1]); - spill[l] = MathF.Max(mx, MathF.Max(sbase[r1 + xx], sbase[r1 + xx + 1])); - } - m = Ld(spill); - } - if (fuseRelu) m = V256.Max(m, F32x8.Zero); - if (xw == 8) - { - St(drow + x0, m); - } - else - { - St(spill, m); - for (int l = 0; l < xw; l++) drow[x0 + l] = spill[l]; - } - } - }); - } - } - - // ======================== GEMM (B row-major [K,N]) ======================== - public static void GemmKN(ReadOnlySpan a, int M, int K, - ReadOnlySpan b, int N, ReadOnlySpan bias, Span dst) - { - bool hasBias = !bias.IsEmpty; - fixed (float* ap0 = a, bp0 = b, cp0 = bias, dp0 = dst) - { - float* ap = ap0, bp = bp0, cp = cp0, dp = dp0; - Parallel.For(0, (M + 3) / 4, PO, t => - { - int m0 = t * 4; - GemmTile(ap, m0, Math.Min(4, M - m0), K, bp, N, hasBias ? cp : null, dp); - }); - } - } - - private static void GemmTile(float* ap, int m0, int mc, int K, - float* bp, int N, float* cp, float* dp) - { - int nv = N - N % 16; - for (int n0 = 0; n0 < nv; n0 += 16) - { - F32x8 c00, c01; - if (cp is null) - { - c00 = c01 = F32x8.Zero; - } - else - { - c00 = Ld(cp + n0); - c01 = Ld(cp + n0 + 8); - } - F32x8 c10 = c00, c11 = c01, c20 = c00, c21 = c01, c30 = c00, c31 = c01; - for (int k = 0; k < K; k++) - { - F32x8 bv0 = Ld(bp + k * N + n0); - F32x8 bv1 = Ld(bp + k * N + n0 + 8); - F32x8 av = Bc(ap + m0 * K + k); - c00 = Fma(av, bv0, c00); c01 = Fma(av, bv1, c01); - if (mc > 1) { av = Bc(ap + (m0 + 1) * K + k); c10 = Fma(av, bv0, c10); c11 = Fma(av, bv1, c11); } - if (mc > 2) { av = Bc(ap + (m0 + 2) * K + k); c20 = Fma(av, bv0, c20); c21 = Fma(av, bv1, c21); } - if (mc > 3) { av = Bc(ap + (m0 + 3) * K + k); c30 = Fma(av, bv0, c30); c31 = Fma(av, bv1, c31); } - } - float* db = dp + m0 * N + n0; - St(db, c00); St(db + 8, c01); - if (mc > 1) { St(db + N, c10); St(db + N + 8, c11); } - if (mc > 2) { St(db + 2 * N, c20); St(db + 2 * N + 8, c21); } - if (mc > 3) { St(db + 3 * N, c30); St(db + 3 * N + 8, c31); } - } - for (int n0 = nv; n0 < N; n0++) - for (int m = 0; m < mc; m++) - { - float s = cp is null ? 0f : cp[n0]; - for (int k = 0; k < K; k++) - s += ap[(m0 + m) * K + k] * bp[k * N + n0]; - dp[(m0 + m) * N + n0] = s; - } - } - - // ======================== LayerNorm ======================== - public static void LayerNorm(ReadOnlySpan x, int rows, int c, - ReadOnlySpan w, ReadOnlySpan b, Span dst, float eps) - { - fixed (float* xp0 = x, wp0 = w, bp0 = b, dp0 = dst) - { - float* xp = xp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, rows, PO, r => - { - float* xr = xp + r * c; - float* dr = dp + r * c; - F32x8 sum = F32x8.Zero, sq = F32x8.Zero; - int i = 0; - for (; i + 8 <= c; i += 8) - { - F32x8 v = Ld(xr + i); - sum = V256.Add(sum, v); - sq = Fma(v, v, sq); - } - float s = 0, ss = 0; - for (int l = 0; l < 8; l++) { s += sum[l]; ss += sq[l]; } - for (; i < c; i++) { float v = xr[i]; s += v; ss += v * v; } - float mean = s / c; - float inv = 1f / MathF.Sqrt(MathF.Max(ss / c - mean * mean, 0f) + eps); - F32x8 vm = V256.Create(mean), vi = V256.Create(inv); - for (i = 0; i + 8 <= c; i += 8) - { - F32x8 v = Ld(xr + i); - F32x8 wv = Ld(wp + i); - F32x8 bv = Ld(bp + i); - F32x8 t = V256.Subtract(v, vm); - St(dr + i, Fma(t, V256.Multiply(vi, wv), bv)); - } - for (; i < c; i++) dr[i] = (xr[i] - mean) * inv * wp[i] + bp[i]; - }); - } - } - - // ======================== parallel unary ======================== - // op: 0 relu, 1 sigmoid, 2 gelu, 3 erf - public static void Unary(ReadOnlySpan src, Span dst, int n, int op) - { - int chunks = Math.Clamp(n / 8192, 1, Threads * 4); - int stride = (n + chunks - 1) / chunks; - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - Parallel.For(0, chunks, PO, t => - { - int i0 = t * stride, i1 = Math.Min(n, i0 + stride); - int i = i0; - for (; i + 8 <= i1; i += 8) - { - F32x8 v = Ld(sp + i); - ApplyUnary(ref v, op); - St(dp + i, v); - } - for (; i < i1; i++) dp[i] = UnaryScalar(sp[i], op); - }); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void ApplyUnary(ref F32x8 v, int op) - { - if (op == 0) v = V256.Max(v, F32x8.Zero); - else if (op == 1) v = SimdMath.SigmoidCore(v); - else if (op == 2) v = SimdMath.GeluCore(v); - else v = SimdMath.ErfCore(v); - } - - private static float UnaryScalar(float x, int op) => op switch - { - 0 => MathF.Max(x, 0f), - 1 => 1f / (1f + MathF.Exp(-x)), - 2 => 0.5f * x * (1f + SimdMath.ErfScalar(x * 0.70710678118654752f)), - _ => SimdMath.ErfScalar(x), - }; - - // ======================== parallel binary elementwise ======================== - // op: 0 add, 1 sub, 2 mul, 3 div - public static void Ew2(ReadOnlySpan a, ReadOnlySpan b, - Span dst, int n, int op, ActivationKind act = ActivationKind.None) - { - int chunks = Math.Clamp(n / 8192, 1, Threads * 4); - int stride = (n + chunks - 1) / chunks; - fixed (float* ap0 = a, bp0 = b, dp0 = dst) - { - float* ap = ap0, bp = bp0, dp = dp0; - Parallel.For(0, chunks, PO, t => - { - int i0 = t * stride, i1 = Math.Min(n, i0 + stride); - int i = i0; - for (; i + 8 <= i1; i += 8) - { - F32x8 v = EwCore(Ld(ap + i), Ld(bp + i), op); - ApplyAct(ref v, act); - St(dp + i, v); - } - for (; i < i1; i++) - { - float v = EwScalar(ap[i], bp[i], op); - dp[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - }); - } - } - - // scalar broadcast: dst[i] = src[i] op s (rev: dst[i] = s op src[i]) - public static void Ew1(ReadOnlySpan src, Span dst, - int n, float s, int op, bool rev = false, ActivationKind act = ActivationKind.None) - { - int chunks = Math.Clamp(n / 8192, 1, Threads * 4); - int stride = (n + chunks - 1) / chunks; - fixed (float* ap0 = src, dp0 = dst) - { - float* ap = ap0, dp = dp0; - F32x8 vs = V256.Create(s); - Parallel.For(0, chunks, PO, t => - { - int i0 = t * stride, i1 = Math.Min(n, i0 + stride); - int i = i0; - for (; i + 8 <= i1; i += 8) - { - F32x8 v = Ld(ap + i); - v = rev ? EwCore(vs, v, op) : EwCore(v, vs, op); - ApplyAct(ref v, act); - St(dp + i, v); - } - for (; i < i1; i++) - { - float v = rev ? EwScalar(s, ap[i], op) : EwScalar(ap[i], s, op); - dp[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - }); - } - } - - // broadcast over a contiguous inner block: b[bo] constant per outer row - public static void EwInner(ReadOnlySpan a, ReadOnlySpan b, - int[] bStr, int[] outShape, int[] outStr, int rank, int inner, - Span dst, int n, int op, ActivationKind act = ActivationKind.None) - { - int outer = n / inner; - fixed (float* ap0 = a, bp0 = b, dp0 = dst) - { - float* ap = ap0, bp = bp0, dp = dp0; - Parallel.For(0, outer, PO, e => - { - int rem = e, bo = 0; - for (int d = 0; d < rank - 1; d++) - { - int cc = rem / (outStr[d] / inner); - rem %= outStr[d] / inner; - bo += cc * bStr[d]; - } - float s = bp[bo]; - F32x8 vs = V256.Create(s); - float* ar = ap + e * inner, dr = dp + e * inner; - int i = 0; - for (; i + 8 <= inner; i += 8) - { - F32x8 v = EwCore(Ld(ar + i), vs, op); - ApplyAct(ref v, act); - St(dr + i, v); - } - for (; i < inner; i++) - { - float v = EwScalar(ar[i], s, op); - dr[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - }); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 EwCore(F32x8 a, F32x8 b, int op) => op switch - { - 0 => V256.Add(a, b), - 1 => V256.Subtract(a, b), - 2 => V256.Multiply(a, b), - _ => V256.Divide(a, b), - }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static float EwScalar(float a, float b, int op) => op switch - { - 0 => a + b, 1 => a - b, 2 => a * b, _ => a / b, - }; - - // ======================== bilinear resize (half-pixel) ======================== - public static void Bilinear(ReadOnlySpan src, int c, int ih, int iw, - Span dst, int oh, int ow, - int[] y0, int[] y1, float[] wy, int[] x0, int[] x1, float[] wx, - ActivationKind act = ActivationKind.None) - { - fixed (float* sp0 = src, dp0 = dst) - fixed (int* y0p = y0, y1p = y1, x0p = x0, x1p = x1) - fixed (float* wyp = wy, wxp = wx) - { - float* sp = sp0, dp = dp0; - int* y0a = y0p, y1a = y1p, x0a = x0p, x1a = x1p; - float* wya = wyp, wxa = wxp; - Parallel.For(0, c * oh, PO, t => - { - int ch = t / oh, y = t % oh; - float* r0 = sp + (ch * ih + y0a[y]) * iw; - float* r1 = sp + (ch * ih + y1a[y]) * iw; - float* dr = dp + (ch * oh + y) * ow; - F32x8 wyv = V256.Create(wya[y]); - int x = 0; - for (; x + 8 <= ow; x += 8) - { - var i0 = Avx.LoadVector256(x0a + x); - var i1 = Avx.LoadVector256(x1a + x); - F32x8 wxv = Ld(wxa + x); - F32x8 t0 = Avx2.GatherVector256(r0, i0, 4); - F32x8 t1 = Avx2.GatherVector256(r0, i1, 4); - F32x8 b0 = Avx2.GatherVector256(r1, i0, 4); - F32x8 b1 = Avx2.GatherVector256(r1, i1, 4); - F32x8 top = Fma(V256.Subtract(t1, t0), wxv, t0); - F32x8 bot = Fma(V256.Subtract(b1, b0), wxv, b0); - F32x8 v = Fma(V256.Subtract(bot, top), wyv, top); - ApplyAct(ref v, act); - St(dr + x, v); - } - for (; x < ow; x++) - { - float t0 = r0[x0a[x]], t1v = r0[x1a[x]]; - float b0 = r1[x0a[x]], b1v = r1[x1a[x]]; - float top = t0 + (t1v - t0) * wxa[x]; - float bot = b0 + (b1v - b0) * wxa[x]; - float v = top + (bot - top) * wya[y]; - dr[x] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - }); - } - } - - // ======================== tiled transpose [a,b] -> [b,a] ======================== - public static void Transpose2D(ReadOnlySpan src, int a, int b, - Span dst, int copies = 1, int srcStride = 0, int dstStride = 0) - { - if (srcStride == 0) srcStride = a * b; - if (dstStride == 0) dstStride = a * b; - int ya = (a + 7) / 8, xb = (b + 7) / 8; - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - Parallel.For(0, copies * ya, PO, t => - { - int o = t / ya, ty = t % ya; - float* sb = sp + o * srcStride, db = dp + o * dstStride; - for (int tx = 0; tx < xb; tx++) - { - int y0 = ty * 8, x0 = tx * 8; - if (y0 + 8 <= a && x0 + 8 <= b) - { - T8(sb + y0 * b + x0, b, db + x0 * a + y0, a); - } - else - { - int ye = Math.Min(y0 + 8, a), xe = Math.Min(x0 + 8, b); - for (int y = y0; y < ye; y++) - for (int x = x0; x < xe; x++) - db[x * a + y] = sb[y * b + x]; - } - } - }); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void T8(float* s, int ls, float* d, int ld) - { - F32x8 r0 = Ld(s), r1 = Ld(s + ls), r2 = Ld(s + 2 * ls), r3 = Ld(s + 3 * ls); - F32x8 r4 = Ld(s + 4 * ls), r5 = Ld(s + 5 * ls), r6 = Ld(s + 6 * ls), r7 = Ld(s + 7 * ls); - F32x8 t0 = Avx.UnpackLow(r0, r1), t1 = Avx.UnpackHigh(r0, r1); - F32x8 t2 = Avx.UnpackLow(r2, r3), t3 = Avx.UnpackHigh(r2, r3); - F32x8 t4 = Avx.UnpackLow(r4, r5), t5 = Avx.UnpackHigh(r4, r5); - F32x8 t6 = Avx.UnpackLow(r6, r7), t7 = Avx.UnpackHigh(r6, r7); - const byte LO = 0x44, HI = 0xEE; - F32x8 u0 = Avx.Shuffle(t0, t2, LO), u1 = Avx.Shuffle(t0, t2, HI); - F32x8 u2 = Avx.Shuffle(t1, t3, LO), u3 = Avx.Shuffle(t1, t3, HI); - F32x8 u4 = Avx.Shuffle(t4, t6, LO), u5 = Avx.Shuffle(t4, t6, HI); - F32x8 u6 = Avx.Shuffle(t5, t7, LO), u7 = Avx.Shuffle(t5, t7, HI); - const byte L = 0x20, H = 0x31; - St(d, Avx.Permute2x128(u0, u4, L)); - St(d + ld, Avx.Permute2x128(u1, u5, L)); - St(d + 2 * ld, Avx.Permute2x128(u2, u6, L)); - St(d + 3 * ld, Avx.Permute2x128(u3, u7, L)); - St(d + 4 * ld, Avx.Permute2x128(u0, u4, H)); - St(d + 5 * ld, Avx.Permute2x128(u1, u5, H)); - St(d + 6 * ld, Avx.Permute2x128(u2, u6, H)); - St(d + 7 * ld, Avx.Permute2x128(u3, u7, H)); - } - - // ======================== pad ======================== - public static void Pad4(ReadOnlySpan src, int nc, int ih, int iw, - Span dst, int oh, int ow, int[] mapY, int[] mapX, float padVal) - { - // contiguous interior run of mapX (identity-mapped region) - int xl = 0; while (xl < ow && mapX[xl] < 0) xl++; - int xr = ow; while (xr > xl && mapX[xr - 1] < 0) xr--; - bool interior = true; - for (int x = xl; x < xr; x++) if (mapX[x] != x - xl + mapX[xl]) { interior = false; break; } - int srcX0 = xl < xr ? mapX[xl] : 0; - fixed (float* sp0 = src, dp0 = dst) - fixed (int* myp = mapY, mxp = mapX) - { - float* sp = sp0, dp = dp0; - int* mya = myp, mxa = mxp; - Parallel.For(0, nc * oh, PO, t => - { - int ch = t / oh, y = t % oh; - float* dr = dp + (ch * oh + y) * ow; - int my = mya[y]; - if (my < 0) - { - F32x8 pv = V256.Create(padVal); - int x = 0; - for (; x + 8 <= ow; x += 8) St(dr + x, pv); - for (; x < ow; x++) dr[x] = padVal; - return; - } - float* sr = sp + (ch * ih + my) * iw; - for (int x = 0; x < xl; x++) dr[x] = mxa[x] < 0 ? padVal : sr[mxa[x]]; - for (int x = xr; x < ow; x++) dr[x] = mxa[x] < 0 ? padVal : sr[mxa[x]]; - if (interior) - { - float* s = sr + srcX0; - int i = 0, len = xr - xl; - for (; i + 8 <= len; i += 8) St(dr + xl + i, Ld(s + i)); - for (; i < len; i++) dr[xl + i] = s[i]; - } - else - { - for (int x = xl; x < xr; x++) dr[x] = sr[mxa[x]]; - } - }); - } - } - - // ======================== channel shift ======================== - public static void ChannelShift(ReadOnlySpan src, int c, int h, int w, - Span dst, int axis, int[] offsets, int[] groupEnds) - { - fixed (float* sp0 = src, dp0 = dst) - fixed (int* offp = offsets, endp = groupEnds) - { - float* sp = sp0, dp = dp0; - int* offa = offp, enda = endp; - int groups = offsets.Length; - Parallel.For(0, c * h, PO, t => - { - int ch = t / h, y = t % h; - int off = 0; - for (int g = 0, g0 = 0; g < groups; g++) - { - int g1 = enda[g]; - if (ch >= g0 && ch < g1) { off = offa[g]; break; } - g0 = g1; - } - float* sr = sp + (ch * h + y) * w; - float* dr = dp + (ch * h + y) * w; - if (axis == 2) - { - int sy = y + off; - if ((uint)sy < (uint)h) - Buffer.MemoryCopy(sp + (ch * h + sy) * w, dr, w * 4L, w * 4L); - else - { - F32x8 z = F32x8.Zero; - int x = 0; - for (; x + 8 <= w; x += 8) St(dr + x, z); - for (; x < w; x++) dr[x] = 0f; - } - } - else if (Math.Abs(off) >= w) - { - int x = 0; - for (; x + 8 <= w; x += 8) St(dr + x, F32x8.Zero); - for (; x < w; x++) dr[x] = 0f; - } - else if (off >= 0) - { - // dr[x] = sr[x+off]; tail zero - Buffer.MemoryCopy(sr + off, dr, (w - off) * 4L, (w - off) * 4L); - int x = w - off; - for (; x < w; x++) dr[x] = 0f; - } - else - { - // dr[x] = sr[x+off] with off<0: head zero, then copy - int x = 0; - for (; x + 8 <= -off; x += 8) St(dr + x, F32x8.Zero); - for (; x < -off; x++) dr[x] = 0f; - Buffer.MemoryCopy(sr, dr - off, (w + off) * 4L, (w + off) * 4L); - } - }); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/NeonKernels.cs b/src/Sdcb.SimdGcdrNet/Kernels/NeonKernels.cs deleted file mode 100644 index c578885..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/NeonKernels.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; - -using V4 = System.Runtime.Intrinsics.Vector128; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// AArch64 AdvSimd (NEON) versions of the non-conv ops that have AVX2 kernels. -internal static unsafe class NeonKernels -{ - private static readonly ParallelOptions PO = new() { MaxDegreeOfParallelism = Avx2Kernels.Threads }; - - /// Row-wise LayerNorm over the last axis (c); rows split across threads in groups. - public static void LayerNorm(ReadOnlySpan x, int rows, int c, - ReadOnlySpan w, ReadOnlySpan b, Span dst, float eps) - { - int per = Math.Max(1, 16384 / c), tasks = (rows + per - 1) / per; - fixed (float* xp0 = x, wp0 = w, bp0 = b, dp0 = dst) - { - float* xp = xp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, tasks, PO, t => - { - for (int r = t * per, re = Math.Min(rows, r + per); r < re; r++) - LayerNormRow(xp + (long)r * c, dp + (long)r * c, c, wp, bp, eps); - }); - } - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void LayerNormRow(float* xr, float* dr, int c, float* wp, float* bp, float eps) - { - V4 sum = V4.Zero, sq = V4.Zero; - int i = 0; - for (; i + 4 <= c; i += 4) - { - V4 v = AdvSimd.LoadVector128(xr + i); - sum = AdvSimd.Add(sum, v); - sq = AdvSimd.FusedMultiplyAdd(sq, v, v); - } - float s = Vector128.Sum(sum), ss = Vector128.Sum(sq); - for (; i < c; i++) { float v = xr[i]; s += v; ss += v * v; } - float mean = s / c; - float inv = 1f / MathF.Sqrt(MathF.Max(ss / c - mean * mean, 0f) + eps); - V4 vm = Vector128.Create(mean), vi = Vector128.Create(inv); - for (i = 0; i + 4 <= c; i += 4) - { - V4 t = AdvSimd.Subtract(AdvSimd.LoadVector128(xr + i), vm); - AdvSimd.Store(dr + i, AdvSimd.FusedMultiplyAdd(AdvSimd.LoadVector128(bp + i), t, - AdvSimd.Multiply(vi, AdvSimd.LoadVector128(wp + i)))); - } - for (; i < c; i++) dr[i] = (xr[i] - mean) * inv * wp[i] + bp[i]; - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Avx512.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Avx512.cs deleted file mode 100644 index c379e04..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Avx512.cs +++ /dev/null @@ -1,392 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; - -using F32x8 = System.Runtime.Intrinsics.Vector256; -using F32x16 = System.Runtime.Intrinsics.Vector512; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// AVX-512 tier of the NHWC conv / Gemm micro-kernel: pixels × 32 oc -/// (two packed 16-oc blocks, one zmm each) per pass, so every broadcast feeds two FMAs and -/// the 32 zmm registers hold the whole accumulator tile. Per output element the reduction -/// is the AVX2 one (bias-init, taps then channels ascending, one FMA each) and the -/// epilogue reuses 's math, so results are bit-identical to AVX2. -/// The k loops store to a scratch tile and never call out (calls make RyuJIT spill the -/// accumulators); the residual/activation epilogue runs afterwards. -/// -internal static unsafe partial class Nhwc -{ - private const int Tile5 = 10; - - /// AVX-512 tier switch: Avx512F.IsSupported, off with GCDRNET_AVX512=0 or the - /// .NET 8/9-style DOTNET_EnableAVX512F=0 (the .NET 10 JIT only honours DOTNET_EnableAVX512). - /// Mutable for in-process A/B tests. - internal static bool UseAvx512 = Avx512F.IsSupported - && Environment.GetEnvironmentVariable("GCDRNET_AVX512") != "0" - && Environment.GetEnvironmentVariable("DOTNET_EnableAVX512F") != "0"; - - /// Tile5 rows × 32 oc: dst[r*ds + 0..15] ← block wb, dst[r*ds + 16..31] ← block wb + wStep. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void Tile512x32(float** ptrs, int taps, int ic, float* wb, long wStep, float* bb, float* dst, int ds) - { - F32x16 a0 = Avx512F.LoadVector512(bb), b0 = Avx512F.LoadVector512(bb + Ob); - F32x16 a1 = a0, a2 = a0, a3 = a0, a4 = a0, a5 = a0, a6 = a0, a7 = a0, a8 = a0, a9 = a0; - F32x16 b1 = b0, b2 = b0, b3 = b0, b4 = b0, b5 = b0, b6 = b0, b7 = b0, b8 = b0, b9 = b0; - for (int tap = 0; tap < taps; tap++) - { - float** pp = ptrs + tap * Tile5; - float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3], p4 = pp[4], p5 = pp[5], p6 = pp[6], p7 = pp[7], p8 = pp[8], p9 = pp[9]; - float* w = wb + (long)tap * ic * Ob; - float* x = w + wStep; - for (int c = 0; c < ic; c++, w += Ob, x += Ob) - { - F32x16 wa = Avx512F.LoadVector512(w), wx = Avx512F.LoadVector512(x); - F32x16 v = Vector512.Create(p0[c]); - a0 = Avx512F.FusedMultiplyAdd(v, wa, a0); b0 = Avx512F.FusedMultiplyAdd(v, wx, b0); - v = Vector512.Create(p1[c]); - a1 = Avx512F.FusedMultiplyAdd(v, wa, a1); b1 = Avx512F.FusedMultiplyAdd(v, wx, b1); - v = Vector512.Create(p2[c]); - a2 = Avx512F.FusedMultiplyAdd(v, wa, a2); b2 = Avx512F.FusedMultiplyAdd(v, wx, b2); - v = Vector512.Create(p3[c]); - a3 = Avx512F.FusedMultiplyAdd(v, wa, a3); b3 = Avx512F.FusedMultiplyAdd(v, wx, b3); - v = Vector512.Create(p4[c]); - a4 = Avx512F.FusedMultiplyAdd(v, wa, a4); b4 = Avx512F.FusedMultiplyAdd(v, wx, b4); - v = Vector512.Create(p5[c]); - a5 = Avx512F.FusedMultiplyAdd(v, wa, a5); b5 = Avx512F.FusedMultiplyAdd(v, wx, b5); - v = Vector512.Create(p6[c]); - a6 = Avx512F.FusedMultiplyAdd(v, wa, a6); b6 = Avx512F.FusedMultiplyAdd(v, wx, b6); - v = Vector512.Create(p7[c]); - a7 = Avx512F.FusedMultiplyAdd(v, wa, a7); b7 = Avx512F.FusedMultiplyAdd(v, wx, b7); - v = Vector512.Create(p8[c]); - a8 = Avx512F.FusedMultiplyAdd(v, wa, a8); b8 = Avx512F.FusedMultiplyAdd(v, wx, b8); - v = Vector512.Create(p9[c]); - a9 = Avx512F.FusedMultiplyAdd(v, wa, a9); b9 = Avx512F.FusedMultiplyAdd(v, wx, b9); - } - } - Avx512F.Store(dst, a0); Avx512F.Store(dst + Ob, b0); dst += ds; - Avx512F.Store(dst, a1); Avx512F.Store(dst + Ob, b1); dst += ds; - Avx512F.Store(dst, a2); Avx512F.Store(dst + Ob, b2); dst += ds; - Avx512F.Store(dst, a3); Avx512F.Store(dst + Ob, b3); dst += ds; - Avx512F.Store(dst, a4); Avx512F.Store(dst + Ob, b4); dst += ds; - Avx512F.Store(dst, a5); Avx512F.Store(dst + Ob, b5); dst += ds; - Avx512F.Store(dst, a6); Avx512F.Store(dst + Ob, b6); dst += ds; - Avx512F.Store(dst, a7); Avx512F.Store(dst + Ob, b7); dst += ds; - Avx512F.Store(dst, a8); Avx512F.Store(dst + Ob, b8); dst += ds; - Avx512F.Store(dst, a9); Avx512F.Store(dst + Ob, b9); - } - - /// Tile5 rows × 16 oc (odd block count tail): dst[r*ds + 0..15]. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void Tile512x16(float** ptrs, int taps, int ic, float* wb, float* bb, float* dst, int ds) - { - F32x16 a0 = Avx512F.LoadVector512(bb); - F32x16 a1 = a0, a2 = a0, a3 = a0, a4 = a0, a5 = a0, a6 = a0, a7 = a0, a8 = a0, a9 = a0; - for (int tap = 0; tap < taps; tap++) - { - float** pp = ptrs + tap * Tile5; - float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3], p4 = pp[4], p5 = pp[5], p6 = pp[6], p7 = pp[7], p8 = pp[8], p9 = pp[9]; - float* w = wb + (long)tap * ic * Ob; - for (int c = 0; c < ic; c++, w += Ob) - { - F32x16 wa = Avx512F.LoadVector512(w); - a0 = Avx512F.FusedMultiplyAdd(Vector512.Create(p0[c]), wa, a0); - a1 = Avx512F.FusedMultiplyAdd(Vector512.Create(p1[c]), wa, a1); - a2 = Avx512F.FusedMultiplyAdd(Vector512.Create(p2[c]), wa, a2); - a3 = Avx512F.FusedMultiplyAdd(Vector512.Create(p3[c]), wa, a3); - a4 = Avx512F.FusedMultiplyAdd(Vector512.Create(p4[c]), wa, a4); - a5 = Avx512F.FusedMultiplyAdd(Vector512.Create(p5[c]), wa, a5); - a6 = Avx512F.FusedMultiplyAdd(Vector512.Create(p6[c]), wa, a6); - a7 = Avx512F.FusedMultiplyAdd(Vector512.Create(p7[c]), wa, a7); - a8 = Avx512F.FusedMultiplyAdd(Vector512.Create(p8[c]), wa, a8); - a9 = Avx512F.FusedMultiplyAdd(Vector512.Create(p9[c]), wa, a9); - } - } - Avx512F.Store(dst, a0); dst += ds; - Avx512F.Store(dst, a1); dst += ds; - Avx512F.Store(dst, a2); dst += ds; - Avx512F.Store(dst, a3); dst += ds; - Avx512F.Store(dst, a4); dst += ds; - Avx512F.Store(dst, a5); dst += ds; - Avx512F.Store(dst, a6); dst += ds; - Avx512F.Store(dst, a7); dst += ds; - Avx512F.Store(dst, a8); dst += ds; - Avx512F.Store(dst, a9); - } - - /// Runs the 32- or 16-wide kernel for one tile into ([Tile5][32]). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Tile512(float** ptrs, int nblk, int taps, int ic, float* wb, long wStep, float* bb, float* part) - { - if (nblk == 2) Tile512x32(ptrs, taps, ic, wb, wStep, bb, part, 2 * Ob); - else Tile512x16(ptrs, taps, ic, wb, bb, part, 2 * Ob); - } - - /// Epilogue of one tile: part ([Tile5][32]) → rows × (lanes0 + lanes1) outputs, + residual, act. - [MethodImpl(MethodImplOptions.NoInlining)] - private static void Epilogue512(float* part, int rows, int nblk, float* outp, float* addp, int oc, - int lanes0, int lanes1, ActivationKind act) - { - bool simple = act is ActivationKind.None or ActivationKind.Relu; - for (int r = 0; r < rows; r++, part += 2 * Ob) - { - float* o = outp + (long)r * oc; - float* ad = addp == null ? null : addp + (long)r * oc; - for (int h = 0; h < nblk; h++) - { - int lanes = h == 0 ? lanes0 : lanes1; - float* s = part + h * Ob, oh = o + h * Ob, ah = ad == null ? null : ad + h * Ob; - if (lanes == Ob && simple) - { - F32x16 v = Avx512F.LoadVector512(s); - if (ah != null) v = Avx512F.Add(v, Avx512F.LoadVector512(ah)); - if (act == ActivationKind.Relu) v = Avx512F.Max(v, F32x16.Zero); - Avx512F.Store(oh, v); - } - else - Store16(oh, ah, Avx.LoadVector256(s), Avx.LoadVector256(s + 8), lanes, act); - } - } - } - - private static void Dense512(ReadOnlySpan src, int iw, int ic, Span dst, - int oh, int ow, int oc, int kh, int kw, int[] rm, int[] cm, float[] padVec, - NhwcConvPlan kp, ReadOnlySpan add, ActivationKind act) - { - int taps = kh * kw, nb = kp.Blocks, np = (nb + 1) / 2; - bool hasAdd = !add.IsEmpty; - fixed (float* sp0 = src, dp0 = dst, ap0 = add, wp0 = kp.W, bp0 = kp.Bias, pv0 = padVec) - fixed (int* rm0 = rm, cm0 = cm) - { - var a = new DenseArgs(sp0, wp0, bp0, pv0, rm0, cm0, iw, ic, ow, oc, kh, kw, nb, act, true); - float* dp = dp0, ap = ap0; - For(oh * np, (long)oh * ow * oc * taps * ic / 64, t => - { - int y = t / np, blk = t % np * 2; - long o = (long)y * ow * oc; - Dense512Row(a, y, blk, dp + o, hasAdd ? ap + o : null); - }); - } - } - - /// All oc blocks of direct-conv output row (the row-level entry of - /// the fused paths: conv+pool bands, UpHead skip recompute). AVX-512 tier: block pairs through - /// (needs a padVec of ic*kw + 16 floats); else . - private static void ConvRow(in DenseArgs a, int y, float* orow, float* arow) - { - if (UseAvx512) - for (int blk = 0; blk < a.Nb; blk += 2) Dense512Row(a, y, blk, orow, arow); - else - for (int blk = 0; blk < a.Nb; blk++) DenseRow(a, y, blk, orow, arow); - } - - /// Output row , oc blocks blk and blk+1 (if present) of a direct conv - /// into (the row's first pixel; addend row likewise). - private static void Dense512Row(in DenseArgs a, int y, int blk, float* orow, float* arow) - { - int kh = a.Kh, kw = a.Kw, taps = kh * kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, nblk = Math.Min(2, a.Nb - blk); - long wStep = (long)taps * ic * Ob; - float** ptrs = stackalloc float*[Tile5 * taps]; - float* part = stackalloc float[Tile5 * 2 * Ob]; - float* wb = a.W + blk * wStep; - float* bb = a.Bias + blk * Ob; - int lanes0 = Math.Min(Ob, oc - blk * Ob), lanes1 = nblk == 2 ? Math.Min(Ob, oc - (blk + 1) * Ob) : 0; - int* rm = a.Rm, cm = a.Cm; - for (int x0 = 0; x0 < ow; x0 += Tile5) - { - int rows = Math.Min(Tile5, ow - x0); - // every column of the tile reads kw adjacent input pixels (no column pad/reflect): one - // kernel row is then a single kw*ic float run in the order of the packed [tap][ic] - // weights, so the tile walks kh "taps" of kw*ic channels (fewer pointers, longer k loops) - bool merged = kw > 1; - for (int i = 0; i < rows && merged; i++) - { - int* cx = cm + (x0 + i) * kw; - merged = cx[0] >= 0; - for (int kx = 1; kx < kw; kx++) merged &= cx[kx] == cx[0] + kx; - } - if (merged) - { - for (int ky = 0; ky < kh; ky++) - { - int r = rm[y * kh + ky]; - for (int i = 0; i < Tile5; i++) - { - int c = cm[(x0 + Math.Min(i, rows - 1)) * kw]; - ptrs[ky * Tile5 + i] = r < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic; - } - } - Tile512(ptrs, nblk, kh, kw * ic, wb, wStep, bb, part); - } - else - { - for (int tap = 0; tap < taps; tap++) - { - int r = rm[y * kh + tap / kw]; - for (int i = 0; i < Tile5; i++) - { - int c = cm[(x0 + Math.Min(i, rows - 1)) * kw + tap % kw]; - ptrs[tap * Tile5 + i] = r < 0 || c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic; - } - } - Tile512(ptrs, nblk, taps, ic, wb, wStep, bb, part); - } - long o = (long)x0 * oc + blk * Ob; - Epilogue512(part, rows, nblk, orow + o, arow != null ? arow + o : null, oc, lanes0, lanes1, a.Act); - } - } - - private static void Gemm512(ReadOnlySpan a, int m, int k, Span dst, int n, - NhwcConvPlan kp, ActivationKind act) - { - const int G = 8; // row tiles per task sharing one weight panel walk - int nb = kp.Blocks, np = (nb + 1) / 2, tiles = (m + Tile5 - 1) / Tile5, groups = (tiles + G - 1) / G; - long wStep = (long)k * Ob; - fixed (float* ap0 = a, dp0 = dst, wp0 = kp.W, bp0 = kp.Bias) - { - float* ap = ap0, dp = dp0, wp = wp0, bp = bp0; - For(groups * np, (long)m * n * k / 64, t => - { - int g = t / np, blk = t % np * 2, nblk = Math.Min(2, nb - blk); - float** ptrs = stackalloc float*[Tile5]; - float* part = stackalloc float[Tile5 * 2 * Ob]; - float* wb = wp + blk * wStep, bb = bp + blk * Ob; - int lanes0 = Math.Min(Ob, n - blk * Ob), lanes1 = nblk == 2 ? Math.Min(Ob, n - (blk + 1) * Ob) : 0; - for (int tile = g * G, te = Math.Min(tiles, tile + G); tile < te; tile++) - { - int m0 = tile * Tile5, rows = Math.Min(Tile5, m - m0); - for (int i = 0; i < Tile5; i++) ptrs[i] = ap + (long)(m0 + Math.Min(i, rows - 1)) * k; - Tile512(ptrs, nblk, 1, k, wb, wStep, bb, part); - Epilogue512(part, rows, nblk, dp + (long)m0 * n + blk * Ob, null, n, lanes0, lanes1, act); - } - }); - } - } - - // ======================== memory-bound NHWC ops (c % 16 == 0) ======================== - - /// with 16-lane vectors: same per-lane ops (and IEEE Max) → same bits. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void ResizePixel512(float* a, float* b, float* e, float* f, F32x16 vX, F32x16 vY, - int c, bool relu, float* add, bool addRelu, float* o) - { - for (int ch = 0; ch < c; ch += 16) - { - var va = Vector512.Load(a + ch); var ve = Vector512.Load(e + ch); - var top = va + (Vector512.Load(b + ch) - va) * vX; - var bot = ve + (Vector512.Load(f + ch) - ve) * vX; - var v = top + (bot - top) * vY; - if (relu) v = Vector512.Max(v, F32x16.Zero); - if (add != null) - { - var sk = Vector512.Load(add + ch); - v += addRelu ? Vector512.Max(sk, F32x16.Zero) : sk; - } - v.Store(o + ch); - } - } - - /// MaxPool, one output pixel per step: max over the valid taps in registers, one store - /// (the generic path copies the first tap and read-modify-writes the output per tap). - private static void MaxPool512(float* sp, int iw, int c, float* dp, int oh, int ow, int kh, int kw, - int[] rm, int[] cm, bool relu) - { - For(oh, (long)oh * ow * c * kh * kw, y => - { - float** taps = stackalloc float*[kh * kw]; - for (int x = 0; x < ow; x++) - { - int n = 0; - for (int ky = 0; ky < kh; ky++) - { - int r = rm[y * kh + ky]; - if (r < 0) continue; - for (int kx = 0; kx < kw; kx++) - { - int cc = cm[x * kw + kx]; - if (cc >= 0) taps[n++] = sp + ((long)r * iw + cc) * c; - } - } - float* o = dp + ((long)y * ow + x) * c; - if (n == 0) { Fill(o, c, 0f); continue; } - for (int ch = 0; ch < c; ch += 16) - { - var v = Vector512.Load(taps[0] + ch); - for (int k = 1; k < n; k++) v = Vector512.Max(v, Vector512.Load(taps[k] + ch)); - if (relu) v = Vector512.Max(v, F32x16.Zero); - v.Store(o + ch); - } - } - }); - } - - /// - /// for the AVX-512 tier: 16-lane resize, and the c-long oc dot runs as four - /// interleaved per-pixel FMA chains (one chain per pixel was FMA-latency bound). Each pixel keeps - /// its bias-initialized, channel-ascending chain, so outputs match the AVX2 path bit for bit. - /// - private static void UpHead512(float* sp, int iw, int c, float* kpp, bool skipConv, DenseArgs ca, float* dp, int oh, int ow, - int[] y0, int[] y1, int[] x0, int[] x1, float[] wy, float[] wx, bool relu, bool skipRelu, - float* wp, float* bp, int oc, ActivationKind act) - { - bool v512 = c % 16 == 0; - For(oh, (long)oh * ow * c * (oc + 4), y => - { - float* s = stackalloc float[4 * c + 8]; - float* acc = stackalloc float[4 * 8]; - float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c; - var vY = new System.Numerics.Vector(wy[y]); - var vY16 = Vector512.Create(wy[y]); - float wY = wy[y]; - float* orow = dp + (long)y * ow * oc; - float* krow = kpp + (long)y * ow * c; - float[]? band = skipConv ? BandBuf((long)ow * c) : null; - fixed (float* sb = band) - { - if (skipConv) - { - ConvRow(ca, y, sb, null); // the skip row, recomputed by the same conv kernel - krow = sb; - } - for (int x = 0; x < ow; x += 4) - { - int n = Math.Min(4, ow - x); - for (int i = 0; i < n; i++) - { - int xx = x + i; - float* a = r0 + (long)x0[xx] * c, b = r0 + (long)x1[xx] * c, e = r1 + (long)x0[xx] * c, f = r1 + (long)x1[xx] * c; - if (v512) ResizePixel512(a, b, e, f, Vector512.Create(wx[xx]), vY16, c, relu, krow + (long)xx * c, skipRelu, s + i * c); - else ResizePixel(a, b, e, f, wx[xx], wY, new System.Numerics.Vector(wx[xx]), vY, c, relu, krow + (long)xx * c, skipRelu, s + i * c); - } - UpHeadDot4(s, c, wp, bp, acc, n); - for (int i = 0; i < n; i++) - { - float* o = orow + (long)(x + i) * oc; - for (int l = 0; l < oc; l++) o[l] = acc[i * 8 + l]; - } - } - if (act != ActivationKind.None) ActRow(orow, ow * oc, act, true); - } - }); - } - - /// acc[i*8 + l] = bias[l] + Σ_ch s_i[ch]·w[ch][l] for 4 pixels (s_i = s + i*c), chains interleaved. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void UpHeadDot4(float* s, int c, float* wp, float* bp, float* acc, int n) - { - float* s0 = s, s1 = n > 1 ? s + c : s, s2 = n > 2 ? s + 2 * c : s, s3 = n > 3 ? s + 3 * c : s; - F32x8 a0 = Avx.LoadVector256(bp), a1 = a0, a2 = a0, a3 = a0; - float* w = wp; - for (int ch = 0; ch < c; ch++, w += 8) - { - F32x8 wv = Avx.LoadVector256(w); - a0 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s0 + ch), wv, a0); - a1 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s1 + ch), wv, a1); - a2 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s2 + ch), wv, a2); - a3 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s3 + ch), wv, a3); - } - Avx.Store(acc, a0); Avx.Store(acc + 8, a1); Avx.Store(acc + 16, a2); Avx.Store(acc + 24, a3); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Neon.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Neon.cs deleted file mode 100644 index 277f774..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Neon.cs +++ /dev/null @@ -1,373 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using Sdcb.SimdGcdrNet.Engine; - -using V4 = System.Runtime.Intrinsics.Vector128; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// AArch64 AdvSimd (NEON) tier: the 6 × 16 conv / Gemm micro-kernel as fmla-by-element on -/// 4-channel input loads, 24 register accumulators (of 32 vector registers), used wherever the -/// x86 tiers take ; direct convs feed it kernel-row runs (kw·ic contiguous -/// NHWC floats per tap row) so the 3- / 6-channel stems run the same loop. -/// -internal static unsafe partial class Nhwc -{ - /// NEON tier switch: AdvSimd.Arm64.IsSupported (DOTNET_EnableAdvSimd=0 drops to the - /// scalar kernels). Settable only for the in-process tier comparison tests. - internal static bool UseNeon = AdvSimd.Arm64.IsSupported; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 Ld4(float* p) => AdvSimd.LoadVector128(p); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 Fma4(V4 acc, V4 a, V4 b) => AdvSimd.FusedMultiplyAdd(acc, a, b); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 Act4(V4 v, ActivationKind act) => act switch - { - ActivationKind.Relu => AdvSimd.Max(v, V4.Zero), - ActivationKind.Sigmoid => SimdMath.SigmoidCore(v), - ActivationKind.Gelu => SimdMath.GeluCore(v), - _ => v, - }; - - /// 6 × 16 tile: into a scratch tile, then the residual / - /// activation epilogue per row. - private static void TileNeon(float** ptrs, int rows, int taps, int ic, float* wb, float* bb, - float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - float* part = stackalloc float[Tile * Ob]; - TileNeonK(ptrs, taps, ic, wb, bb, part, Ob); - TileNeonStore(part, rows, outp, addp, oc, lanes, act); - } - - /// Residual / activation epilogue of a finished 6 × 16 scratch tile. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void TileNeonStore(float* part, int rows, float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - for (int r = 0; r < rows; r++, part += Ob) - { - float* o = outp + (long)r * oc; - float* ad = addp == null ? null : addp + (long)r * oc; - if (lanes == Ob) - { - for (int l = 0; l < Ob; l += 4) - { - V4 v = Ld4(part + l); - if (ad != null) v = AdvSimd.Add(v, Ld4(ad + l)); - AdvSimd.Store(o + l, Act4(v, act)); - } - continue; - } - for (int l = 0; l < lanes; l++) - o[l] = ActS(ad != null ? part[l] + ad[l] : part[l], act); - } - } - - /// 6 × 16 k loop over all ic channels of every tap (weights [tap][ic][16]), bias-initialized. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void TileNeonK(float** ptrs, int taps, int ic, float* wb, float* bb, float* dst, int ds) => - TileNeonK(ptrs, taps, 0, ic, (long)ic * Ob, wb, bb, dst, ds); - - /// - /// The 6 × 16 k loop: dst[r*ds + 0..15] = init + Σ x·w for all rows, where - /// ptrs[tap*Tile + r] + coff = row r's ic inputs of tap and wb + tap·wTap its [ic][16] weights; - /// init = bias , or (bb == null) the partial sums already in dst. Four - /// channels per step (one 128-bit load per pixel, fmla by element; the next weight vector is - /// loaded one group ahead for the in-order A53), ic % 4 leftovers by broadcast. Never calls - /// out, so the 24 accumulators stay in registers. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void TileNeonK(float** ptrs, int taps, int coff, int ic, long wTap, float* wb, float* bb, float* dst, int ds, - bool relu = false) - { - V4 a0, a1, a2, a3, a4, a5, b0, b1, b2, b3, b4, b5, c0, c1, c2, c3, c4, c5, d0, d1, d2, d3, d4, d5; - if (bb != null) - { - a0 = Ld4(bb); b0 = Ld4(bb + 4); c0 = Ld4(bb + 8); d0 = Ld4(bb + 12); - a1 = a2 = a3 = a4 = a5 = a0; - b1 = b2 = b3 = b4 = b5 = b0; - c1 = c2 = c3 = c4 = c5 = c0; - d1 = d2 = d3 = d4 = d5 = d0; - } - else - { - float* s = dst; - a0 = Ld4(s); b0 = Ld4(s + 4); c0 = Ld4(s + 8); d0 = Ld4(s + 12); s += ds; - a1 = Ld4(s); b1 = Ld4(s + 4); c1 = Ld4(s + 8); d1 = Ld4(s + 12); s += ds; - a2 = Ld4(s); b2 = Ld4(s + 4); c2 = Ld4(s + 8); d2 = Ld4(s + 12); s += ds; - a3 = Ld4(s); b3 = Ld4(s + 4); c3 = Ld4(s + 8); d3 = Ld4(s + 12); s += ds; - a4 = Ld4(s); b4 = Ld4(s + 4); c4 = Ld4(s + 8); d4 = Ld4(s + 12); s += ds; - a5 = Ld4(s); b5 = Ld4(s + 4); c5 = Ld4(s + 8); d5 = Ld4(s + 12); - } - int ic4 = ic & ~3; - for (int tap = 0; tap < taps; tap++) - { - float** pp = ptrs + tap * Tile; - float* p0 = pp[0] + coff, p1 = pp[1] + coff, p2 = pp[2] + coff, p3 = pp[3] + coff, p4 = pp[4] + coff, p5 = pp[5] + coff; - float* w = wb + tap * wTap; - int c = 0; - for (; c < ic4; c += 4, w += 4 * Ob) - { - V4 x0 = Ld4(p0 + c), x1 = Ld4(p1 + c), x2 = Ld4(p2 + c); - V4 x3 = Ld4(p3 + c), x4 = Ld4(p4 + c), x5 = Ld4(p5 + c); - V4 wv0 = Ld4(w), wv1; - wv1 = Ld4(w + 4); - a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 0); - a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 0); - a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 0); - a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 0); - a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 0); - a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 0); - wv0 = Ld4(w + 8); - b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 0); - b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 0); - b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 0); - b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 0); - b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 0); - b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 0); - wv1 = Ld4(w + 12); - c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 0); - c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 0); - c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 0); - c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 0); - c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 0); - c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 0); - wv0 = Ld4(w + 16); - d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 0); - d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 0); - d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 0); - d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 0); - d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 0); - d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 0); - wv1 = Ld4(w + 20); - a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 1); - a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 1); - a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 1); - a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 1); - a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 1); - a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 1); - wv0 = Ld4(w + 24); - b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 1); - b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 1); - b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 1); - b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 1); - b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 1); - b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 1); - wv1 = Ld4(w + 28); - c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 1); - c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 1); - c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 1); - c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 1); - c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 1); - c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 1); - wv0 = Ld4(w + 32); - d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 1); - d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 1); - d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 1); - d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 1); - d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 1); - d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 1); - wv1 = Ld4(w + 36); - a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 2); - a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 2); - a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 2); - a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 2); - a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 2); - a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 2); - wv0 = Ld4(w + 40); - b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 2); - b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 2); - b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 2); - b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 2); - b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 2); - b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 2); - wv1 = Ld4(w + 44); - c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 2); - c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 2); - c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 2); - c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 2); - c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 2); - c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 2); - wv0 = Ld4(w + 48); - d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 2); - d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 2); - d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 2); - d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 2); - d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 2); - d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 2); - wv1 = Ld4(w + 52); - a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 3); - a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 3); - a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 3); - a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 3); - a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 3); - a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 3); - wv0 = Ld4(w + 56); - b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 3); - b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 3); - b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 3); - b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 3); - b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 3); - b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 3); - wv1 = Ld4(w + 60); - c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 3); - c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 3); - c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 3); - c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 3); - c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 3); - c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 3); - d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 3); - d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 3); - d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 3); - d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 3); - d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 3); - d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 3); - } - for (; c < ic; c++, w += Ob) - { - V4 w0 = Ld4(w), w1 = Ld4(w + 4), w2 = Ld4(w + 8), w3 = Ld4(w + 12); - V4 x = AdvSimd.LoadAndReplicateToVector128(p0 + c); - a0 = Fma4(a0, w0, x); b0 = Fma4(b0, w1, x); c0 = Fma4(c0, w2, x); d0 = Fma4(d0, w3, x); - x = AdvSimd.LoadAndReplicateToVector128(p1 + c); - a1 = Fma4(a1, w0, x); b1 = Fma4(b1, w1, x); c1 = Fma4(c1, w2, x); d1 = Fma4(d1, w3, x); - x = AdvSimd.LoadAndReplicateToVector128(p2 + c); - a2 = Fma4(a2, w0, x); b2 = Fma4(b2, w1, x); c2 = Fma4(c2, w2, x); d2 = Fma4(d2, w3, x); - x = AdvSimd.LoadAndReplicateToVector128(p3 + c); - a3 = Fma4(a3, w0, x); b3 = Fma4(b3, w1, x); c3 = Fma4(c3, w2, x); d3 = Fma4(d3, w3, x); - x = AdvSimd.LoadAndReplicateToVector128(p4 + c); - a4 = Fma4(a4, w0, x); b4 = Fma4(b4, w1, x); c4 = Fma4(c4, w2, x); d4 = Fma4(d4, w3, x); - x = AdvSimd.LoadAndReplicateToVector128(p5 + c); - a5 = Fma4(a5, w0, x); b5 = Fma4(b5, w1, x); c5 = Fma4(c5, w2, x); d5 = Fma4(d5, w3, x); - } - } - if (relu) - { - V4 z = V4.Zero; - a0 = AdvSimd.Max(a0, z); b0 = AdvSimd.Max(b0, z); c0 = AdvSimd.Max(c0, z); d0 = AdvSimd.Max(d0, z); - a1 = AdvSimd.Max(a1, z); b1 = AdvSimd.Max(b1, z); c1 = AdvSimd.Max(c1, z); d1 = AdvSimd.Max(d1, z); - a2 = AdvSimd.Max(a2, z); b2 = AdvSimd.Max(b2, z); c2 = AdvSimd.Max(c2, z); d2 = AdvSimd.Max(d2, z); - a3 = AdvSimd.Max(a3, z); b3 = AdvSimd.Max(b3, z); c3 = AdvSimd.Max(c3, z); d3 = AdvSimd.Max(d3, z); - a4 = AdvSimd.Max(a4, z); b4 = AdvSimd.Max(b4, z); c4 = AdvSimd.Max(c4, z); d4 = AdvSimd.Max(d4, z); - a5 = AdvSimd.Max(a5, z); b5 = AdvSimd.Max(b5, z); c5 = AdvSimd.Max(c5, z); d5 = AdvSimd.Max(d5, z); - } - StoreRow(dst, a0, b0, c0, d0); dst += ds; - StoreRow(dst, a1, b1, c1, d1); dst += ds; - StoreRow(dst, a2, b2, c2, d2); dst += ds; - StoreRow(dst, a3, b3, c3, d3); dst += ds; - StoreRow(dst, a4, b4, c4, d4); dst += ds; - StoreRow(dst, a5, b5, c5, d5); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void StoreRow(float* d, V4 a, V4 b, V4 c, V4 e) - { - AdvSimd.Store(d, a); AdvSimd.Store(d + 4, b); AdvSimd.Store(d + 8, c); AdvSimd.Store(d + 12, e); - } - - [ThreadStatic] private static float[]? t_runs; - - /// - /// One output row for oc <= 4, c % 4 == 0: per 4 channels the bilinear - /// lerps (fma), [relu], + [relu](skip), then fmla by element into one 4-lane accumulator against - /// the packed [c][8] weights — the resized pixel never leaves registers. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void UpHeadRowNeon(float* r0, float* r1, int* x0, int* x1, float* wx, float wy, float* krow, - int c, int ow, float* wp, float* bp, float* orow, int oc, bool relu, bool skipRelu, ActivationKind act) - { - V4 vY = Vector128.Create(wy), bias = Ld4(bp); - for (int x = 0; x < ow; x++) - { - float* a = r0 + (long)x0[x] * c, b = r0 + (long)x1[x] * c, e = r1 + (long)x0[x] * c, f = r1 + (long)x1[x] * c; - float* k = krow + (long)x * c; - V4 vX = Vector128.Create(wx[x]), acc = bias; - float* w = wp; - for (int ch = 0; ch < c; ch += 4, w += 32) - { - V4 va = Ld4(a + ch), ve = Ld4(e + ch); - V4 top = Fma4(va, AdvSimd.Subtract(Ld4(b + ch), va), vX); - V4 bot = Fma4(ve, AdvSimd.Subtract(Ld4(f + ch), ve), vX); - V4 v = Fma4(top, AdvSimd.Subtract(bot, top), vY); - if (relu) v = AdvSimd.Max(v, V4.Zero); - V4 s = Ld4(k + ch); - v = AdvSimd.Add(v, skipRelu ? AdvSimd.Max(s, V4.Zero) : s); - acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w), v, 0); - acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 8), v, 1); - acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 16), v, 2); - acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 24), v, 3); - } - acc = Act4(acc, act); - float* o = orow + (long)x * oc; - o[0] = acc.ToScalar(); - if (oc > 1) o[1] = acc.GetElement(1); - if (oc > 2) o[2] = acc.GetElement(2); - if (oc > 3) o[3] = acc.GetElement(3); - } - } - - /// Thread-local pinned scratch (the pointer outlives any fixed scope). - private static T* Pinned(ref T[]? slot, int n) where T : unmanaged - { - if (slot is not { } b || b.Length < n) slot = b = GC.AllocateUninitializedArray(n, pinned: true); - return (T*)Unsafe.AsPointer(ref System.Runtime.InteropServices.MemoryMarshal.GetArrayDataReference(b)); - } - - /// - /// on NEON: a k×k conv as kh taps over kernel-row runs. For each tap row - /// the kw·ic inputs of a pixel are contiguous in NHWC (weights [ky][kx][ic][16] are already - /// [ky][kx·ic][16]), so interior pixels point straight into the source; pixels whose run - /// crosses a padded / reflected border are gathered into a thread-local buffer, constant-pad - /// rows read . - /// - private static void DenseRowNeon(in DenseArgs a, int y, int blk, float* orow, float* arow) - { - int kh = a.Kh, kw = a.Kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, run = kw * ic; - float** ptrs = stackalloc float*[Tile * kh]; - float* part = stackalloc float[Tile * Ob]; - float* wb = a.W + (long)blk * kh * run * Ob; - float* bb = a.Bias + blk * Ob; - int lanes = Math.Min(Ob, oc - blk * Ob); - bool direct = lanes == Ob && arow == null && a.Act is ActivationKind.None or ActivationKind.Relu; - float* gather = null; - for (int x0 = 0; x0 < ow; x0 += Tile) - { - int rows = Math.Min(Tile, ow - x0); - for (int ky = 0; ky < kh; ky++) - { - int r = a.Rm[y * kh + ky]; - for (int i = 0; i < Tile; i++) - { - int* cmx = a.Cm + (x0 + Math.Min(i, rows - 1)) * kw; - int cx = cmx[0]; - float* p; - if (r < 0) p = a.PadVec; - else if (cx >= 0 && cmx[kw - 1] == cx + kw - 1) p = a.Src + ((long)r * a.Iw + cx) * ic; - else - { - if (gather == null) gather = Pinned(ref t_runs, Tile * kh * run); - p = gather + (ky * Tile + i) * run; - for (int kx = 0; kx < kw; kx++) - { - int c = cmx[kx]; - Buffer.MemoryCopy(c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic, p + kx * ic, ic * 4L, ic * 4L); - } - } - ptrs[ky * Tile + i] = p; - } - } - long o = (long)x0 * oc + blk * Ob; - if (direct && rows == Tile) - { - // full tile, no residual / transcendental epilogue: store straight from the accumulators - TileNeonK(ptrs, kh, 0, run, (long)run * Ob, wb, bb, orow + o, oc, a.Act == ActivationKind.Relu); - continue; - } - TileNeonK(ptrs, kh, run, wb, bb, part, Ob); - TileNeonStore(part, rows, orow + o, arow != null ? arow + o : null, oc, lanes, a.Act); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.PackedSimd.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.PackedSimd.cs deleted file mode 100644 index ba2568a..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.PackedSimd.cs +++ /dev/null @@ -1,251 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Wasm; -using Sdcb.SimdGcdrNet.Engine; - -using V4 = System.Runtime.Intrinsics.Vector128; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// WebAssembly SIMD128 (PackedSimd) tier, after SimdPaddleOCR's Nhwc.Conv.PackedSimd. WASM SIMD has -/// no FMA (relaxed SIMD is off) and V8 maps v128 values onto at most 15 allocatable XMM registers -/// on x64, so the conv / Gemm micro-kernel is pixels × 8 oc (8 accumulators + -/// 2 weights + 1 splat), run twice per 16-oc packed block; each input float is broadcast straight -/// from memory (v128.load32_splat). Every reduction keeps the Vector<float> tier's per-lane -/// order (bias-init, taps then channels ascending, separate multiply then add), so conv sums are -/// bit-identical to it; direct convs read NHWC kernel-row runs like the NEON tier (same k order). -/// -internal static unsafe partial class Nhwc -{ - /// PackedSimd tier switch: PackedSimd.IsSupported (browser-wasm with SIMD). - /// Settable only for in-process tier comparisons (WasmBench --tiers / --vector). - internal static bool UsePacked = PackedSimd.IsSupported; - - private const int PkTile = 4; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 LdP(float* p) => PackedSimd.LoadVector128(p); - - /// acc + a·b, rounded twice (no FMA): the Vector<float> tier's acc += a * b. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 MadP(V4 acc, V4 a, V4 b) => PackedSimd.Add(acc, PackedSimd.Multiply(a, b)); - - // pmax(0, v): one maxps in V8 (f32x4.max needs a NaN / -0 fix-up); equals Max(v, 0) for finite v - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 ActP(V4 v, ActivationKind act) => act switch - { - ActivationKind.Relu => PackedSimd.PseudoMax(V4.Zero, v), - ActivationKind.Sigmoid => SimdMath.SigmoidPk(v), - ActivationKind.Gelu => SimdMath.GeluPk(v), - _ => v, - }; - - /// - /// rows × 16 oc: dst[r*ds + 0..15] = bias + Σ x·w, where ptrs[tap*ps + r] are - /// row r's ic inputs of tap and wb + tap·wTap its [ic][16] weights. Two 8-oc passes; the k loop - /// never calls out, so the accumulators stay in registers. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void TilePkK(float** ptrs, int ps, int taps, int ic, long wTap, float* wb, float* bb, float* dst, int ds) - { - for (int h = 0; h < Ob; h += 8) - { - V4 a0 = LdP(bb + h), b0 = LdP(bb + h + 4); - V4 a1 = a0, a2 = a0, a3 = a0, b1 = b0, b2 = b0, b3 = b0; - for (int tap = 0; tap < taps; tap++) - { - float** pp = ptrs + tap * ps; - float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3]; - float* w = wb + tap * wTap + h; - for (int c = 0; c < ic; c++, w += Ob) - { - V4 w0 = LdP(w), w1 = LdP(w + 4); - V4 v = PackedSimd.LoadScalarAndSplatVector128(p0 + c); - a0 = MadP(a0, v, w0); b0 = MadP(b0, v, w1); - v = PackedSimd.LoadScalarAndSplatVector128(p1 + c); - a1 = MadP(a1, v, w0); b1 = MadP(b1, v, w1); - v = PackedSimd.LoadScalarAndSplatVector128(p2 + c); - a2 = MadP(a2, v, w0); b2 = MadP(b2, v, w1); - v = PackedSimd.LoadScalarAndSplatVector128(p3 + c); - a3 = MadP(a3, v, w0); b3 = MadP(b3, v, w1); - } - } - float* d = dst + h; - PackedSimd.Store(d, a0); PackedSimd.Store(d + 4, b0); d += ds; - PackedSimd.Store(d, a1); PackedSimd.Store(d + 4, b1); d += ds; - PackedSimd.Store(d, a2); PackedSimd.Store(d + 4, b2); d += ds; - PackedSimd.Store(d, a3); PackedSimd.Store(d + 4, b3); - } - } - - /// Residual / activation epilogue of a finished [rows][16] scratch tile. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void StorePk(float* part, int rows, float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - for (int r = 0; r < rows; r++, part += Ob) - { - float* o = outp + (long)r * oc; - float* ad = addp == null ? null : addp + (long)r * oc; - if (lanes == Ob) - { - for (int l = 0; l < Ob; l += 4) - { - V4 v = LdP(part + l); - if (ad != null) v = PackedSimd.Add(v, LdP(ad + l)); - PackedSimd.Store(o + l, ActP(v, act)); - } - continue; - } - for (int l = 0; l < lanes; l++) - o[l] = ActS(ad != null ? part[l] + ad[l] : part[l], act); - } - } - - /// on PackedSimd: kh taps over NHWC kernel-row runs (kw·ic contiguous - /// floats, see ); border-crossing runs are gathered, constant-pad rows read PadVec. - private static void DenseRowPk(in DenseArgs a, int y, int blk, float* orow, float* arow) - { - int kh = a.Kh, kw = a.Kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, run = kw * ic; - float** ptrs = stackalloc float*[PkTile * kh]; - float* part = stackalloc float[PkTile * Ob]; - float* wb = a.W + (long)blk * kh * run * Ob; - float* bb = a.Bias + blk * Ob; - int lanes = Math.Min(Ob, oc - blk * Ob); - float* gather = null; - for (int x0 = 0; x0 < ow; x0 += PkTile) - { - int rows = Math.Min(PkTile, ow - x0); - for (int ky = 0; ky < kh; ky++) - { - int r = a.Rm[y * kh + ky]; - for (int i = 0; i < PkTile; i++) - { - int* cmx = a.Cm + (x0 + Math.Min(i, rows - 1)) * kw; - int cx = cmx[0]; - float* p; - if (r < 0) p = a.PadVec; - else if (cx >= 0 && cmx[kw - 1] == cx + kw - 1) p = a.Src + ((long)r * a.Iw + cx) * ic; - else - { - if (gather == null) gather = Pinned(ref t_runs, PkTile * kh * run); - p = gather + (ky * PkTile + i) * run; - for (int kx = 0; kx < kw; kx++) - { - int c = cmx[kx]; - Buffer.MemoryCopy(c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic, p + kx * ic, ic * 4L, ic * 4L); - } - } - ptrs[ky * PkTile + i] = p; - } - } - long o = (long)x0 * oc + blk * Ob; - TilePkK(ptrs, PkTile, kh, run, (long)run * Ob, wb, bb, part, Ob); - StorePk(part, rows, orow + o, arow != null ? arow + o : null, oc, lanes, a.Act); - } - } - - /// on PackedSimd: -row tiles, 12 per task sharing one weight panel walk. - private static void GemmPk(ReadOnlySpan a, int m, int k, Span dst, int n, NhwcConvPlan kp, ActivationKind act) - { - const int G = 12; - int nb = kp.Blocks, tiles = (m + PkTile - 1) / PkTile, groups = (tiles + G - 1) / G; - fixed (float* ap0 = a, dp0 = dst, wp0 = kp.W, bp0 = kp.Bias) - { - float* ap = ap0, dp = dp0, wp = wp0, bp = bp0; - For(groups * nb, (long)m * n * k / 64, t => - { - int g = t / nb, blk = t % nb; - float** ptrs = stackalloc float*[PkTile]; - float* part = stackalloc float[PkTile * Ob]; - float* wb = wp + (long)blk * k * Ob, bb = bp + blk * Ob; - int lanes = Math.Min(Ob, n - blk * Ob); - for (int tile = g * G, te = Math.Min(tiles, tile + G); tile < te; tile++) - { - int m0 = tile * PkTile, rows = Math.Min(PkTile, m - m0); - for (int i = 0; i < PkTile; i++) ptrs[i] = ap + (long)(m0 + Math.Min(i, rows - 1)) * k; - TilePkK(ptrs, PkTile, 1, k, (long)k * Ob, wb, bb, part, Ob); - StorePk(part, rows, dp + (long)m0 * n + blk * Ob, null, n, lanes, act); - } - }); - } - } - - /// - /// One depthwise output pixel on PackedSimd: 16 then 4 channels per step, bias-initialized, taps - /// ascending, + residual, activation. Returns the first channel left for the scalar tail. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static int DepthwisePixelPk(float** ptrs, int taps, int c, float* wp, float* bp, float* ad, float* o, ActivationKind act) - { - int ch = 0; - for (; ch + 16 <= c; ch += 16) - { - V4 a0 = LdP(bp + ch), a1 = LdP(bp + ch + 4), a2 = LdP(bp + ch + 8), a3 = LdP(bp + ch + 12); - float* w = wp + ch; - for (int tap = 0; tap < taps; tap++, w += c) - { - float* s = ptrs[tap] + ch; - a0 = MadP(a0, LdP(s), LdP(w)); - a1 = MadP(a1, LdP(s + 4), LdP(w + 4)); - a2 = MadP(a2, LdP(s + 8), LdP(w + 8)); - a3 = MadP(a3, LdP(s + 12), LdP(w + 12)); - } - if (ad != null) - { - a0 = PackedSimd.Add(a0, LdP(ad + ch)); a1 = PackedSimd.Add(a1, LdP(ad + ch + 4)); - a2 = PackedSimd.Add(a2, LdP(ad + ch + 8)); a3 = PackedSimd.Add(a3, LdP(ad + ch + 12)); - } - PackedSimd.Store(o + ch, ActP(a0, act)); PackedSimd.Store(o + ch + 4, ActP(a1, act)); - PackedSimd.Store(o + ch + 8, ActP(a2, act)); PackedSimd.Store(o + ch + 12, ActP(a3, act)); - } - for (; ch + 4 <= c; ch += 4) - { - V4 acc = LdP(bp + ch); - for (int tap = 0; tap < taps; tap++) - acc = MadP(acc, LdP(ptrs[tap] + ch), LdP(wp + tap * c + ch)); - if (ad != null) acc = PackedSimd.Add(acc, LdP(ad + ch)); - PackedSimd.Store(o + ch, ActP(acc, act)); - } - return ch; - } - - /// - /// One output row for oc <= 4, c % 4 == 0 (see ): - /// per 4 channels the bilinear lerps, [relu], + [relu](skip), then each channel is splatted back - /// from a 16-byte spill (WASM SIMD has no by-element multiply) into one 4-lane accumulator. - /// Same per-element ops as the unfused Vector<float> path. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void UpHeadRowPk(float* r0, float* r1, int* x0, int* x1, float* wx, float wy, float* krow, - int c, int ow, float* wp, float* bp, float* orow, int oc, bool relu, bool skipRelu, ActivationKind act) - { - V4 vY = PackedSimd.Splat(wy), bias = LdP(bp); - float* s = stackalloc float[4]; - for (int x = 0; x < ow; x++) - { - float* a = r0 + (long)x0[x] * c, b = r0 + (long)x1[x] * c, e = r1 + (long)x0[x] * c, f = r1 + (long)x1[x] * c; - float* k = krow + (long)x * c; - V4 vX = PackedSimd.Splat(wx[x]), acc = bias; - float* w = wp; - for (int ch = 0; ch < c; ch += 4, w += 32) - { - V4 va = LdP(a + ch), ve = LdP(e + ch); - V4 top = MadP(va, PackedSimd.Subtract(LdP(b + ch), va), vX); - V4 bot = MadP(ve, PackedSimd.Subtract(LdP(f + ch), ve), vX); - V4 v = MadP(top, PackedSimd.Subtract(bot, top), vY); - if (relu) v = PackedSimd.PseudoMax(V4.Zero, v); - V4 sk = LdP(k + ch); - v = PackedSimd.Add(v, skipRelu ? PackedSimd.PseudoMax(V4.Zero, sk) : sk); - PackedSimd.Store(s, v); - acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s), LdP(w)); - acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 1), LdP(w + 8)); - acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 2), LdP(w + 16)); - acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 3), LdP(w + 24)); - } - PackedSimd.Store(s, ActP(acc, act)); - float* o = orow + (long)x * oc; - for (int l = 0; l < oc; l++) o[l] = s[l]; - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.cs deleted file mode 100644 index e3122af..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.cs +++ /dev/null @@ -1,904 +0,0 @@ -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.Wasm; -using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; - -using F32x8 = System.Runtime.Intrinsics.Vector256; -using V128 = System.Runtime.Intrinsics.Vector128; -using V256 = System.Runtime.Intrinsics.Vector256; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// Backend-private packed conv weights for the NHWC CPU kernels (lives in ). -internal sealed class NhwcConvPlan -{ - public required float[] W; // dense: [oc/16][tap][ic][16] (oc zero-padded); depthwise: [tap][c] - public required float[] Bias; // dense: padded to blocks*16; depthwise: [c] - public bool Depthwise; - public int Blocks; - public float[]? WinoU; // Winograd F(4x4,3x3) weights [36][oc/16][ic][16], null = direct conv -} - -/// Packed weights: [c][8] oc lanes (zero-padded) + bias[8]. -internal sealed class UpHeadPlan -{ - public required float[] W; - public required float[] Bias; - public int Oc, C; -} - -/// -/// Channels-last (NHWC, N==1) kernels. Shapes are passed in storage order -/// ([h,w,c]). Every pad mode goes through precomputed source row/col maps -/// (-1 = constant pad → a pad-value pixel vector), so the inner loops never -/// branch on borders. Dense/pointwise conv: register-tiled 6 pixels × 16 oc -/// AVX2 micro-kernel (after SimdPaddleOCR's Nhwc.Conv) with bias-init and a -/// fused residual + activation epilogue; AVX-512 widens it to 10 × 32 (Nhwc.Avx512.cs), -/// NEON keeps 6 × 16 (Nhwc.Neon.cs), WASM SIMD128 narrows it to 4 × 8 (Nhwc.PackedSimd.cs); -/// other tiers use Vector<float>. -/// -internal static unsafe partial class Nhwc -{ - private const int Ob = 16, Tile = 6; - private static readonly ParallelOptions PO = new() { MaxDegreeOfParallelism = Avx2Kernels.Threads }; - private static bool UseAvx => Avx2.IsSupported && Fma.IsSupported; - - private static void For(int n, long work, Action body) - { - if (work < 32768 || n == 1) { for (int i = 0; i < n; i++) body(i); return; } - Parallel.For(0, n, PO, body); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int Reflect(int i, int dim) - { - if (dim <= 1) return 0; - while (i < 0 || i >= dim) i = i < 0 ? -i : 2 * dim - 2 - i; - return i; - } - - /// map[o*k+kk] = source index of output o, tap kk, or -1 for constant pad. - private static int[] Map(int outLen, int k, int stride, int pad, int inLen, PadMode mode) - { - int[] m = new int[outLen * k]; - for (int o = 0; o < outLen; o++) - for (int kk = 0; kk < k; kk++) - { - int s = o * stride + kk - pad; - m[o * k + kk] = (uint)s < (uint)inLen ? s : mode == PadMode.Reflect ? Reflect(s, inLen) : -1; - } - return m; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static float ActS(float v, ActivationKind act) => - act == ActivationKind.Relu ? MathF.Max(v, 0f) : - act == ActivationKind.Sigmoid ? 1f / (1f + MathF.Exp(-v)) : - act == ActivationKind.Gelu ? 0.5f * v * (1f + SimdMath.ErfScalar(v * 0.70710678f)) : v; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 Act8(F32x8 v, ActivationKind act) => act switch - { - ActivationKind.Relu => V256.Max(v, F32x8.Zero), - ActivationKind.Sigmoid => SimdMath.SigmoidCore(v), - ActivationKind.Gelu => SimdMath.GeluCore(v), - _ => v, - }; - - // ======================== weight packing ======================== - public static NhwcConvPlan Pack(ReadOnlySpan w, int[] wShape, ReadOnlySpan bias, int group, int ic) - { - int oc = wShape[0], icg = wShape[1], taps = wShape[2] * wShape[3]; - if (group > 1) - { - if (group != ic || icg != 1 || oc != ic) - throw new NotSupportedException($"NHWC grouped conv supports depthwise only (g={group}, ic={ic}, oc={oc})."); - float[] pw = new float[taps * oc]; - for (int c = 0; c < oc; c++) - for (int t = 0; t < taps; t++) pw[t * oc + c] = w[c * taps + t]; - float[] pb = new float[oc]; - if (!bias.IsEmpty) bias.CopyTo(pb); - return new NhwcConvPlan { W = pw, Bias = pb, Depthwise = true }; - } - int nb = (oc + Ob - 1) / Ob; - float[] p = new float[checked(nb * taps * ic * Ob)]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < taps; t++) - p[(((o / Ob) * taps + t) * ic + c) * Ob + o % Ob] = w[(o * ic + c) * taps + t]; - float[] b = new float[nb * Ob]; - if (!bias.IsEmpty) bias.CopyTo(b); - return new NhwcConvPlan { W = p, Bias = b, Blocks = nb }; - } - - /// Packs Gemm B ([K,N], or [N,K] when transB) as [N/16][K][16] — a 1x1 conv panel. - public static NhwcConvPlan PackGemm(ReadOnlySpan b, int k, int n, bool transB, ReadOnlySpan bias) - { - int nb = (n + Ob - 1) / Ob; - float[] p = new float[checked(nb * k * Ob)]; - for (int o = 0; o < n; o++) - for (int c = 0; c < k; c++) - p[((o / Ob) * k + c) * Ob + o % Ob] = transB ? b[o * k + c] : b[c * n + o]; - float[] pb = new float[nb * Ob]; - if (!bias.IsEmpty) bias.CopyTo(pb); - return new NhwcConvPlan { W = p, Bias = pb, Blocks = nb }; - } - - /// dst[M,N] = act(a[M,K]·B + bias) through the 6x16 conv micro-kernel. - public static void Gemm(ReadOnlySpan a, int m, int k, Span dst, int n, - NhwcConvPlan kp, ActivationKind act) - { - if (UseAvx512) { Gemm512(a, m, k, dst, n, kp, act); return; } - if (UsePacked) { GemmPk(a, m, k, dst, n, kp, act); return; } - const int G = 8; // row tiles per task sharing one weight panel walk - int nb = kp.Blocks, tiles = (m + Tile - 1) / Tile, groups = (tiles + G - 1) / G; - bool avx = UseAvx; - fixed (float* ap0 = a, dp0 = dst, wp0 = kp.W, bp0 = kp.Bias) - { - float* ap = ap0, dp = dp0, wp = wp0, bp = bp0; - For(groups * nb, (long)m * n * k / 64, t => - { - int g = t / nb, blk = t % nb; - float** ptrs = stackalloc float*[Tile]; - float* wb = wp + (long)blk * k * Ob, bb = bp + blk * Ob; - int lanes = Math.Min(Ob, n - blk * Ob); - for (int tile = g * G, te = Math.Min(tiles, tile + G); tile < te; tile++) - { - int m0 = tile * Tile, rows = Math.Min(Tile, m - m0); - for (int i = 0; i < Tile; i++) ptrs[i] = ap + (long)(m0 + Math.Min(i, rows - 1)) * k; - float* outp = dp + (long)m0 * n + blk * Ob; - if (avx) TileAvx(ptrs, rows, 1, k, wb, bb, outp, null, n, lanes, act); - else TileVec(ptrs, rows, 1, k, wb, bb, outp, null, n, lanes, act); - } - }); - } - } - - // ======================== conv ======================== - public static void Conv(ReadOnlySpan src, int ih, int iw, int ic, - Span dst, int oh, int ow, int oc, ConvP p, NhwcConvPlan kp, ReadOnlySpan add) - { - if (!add.IsEmpty && add.Length != dst.Length) - throw new NotSupportedException($"conv residual addend has {add.Length} elems, expected {dst.Length}."); - if (kp.WinoU is not null) - { - WinogradConv(src, ih, iw, ic, dst, oh, ow, oc, p, kp, add); - return; - } - int kh = p.Kernel[0], kw = p.Kernel[1]; - int[] rm = Map(oh, kh, p.Strides[0], p.Pads[0], ih, p.PadMode); - int[] cm = Map(ow, kw, p.Strides[1], p.Pads[1], iw, p.PadMode); - float[] padVec = new float[ic * kw + Ob]; // one constant-pad kernel row (Dense512 row runs) - Array.Fill(padVec, p.PadValue); - if (kp.Depthwise) - Depthwise(src, iw, ic, dst, oh, ow, kh, kw, rm, cm, padVec, kp, add, p.Act); - else if (UseAvx512) - Dense512(src, iw, ic, dst, oh, ow, oc, kh, kw, rm, cm, padVec, kp, add, p.Act); - else - Dense(src, iw, ic, dst, oh, ow, oc, kh, kw, rm, cm, padVec, kp, add, p.Act); - } - - private static void Dense(ReadOnlySpan src, int iw, int ic, Span dst, - int oh, int ow, int oc, int kh, int kw, int[] rm, int[] cm, float[] padVec, - NhwcConvPlan kp, ReadOnlySpan add, ActivationKind act) - { - int taps = kh * kw, nb = kp.Blocks; - bool hasAdd = !add.IsEmpty, avx = UseAvx; - fixed (float* sp0 = src, dp0 = dst, ap0 = add, wp0 = kp.W, bp0 = kp.Bias, pv0 = padVec) - fixed (int* rm0 = rm, cm0 = cm) - { - var a = new DenseArgs(sp0, wp0, bp0, pv0, rm0, cm0, iw, ic, ow, oc, kh, kw, nb, act, avx); - float* dp = dp0, ap = ap0; - For(oh * nb, (long)oh * ow * oc * taps * ic / 64, t => - { - int y = t / nb, blk = t % nb; - long o = (long)y * ow * oc; - DenseRow(a, y, blk, dp + o, hasAdd ? ap + o : null); - }); - } - } - - /// Per-call direct-conv constants (pointers into pinned/fixed storage). - private readonly unsafe struct DenseArgs(float* src, float* w, float* bias, float* padVec, int* rm, int* cm, - int iw, int ic, int ow, int oc, int kh, int kw, int nb, ActivationKind act, bool avx) - { - public readonly float* Src = src, W = w, Bias = bias, PadVec = padVec; - public readonly int* Rm = rm, Cm = cm; - public readonly int Iw = iw, Ic = ic, Ow = ow, Oc = oc, Kh = kh, Kw = kw, Nb = nb; - public readonly ActivationKind Act = act; - public readonly bool Avx = avx; - } - - /// Output row , oc block of a direct conv - /// into (the row's first pixel; addend row likewise). - private static void DenseRow(in DenseArgs a, int y, int blk, float* orow, float* arow) - { - if (UseNeon) - { - DenseRowNeon(a, y, blk, orow, arow); - return; - } - if (UsePacked) - { - DenseRowPk(a, y, blk, orow, arow); - return; - } - int taps = a.Kh * a.Kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, kw = a.Kw; - float** ptrs = stackalloc float*[Tile * taps]; - float* wb = a.W + (long)blk * taps * ic * Ob; - float* bb = a.Bias + blk * Ob; - int lanes = Math.Min(Ob, oc - blk * Ob); - for (int x0 = 0; x0 < ow; x0 += Tile) - { - int rows = Math.Min(Tile, ow - x0); - for (int tap = 0; tap < taps; tap++) - { - int r = a.Rm[y * a.Kh + tap / kw]; - for (int i = 0; i < Tile; i++) - { - int c = a.Cm[(x0 + Math.Min(i, rows - 1)) * kw + tap % kw]; - ptrs[tap * Tile + i] = r < 0 || c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic; - } - } - long o = (long)x0 * oc + blk * Ob; - float* outp = orow + o; - float* addp = arow != null ? arow + o : null; - if (a.Avx) TileAvx(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, a.Act); - else TileVec(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, a.Act); - } - } - - /// 6 × 16 tile: into a scratch tile, then the - /// residual / activation epilogue () per row. - private static void TileAvx(float** ptrs, int rows, int taps, int ic, float* wb, float* bb, - float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - float* part = stackalloc float[Tile * Ob]; - TileAvxK(ptrs, taps, ic, wb, bb, part, Ob); - for (int r = 0; r < rows; r++, part += Ob) - Store16(outp + (long)r * oc, addp == null ? null : addp + (long)r * oc, - Avx.LoadVector256(part), Avx.LoadVector256(part + 8), lanes, act); - } - - /// - /// The 6 × 16 k loop: dst[r*ds + 0..15] = bias + Σ x·w for all rows. It never - /// calls out — passing the accumulators to a helper (by reference under the x64 ABIs) makes - /// RyuJIT keep all twelve on the stack, a load + store around every FMA. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void TileAvxK(float** ptrs, int taps, int ic, float* wb, float* bb, float* dst, int ds) - { - F32x8 a0 = Avx.LoadVector256(bb), b0 = Avx.LoadVector256(bb + 8); - F32x8 a1 = a0, a2 = a0, a3 = a0, a4 = a0, a5 = a0; - F32x8 b1 = b0, b2 = b0, b3 = b0, b4 = b0, b5 = b0; - for (int tap = 0; tap < taps; tap++) - { - float** pp = ptrs + tap * Tile; - float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3], p4 = pp[4], p5 = pp[5]; - float* w = wb + (long)tap * ic * Ob; - for (int c = 0; c < ic; c++, w += Ob) - { - F32x8 w0 = Avx.LoadVector256(w), w1 = Avx.LoadVector256(w + 8); - F32x8 v0 = Avx.BroadcastScalarToVector256(p0 + c), v1 = Avx.BroadcastScalarToVector256(p1 + c); - a0 = Fma.MultiplyAdd(v0, w0, a0); b0 = Fma.MultiplyAdd(v0, w1, b0); - a1 = Fma.MultiplyAdd(v1, w0, a1); b1 = Fma.MultiplyAdd(v1, w1, b1); - F32x8 v2 = Avx.BroadcastScalarToVector256(p2 + c), v3 = Avx.BroadcastScalarToVector256(p3 + c); - a2 = Fma.MultiplyAdd(v2, w0, a2); b2 = Fma.MultiplyAdd(v2, w1, b2); - a3 = Fma.MultiplyAdd(v3, w0, a3); b3 = Fma.MultiplyAdd(v3, w1, b3); - F32x8 v4 = Avx.BroadcastScalarToVector256(p4 + c), v5 = Avx.BroadcastScalarToVector256(p5 + c); - a4 = Fma.MultiplyAdd(v4, w0, a4); b4 = Fma.MultiplyAdd(v4, w1, b4); - a5 = Fma.MultiplyAdd(v5, w0, a5); b5 = Fma.MultiplyAdd(v5, w1, b5); - } - } - Avx.Store(dst, a0); Avx.Store(dst + 8, b0); dst += ds; - Avx.Store(dst, a1); Avx.Store(dst + 8, b1); dst += ds; - Avx.Store(dst, a2); Avx.Store(dst + 8, b2); dst += ds; - Avx.Store(dst, a3); Avx.Store(dst + 8, b3); dst += ds; - Avx.Store(dst, a4); Avx.Store(dst + 8, b4); dst += ds; - Avx.Store(dst, a5); Avx.Store(dst + 8, b5); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Store16(float* o, float* add, F32x8 a, F32x8 b, int lanes, ActivationKind act) - { - if (lanes == Ob) - { - if (add != null) { a = Avx.Add(a, Avx.LoadVector256(add)); b = Avx.Add(b, Avx.LoadVector256(add + 8)); } - Avx.Store(o, Act8(a, act)); Avx.Store(o + 8, Act8(b, act)); - return; - } - // channel tail (e.g. the final oc=3 conv): only the valid lanes touch memory - float* t = stackalloc float[Ob]; - Avx.Store(t, a); Avx.Store(t + 8, b); - for (int l = 0; l < lanes; l++) - o[l] = ActS(add != null ? t[l] + add[l] : t[l], act); - } - - private static void TileVec(float** ptrs, int rows, int taps, int ic, float* wb, float* bb, - float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - if (UseNeon) - { - TileNeon(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, act); - return; - } - int VL = Vector.Count; - // software Vector (DOTNET_EnableHWIntrinsic=0) is far slower than plain scalar - bool vec = Vector.IsHardwareAccelerated && Ob % VL == 0; - if (vec && VL == 4) - { - TileVec4(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, act); - return; - } - float* acc = stackalloc float[Ob]; - for (int i = 0; i < rows; i++) - { - for (int l = 0; l < Ob; l++) acc[l] = bb[l]; - for (int tap = 0; tap < taps; tap++) - { - float* p = ptrs[tap * Tile + i]; - float* w = wb + (long)tap * ic * Ob; - for (int c = 0; c < ic; c++, w += Ob) - { - float v = p[c]; - if (vec) - { - var vv = new Vector(v); - for (int l = 0; l < Ob; l += VL) - Simd.VectorStore(acc + l, Simd.VectorLoad(acc + l) + Simd.VectorLoad(w + l) * vv); - } - else - for (int l = 0; l < Ob; l++) acc[l] += v * w[l]; - } - } - float* o = outp + (long)i * oc; - float* ad = addp == null ? null : addp + (long)i * oc; - for (int l = 0; l < lanes; l++) - o[l] = ActS(ad != null ? acc[l] + ad[l] : acc[l], act); - } - } - - /// - /// 128-bit Vector<float> tile (AVX2 disabled): two pixel rows × 16 oc in eight register - /// accumulators (+4 weight vectors), instead of a stack accumulator round-tripped through - /// memory on every multiply-add. Same per-lane op order (acc + w·x) as the generic loop. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void TileVec4(float** ptrs, int rows, int taps, int ic, float* wb, float* bb, - float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - float* tmp = stackalloc float[2 * Ob]; - for (int i = 0; i < rows; i += 2) - { - Vector a0 = Simd.VectorLoad(bb), a1 = Simd.VectorLoad(bb + 4), - a2 = Simd.VectorLoad(bb + 8), a3 = Simd.VectorLoad(bb + 12); - Vector c0 = a0, c1 = a1, c2 = a2, c3 = a3; - for (int tap = 0; tap < taps; tap++) - { - // ptrs holds Tile entries per tap (short tiles repeat their last pixel) - float* p = ptrs[tap * Tile + i], q = ptrs[tap * Tile + i + 1]; - float* w = wb + (long)tap * ic * Ob; - for (int c = 0; c < ic; c++, w += Ob) - { - Vector w0 = Simd.VectorLoad(w), w1 = Simd.VectorLoad(w + 4), - w2 = Simd.VectorLoad(w + 8), w3 = Simd.VectorLoad(w + 12); - var vp = new Vector(p[c]); - a0 += w0 * vp; a1 += w1 * vp; a2 += w2 * vp; a3 += w3 * vp; - var vq = new Vector(q[c]); - c0 += w0 * vq; c1 += w1 * vq; c2 += w2 * vq; c3 += w3 * vq; - } - } - Simd.VectorStore(tmp, a0); Simd.VectorStore(tmp + 4, a1); - Simd.VectorStore(tmp + 8, a2); Simd.VectorStore(tmp + 12, a3); - Simd.VectorStore(tmp + 16, c0); Simd.VectorStore(tmp + 20, c1); - Simd.VectorStore(tmp + 24, c2); Simd.VectorStore(tmp + 28, c3); - for (int r = 0; r < 2 && i + r < rows; r++) - { - float* o = outp + (long)(i + r) * oc; - float* ad = addp == null ? null : addp + (long)(i + r) * oc; - float* t = tmp + r * Ob; - for (int l = 0; l < lanes; l++) - o[l] = ActS(ad != null ? t[l] + ad[l] : t[l], act); - } - } - } - - private static void Depthwise(ReadOnlySpan src, int iw, int c, Span dst, - int oh, int ow, int kh, int kw, int[] rm, int[] cm, float[] padVec, - NhwcConvPlan kp, ReadOnlySpan add, ActivationKind act) - { - int taps = kh * kw; - bool hasAdd = !add.IsEmpty, avx = UseAvx; - int VL = Vector.Count; - fixed (float* sp0 = src, dp0 = dst, ap0 = add, wp0 = kp.W, bp0 = kp.Bias, pv0 = padVec) - fixed (int* rm0 = rm, cm0 = cm) - { - float* sp = sp0, dp = dp0, ap = ap0, wp = wp0, bp = bp0, pv = pv0; - int* rmp = rm0, cmp = cm0; - For(oh, (long)oh * ow * c * taps, y => - { - float** ptrs = stackalloc float*[taps]; - for (int x = 0; x < ow; x++) - { - for (int tap = 0; tap < taps; tap++) - { - int r = rmp[y * kh + tap / kw], cc = cmp[x * kw + tap % kw]; - ptrs[tap] = r < 0 || cc < 0 ? pv : sp + ((long)r * iw + cc) * c; - } - long ob = ((long)y * ow + x) * c; - float* o = dp + ob; - float* ad = hasAdd ? ap + ob : null; - int ch = 0; - if (UseNeon) - { - for (; ch + 4 <= c; ch += 4) - { - V128 acc = AdvSimd.LoadVector128(bp + ch); - for (int tap = 0; tap < taps; tap++) - acc = AdvSimd.FusedMultiplyAdd(acc, AdvSimd.LoadVector128(ptrs[tap] + ch), AdvSimd.LoadVector128(wp + tap * c + ch)); - if (ad != null) acc = AdvSimd.Add(acc, AdvSimd.LoadVector128(ad + ch)); - AdvSimd.Store(o + ch, Act4(acc, act)); - } - } - else if (avx) - { - for (; ch + 8 <= c; ch += 8) - { - F32x8 acc = Avx.LoadVector256(bp + ch); - for (int tap = 0; tap < taps; tap++) - acc = Fma.MultiplyAdd(Avx.LoadVector256(ptrs[tap] + ch), Avx.LoadVector256(wp + tap * c + ch), acc); - if (ad != null) acc = Avx.Add(acc, Avx.LoadVector256(ad + ch)); - Avx.Store(o + ch, Act8(acc, act)); - } - } - else if (UsePacked) - ch = DepthwisePixelPk(ptrs, taps, c, wp, bp, ad, o, act); - else if (Vector.IsHardwareAccelerated && act is ActivationKind.None or ActivationKind.Relu) - { - for (; ch + VL <= c; ch += VL) - { - var acc = Simd.VectorLoad(bp + ch); - for (int tap = 0; tap < taps; tap++) - acc += Simd.VectorLoad(ptrs[tap] + ch) * Simd.VectorLoad(wp + tap * c + ch); - if (ad != null) acc += Simd.VectorLoad(ad + ch); - if (act == ActivationKind.Relu) acc = Vector.Max(acc, Vector.Zero); - Simd.VectorStore(o + ch, acc); - } - } - for (; ch < c; ch++) - { - float acc = bp[ch]; - for (int tap = 0; tap < taps; tap++) acc += ptrs[tap][ch] * wp[tap * c + ch]; - if (ad != null) acc += ad[ch]; - o[ch] = ActS(acc, act); - } - } - }); - } - } - - // ======================== pixel-vector helpers ======================== - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Copy(float* s, float* d, int n) => - Buffer.MemoryCopy(s, d, (long)n * 4, (long)n * 4); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Fill(float* d, int n, float v) => new Span(d, n).Fill(v); - - // ======================== conv + 2x2 max pool ======================== - [ThreadStatic] private static float[]? t_band; - - private static float[] BandBuf(long floats) => - t_band is { } b && b.Length >= floats ? b : (t_band = new float[checked((int)floats)]); - - /// - /// Dense / Winograd conv with a fused 2x2 stride-2 max pool (): - /// each task produces a band of conv rows (2 direct, 4 = one Winograd tile row) in a - /// thread-local buffer and pools it straight into ([ph, pw, oc]), - /// so the full-resolution conv output is never stored. Per-element math is the same as - /// the separate Conv → MaxPool nodes. - /// - public static void ConvPool(ReadOnlySpan src, int ih, int iw, int ic, - Span dst, int ph, int pw, int oc, ConvP p, NhwcConvPlan kp) - { - int oh = ph * 2, ow = pw * 2; - if (kp.Depthwise || p.AddValue >= 0 || dst.Length != (long)ph * pw * oc) - throw new NotSupportedException("ConvPool shape/kind mismatch."); - bool relu = p.Pool!.FuseRelu, wino = kp.WinoU is not null; - int band = wino ? 4 : 2, tasks = (oh + band - 1) / band; - int kh = p.Kernel[0], kw = p.Kernel[1]; - float[] padVec = new float[ic * kw + Ob]; // one constant-pad kernel row (Dense512Row) - Array.Fill(padVec, p.PadValue); - int[] rm = Map(oh, kh, p.Strides[0], p.Pads[0], ih, p.PadMode); - int[] cm = Map(ow, kw, p.Strides[1], p.Pads[1], iw, p.PadMode); - fixed (float* sp = src, dp0 = dst, wp = kp.W, up = kp.WinoU, bp = kp.Bias, pv = padVec) - fixed (int* rmp = rm, cmp = cm) - { - var da = new DenseArgs(sp, wp, bp, pv, rmp, cmp, iw, ic, ow, oc, kh, kw, kp.Blocks, p.Act, UseAvx); - var wa = wino ? new WinoArgs(sp, up, bp, pv, ih, iw, ic, oh, ow, oc, kp.Blocks, p) : default; - float* dp = dp0; - For(tasks, (long)oh * ow * oc * kh * kw * ic / 64, t => - { - int y0 = t * band, rows = Math.Min(band, oh - y0); - float[] buf = BandBuf((long)band * ow * oc); - fixed (float* sb = buf) - { - if (wino) - for (int t0 = y0 / 4 * wa.Tw, te = t0 + wa.Tw; t0 < te; t0 += wa.T) - WinoTiles(wa, t0, Math.Min(wa.T, te - t0), sb, null, y0); - else - for (int r = 0; r < rows; r++) - ConvRow(da, y0 + r, sb + (long)r * ow * oc, null); - PoolBand(sb, rows / 2, ow, oc, dp + (long)(y0 / 2) * pw * oc, pw, relu); - } - }); - } - } - - /// 2x2/2 max over conv row pairs of a band; same op order as . - private static void PoolBand(float* sb, int prows, int ow, int c, float* drow, int pw, bool relu) - { - int VL = Vector.Count; - for (int r = 0; r < prows; r++) - { - float* top = sb + (long)(2 * r) * ow * c, bot = top + (long)ow * c; - float* orow = drow + (long)r * pw * c; - for (int x = 0; x < pw; x++) - { - float* o = orow + (long)x * c; - float* s0 = top + (long)(2 * x) * c, s1 = s0 + c, s2 = bot + (long)(2 * x) * c, s3 = s2 + c; - int ch = 0; - for (; ch + VL <= c; ch += VL) - { - var v = Vector.Max(Simd.VectorLoad(s0 + ch), Simd.VectorLoad(s1 + ch)); - v = Vector.Max(v, Simd.VectorLoad(s2 + ch)); - Simd.VectorStore(o + ch, Vector.Max(v, Simd.VectorLoad(s3 + ch))); - } - for (; ch < c; ch++) o[ch] = MathF.Max(MathF.Max(MathF.Max(s0[ch], s1[ch]), s2[ch]), s3[ch]); - if (relu) - for (ch = 0; ch < c; ch++) o[ch] = MathF.Max(o[ch], 0f); - } - } - } - - // ======================== maxpool ======================== - public static void MaxPool(ReadOnlySpan src, int ih, int iw, int c, - Span dst, int oh, int ow, PoolP p) - { - int kh = p.Kernel[0], kw = p.Kernel[1]; - int[] rm = Map(oh, kh, p.Strides[0], p.Pads[0], ih, PadMode.Constant); - int[] cm = Map(ow, kw, p.Strides[1], p.Pads[1], iw, PadMode.Constant); - bool relu = p.FuseRelu; - int VL = Vector.Count; - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - if (UseAvx512 && c % 16 == 0) { MaxPool512(sp, iw, c, dp, oh, ow, kh, kw, rm, cm, relu); return; } - For(oh, (long)oh * ow * c * kh * kw, y => - { - for (int x = 0; x < ow; x++) - { - float* o = dp + ((long)y * ow + x) * c; - bool any = false; - for (int ky = 0; ky < kh; ky++) - { - int r = rm[y * kh + ky]; - if (r < 0) continue; - for (int kx = 0; kx < kw; kx++) - { - int cc = cm[x * kw + kx]; - if (cc < 0) continue; - float* s = sp + ((long)r * iw + cc) * c; - if (!any) { Copy(s, o, c); any = true; continue; } - int ch = 0; - for (; ch + VL <= c; ch += VL) - Simd.VectorStore(o + ch, Vector.Max(Simd.VectorLoad(o + ch), Simd.VectorLoad(s + ch))); - for (; ch < c; ch++) o[ch] = MathF.Max(o[ch], s[ch]); - } - } - if (!any) Fill(o, c, 0f); - else if (relu) - for (int ch = 0; ch < c; ch++) o[ch] = MathF.Max(o[ch], 0f); - } - }); - } - } - - // ======================== bilinear resize (half-pixel) ======================== - private static void ResizeAxis(int outLen, int inLen, float scale, int[] i0, int[] i1, float[] wt) - { - for (int o = 0; o < outLen; o++) - { - float f = (o + 0.5f) / scale - 0.5f; - float fl = MathF.Floor(f); - i0[o] = Math.Clamp((int)fl, 0, inLen - 1); - i1[o] = Math.Clamp((int)fl + 1, 0, inLen - 1); - wt[o] = f - fl; - } - } - - /// One output pixel of the bilinear resize into (c channels): - /// v = [relu](lerp) + [relu](add) — the same per-element ops as separate Resize/Relu/Add nodes. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void ResizePixel(float* a, float* b, float* e, float* f, float wX, float wY, - Vector vX, Vector vY, int c, bool relu, float* add, bool addRelu, float* o) - { - int VL = Vector.Count, ch = 0; - for (; ch + VL <= c; ch += VL) - { - var va = Simd.VectorLoad(a + ch); var ve = Simd.VectorLoad(e + ch); - var top = va + (Simd.VectorLoad(b + ch) - va) * vX; - var bot = ve + (Simd.VectorLoad(f + ch) - ve) * vX; - var v = top + (bot - top) * vY; - if (relu) v = Vector.Max(v, Vector.Zero); - if (add != null) - { - var s = Simd.VectorLoad(add + ch); - v += addRelu ? Vector.Max(s, Vector.Zero) : s; - } - Simd.VectorStore(o + ch, v); - } - for (; ch < c; ch++) - { - float top = a[ch] + (b[ch] - a[ch]) * wX; - float bot = e[ch] + (f[ch] - e[ch]) * wX; - float v = top + (bot - top) * wY; - if (relu) v = MathF.Max(v, 0f); - if (add != null) v += addRelu ? MathF.Max(add[ch], 0f) : add[ch]; - o[ch] = v; - } - } - - /// NHWC bilinear resize; optional same-shape addend (out = [relu](resize) + [relu](add)). - public static void ResizeBilinear(ReadOnlySpan src, int ih, int iw, int c, - Span dst, int oh, int ow, float sy, float sx, ActivationKind act, - ReadOnlySpan add = default, bool addRelu = false) - { - if (!add.IsEmpty && add.Length != dst.Length) - throw new NotSupportedException($"resize addend has {add.Length} elems, expected {dst.Length}."); - int[] y0 = new int[oh], y1 = new int[oh], x0 = new int[ow], x1 = new int[ow]; - float[] wy = new float[oh], wx = new float[ow]; - ResizeAxis(oh, ih, sy, y0, y1, wy); - ResizeAxis(ow, iw, sx, x0, x1, wx); - bool relu = act == ActivationKind.Relu, hasAdd = !add.IsEmpty, v512 = UseAvx512 && c % 16 == 0; - fixed (float* sp0 = src, dp0 = dst, ap0 = add) - { - float* sp = sp0, dp = dp0, ap = ap0; - For(oh, (long)oh * ow * c, y => - { - float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c; - var vY = new Vector(wy[y]); - float wY = wy[y]; - if (v512) - { - var vY16 = Vector512.Create(wY); - for (int x = 0; x < ow; x++) - { - long ob = ((long)y * ow + x) * c; - ResizePixel512(r0 + (long)x0[x] * c, r0 + (long)x1[x] * c, r1 + (long)x0[x] * c, r1 + (long)x1[x] * c, - Vector512.Create(wx[x]), vY16, c, relu, hasAdd ? ap + ob : null, addRelu, dp + ob); - } - return; - } - for (int x = 0; x < ow; x++) - { - long ob = ((long)y * ow + x) * c; - ResizePixel(r0 + (long)x0[x] * c, r0 + (long)x1[x] * c, r1 + (long)x0[x] * c, r1 + (long)x1[x] * c, - wx[x], wY, new Vector(wx[x]), vY, c, relu, hasAdd ? ap + ob : null, addRelu, dp + ob); - } - }); - } - } - - // ======================== fused upsample tail ======================== - /// Packs UpHead weights [oc, c, 1, 1] as [c][8] (oc lanes, zero-padded) + bias[8]. - public static UpHeadPlan PackUpHead(ReadOnlySpan w, int oc, int c, ReadOnlySpan bias) - { - if (oc > 8) throw new NotSupportedException($"UpHead supports oc <= 8 (got {oc})."); - float[] pw = new float[c * 8], pb = new float[8]; - for (int o = 0; o < oc; o++) - for (int ch = 0; ch < c; ch++) pw[ch * 8 + o] = w[o * c + ch]; - if (!bias.IsEmpty) bias.CopyTo(pb); - return new UpHeadPlan { W = pw, Bias = pb, Oc = oc, C = c }; - } - - /// - /// out[y,x,o] = act(bias[o] + Σ_c w[o,c]·s[c]), s = [relu](resize(src)) + [relu](skip): the - /// full-resolution resize/add tensors live only in a per-pixel stack buffer. Per-element - /// math follows the unfused nodes (bias-initialized, channel-ascending accumulation). - /// - public static void UpHead(ReadOnlySpan src, int ih, int iw, int c, ReadOnlySpan skip, - Span dst, int oh, int ow, float sy, float sx, UpHeadP p, UpHeadPlan kp, - int skipH = 0, int skipW = 0, int skipC = 0) - { - // SkipConv: `skip` is the conv's input [skipH, skipW, skipC]; each task recomputes - // its skip row (same direct-conv kernel → same values) instead of reading a stored one - ConvP? sc = p.SkipConv; - var skp = sc?.KernelPlan as NhwcConvPlan; - if (kp.C != c || dst.Length != (long)oh * ow * kp.Oc - || (sc is null ? skip.Length != (long)oh * ow * c - : skp is null || skp.Depthwise || skp.WinoU is not null || sc.AddValue >= 0 || skip.Length != (long)skipH * skipW * skipC)) - throw new NotSupportedException("UpHead shape mismatch."); - int[] y0 = new int[oh], y1 = new int[oh], x0 = new int[ow], x1 = new int[ow]; - float[] wy = new float[oh], wx = new float[ow]; - ResizeAxis(oh, ih, sy, y0, y1, wy); - ResizeAxis(ow, iw, sx, x0, x1, wx); - bool relu = p.ResizeRelu, skipRelu = p.SkipRelu, avx = UseAvx; - int oc = kp.Oc; - bool neonUp = UseNeon && c % 4 == 0 && oc <= 4; - bool pkUp = UsePacked && c % 4 == 0 && oc <= 4; - ActivationKind act = p.Act; - int[] rm = [], cm = []; - float[] padVec = []; - if (sc is not null) - { - rm = Map(oh, sc.Kernel[0], sc.Strides[0], sc.Pads[0], skipH, sc.PadMode); - cm = Map(ow, sc.Kernel[1], sc.Strides[1], sc.Pads[1], skipW, sc.PadMode); - padVec = new float[skipC * sc.Kernel[1] + Ob]; // one constant-pad kernel row (Dense512Row) - Array.Fill(padVec, sc.PadValue); - } - fixed (float* sp0 = src, dp0 = dst, kp0 = skip, wp0 = kp.W, bp0 = kp.Bias) - fixed (float* cw = skp?.W, cb = skp?.Bias, pv = padVec) - fixed (int* rmp = rm, cmp = cm) - { - float* sp = sp0, dp = dp0, kpp = kp0, wp = wp0, bp = bp0; - var ca = sc is null ? default - : new DenseArgs(kp0, cw, cb, pv, rmp, cmp, skipW, skipC, ow, c, sc.Kernel[0], sc.Kernel[1], skp!.Blocks, sc.Act, avx); - if (UseAvx512) - { - UpHead512(sp, iw, c, kpp, sc is not null, ca, dp, oh, ow, y0, y1, x0, x1, wy, wx, relu, skipRelu, wp, bp, oc, act); - return; - } - For(oh, (long)oh * ow * c * (oc + 4), y => - { - float* s = stackalloc float[c + 8]; - float* acc = stackalloc float[8]; - float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c; - var vY = new Vector(wy[y]); - float wY = wy[y]; - float* orow = dp + (long)y * ow * oc; - float[]? band = sc is null ? null : BandBuf((long)ow * c); - fixed (float* sb = band) - { - float* krow = kpp + (long)y * ow * c; - if (sc is not null) - { - ConvRow(ca, y, sb, null); - krow = sb; - } - if (neonUp) - { - fixed (int* xa = x0, xb = x1) - fixed (float* wxp = wx) - UpHeadRowNeon(r0, r1, xa, xb, wxp, wY, krow, c, ow, wp, bp, orow, oc, relu, skipRelu, act); - return; - } - if (pkUp) - { - fixed (int* xa = x0, xb = x1) - fixed (float* wxp = wx) - UpHeadRowPk(r0, r1, xa, xb, wxp, wY, krow, c, ow, wp, bp, orow, oc, relu, skipRelu, act); - return; - } - for (int x = 0; x < ow; x++) - { - ResizePixel(r0 + (long)x0[x] * c, r0 + (long)x1[x] * c, r1 + (long)x0[x] * c, r1 + (long)x1[x] * c, - wx[x], wY, new Vector(wx[x]), vY, c, relu, krow + (long)x * c, skipRelu, s); - float* o = orow + (long)x * oc; - if (avx) - { - F32x8 a = Avx.LoadVector256(bp); - float* w = wp; - for (int ch = 0; ch < c; ch++, w += 8) - a = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s + ch), Avx.LoadVector256(w), a); - Avx.Store(acc, a); - } - else if (UseNeon) - { - V128 a = AdvSimd.LoadVector128(bp), b = AdvSimd.LoadVector128(bp + 4); - float* w = wp; - for (int ch = 0; ch < c; ch++, w += 8) - { - V128 v = AdvSimd.LoadAndReplicateToVector128(s + ch); - a = AdvSimd.FusedMultiplyAdd(a, AdvSimd.LoadVector128(w), v); - b = AdvSimd.FusedMultiplyAdd(b, AdvSimd.LoadVector128(w + 4), v); - } - AdvSimd.Store(acc, a); AdvSimd.Store(acc + 4, b); - } - else - { - for (int l = 0; l < 8; l++) acc[l] = bp[l]; - float* w = wp; - for (int ch = 0; ch < c; ch++, w += 8) - { - float v = s[ch]; - for (int l = 0; l < oc; l++) acc[l] += v * w[l]; - } - } - for (int l = 0; l < oc; l++) o[l] = acc[l]; - } - if (act != ActivationKind.None) ActRow(orow, ow * oc, act, avx); - } - }); - } - } - - private static void ActRow(float* p, int n, ActivationKind act, bool avx) - { - int i = 0; - if (avx) - for (; i + 8 <= n; i += 8) Avx.Store(p + i, Act8(Avx.LoadVector256(p + i), act)); - else if (UseNeon) - for (; i + 4 <= n; i += 4) AdvSimd.Store(p + i, Act4(AdvSimd.LoadVector128(p + i), act)); - else if (UsePacked) - for (; i + 4 <= n; i += 4) PackedSimd.Store(p + i, ActP(LdP(p + i), act)); - else if (act == ActivationKind.Sigmoid) - { - var row = new Span(p + i, n - i); - SimdMath.Sigmoid(row, row); - return; - } - for (; i < n; i++) p[i] = ActS(p[i], act); - } - - // ======================== pad (spatial only) ======================== - public static void Pad(ReadOnlySpan src, int ih, int iw, int c, - Span dst, int oh, int ow, PadP p) - { - int rank = p.Pads.Length / 2; - if (rank != 4) throw new NotSupportedException("NHWC Pad supports rank-4 only."); - int[] rm = Map(oh, 1, 1, p.Pads[2], ih, p.Mode); - int[] cm = Map(ow, 1, 1, p.Pads[3], iw, p.Mode); - float val = p.Value; - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - For(oh, (long)oh * ow * c, y => - { - int r = rm[y]; - for (int x = 0; x < ow; x++) - { - float* o = dp + ((long)y * ow + x) * c; - int cc = cm[x]; - if (r < 0 || cc < 0) Fill(o, c, val); - else Copy(sp + ((long)r * iw + cc) * c, o, c); - } - }); - } - } - - // ======================== channel shift ======================== - // out[y,x,c] = in[y+off_g, x, c] (logical axis 2) or in[y, x+off_g, c] (axis 3), zero-fill - public static void ChannelShift(ReadOnlySpan src, int h, int w, int c, - Span dst, ShiftP p) - { - int[] ends = p.GroupEnds, offs = p.Offsets; - bool alongH = p.Axis == 2; - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - For(h, (long)h * w * c, y => - { - for (int x = 0; x < w; x++) - { - float* o = dp + ((long)y * w + x) * c; - int g0 = 0; - for (int g = 0; g < offs.Length; g++) - { - int g1 = ends[g]; - int sy = alongH ? y + offs[g] : y, sx = alongH ? x : x + offs[g]; - if ((uint)sy < (uint)h && (uint)sx < (uint)w) - Copy(sp + ((long)sy * w + sx) * c + g0, o + g0, g1 - g0); - else - Fill(o + g0, g1 - g0, 0f); - g0 = g1; - } - if (g0 < c) Copy(sp + ((long)y * w + x) * c + g0, o + g0, c - g0); - } - }); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.Neon.cs b/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.Neon.cs deleted file mode 100644 index d70409a..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.Neon.cs +++ /dev/null @@ -1,75 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// NEON Winograd schedule for layers whose transformed weights U (36·oc·ic floats) outgrow the L2 -/// caches (the deep, low-resolution layers): the per-task tile blocks of stream -/// every U panel once per block of tiles — DRAM-bound on the RK3399 (512 KB A53 / 1 MB A72 L2). -/// Instead tiles are taken in large groups and each group runs three parallel phases: input -/// transforms into V[36][Tg][ic], one GEMM task per (xi, oc block) that loads its U panel once for -/// the whole group, then the output transforms with the usual epilogue. Same per-element math as -/// (same transforms, same k loop), so the outputs are bit-identical. -/// -internal static unsafe partial class Nhwc -{ - /// U sizes (floats) above this take on NEON (512 KB). - private const long NeonWinoUFloats = 128 * 1024; - - [ThreadStatic] private static float[]? t_wgV, t_wgM; - - private static void WinogradGrouped(in WinoArgs a, float* dp, float* ap) - { - int ic = a.Ic, nb = a.Nb, tw = a.Tw, oh = a.Oh, ow = a.Ow, oc = a.Oc; - int nt = (oh + 3) / 4 * tw; - // V group ~2 MB (spread over the L2s, re-read nb times), whole Tile-row GEMM tiles - int tg = Math.Clamp(524288 / (36 * ic) / Tile * Tile, Tile, (nt + Tile - 1) / Tile * Tile); - float* V = Pinned(ref t_wgV, 36 * tg * ic), M = Pinned(ref t_wgM, 36 * nb * tg * Ob); - WinoArgs ca = a; - for (int g0 = 0; g0 < nt; g0 += tg) - { - int tn = Math.Min(tg, nt - g0); - Parallel.For(0, (tn + 7) / 8, PO, k => - { - float** dp6 = stackalloc float*[36]; - for (int t = k * 8, te = Math.Min(tn, t + 8); t < te; t++) - { - int ty = (g0 + t) / tw, tx = (g0 + t) % tw; - int iy0 = ty * 4 - ca.PadT, ix0 = tx * 4 - ca.PadL; - for (int i = 0; i < 6; i++) - { - int r = SrcIdx(iy0 + i, ca.Ih, ca.Reflect); - for (int j = 0; j < 6; j++) - { - int c = SrcIdx(ix0 + j, ca.Iw, ca.Reflect); - dp6[i * 6 + j] = r < 0 || c < 0 ? ca.PadVec : ca.Src + ((long)r * ca.Iw + c) * ic; - } - } - InputTransform(dp6, ic, V + (long)t * ic, (long)tg * ic); - } - }); - Parallel.For(0, 36 * nb, PO, j => - { - int xi = j / nb, blk = j % nb; - float** ptrs = stackalloc float*[Tile]; - float* zero = stackalloc float[Ob]; - new Span(zero, Ob).Clear(); - float* wb = ca.U + ((long)xi * nb + blk) * ic * Ob; - float* vx = V + (long)xi * tg * ic, mx = M + ((long)xi * nb + blk) * tg * Ob; - // all Tile rows land in M (tg is a multiple of Tile); rows past tn are never read - for (int r0 = 0; r0 < tn; r0 += Tile) - { - int rows = Math.Min(Tile, tn - r0); - for (int i = 0; i < Tile; i++) ptrs[i] = vx + (long)(r0 + Math.Min(i, rows - 1)) * ic; - TileNeonK(ptrs, 1, ic, wb, zero, mx + (long)r0 * Ob, Ob); - } - }); - int chunks = (tn + 15) / 16; - Parallel.For(0, chunks * nb, PO, j => - { - int blk = j % nb, t0 = j / nb * 16; - float* tmp = stackalloc float[16 * Ob]; - WinoOutput(M + ((long)blk * tg + t0) * Ob, (long)nb * tg * Ob, Ob, Math.Min(16, tn - t0), g0 + t0, - tw, oh, ow, oc, blk * Ob, ca.Bias + blk * Ob, dp, ap, 0, ca.Act, tmp, false, false); - }); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.cs b/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.cs deleted file mode 100644 index aea5029..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.cs +++ /dev/null @@ -1,399 +0,0 @@ -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.Wasm; -using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; -using F32x8 = System.Runtime.Intrinsics.Vector256; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// Winograd F(4x4, 3x3) for dense 3x3 stride-1 NHWC convs: 36 multiplies per -/// 4x4 output tile and (ic, oc) pair instead of 144. Tiles are processed in -/// cache-sized blocks per task — input transform into a block-local -/// V[36][T][ic], then per 16-oc block 36 small GEMMs through the 6x16 conv -/// micro-kernel into M[36][T][16], then the output transform with the usual -/// bias / residual / activation epilogue — so the 2.25x-expanded transform -/// tensors never leave L2. -/// -internal static unsafe partial class Nhwc -{ - private static readonly bool s_winograd = Environment.GetEnvironmentVariable("GCDRNET_WINOGRAD") != "0"; - - [ThreadStatic] private static float[]? t_wino; - - /// Dense 3x3 s1 convs with enough channels for the transforms to amortize. - internal static bool WinogradEligible(ConvP p, int[] wShape, int ic) => - s_winograd && Vector.IsHardwareAccelerated && p.Group == 1 - && wShape[2] == 3 && wShape[3] == 3 && p.Strides[0] == 1 && p.Strides[1] == 1 - && p.Dilations[0] == 1 && p.Dilations[1] == 1 - && ic >= 16 && ic % Vector.Count == 0 && ic % 8 == 0; - - // G (6x3) of F(4,3): U = G g G^T - private static readonly double[,] s_g = - { - { 1.0 / 4, 0, 0 }, - { -1.0 / 6, -1.0 / 6, -1.0 / 6 }, - { -1.0 / 6, 1.0 / 6, -1.0 / 6 }, - { 1.0 / 24, 1.0 / 12, 1.0 / 6 }, - { 1.0 / 24, -1.0 / 12, 1.0 / 6 }, - { 0, 0, 1 }, - }; - - /// Transformed weights [36][oc/16][ic][16] (oc zero-padded), from ONNX [oc][ic][3][3]. - internal static float[] PackWinograd(ReadOnlySpan w, int oc, int ic) - { - int nb = (oc + Ob - 1) / Ob; - float[] u = new float[checked(36 * nb * ic * Ob)]; - double* gg = stackalloc double[18]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - { - ReadOnlySpan k = w.Slice((o * ic + c) * 9, 9); - // gg = G·k (6x3) - for (int i = 0; i < 6; i++) - for (int j = 0; j < 3; j++) - gg[i * 3 + j] = s_g[i, 0] * k[j] + s_g[i, 1] * k[3 + j] + s_g[i, 2] * k[6 + j]; - // U = gg·G^T (6x6) - for (int i = 0; i < 6; i++) - for (int j = 0; j < 6; j++) - { - double v = gg[i * 3] * s_g[j, 0] + gg[i * 3 + 1] * s_g[j, 1] + gg[i * 3 + 2] * s_g[j, 2]; - u[(((i * 6 + j) * nb + o / Ob) * ic + c) * Ob + o % Ob] = (float)v; - } - } - return u; - } - - /// Per-call Winograd constants shared by every tile block. - private readonly unsafe struct WinoArgs - { - public readonly float* Src, U, Bias, PadVec; - public readonly int Ih, Iw, Ic, Oh, Ow, Oc, Tw, T, Nb, PadT, PadL; - public readonly bool Reflect, Avx, Avx512, Pk; - public readonly ActivationKind Act; - - public WinoArgs(float* src, float* u, float* bias, float* padVec, int ih, int iw, int ic, - int oh, int ow, int oc, int nb, ConvP p) - { - Src = src; U = u; Bias = bias; PadVec = padVec; - Ih = ih; Iw = iw; Ic = ic; Oh = oh; Ow = ow; Oc = oc; Nb = nb; - Tw = (ow + 3) / 4; - Avx512 = UseAvx512; - Avx = UseAvx; - Pk = UsePacked; - // V block ~256 KB (AVX2); the AVX-512 tier (Tile5-row x 32-oc GEMM tiles) measured - // best at ~64 KB on Zen 5, NEON at ~16 KB (one Tile row for ic >= 32: four RK3399 A53s share 512 KB L2) - int tr = Avx512 ? Tile5 : Pk ? PkTile : Tile, vBudget = Avx512 ? 16384 : UseNeon ? 4096 : 65536; - T = Math.Clamp(vBudget / (36 * ic) / tr * tr, tr, 60 / tr * tr); - PadT = p.Pads[0]; PadL = p.Pads[1]; - Reflect = p.PadMode == PadMode.Reflect; - Act = p.Act; - } - } - - private static void WinogradConv(ReadOnlySpan src, int ih, int iw, int ic, - Span dst, int oh, int ow, int oc, ConvP p, NhwcConvPlan kp, ReadOnlySpan add) - { - float[] padVec = new float[ic]; - Array.Fill(padVec, p.PadValue); - bool hasAdd = !add.IsEmpty; - fixed (float* sp = src, dp0 = dst, ap0 = add, up = kp.WinoU, bp = kp.Bias, pv = padVec) - { - var a = new WinoArgs(sp, up, bp, pv, ih, iw, ic, oh, ow, oc, kp.Blocks, p); - int nt = (oh + 3) / 4 * a.Tw, chunks = (nt + a.T - 1) / a.T; - float* dp = dp0, ap = hasAdd ? ap0 : null; - if (UseNeon && 36L * a.Nb * ic * Ob > NeonWinoUFloats) - { - WinogradGrouped(a, dp, ap); - return; - } - For(chunks, (long)oh * ow * oc * ic, chunk => - { - int t0 = chunk * a.T; - WinoTiles(a, t0, Math.Min(a.T, nt - t0), dp, ap, 0); - }); - } - } - - /// - /// Tiles [t0, t0+tn) (raster order, tn <= T) of one Winograd conv, written to - /// whose row 0 is output row (rows past - /// Oh are skipped). Every tile's math is independent of how tiles are grouped. - /// - private static void WinoTiles(in WinoArgs a, int t0, int tn, float* dp, float* ap, int rowBase) - { - int ic = a.Ic, T = a.T, tw = a.Tw, nb = a.Nb, oc = a.Oc, ow = a.Ow, oh = a.Oh; - bool avx = a.Avx, avx512 = a.Avx512, pk = a.Pk; - // AVX-512: Tile5-row x 32-oc GEMM tiles over block pairs; M rows hold both blocks - int tr = avx512 ? Tile5 : pk ? PkTile : Tile, step = avx512 ? 2 : 1, mRow = step * Ob; - int vSize = 36 * T * ic, mSize = 36 * T * mRow; - int need = vSize + mSize + 16 * Ob + 2 * Ob; - float[] buf = t_wino is { } tb && tb.Length >= need ? tb : (t_wino = new float[need]); - fixed (float* b0 = buf) - { - float* V = b0, M = b0 + vSize, tmp = M + mSize, zero = tmp + 16 * Ob; - new Span(zero, 2 * Ob).Clear(); - float** dp6 = stackalloc float*[36]; - for (int t = 0; t < tn; t++) - { - int ty = (t0 + t) / tw, tx = (t0 + t) % tw; - int iy0 = ty * 4 - a.PadT, ix0 = tx * 4 - a.PadL; - for (int i = 0; i < 6; i++) - { - int r = SrcIdx(iy0 + i, a.Ih, a.Reflect); - for (int j = 0; j < 6; j++) - { - int c = SrcIdx(ix0 + j, a.Iw, a.Reflect); - dp6[i * 6 + j] = r < 0 || c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic; - } - } - if (avx512 && ic % 16 == 0) InputTransform512(dp6, ic, V + (long)t * ic, (long)T * ic); - else InputTransform(dp6, ic, V + (long)t * ic, (long)T * ic); - } - - float** ptrs = stackalloc float*[Math.Max(Tile, Tile5)]; - for (int blk0 = 0; blk0 < nb; blk0 += step) - { - int nblk = Math.Min(step, nb - blk0); - for (int xi = 0; xi < 36; xi++) - { - float* wb = a.U + ((long)xi * nb + blk0) * ic * Ob; - float* vx = V + (long)xi * T * ic, mx = M + (long)xi * T * mRow; - for (int r0 = 0; r0 < tn; r0 += tr) - { - int rows = Math.Min(tr, tn - r0); - for (int i = 0; i < tr; i++) ptrs[i] = vx + (long)(r0 + Math.Min(i, rows - 1)) * ic; - // AVX2 / NEON / PackedSimd write all tr rows into M (T is a multiple of tr); rows past tn are never read - if (avx512) Tile512(ptrs, nblk, 1, ic, wb, (long)ic * Ob, zero, mx + (long)r0 * mRow); - else if (avx) TileAvxK(ptrs, 1, ic, wb, zero, mx + (long)r0 * Ob, Ob); - else if (UseNeon) TileNeonK(ptrs, 1, ic, wb, zero, mx + (long)r0 * Ob, Ob); - else if (pk) TilePkK(ptrs, PkTile, 1, ic, (long)ic * Ob, wb, zero, mx + (long)r0 * Ob, Ob); - else TileVec(ptrs, rows, 1, ic, wb, zero, mx + (long)r0 * Ob, null, Ob, Ob, ActivationKind.None); - } - } - for (int h = 0; h < nblk; h++) - { - int blk = blk0 + h; - WinoOutput(M + h * Ob, (long)T * mRow, mRow, tn, t0, tw, oh, ow, oc, blk * Ob, a.Bias + blk * Ob, - dp, ap, rowBase, a.Act, tmp, avx, avx512); - } - } - } - } - - /// Output transform + bias/residual/activation epilogue of one 16-oc block for tn tiles; - /// tile t's 36 products are m[xi*mStride + t*mRow + 0..15]. - private static void WinoOutput(float* m, long mStride, int mRow, int tn, int t0, int tw, int oh, int ow, int oc, - int ocOff, float* bb, float* dp, float* ap, int rowBase, ActivationKind act, float* tmp, bool avx, bool avx512) - { - int lanes = Math.Min(Ob, oc - ocOff); - bool fast512 = avx512 && lanes == Ob && act is ActivationKind.None or ActivationKind.Relu; - bool fastNeon = UseNeon && lanes == Ob; - bool fastPk = UsePacked && lanes == Ob; - for (int t = 0; t < tn; t++) - { - int ty = (t0 + t) / tw, tx = (t0 + t) % tw; - if (avx512) OutputTransform512(m + (long)t * mRow, mStride, tmp); - else OutputTransform(m + (long)t * mRow, mStride, tmp); - for (int py = 0; py < 4; py++) - { - int oy = ty * 4 + py; - if (oy >= oh) break; - for (int px = 0; px < 4; px++) - { - int ox = tx * 4 + px; - if (ox >= ow) break; - long o = ((long)(oy - rowBase) * ow + ox) * oc + ocOff; - float* tp = tmp + (py * 4 + px) * Ob; - if (fast512) - { - var v = Avx512F.Add(Avx512F.LoadVector512(tp), Avx512F.LoadVector512(bb)); - if (ap != null) v = Avx512F.Add(v, Avx512F.LoadVector512(ap + o)); - if (act == ActivationKind.Relu) v = Avx512F.Max(v, Vector512.Zero); - Avx512F.Store(dp + o, v); - } - else if (fastNeon) - for (int l = 0; l < Ob; l += 4) - { - var v = AdvSimd.Add(AdvSimd.LoadVector128(tp + l), AdvSimd.LoadVector128(bb + l)); - if (ap != null) v = AdvSimd.Add(v, AdvSimd.LoadVector128(ap + o + l)); - AdvSimd.Store(dp + o + l, Act4(v, act)); - } - else if (fastPk) - for (int l = 0; l < Ob; l += 4) - { - var v = PackedSimd.Add(LdP(tp + l), LdP(bb + l)); - if (ap != null) v = PackedSimd.Add(v, LdP(ap + o + l)); - PackedSimd.Store(dp + o + l, ActP(v, act)); - } - else if (avx) - Store16(dp + o, ap != null ? ap + o : null, - Avx.Add(Avx.LoadVector256(tp), Avx.LoadVector256(bb)), - Avx.Add(Avx.LoadVector256(tp + 8), Avx.LoadVector256(bb + 8)), lanes, act); - else - for (int l = 0; l < lanes; l++) - { - float v = tp[l] + bb[l]; - if (ap != null) v += ap[o + l]; - dp[o + l] = ActS(v, act); - } - } - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int SrcIdx(int i, int n, bool reflect) => - (uint)i < (uint)n ? i : reflect ? Reflect(i, n) : -1; - - /// V[xi] = (B^T d B)[xi] for one tile, all channels; V row stride per xi = vStride. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void InputTransform(float** d, int ic, float* v, long vStride) - { - int VL = Vector.Count; - var t = stackalloc Vector[36]; - var four = new Vector(4f); - var five = new Vector(5f); - var two = new Vector(2f); - for (int c = 0; c < ic; c += VL) - { - // columns: t = B^T d - for (int j = 0; j < 6; j++) - { - var d0 = Simd.VectorLoad(d[j] + c); var d1 = Simd.VectorLoad(d[6 + j] + c); - var d2 = Simd.VectorLoad(d[12 + j] + c); var d3 = Simd.VectorLoad(d[18 + j] + c); - var d4 = Simd.VectorLoad(d[24 + j] + c); var d5 = Simd.VectorLoad(d[30 + j] + c); - t[j] = four * d0 - five * d2 + d4; - t[6 + j] = d3 + d4 - four * (d1 + d2); - t[12 + j] = d4 - d3 + four * (d1 - d2); - t[18 + j] = d4 - d2 + two * (d3 - d1); - t[24 + j] = d4 - d2 - two * (d3 - d1); - t[30 + j] = four * d1 - five * d3 + d5; - } - // rows: V = t B - for (int i = 0; i < 6; i++) - { - var r0 = t[i * 6]; var r1 = t[i * 6 + 1]; var r2 = t[i * 6 + 2]; - var r3 = t[i * 6 + 3]; var r4 = t[i * 6 + 4]; var r5 = t[i * 6 + 5]; - float* o = v + c; - long b = (long)(i * 6) * vStride; - Simd.VectorStore(o + b, four * r0 - five * r2 + r4); - Simd.VectorStore(o + b + vStride, r3 + r4 - four * (r1 + r2)); - Simd.VectorStore(o + b + 2 * vStride, r4 - r3 + four * (r1 - r2)); - Simd.VectorStore(o + b + 3 * vStride, r4 - r2 + two * (r3 - r1)); - Simd.VectorStore(o + b + 4 * vStride, r4 - r2 - two * (r3 - r1)); - Simd.VectorStore(o + b + 5 * vStride, four * r1 - five * r3 + r5); - } - } - } - - /// tmp[(py*4+px)*16 + l] = (A^T m A)[py,px] for one tile's 16 oc lanes; m row stride per xi = mStride. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void OutputTransform(float* m, long mStride, float* tmp) - { - int VL = Vector.Count; - var s = stackalloc Vector[24]; - var two = new Vector(2f); - var four = new Vector(4f); - var eight = new Vector(8f); - for (int l = 0; l < Ob; l += VL) - { - // columns: s = A^T m (4x6) - for (int j = 0; j < 6; j++) - { - var m0 = Simd.VectorLoad(m + j * mStride + l); var m1 = Simd.VectorLoad(m + (6 + j) * mStride + l); - var m2 = Simd.VectorLoad(m + (12 + j) * mStride + l); var m3 = Simd.VectorLoad(m + (18 + j) * mStride + l); - var m4 = Simd.VectorLoad(m + (24 + j) * mStride + l); var m5 = Simd.VectorLoad(m + (30 + j) * mStride + l); - var a = m1 + m2; var b = m1 - m2; var c = m3 + m4; var e = m3 - m4; - s[j] = m0 + a + c; - s[6 + j] = b + two * e; - s[12 + j] = a + four * c; - s[18 + j] = b + eight * e + m5; - } - // rows: y = s A (4x4) - for (int i = 0; i < 4; i++) - { - var r0 = s[i * 6]; var r1 = s[i * 6 + 1]; var r2 = s[i * 6 + 2]; - var r3 = s[i * 6 + 3]; var r4 = s[i * 6 + 4]; var r5 = s[i * 6 + 5]; - var a = r1 + r2; var b = r1 - r2; var c = r3 + r4; var e = r3 - r4; - float* o = tmp + i * 4 * Ob + l; - Simd.VectorStore(o, r0 + a + c); - Simd.VectorStore(o + Ob, b + two * e); - Simd.VectorStore(o + 2 * Ob, a + four * c); - Simd.VectorStore(o + 3 * Ob, b + eight * e + r5); - } - } - } - - /// with 16 lanes (AVX-512; ic % 16 == 0). Same per-lane ops → same bits. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void InputTransform512(float** d, int ic, float* v, long vStride) - { - var t = stackalloc Vector512[36]; - var four = Vector512.Create(4f); - var five = Vector512.Create(5f); - var two = Vector512.Create(2f); - for (int c = 0; c < ic; c += 16) - { - for (int j = 0; j < 6; j++) - { - var d0 = Vector512.Load(d[j] + c); var d1 = Vector512.Load(d[6 + j] + c); - var d2 = Vector512.Load(d[12 + j] + c); var d3 = Vector512.Load(d[18 + j] + c); - var d4 = Vector512.Load(d[24 + j] + c); var d5 = Vector512.Load(d[30 + j] + c); - t[j] = four * d0 - five * d2 + d4; - t[6 + j] = d3 + d4 - four * (d1 + d2); - t[12 + j] = d4 - d3 + four * (d1 - d2); - t[18 + j] = d4 - d2 + two * (d3 - d1); - t[24 + j] = d4 - d2 - two * (d3 - d1); - t[30 + j] = four * d1 - five * d3 + d5; - } - for (int i = 0; i < 6; i++) - { - var r0 = t[i * 6]; var r1 = t[i * 6 + 1]; var r2 = t[i * 6 + 2]; - var r3 = t[i * 6 + 3]; var r4 = t[i * 6 + 4]; var r5 = t[i * 6 + 5]; - float* o = v + c + (long)(i * 6) * vStride; - (four * r0 - five * r2 + r4).Store(o); - (r3 + r4 - four * (r1 + r2)).Store(o + vStride); - (r4 - r3 + four * (r1 - r2)).Store(o + 2 * vStride); - (r4 - r2 + two * (r3 - r1)).Store(o + 3 * vStride); - (r4 - r2 - two * (r3 - r1)).Store(o + 4 * vStride); - (four * r1 - five * r3 + r5).Store(o + 5 * vStride); - } - } - } - - /// with one 16-lane vector per element (AVX-512). - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void OutputTransform512(float* m, long mStride, float* tmp) - { - var s = stackalloc Vector512[24]; - var two = Vector512.Create(2f); - var four = Vector512.Create(4f); - var eight = Vector512.Create(8f); - for (int j = 0; j < 6; j++) - { - var m0 = Vector512.Load(m + j * mStride); var m1 = Vector512.Load(m + (6 + j) * mStride); - var m2 = Vector512.Load(m + (12 + j) * mStride); var m3 = Vector512.Load(m + (18 + j) * mStride); - var m4 = Vector512.Load(m + (24 + j) * mStride); var m5 = Vector512.Load(m + (30 + j) * mStride); - var a = m1 + m2; var b = m1 - m2; var c = m3 + m4; var e = m3 - m4; - s[j] = m0 + a + c; - s[6 + j] = b + two * e; - s[12 + j] = a + four * c; - s[18 + j] = b + eight * e + m5; - } - for (int i = 0; i < 4; i++) - { - var r0 = s[i * 6]; var r1 = s[i * 6 + 1]; var r2 = s[i * 6 + 2]; - var r3 = s[i * 6 + 3]; var r4 = s[i * 6 + 4]; var r5 = s[i * 6 + 5]; - var a = r1 + r2; var b = r1 - r2; var c = r3 + r4; var e = r3 - r4; - float* o = tmp + i * 4 * Ob; - (r0 + a + c).Store(o); - (b + two * e).Store(o + Ob); - (a + four * c).Store(o + 2 * Ob); - (b + eight * e + r5).Store(o + 3 * Ob); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Ops.cs b/src/Sdcb.SimdGcdrNet/Kernels/Ops.cs deleted file mode 100644 index 3f1eabe..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Ops.cs +++ /dev/null @@ -1,1005 +0,0 @@ -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// Flat-float NCHW kernel library. Dispatch is a plain IsSupported chain: -/// AVX2+FMA hot paths, then the portable Vector<float> tier, else scalar -/// (the reference). Runtime knobs: DOTNET_EnableAVX2 / DOTNET_EnableAVX / -/// DOTNET_EnableHWIntrinsic. -/// -internal static unsafe class Ops -{ - // ======================== Conv ======================== - // src [1,IC,H,W]; w [OC,IC/g,kh,kw]; dst [1,OC,OH,OW]. N==1 assumed. - // `add` is an optional same-shape residual addend, applied before the activation. - public static void Conv(ReadOnlySpan src, int[] inShape, - ReadOnlySpan w, int[] wShape, ReadOnlySpan bias, - Span dst, int[] outShape, ConvP p, ReadOnlySpan add = default) - { - if (!add.IsEmpty && add.Length != dst.Length) - throw new NotSupportedException( - $"conv residual addend has {add.Length} elems, expected {dst.Length}."); - if (!add.IsEmpty) - { - // activation must run after the residual add: conv without act, then fused add+act - var q = new ConvP - { - Pads = p.Pads, Strides = p.Strides, Dilations = p.Dilations, Kernel = p.Kernel, - Group = p.Group, Act = ActivationKind.None, PadMode = p.PadMode, - PadValue = p.PadValue, BiasValue = p.BiasValue, KernelPlan = p.KernelPlan, - }; - Conv(src, inShape, w, wShape, bias, dst, outShape, q); - AddInPlace(dst, add, p.Act); - return; - } - int ic = inShape[1], ih = inShape[2], iw = inShape[3]; - int oc = wShape[0], kh = wShape[2], kw = wShape[3]; - int oh = outShape[2], ow = outShape[3]; - int sh = p.Strides[0], sw = p.Strides[1]; - int pt = p.Pads[0], pl = p.Pads[1], pb = p.Pads[2], pr = p.Pads[3]; - int group = p.Group; - - bool symmetricPad = pt == pb && pl == pr; - if (Avx2.IsSupported && Fma.IsSupported) - { - if (group > 1 && group == ic) - { - Avx2Kernels.Depthwise(src, ic, ih, iw, w, bias, dst, oh, ow, - kh, kw, pt, pl, p.PadMode, p.PadValue, p.Act); - return; - } - if (group == 1 && kh == 3 && kw == 3 && oc % 4 == 0 && symmetricPad) - { - if (sh == 1 && sw == 1) - { - Avx2Kernels.Conv3x3S1(src, ic, ih, iw, w, bias, dst, oc, oh, ow, - pt, pl, p.PadMode, p.PadValue, p.Act); - return; - } - if (sh == 2 && sw == 2) - { - Avx2Kernels.Conv3x3S2(src, ic, ih, iw, w, bias, dst, oc, oh, ow, - pt, pl, p.PadMode, p.PadValue, p.Act); - return; - } - } - if (group == 1 && kh == 1 && kw == 1 && sh == 1 && sw == 1 && pt == 0 && pl == 0 && pb == 0 && pr == 0) - { - Avx2Kernels.Conv1x1(src, ic, ih * iw, w, bias, dst, oc, p.Act); - return; - } - // unspecialized shape: fall through to the Vector tier - } - else if (!Vector.IsHardwareAccelerated) - { - ConvScalar(src, ic, ih, iw, w, bias, dst, oc, oh, ow, kh, kw, sh, sw, - pt, pl, group, p.PadMode, p.PadValue, p.Act); - return; - } - - // Vector tier - if (group > 1) - { - if (group != ic) // grouped (non-depthwise) conv: scalar reference handles groups correctly - { - ConvScalar(src, ic, ih, iw, w, bias, dst, oc, oh, ow, kh, kw, sh, sw, - pt, pl, group, p.PadMode, p.PadValue, p.Act); - return; - } - ConvDepthwiseVec(src, ic, ih, iw, w, bias, dst, oh, ow, kh, kw, - pt, pl, p.PadMode, p.PadValue, p.Act); - return; - } - if (kh == 1 && kw == 1 && sh == 1 && sw == 1 && pt == 0 && pl == 0) - { - Conv1x1Vec(src, ic, ih, iw, w, bias, dst, oc, oh, ow, p.Act); - return; - } - ConvVecW(src, ic, ih, iw, w, bias, dst, oc, oh, ow, kh, kw, sh, sw, - pt, pl, group, p.PadMode, p.PadValue, p.Act); - } - - /// dst = act(dst + add) — residual epilogue for fused Conv+Add. - private static void AddInPlace(Span dst, ReadOnlySpan add, ActivationKind act) - { - int n = dst.Length, i = 0, VL = Vector.Count; - var zero = Vector.Zero; - for (; i + VL <= n; i += VL) - { - var v = Simd.VectorLoad(dst, i) + Simd.VectorLoad(add, i); - if (act == ActivationKind.Relu) v = Vector.Max(v, zero); - Simd.VectorStore(dst, i, v); - } - for (; i < n; i++) - { - float v = dst[i] + add[i]; - dst[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - if (act != ActivationKind.None && act != ActivationKind.Relu) - ApplyActVec(dst, act); - } - - // ---- scalar reference conv (N==1, dense or grouped) ---- - private static void ConvScalar(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, - Span dst, int oc, int oh, int ow, int kh, int kw, - int sh, int sw, int pt, int pl, int group, - PadMode padMode, float padValue, ActivationKind act) - { - int icg = ic / group, ocg = oc / group; - for (int g = 0; g < group; g++) - for (int o = g * ocg; o < (g + 1) * ocg; o++) - for (int y = 0; y < oh; y++) - for (int x = 0; x < ow; x++) - { - float acc = bias.IsEmpty ? 0f : bias[o]; - for (int c = g * icg; c < (g + 1) * icg; c++) - for (int ky = 0; ky < kh; ky++) - { - int sy = y * sh + ky - pt; - if ((uint)sy >= (uint)ih) - { - if (padMode == PadMode.Reflect) sy = Reflect(sy, ih); - else continue; - } - for (int kx = 0; kx < kw; kx++) - { - int sx = x * sw + kx - pl; - if ((uint)sx >= (uint)iw) - { - if (padMode == PadMode.Reflect) sx = Reflect(sx, iw); - else continue; - } - acc += src[(c * ih + sy) * iw + sx] * w[((o * icg + c - g * icg) * kh + ky) * kw + kx]; - } - } - dst[(o * oh + y) * ow + x] = ApplyAct(acc, act); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int Reflect(int i, int dim) - { - // ONNX reflect (half-sample symmetric, no edge repeat): -1→1, dim→dim-2 - if (dim <= 1) return 0; // reflect is undefined on a singleton axis; don't spin - while (i < 0 || i >= dim) i = i < 0 ? -i : 2 * dim - 2 - i; - return i; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static float ApplyAct(float v, ActivationKind act) => - act == ActivationKind.Relu ? MathF.Max(v, 0f) : - act == ActivationKind.Sigmoid ? 1f / (1f + MathF.Exp(-v)) : - act == ActivationKind.Gelu ? 0.5f * v * (1f + SimdMath.ErfScalar(v * 0.70710678f)) : v; - - // ---- vector-over-output-W conv: one output row vector per (oc,y) ---- - private static void ConvVecW(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, - Span dst, int oc, int oh, int ow, int kh, int kw, - int sh, int sw, int pt, int pl, int group, - PadMode padMode, float padValue, ActivationKind act) - { - int icg = ic / group; - int VL = Vector.Count; - // interior x-range where all taps in-bounds (no pad needed) - for (int o = 0; o < oc; o++) - { - int g = o / (oc / group); // output channel -> group index - for (int y = 0; y < oh; y++) - { - Span drow = dst.Slice((o * oh + y) * ow, ow); - if (!bias.IsEmpty) drow.Fill(bias[o]); else drow.Clear(); - for (int c = g * icg; c < (g + 1) * icg; c++) - { - ReadOnlySpan sbase = src.Slice(c * ih * iw, ih * iw); - ReadOnlySpan wk = w.Slice((o * icg + (c - g * icg)) * kh * kw, kh * kw); - for (int ky = 0; ky < kh; ky++) - { - int sy = y * sh + ky - pt; - bool yOk = (uint)sy < (uint)ih; - int syR = yOk ? sy : (padMode == PadMode.Reflect ? Reflect(sy, ih) : -1); - for (int kx = 0; kx < kw; kx++) - { - float wv = wk[ky * kw + kx]; - if (wv == 0f) continue; - if (sw == 1) - { - // contiguous reads: dst[x] += wv * src[sy][x + kx - pl] - int xBegin = Math.Max(0, pl - kx); - int xEnd = Math.Min(ow, iw + pl - kx); - if (yOk || padMode == PadMode.Reflect) - { - VecAccMul(drow.Slice(xBegin, xEnd - xBegin), - sbase.Slice(syR * iw + xBegin + kx - pl, xEnd - xBegin), wv); - } - else if (padValue != 0f) - { - for (int x = xBegin; x < xEnd; x++) drow[x] += wv * padValue; - } - // pad regions (constant → add wv*padValue; reflect → gather) - for (int x = 0; x < xBegin; x++) drow[x] += wv * PadFetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue, syR, sy); - for (int x = xEnd; x < ow; x++) drow[x] += wv * PadFetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue, syR, sy); - } - else - { - for (int x = 0; x < ow; x++) - drow[x] += wv * PadFetch(sbase, sy, x * sw + kx - pl, ih, iw, padMode, padValue, syR, sy); - } - } - } - } - ApplyActVec(drow, act); - } - } - - static float PadFetch(ReadOnlySpan sbase, int sy, int sx, int ih, int iw, - PadMode mode, float padValue, int syR, int syOrig) - { - int yy = syOrig, xx = sx; - if ((uint)yy >= (uint)ih || (uint)xx >= (uint)iw) - { - if (mode == PadMode.Constant) return padValue; - yy = Reflect(yy, ih); xx = Reflect(xx, iw); - } - return sbase[yy * iw + xx]; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void VecAccMul(Span dst, ReadOnlySpan src, float w) - { - int i = 0, VL = Vector.Count; - var vw = new Vector(w); - for (; i + VL <= dst.Length; i += VL) - Simd.VectorStore(dst, i, Simd.VectorAddMul(Simd.VectorLoad(dst, i), Simd.VectorLoad(src, i), vw)); - for (; i < dst.Length; i++) dst[i] += src[i] * w; - } - - private static void ApplyActVec(Span row, ActivationKind act) - { - if (act == ActivationKind.None) return; - if (act == ActivationKind.Relu) { Relu(row, row); return; } - if (act == ActivationKind.Gelu) { SimdMath.Gelu(row, row); return; } - if (act == ActivationKind.Sigmoid) { SimdMath.Sigmoid(row, row); return; } - for (int i = 0; i < row.Length; i++) row[i] = ApplyAct(row[i], act); - } - - // ---- 1x1 conv = GEMM over pixels: dst[oc, HW] = W[oc, ic] * src[ic, HW] ---- - private static void Conv1x1Vec(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, - Span dst, int oc, int oh, int ow, ActivationKind act) - { - int hw = ih * iw; - int VL = Vector.Count; - for (int o = 0; o < oc; o++) - { - Span drow = dst.Slice(o * hw, hw); - float b = bias.IsEmpty ? 0f : bias[o]; - for (int i = 0; i < hw; i++) drow[i] = b; - ReadOnlySpan wo = w.Slice(o * ic, ic); - for (int c = 0; c < ic; c++) - { - float wv = wo[c]; - if (wv == 0f) continue; - VecAccMul(drow, src.Slice(c * hw, hw), wv); - } - ApplyActVec(drow, act); - } - } - - // ---- depthwise conv (group == C): dst[c,y,x] ---- - private static void ConvDepthwiseVec(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, - Span dst, int oh, int ow, int kh, int kw, - int pt, int pl, PadMode padMode, float padValue, ActivationKind act) - { - int VL = Vector.Count; - for (int c = 0; c < ic; c++) - { - ReadOnlySpan sbase = src.Slice(c * ih * iw, ih * iw); - ReadOnlySpan wk = w.Slice(c * kh * kw, kh * kw); - for (int y = 0; y < oh; y++) - { - Span drow = dst.Slice((c * oh + y) * ow, ow); - float b = bias.IsEmpty ? 0f : bias[c]; - for (int i = 0; i < ow; i++) drow[i] = b; - for (int ky = 0; ky < kh; ky++) - { - int sy = y + ky - pt; - for (int kx = 0; kx < kw; kx++) - { - float wv = wk[ky * kw + kx]; - if (wv == 0f) continue; - int xBegin = Math.Max(0, pl - kx); - int xEnd = Math.Min(ow, iw + pl - kx); - if ((uint)sy < (uint)ih) - { - VecAccMul(drow.Slice(xBegin, xEnd - xBegin), - sbase.Slice(sy * iw + xBegin + kx - pl, xEnd - xBegin), wv); - } - for (int x = 0; x < xBegin; x++) drow[x] += wv * Fetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue); - for (int x = xEnd; x < ow; x++) drow[x] += wv * Fetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue); - if ((uint)sy >= (uint)ih && padMode == PadMode.Constant && padValue != 0f) - for (int x = xBegin; x < xEnd; x++) drow[x] += wv * padValue; - else if ((uint)sy >= (uint)ih && padMode == PadMode.Reflect) - for (int x = xBegin; x < xEnd; x++) drow[x] += wv * sbase[Reflect(sy, ih) * iw + x + kx - pl]; - } - } - ApplyActVec(drow, act); - } - } - - static float Fetch(ReadOnlySpan s, int y, int x, int ih, int iw, PadMode m, float pv) - { - if ((uint)y >= (uint)ih || (uint)x >= (uint)iw) - { - if (m == PadMode.Constant) return pv; - y = Reflect(y, ih); x = Reflect(x, iw); - } - return s[y * iw + x]; - } - } - - // ======================== MaxPool ======================== - public static void MaxPool(ReadOnlySpan src, int[] inShape, Span dst, int[] outShape, - PoolP p) - { - int c = inShape[1], ih = inShape[2], iw = inShape[3]; - int oh = outShape[2], ow = outShape[3]; - int kh = p.Kernel[0], kw = p.Kernel[1], sh = p.Strides[0], sw = p.Strides[1]; - int pt = p.Pads[0], pl = p.Pads[1]; - if (Avx2.IsSupported && Fma.IsSupported && - kh == 2 && kw == 2 && sh == 2 && sw == 2 && pt == 0 && pl == 0) - { - Avx2Kernels.MaxPool2x2(src, c, ih, iw, dst, oh, ow, p.FuseRelu); - return; - } - for (int ch = 0; ch < c; ch++) - { - ReadOnlySpan s = src.Slice(ch * ih * iw, ih * iw); - Span d = dst.Slice(ch * oh * ow, oh * ow); - for (int y = 0; y < oh; y++) - for (int x = 0; x < ow; x++) - { - float m = float.NegativeInfinity; - for (int ky = 0; ky < kh; ky++) - { - int sy = y * sh + ky - pt; - if ((uint)sy >= (uint)ih) continue; - for (int kx = 0; kx < kw; kx++) - { - int sx = x * sw + kx - pl; - if ((uint)sx >= (uint)iw) continue; - float v = s[sy * iw + sx]; - if (v > m) m = v; - } - } - if (m == float.NegativeInfinity) m = 0f; - d[y * ow + x] = p.FuseRelu && m < 0f ? 0f : m; - } - } - } - - // ======================== elementwise ======================== - public static void Elementwise(ReadOnlySpan a, int[] aShape, - ReadOnlySpan b, int[] bShape, Span dst, int[] outShape, EwOp op, - ActivationKind act = ActivationKind.None) - { - int n = outShape.Aggregate(1, (x, y) => x * y); - int oi = (int)op; - bool avx = Avx2.IsSupported && Fma.IsSupported && n >= 8192; - // fast path: same shape, contiguous - if (aShape.SequenceEqual(outShape) && bShape.SequenceEqual(outShape)) - { - if (avx) { Avx2Kernels.Ew2(a, b, dst, n, oi, act); return; } - VecBin(a, b, dst, n, oi); goto Done; - } - // scalar broadcast - if (bShape.Aggregate(1, (x, y) => x * y) == 1) - { - if (avx) { Avx2Kernels.Ew1(a, dst, n, b[0], oi, act: act); return; } - VecScalar(a, dst, n, b[0], oi); goto Done; - } - if (aShape.Aggregate(1, (x, y) => x * y) == 1) - { - if (avx) - { - Avx2Kernels.Ew1(b, dst, n, a[0], oi, - rev: op is EwOp.Sub or EwOp.Div, act: act); - return; - } - if (op is EwOp.Sub or EwOp.Div) VecScalarR(b, dst, n, a[0], oi); - else VecScalar(b, dst, n, a[0], oi); - goto Done; - } - // inner-block broadcast: a full-shape, b constant over trailing dims - if (avx && aShape.SequenceEqual(outShape)) - { - int[] bStr = BroadcastStrides(bShape, outShape.Length); - int[] outStr = StridesOf(outShape); - int inner = 1, rank = outShape.Length; - while (rank > 1 && bStr[rank - 1] == 0) - { - inner *= outShape[rank - 1]; - rank--; - } - if (inner > 8 && n % inner == 0) - { - Avx2Kernels.EwInner(a, b, bStr, outShape, outStr, - outShape.Length, inner, dst, n, oi, act); - return; - } - } - // general broadcast (strided) - BroadcastLoop(a, aShape, b, bShape, dst, outShape, op); - Done: - if (act == ActivationKind.Relu) Relu(dst, dst); - } - - private static void VecBin(ReadOnlySpan a, ReadOnlySpan b, Span d, int n, int op) - { - int i = 0, VL = Vector.Count; - for (; i + VL <= n; i += VL) - { - var va = Simd.VectorLoad(a, i); var vb = Simd.VectorLoad(b, i); - var r = op switch { 0 => va + vb, 1 => va - vb, 2 => va * vb, _ => va / vb }; - Simd.VectorStore(d, i, r); - } - for (; i < n; i++) - d[i] = op switch { 0 => a[i] + b[i], 1 => a[i] - b[i], 2 => a[i] * b[i], _ => a[i] / b[i] }; - } - - private static void VecScalar(ReadOnlySpan a, Span d, int n, float s, int op) - { - int i = 0, VL = Vector.Count; - var vs = new Vector(s); - for (; i + VL <= n; i += VL) - { - var va = Simd.VectorLoad(a, i); - var r = op switch { 0 => va + vs, 1 => va - vs, 2 => va * vs, _ => va / vs }; - Simd.VectorStore(d, i, r); - } - for (; i < n; i++) - d[i] = op switch { 0 => a[i] + s, 1 => a[i] - s, 2 => a[i] * s, _ => a[i] / s }; - } - - private static void VecScalarR(ReadOnlySpan b, Span d, int n, float s, int op) - { - for (int i = 0; i < n; i++) - d[i] = op == 1 ? s - b[i] : s / b[i]; - } - - private static void BroadcastLoop(ReadOnlySpan a, int[] aShape, - ReadOnlySpan b, int[] bShape, Span dst, int[] outShape, EwOp op) - { - int rank = outShape.Length; - int[] aStr = BroadcastStrides(aShape, rank), bStr = BroadcastStrides(bShape, rank); - int[] outStr = StridesOf(outShape); - int n = outShape.Aggregate(1, (x, y) => x * y); - for (int e = 0; e < n; e++) - { - int rem = e, ao = 0, bo = 0; - for (int d = 0; d < rank; d++) - { - int c = rem / outStr[d]; - rem %= outStr[d]; - ao += c * aStr[d]; bo += c * bStr[d]; - } - dst[e] = op switch - { - EwOp.Add => a[ao] + b[bo], EwOp.Sub => a[ao] - b[bo], - EwOp.Mul => a[ao] * b[bo], _ => a[ao] / b[bo], - }; - } - } - - private static int[] BroadcastStrides(int[] shape, int rank) - { - int[] s = new int[rank]; - int acc = 1; - for (int i = shape.Length - 1; i >= 0; i--) { s[i + rank - shape.Length] = acc; acc *= shape[i]; } - for (int i = 0; i < rank; i++) - { - int si = i - (rank - shape.Length); - if (si < 0 || shape[si] == 1) s[i] = 0; - } - return s; - } - - private static int[] StridesOf(int[] shape) - { - int[] s = new int[shape.Length]; - int acc = 1; - for (int i = shape.Length - 1; i >= 0; i--) { s[i] = acc; acc *= shape[i]; } - return s; - } - - // ======================== unary ======================== - public static void Relu(ReadOnlySpan src, Span dst) - { - int n = src.Length; - if (Avx2.IsSupported && Fma.IsSupported && n >= 8192) { Avx2Kernels.Unary(src, dst, n, 0); return; } - int i = 0, VL = Vector.Count; - for (; i + VL <= n; i += VL) - Simd.VectorStore(dst, i, Vector.Max(Simd.VectorLoad(src, i), Vector.Zero)); - for (; i < n; i++) dst[i] = MathF.Max(src[i], 0f); - } - - public static void Sigmoid(ReadOnlySpan src, Span dst) - { - if (Avx2.IsSupported && Fma.IsSupported && src.Length >= 8192) { Avx2Kernels.Unary(src, dst, src.Length, 1); return; } - SimdMath.Sigmoid(src, dst); - } - - public static void Erf(ReadOnlySpan src, Span dst) - { - if (Avx2.IsSupported && Fma.IsSupported && src.Length >= 8192) { Avx2Kernels.Unary(src, dst, src.Length, 3); return; } - SimdMath.Erf(src, dst); - } - - public static void Gelu(ReadOnlySpan src, Span dst) - { - if (Avx2.IsSupported && Fma.IsSupported && src.Length >= 8192) { Avx2Kernels.Unary(src, dst, src.Length, 2); return; } - SimdMath.Gelu(src, dst); - } - - // ======================== LayerNorm ======================== - // x [.., rows, C] normalized over last dim; w[C], b[C] - public static void LayerNorm(ReadOnlySpan x, int[] shape, - ReadOnlySpan w, ReadOnlySpan b, Span dst, float eps) - { - int c = shape[^1]; - int rows = (int)(x.Length / c); - if (Avx2.IsSupported && Fma.IsSupported) - { - Avx2Kernels.LayerNorm(x, rows, c, w, b, dst, eps); - return; - } - if (Nhwc.UseNeon) - { - NeonKernels.LayerNorm(x, rows, c, w, b, dst, eps); - return; - } - if (Nhwc.UsePacked) - { - PackedKernels.LayerNorm(x, rows, c, w, b, dst, eps); - return; - } - for (int r = 0; r < rows; r++) - { - ReadOnlySpan xr = x.Slice(r * c, c); - Span dr = dst.Slice(r * c, c); - float s = 0f, s2 = 0f; - for (int i = 0; i < c; i++) { float v = xr[i]; s += v; s2 += v * v; } - float mean = s / c; - float inv = 1f / MathF.Sqrt(s2 / c - mean * mean + eps); - for (int i = 0; i < c; i++) dr[i] = (xr[i] - mean) * inv * w[i] + b[i]; - } - } - - // ======================== Gemm ======================== - // A [M,K] (last-2-dims of A), B [K,N] or [N,K] if transB, bias [N]; dst [M,N] - public static void Gemm(ReadOnlySpan a, int[] aShape, ReadOnlySpan b, - int[] bShape, ReadOnlySpan bias, Span dst, int[] outShape, GemmP p) - { - int m = 1; - for (int i = 0; i < aShape.Length - 1; i++) m *= aShape[i]; - int k = aShape[^1]; - int n = outShape[^1]; - if (!p.TransB && Avx2.IsSupported && Fma.IsSupported) - Avx2Kernels.GemmKN(a, m, k, b, n, bias, dst); - else if (p.TransB) - GemmBT(a, m, k, b, n, bias, dst); - else - GemmKN(a, m, k, b, n, bias, dst); - switch (p.Act) - { - case ActivationKind.Relu: Relu(dst, dst); break; - case ActivationKind.Gelu: SimdMath.Gelu(dst, dst); break; - case ActivationKind.Sigmoid: SimdMath.Sigmoid(dst, dst); break; - } - } - - // B row-major [K,N]: dst[m,n] = Σ_k a[m,k]*b[k,n] — vector over N - private static void GemmKN(ReadOnlySpan a, int M, int K, - ReadOnlySpan b, int N, ReadOnlySpan bias, Span dst) - { - int VL = Vector.Count; - // tile M by 4 for B-row reuse - int m0 = 0; - for (; m0 + 4 <= M; m0 += 4) - { - ReadOnlySpan a0 = a.Slice(m0 * K, K), a1 = a.Slice((m0 + 1) * K, K), - a2 = a.Slice((m0 + 2) * K, K), a3 = a.Slice((m0 + 3) * K, K); - for (int n0 = 0; n0 < N; n0 += VL) - { - int nw = Math.Min(VL, N - n0); - if (nw < VL) break; // tail handled scalar below - var acc0 = bias.IsEmpty ? Vector.Zero : Simd.VectorLoad(bias, n0); - var acc1 = acc0; var acc2 = acc0; var acc3 = acc0; - for (int k = 0; k < K; k++) - { - var bv = Simd.VectorLoad(b, k * N + n0); - acc0 = Simd.VectorAddMul(acc0, bv, a0[k]); - acc1 = Simd.VectorAddMul(acc1, bv, a1[k]); - acc2 = Simd.VectorAddMul(acc2, bv, a2[k]); - acc3 = Simd.VectorAddMul(acc3, bv, a3[k]); - } - Simd.VectorStore(dst, m0 * N + n0, acc0); - Simd.VectorStore(dst, (m0 + 1) * N + n0, acc1); - Simd.VectorStore(dst, (m0 + 2) * N + n0, acc2); - Simd.VectorStore(dst, (m0 + 3) * N + n0, acc3); - } - // N tail - for (int n0 = N - N % VL; n0 < N; n0++) - { - float s0 = bias.IsEmpty ? 0 : bias[n0], s1 = s0, s2 = s0, s3 = s0; - for (int k = 0; k < K; k++) - { - float bv = b[k * N + n0]; - s0 += a0[k] * bv; s1 += a1[k] * bv; s2 += a2[k] * bv; s3 += a3[k] * bv; - } - dst[m0 * N + n0] = s0; dst[(m0 + 1) * N + n0] = s1; - dst[(m0 + 2) * N + n0] = s2; dst[(m0 + 3) * N + n0] = s3; - } - } - for (; m0 < M; m0++) - { - ReadOnlySpan ar = a.Slice(m0 * K, K); - for (int n0 = 0; n0 + VL <= N; n0 += VL) - { - var acc = bias.IsEmpty ? Vector.Zero : Simd.VectorLoad(bias, n0); - for (int k = 0; k < K; k++) - acc = Simd.VectorAddMul(acc, Simd.VectorLoad(b, k * N + n0), ar[k]); - Simd.VectorStore(dst, m0 * N + n0, acc); - } - for (int n0 = N - N % VL; n0 < N; n0++) - { - float s = bias.IsEmpty ? 0 : bias[n0]; - for (int k = 0; k < K; k++) s += ar[k] * b[k * N + n0]; - dst[m0 * N + n0] = s; - } - } - } - - // B row-major [N,K] (transB): dst[m,n] = Σ_k a[m,k]*b[n,k] — dot products - private static void GemmBT(ReadOnlySpan a, int M, int K, - ReadOnlySpan b, int N, ReadOnlySpan bias, Span dst) - { - for (int m = 0; m < M; m++) - { - ReadOnlySpan ar = a.Slice(m * K, K); - for (int n = 0; n < N; n++) - { - float s = bias.IsEmpty ? 0 : bias[n]; - ReadOnlySpan br = b.Slice(n * K, K); - int k = 0, VL = Vector.Count; - var acc = Vector.Zero; - for (; k + VL <= K; k += VL) - acc += Simd.VectorLoad(ar, k) * Simd.VectorLoad(br, k); - for (int i = 0; i < VL; i++) s += acc[i]; - for (; k < K; k++) s += ar[k] * br[k]; - dst[m * N + n] = s; - } - } - } - - // ======================== Resize ======================== - // bilinear, half_pixel, per-axis scale (NCHW; only spatial dims scale != 1) - public static void ResizeBilinear(ReadOnlySpan src, int[] inShape, - Span dst, int[] outShape, float[] scales, - ActivationKind act = ActivationKind.None) - { - int c = inShape[1], ih = inShape[2], iw = inShape[3]; - int oh = outShape[2], ow = outShape[3]; - float sy = scales[2], sx = scales[3]; - // precompute row/col src indices + lerp weights - int[] y0 = new int[oh], y1 = new int[oh]; - float[] wy = new float[oh]; - for (int y = 0; y < oh; y++) - { - float fy = (y + 0.5f) / sy - 0.5f; - float fl = MathF.Floor(fy); - y0[y] = Math.Clamp((int)fl, 0, ih - 1); - y1[y] = Math.Clamp((int)fl + 1, 0, ih - 1); - wy[y] = fy - fl; - } - int[] x0 = new int[ow], x1 = new int[ow]; - float[] wx = new float[ow]; - for (int x = 0; x < ow; x++) - { - float fx = (x + 0.5f) / sx - 0.5f; - float fl = MathF.Floor(fx); - x0[x] = Math.Clamp((int)fl, 0, iw - 1); - x1[x] = Math.Clamp((int)fl + 1, 0, iw - 1); - wx[x] = fx - fl; - } - if (Avx2.IsSupported && Fma.IsSupported && oh * ow >= 8192) - { - Avx2Kernels.Bilinear(src, c, ih, iw, dst, oh, ow, y0, y1, wy, x0, x1, wx, act); - return; - } - for (int ch = 0; ch < c; ch++) - { - ReadOnlySpan s = src.Slice(ch * ih * iw, ih * iw); - Span d = dst.Slice(ch * oh * ow, oh * ow); - for (int y = 0; y < oh; y++) - { - ReadOnlySpan r0 = s.Slice(y0[y] * iw, iw); - ReadOnlySpan r1 = s.Slice(y1[y] * iw, iw); - float wY = wy[y]; - for (int x = 0; x < ow; x++) - { - float wX = wx[x]; - float top = r0[x0[x]] + (r0[x1[x]] - r0[x0[x]]) * wX; - float bot = r1[x0[x]] + (r1[x1[x]] - r1[x0[x]]) * wX; - float v = top + (bot - top) * wY; - d[y * ow + x] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - } - } - } - - // ======================== Pad ======================== - public static void Pad(ReadOnlySpan src, int[] inShape, Span dst, - int[] outShape, PadP p) - { - int rank = inShape.Length; - int n = 0, c = 0, ih = 0, iw = 0, oh = 0, ow = 0; - if (rank == 4) { n = inShape[0]; c = inShape[1]; ih = inShape[2]; iw = inShape[3]; oh = outShape[2]; ow = outShape[3]; } - else throw new NotSupportedException("Pad supports rank-4 only."); - int pt = p.Pads[2], pl = p.Pads[3]; - // index maps - int[] mapY = new int[oh], mapX = new int[ow]; - for (int y = 0; y < oh; y++) - { - int sy = y - pt; - if ((uint)sy < (uint)ih) mapY[y] = sy; - else if (p.Mode == PadMode.Reflect) mapY[y] = Reflect(sy, ih); - else mapY[y] = -1; - } - for (int x = 0; x < ow; x++) - { - int sx = x - pl; - if ((uint)sx < (uint)iw) mapX[x] = sx; - else if (p.Mode == PadMode.Reflect) mapX[x] = Reflect(sx, iw); - else mapX[x] = -1; - } - if (Avx2.IsSupported && Fma.IsSupported && oh * ow >= 4096) - { - Avx2Kernels.Pad4(src, n * c, ih, iw, dst, oh, ow, mapY, mapX, p.Value); - return; - } - for (int nc = 0; nc < n * c; nc++) - { - ReadOnlySpan s = src.Slice(nc * ih * iw, ih * iw); - Span d = dst.Slice(nc * oh * ow, oh * ow); - for (int y = 0; y < oh; y++) - { - int my = mapY[y]; - Span drow = d.Slice(y * ow, ow); - if (my < 0) { drow.Fill(p.Value); continue; } - ReadOnlySpan srow = s.Slice(my * iw, iw); - // x regions: [0,xl) pad, [xl,xr) copy, [xr,ow) pad - int xl = 0; while (xl < ow && mapX[xl] < 0) xl++; - int xr = ow; while (xr > xl && mapX[xr - 1] < 0) xr--; - if (p.Mode == PadMode.Constant) - { - for (int x = 0; x < xl; x++) drow[x] = p.Value; - for (int x = xr; x < ow; x++) drow[x] = p.Value; - } - else - { - for (int x = 0; x < xl; x++) drow[x] = srow[mapX[x]]; - for (int x = xr; x < ow; x++) drow[x] = srow[mapX[x]]; - } - for (int x = xl; x < xr; x++) drow[x] = srow[mapX[x]]; - } - } - } - - // ======================== Transpose ======================== - public static void Transpose(ReadOnlySpan src, int[] inShape, Span dst, - int[] outShape, int[] perm) - { - int rank = inShape.Length; - // fast path: last-two-dim swap on any rank (most common here); leading dims must be identity - bool isSwapLast2 = perm.Length == rank && perm[^1] == rank - 2 && perm[^2] == rank - 1; - for (int i = 0; isSwapLast2 && i < rank - 2; i++) isSwapLast2 = perm[i] == i; - if (isSwapLast2) - { - int a = inShape[rank - 2], b = inShape[rank - 1]; - int outer = 1; - for (int i = 0; i < rank - 2; i++) outer *= inShape[i]; - if (Avx2.IsSupported && Fma.IsSupported && a * b >= 8192) - { - Avx2Kernels.Transpose2D(src, a, b, dst, copies: outer); - return; - } - for (int o = 0; o < outer; o++) - Transpose2D(src.Slice(o * a * b, a * b), dst.Slice(o * a * b, a * b), a, b); - return; - } - if (rank == 4 && perm[0] == 0 && perm[1] == 3 && perm[2] == 1 && perm[3] == 2) - { - // [1,h,w,c] -> [1,c,h,w] == 2D transpose of [hw,c]->[c,hw] - int hw = inShape[1] * inShape[2], cc = inShape[3]; - if (Avx2.IsSupported && Fma.IsSupported && hw * cc >= 8192) - { - Avx2Kernels.Transpose2D(src, hw, cc, dst); - return; - } - Transpose2D(src, dst, hw, cc); - return; - } - if (rank == 4 && perm[0] == 0 && perm[1] == 2 && perm[2] == 3 && perm[3] == 1) - { - // [1,c,h,w] -> [1,h,w,c] == 2D transpose of [c,hw]->[hw,c] - int cc = inShape[1], hw = inShape[2] * inShape[3]; - if (Avx2.IsSupported && Fma.IsSupported && hw * cc >= 8192) - { - Avx2Kernels.Transpose2D(src, cc, hw, dst); - return; - } - Transpose2D(src, dst, cc, hw); - return; - } - // generic permute - int[] inStr = StridesOf(inShape), outStr = StridesOf(outShape); - int[] permutedInStr = new int[rank]; - for (int i = 0; i < rank; i++) permutedInStr[i] = inStr[perm[i]]; - int n = outShape.Aggregate(1, (x, y) => x * y); - for (int e = 0; e < n; e++) - { - int rem = e, so = 0; - for (int d = 0; d < rank; d++) - { - int c = rem / outStr[d]; - rem %= outStr[d]; - so += c * permutedInStr[d]; - } - dst[e] = src[so]; - } - } - - // tiled [a,b] -> [b,a] - private static void Transpose2D(ReadOnlySpan src, Span dst, int a, int b) - { - const int T = 16; - for (int y0 = 0; y0 < a; y0 += T) - for (int x0 = 0; x0 < b; x0 += T) - { - int ye = Math.Min(y0 + T, a), xe = Math.Min(x0 + T, b); - for (int y = y0; y < ye; y++) - for (int x = x0; x < xe; x++) - dst[x * a + y] = src[y * b + x]; - } - } - - // ======================== Slice ======================== - public static void Slice(ReadOnlySpan src, int[] inShape, Span dst, - int[] outShape, SliceP p) - { - int rank = inShape.Length; - int[] inStr = StridesOf(inShape), outStr = StridesOf(outShape); - int[] start = new int[rank], step = Enumerable.Repeat(1, rank).ToArray(); - for (int i = 0; i < p.Axes.Length; i++) { start[p.Axes[i]] = p.Starts[i]; step[p.Axes[i]] = p.Steps[i]; } - int total = outShape.Aggregate(1, (a, b) => a * b); - if (total == 0) return; - // fast path: contiguous tail (last sliced axis is innermost and steps 1) - int innerAxis = rank - 1; - if (step[innerAxis] == 1) - { - int innerLen = outShape[innerAxis]; - int innerOff = start[innerAxis]; - int outer = total / innerLen; - for (int e = 0; e < outer; e++) - { - int rem = e, so = 0; - for (int d = 0; d < rank - 1; d++) - { - int cc = rem / (outStr[d] / innerLen); - rem %= outStr[d] / innerLen; - so += (start[d] + cc * step[d]) * inStr[d]; - } - src.Slice(so + innerOff, innerLen).CopyTo(dst.Slice(e * innerLen, innerLen)); - } - return; - } - int n = outShape.Aggregate(1, (a, b) => a * b); - for (int e = 0; e < n; e++) - { - int rem = e, so = 0; - for (int d = 0; d < rank; d++) - { - int cc = rem / outStr[d]; - rem %= outStr[d]; - so += (start[d] + cc * step[d]) * inStr[d]; - } - dst[e] = src[so]; - } - } - - // ======================== Concat ======================== - public static void Concat(FBuf[] ins, int[][] inShapes, Span dst, - int[] outShape, int axis) - { - int inner = 1; - for (int d = axis + 1; d < outShape.Length; d++) inner *= outShape[d]; - int outer = 1; - for (int d = 0; d < axis; d++) outer *= outShape[d]; - int dstOff = 0; - for (int o = 0; o < outer; o++) - { - for (int i = 0; i < ins.Length; i++) - { - int count = inShapes[i][axis] * inner; - if (count == 0) continue; - ins[i].Span.Slice(o * count, count).CopyTo(dst.Slice(dstOff, count)); - dstOff += count; - } - } - } - - // ======================== ChannelShift ======================== - // out[c,y,x] = in[c, y+dy_g] (axis=2) or in[c,y,x+dx_g] (axis=3), zero-fill - public static void ChannelShift(ReadOnlySpan src, int[] inShape, Span dst, - int[] outShape, ShiftP p) - { - int c = inShape[1], h = inShape[2], w = inShape[3]; - if (Avx2.IsSupported && Fma.IsSupported && c * h * w >= 8192) - { - Avx2Kernels.ChannelShift(src, c, h, w, dst, p.Axis, p.Offsets, p.GroupEnds); - return; - } - int g0 = 0; - for (int g = 0; g < p.Offsets.Length; g++) - { - int g1 = p.GroupEnds[g]; - int off = p.Offsets[g]; - for (int ch = g0; ch < g1; ch++) - { - ReadOnlySpan s = src.Slice(ch * h * w, h * w); - Span d = dst.Slice(ch * h * w, h * w); - if (p.Axis == 2) - { - for (int y = 0; y < h; y++) - { - int sy = y + off; - if ((uint)sy < (uint)h) s.Slice(sy * w, w).CopyTo(d.Slice(y * w, w)); - else d.Slice(y * w, w).Clear(); - } - } - else if (Math.Abs(off) >= w) - { - d.Clear(); // shifted fully out of range -> all zeros - } - else - { - // out[c,y,x] = in[c,y,x+off], zero-fill — same convention as axis 2 - if (off >= 0) - for (int y = 0; y < h; y++) - { - ReadOnlySpan sr = s.Slice(y * w, w); - Span dr = d.Slice(y * w, w); - sr.Slice(off).CopyTo(dr.Slice(0, w - off)); - dr.Slice(w - off).Clear(); - } - else - for (int y = 0; y < h; y++) - { - ReadOnlySpan sr = s.Slice(y * w, w); - Span dr = d.Slice(y * w, w); - sr.Slice(0, w + off).CopyTo(dr.Slice(-off)); - dr.Slice(0, -off).Clear(); - } - } - } - g0 = g1; - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/PackedKernels.cs b/src/Sdcb.SimdGcdrNet/Kernels/PackedKernels.cs deleted file mode 100644 index 069d467..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/PackedKernels.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Wasm; - -using V4 = System.Runtime.Intrinsics.Vector128; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// WebAssembly SIMD128 (PackedSimd) versions of the non-conv ops that have AVX2 / NEON kernels. -internal static unsafe class PackedKernels -{ - private static readonly ParallelOptions PO = new() { MaxDegreeOfParallelism = Avx2Kernels.Threads }; - - /// Row-wise LayerNorm over the last axis (c); rows split across threads in groups - /// ( with separate multiply + add). - public static void LayerNorm(ReadOnlySpan x, int rows, int c, - ReadOnlySpan w, ReadOnlySpan b, Span dst, float eps) - { - int per = Math.Max(1, 16384 / c), tasks = (rows + per - 1) / per; - fixed (float* xp0 = x, wp0 = w, bp0 = b, dp0 = dst) - { - float* xp = xp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, tasks, PO, t => - { - for (int r = t * per, re = Math.Min(rows, r + per); r < re; r++) - LayerNormRow(xp + (long)r * c, dp + (long)r * c, c, wp, bp, eps); - }); - } - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void LayerNormRow(float* xr, float* dr, int c, float* wp, float* bp, float eps) - { - V4 sum = V4.Zero, sq = V4.Zero; - int i = 0; - for (; i + 4 <= c; i += 4) - { - V4 v = PackedSimd.LoadVector128(xr + i); - sum = PackedSimd.Add(sum, v); - sq = PackedSimd.Add(sq, PackedSimd.Multiply(v, v)); - } - float s = Vector128.Sum(sum), ss = Vector128.Sum(sq); - for (; i < c; i++) { float v = xr[i]; s += v; ss += v * v; } - float mean = s / c; - float inv = 1f / MathF.Sqrt(MathF.Max(ss / c - mean * mean, 0f) + eps); - V4 vm = PackedSimd.Splat(mean), vi = PackedSimd.Splat(inv); - for (i = 0; i + 4 <= c; i += 4) - { - V4 t = PackedSimd.Subtract(PackedSimd.LoadVector128(xr + i), vm); - V4 g = PackedSimd.Multiply(vi, PackedSimd.LoadVector128(wp + i)); - PackedSimd.Store(dr + i, PackedSimd.Add(PackedSimd.LoadVector128(bp + i), PackedSimd.Multiply(t, g))); - } - for (; i < c; i++) dr[i] = (xr[i] - mean) * inv * wp[i] + bp[i]; - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/PixelOps.cs b/src/Sdcb.SimdGcdrNet/Kernels/PixelOps.cs index 4e33b58..8ff88dd 100644 --- a/src/Sdcb.SimdGcdrNet/Kernels/PixelOps.cs +++ b/src/Sdcb.SimdGcdrNet/Kernels/PixelOps.cs @@ -1,5 +1,6 @@ using System.Numerics; using System.Runtime.CompilerServices; +using Sdcb.SimdOnnx.Kernels; namespace Sdcb.SimdGcdrNet.Kernels; diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Simd.cs b/src/Sdcb.SimdGcdrNet/Kernels/Simd.cs deleted file mode 100644 index 562903b..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Simd.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// Shared Vector<float> helpers (width-agnostic SIMD). -internal static class Simd -{ - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector VectorLoad(ReadOnlySpan source, int offset) => - Vector.LoadUnsafe(ref Unsafe.Add(ref MemoryMarshal.GetReference(source), offset)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void VectorStore(Span destination, int offset, Vector value) => - value.StoreUnsafe(ref Unsafe.Add(ref MemoryMarshal.GetReference(destination), offset)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe Vector VectorLoad(float* source) => - Vector.LoadUnsafe(ref Unsafe.AsRef(source)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void VectorStore(float* destination, Vector value) => - value.StoreUnsafe(ref Unsafe.AsRef(destination)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector VectorAddMul(Vector acc, Vector value, float weight) => - acc + value * new Vector(weight); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector VectorAddMul(Vector acc, Vector value, Vector weight) => - acc + value * weight; -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/SimdMath.cs b/src/Sdcb.SimdGcdrNet/Kernels/SimdMath.cs deleted file mode 100644 index de100ef..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/SimdMath.cs +++ /dev/null @@ -1,270 +0,0 @@ -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Wasm; -using System.Runtime.Intrinsics.X86; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// Vectorized transcendental math: exp / sigmoid / erf / gelu. -/// Precision targets ~1e-6 absolute so fused activations stay well inside the -/// 2e-3 validation budget. -/// -internal static class SimdMath -{ - // ---------- exp(x): cephes-style 2^n * poly(r) ---------- - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector Exp(Vector x) - { - int VL = Vector.Count; - // clamp to avoid overflow in scale - x = Vector.Max(x, new Vector(-87.3f)); - x = Vector.Min(x, new Vector(88.7f)); - float invLn2 = 1.4426950408889634f; - var fx = x * new Vector(invLn2) + new Vector(0.5f); - // floor(fx) - var n = Vector.ConvertToInt32(fx); - var nf = Vector.ConvertToSingle(n); - // nf > fx (i.e. fx negative half case) fix - var mask = Vector.GreaterThan(nf, fx); - nf -= Vector.BitwiseAnd(Vector.One, Vector.AsVectorSingle(mask)); - // r = x - n*ln2 (hi/lo parts for precision) - const float ln2hi = 0.693359375f, ln2lo = -0.00021219444005469058f; - var r = x - nf * new Vector(ln2hi) - nf * new Vector(ln2lo); - // exp(r) poly, degree 5 (cephes) - var p = new Vector(1.9875691500e-4f); - p = p * r + new Vector(1.3981999507e-3f); - p = p * r + new Vector(8.3334519073e-3f); - p = p * r + new Vector(4.1665795894e-2f); - p = p * r + new Vector(1.6666665459e-1f); - p = p * r + new Vector(5.0000001201e-1f); - var y = p * (r * r) + r + Vector.One; - // scale by 2^n - var ni = Vector.ConvertToInt32(nf) + new Vector(127); - var pow2n = Vector.AsVectorSingle(ni << 23); - return y * pow2n; - } - - public static void Exp(ReadOnlySpan src, Span dst) - { - int n = src.Length, i = 0, VL = Vector.Count; - for (; i + VL <= n; i += VL) - Simd.VectorStore(dst, i, Exp(Simd.VectorLoad(src, i))); - for (; i < n; i++) dst[i] = MathF.Exp(src[i]); - } - - // ---------- sigmoid = 1/(1+e^-x) ---------- - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector SigmoidV(Vector x) => - Vector.One / (Vector.One + Exp(-x)); - - public static void Sigmoid(ReadOnlySpan src, Span dst) - { - int n = src.Length, i = 0, VL = Vector.Count; - for (; i + VL <= n; i += VL) - Simd.VectorStore(dst, i, SigmoidV(Simd.VectorLoad(src, i))); - for (; i < n; i++) dst[i] = 1f / (1f + MathF.Exp(-src[i])); - } - - // ---------- erf(x): Abramowitz–Stegun 7.1.26 (max ~1.5e-7) ---------- - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector ErfV(Vector x) - { - var sign = Vector.ConditionalSelect(Vector.LessThan(x, Vector.Zero), - -Vector.One, Vector.One); - var ax = Vector.Abs(x); - var t = Vector.One / (Vector.One + new Vector(0.3275911f) * ax); - var p = new Vector(1.061405429f); - p = p * t + new Vector(-1.453152027f); - p = p * t + new Vector(1.421413741f); - p = p * t + new Vector(-0.284496736f); - p = p * t + new Vector(0.254829592f); - var y = Vector.One - p * t * Exp(-ax * ax); - // apply sign (sign(0)=0 → y*0 = 0 correct) - return y * sign; - } - - public static void Erf(ReadOnlySpan src, Span dst) - { - int n = src.Length, i = 0, VL = Vector.Count; - for (; i + VL <= n; i += VL) - Simd.VectorStore(dst, i, ErfV(Simd.VectorLoad(src, i))); - for (; i < n; i++) dst[i] = ErfScalar(src[i]); - } - - // ---------- gelu(x) = 0.5*x*(1+erf(x/√2)) ---------- - public static void Gelu(ReadOnlySpan src, Span dst) - { - int n = src.Length, i = 0, VL = Vector.Count; - var invSqrt2 = new Vector(0.70710678118654752f); - var half = new Vector(0.5f); - for (; i + VL <= n; i += VL) - { - var x = Simd.VectorLoad(src, i); - var g = half * x * (Vector.One + ErfV(x * invSqrt2)); - Simd.VectorStore(dst, i, g); - } - for (; i < n; i++) dst[i] = 0.5f * src[i] * (1f + ErfScalar(src[i] * 0.70710678118654752f)); - } - - // ---------- AVX2 cores (used by Avx2Kernels fused epilogues) ---------- - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector256 ExpCore(Vector256 x) - { - x = Vector256.Max(x, Vector256.Create(-87.3f)); - x = Vector256.Min(x, Vector256.Create(88.7f)); - var fx = Vector256.FusedMultiplyAdd(x, Vector256.Create(1.4426950408889634f), Vector256.Create(0.5f)); - var nf = Vector256.Floor(fx); - var r = Vector256.FusedMultiplyAdd(nf, Vector256.Create(-0.693359375f), x); - r = Vector256.FusedMultiplyAdd(nf, Vector256.Create(0.00021219444005469058f), r); - var p = Vector256.Create(1.9875691500e-4f); - p = Vector256.FusedMultiplyAdd(p, r, Vector256.Create(1.3981999507e-3f)); - p = Vector256.FusedMultiplyAdd(p, r, Vector256.Create(8.3334519073e-3f)); - p = Vector256.FusedMultiplyAdd(p, r, Vector256.Create(4.1665795894e-2f)); - p = Vector256.FusedMultiplyAdd(p, r, Vector256.Create(1.6666665459e-1f)); - p = Vector256.FusedMultiplyAdd(p, r, Vector256.Create(5.0000001201e-1f)); - var y = Vector256.Add(Vector256.FusedMultiplyAdd(p, Vector256.Multiply(r, r), r), Vector256.One); - var ni = Avx2.Add(Avx2.ConvertToVector256Int32(nf), Vector256.Create(127)); - var pow2n = Avx2.ShiftLeftLogical(ni, 23).AsSingle(); - return Vector256.Multiply(y, pow2n); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector256 SigmoidCore(Vector256 x) => - Vector256.Divide(Vector256.One, - Vector256.Add(Vector256.One, ExpCore(Vector256.Negate(x)))); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector256 ErfCore(Vector256 x) - { - var sign = Avx.Or(Avx.And(x, Vector256.Create(-0f)), - Vector256.One); - var ax = Avx.AndNot(Vector256.Create(-0f), x); - var t = Vector256.Divide(Vector256.One, - Vector256.Add(Vector256.One, Vector256.Multiply(Vector256.Create(0.3275911f), ax))); - var p = Vector256.Create(1.061405429f); - p = Vector256.FusedMultiplyAdd(p, t, Vector256.Create(-1.453152027f)); - p = Vector256.FusedMultiplyAdd(p, t, Vector256.Create(1.421413741f)); - p = Vector256.FusedMultiplyAdd(p, t, Vector256.Create(-0.284496736f)); - p = Vector256.FusedMultiplyAdd(p, t, Vector256.Create(0.254829592f)); - var y = Vector256.Subtract(Vector256.One, - Vector256.Multiply(Vector256.Multiply(p, t), ExpCore(Vector256.Negate(Vector256.Multiply(ax, ax))))); - return Vector256.Multiply(y, sign); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector256 GeluCore(Vector256 x) => - Vector256.Multiply(Vector256.Multiply(Vector256.Create(0.5f), x), - Vector256.Add(Vector256.One, - ErfCore(Vector256.Multiply(x, Vector256.Create(0.70710678118654752f))))); - - // ---------- 128-bit cores (NEON tier fused epilogues; same formulas as the AVX2 cores) ---------- - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 ExpCore(Vector128 x) - { - x = Vector128.Max(x, Vector128.Create(-87.3f)); - x = Vector128.Min(x, Vector128.Create(88.7f)); - var fx = Vector128.FusedMultiplyAdd(x, Vector128.Create(1.4426950408889634f), Vector128.Create(0.5f)); - var nf = Vector128.Floor(fx); - var r = Vector128.FusedMultiplyAdd(nf, Vector128.Create(-0.693359375f), x); - r = Vector128.FusedMultiplyAdd(nf, Vector128.Create(0.00021219444005469058f), r); - var p = Vector128.Create(1.9875691500e-4f); - p = Vector128.FusedMultiplyAdd(p, r, Vector128.Create(1.3981999507e-3f)); - p = Vector128.FusedMultiplyAdd(p, r, Vector128.Create(8.3334519073e-3f)); - p = Vector128.FusedMultiplyAdd(p, r, Vector128.Create(4.1665795894e-2f)); - p = Vector128.FusedMultiplyAdd(p, r, Vector128.Create(1.6666665459e-1f)); - p = Vector128.FusedMultiplyAdd(p, r, Vector128.Create(5.0000001201e-1f)); - var y = Vector128.FusedMultiplyAdd(p, r * r, r) + Vector128.One; - var ni = Vector128.ConvertToInt32(nf) + Vector128.Create(127); - return y * Vector128.ShiftLeft(ni, 23).AsSingle(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 SigmoidCore(Vector128 x) => - Vector128.One / (Vector128.One + ExpCore(-x)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 ErfCore(Vector128 x) - { - var signBit = Vector128.Create(-0f); - var sign = (x & signBit) | Vector128.One; - var ax = Vector128.AndNot(x, signBit); - var t = Vector128.One / (Vector128.One + Vector128.Create(0.3275911f) * ax); - var p = Vector128.Create(1.061405429f); - p = Vector128.FusedMultiplyAdd(p, t, Vector128.Create(-1.453152027f)); - p = Vector128.FusedMultiplyAdd(p, t, Vector128.Create(1.421413741f)); - p = Vector128.FusedMultiplyAdd(p, t, Vector128.Create(-0.284496736f)); - p = Vector128.FusedMultiplyAdd(p, t, Vector128.Create(0.254829592f)); - var y = Vector128.One - p * t * ExpCore(-(ax * ax)); - return y * sign; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 GeluCore(Vector128 x) => - Vector128.Create(0.5f) * x * (Vector128.One + ErfCore(x * Vector128.Create(0.70710678118654752f))); - - // ---------- WASM SIMD128 (PackedSimd) cores: Exp / SigmoidV / ErfV / Gelu above, op for op ---------- - // WASM SIMD has no FMA (relaxed SIMD is off), and Vector128.FusedMultiplyAdd would fall back to - // scalar fmaf, so these keep the Vector tier's separate multiply + add (same bits for finite x). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 MulAddPk(Vector128 a, Vector128 b, Vector128 c) => - PackedSimd.Add(PackedSimd.Multiply(a, b), c); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 ExpPk(Vector128 x) - { - // pmax/pmin: one maxps/minps in V8 (f32x4.max/min need a NaN fix-up sequence) - x = PackedSimd.PseudoMax(Vector128.Create(-87.3f), x); - x = PackedSimd.PseudoMin(Vector128.Create(88.7f), x); - var fx = MulAddPk(x, Vector128.Create(1.4426950408889634f), Vector128.Create(0.5f)); - var nf = PackedSimd.ConvertToSingle(PackedSimd.ConvertToInt32Saturate(fx)); - nf = PackedSimd.Subtract(nf, PackedSimd.And(Vector128.One, PackedSimd.CompareGreaterThan(nf, fx))); - var r = PackedSimd.Subtract(PackedSimd.Subtract(x, PackedSimd.Multiply(nf, Vector128.Create(0.693359375f))), - PackedSimd.Multiply(nf, Vector128.Create(-0.00021219444005469058f))); - var p = Vector128.Create(1.9875691500e-4f); - p = MulAddPk(p, r, Vector128.Create(1.3981999507e-3f)); - p = MulAddPk(p, r, Vector128.Create(8.3334519073e-3f)); - p = MulAddPk(p, r, Vector128.Create(4.1665795894e-2f)); - p = MulAddPk(p, r, Vector128.Create(1.6666665459e-1f)); - p = MulAddPk(p, r, Vector128.Create(5.0000001201e-1f)); - var y = PackedSimd.Add(PackedSimd.Add(PackedSimd.Multiply(p, PackedSimd.Multiply(r, r)), r), Vector128.One); - var ni = PackedSimd.Add(PackedSimd.ConvertToInt32Saturate(nf), Vector128.Create(127)); - return PackedSimd.Multiply(y, PackedSimd.ShiftLeft(ni, 23).AsSingle()); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 SigmoidPk(Vector128 x) => - PackedSimd.Divide(Vector128.One, PackedSimd.Add(Vector128.One, ExpPk(PackedSimd.Negate(x)))); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 ErfPk(Vector128 x) - { - var one = Vector128.One; - var sign = Vector128.ConditionalSelect(PackedSimd.CompareLessThan(x, Vector128.Zero), Vector128.Create(-1f), one); - var ax = PackedSimd.Abs(x); - var t = PackedSimd.Divide(one, PackedSimd.Add(one, PackedSimd.Multiply(Vector128.Create(0.3275911f), ax))); - var p = Vector128.Create(1.061405429f); - p = MulAddPk(p, t, Vector128.Create(-1.453152027f)); - p = MulAddPk(p, t, Vector128.Create(1.421413741f)); - p = MulAddPk(p, t, Vector128.Create(-0.284496736f)); - p = MulAddPk(p, t, Vector128.Create(0.254829592f)); - var y = PackedSimd.Subtract(one, PackedSimd.Multiply(PackedSimd.Multiply(p, t), - ExpPk(PackedSimd.Multiply(PackedSimd.Negate(ax), ax)))); - return PackedSimd.Multiply(y, sign); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 GeluPk(Vector128 x) => - PackedSimd.Multiply(PackedSimd.Multiply(Vector128.Create(0.5f), x), - PackedSimd.Add(Vector128.One, ErfPk(PackedSimd.Multiply(x, Vector128.Create(0.70710678118654752f))))); - - internal static float ErfScalar(float x) - { - float ax = MathF.Abs(x); - float t = 1f / (1f + 0.3275911f * ax); - float p = ((((1.061405429f * t - 1.453152027f) * t + 1.421413741f) * t - 0.284496736f) * t + 0.254829592f) * t; - float y = 1f - p * MathF.Exp(-ax * ax); - return x < 0 ? -y : y; - } -} diff --git a/src/Sdcb.SimdGcdrNet/Onnx/OnnxModel.cs b/src/Sdcb.SimdGcdrNet/Onnx/OnnxModel.cs deleted file mode 100644 index fa30861..0000000 --- a/src/Sdcb.SimdGcdrNet/Onnx/OnnxModel.cs +++ /dev/null @@ -1,49 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Onnx; - -/// Validated ONNX model: graph nodes, initializers, IO metadata. -public sealed class OnnxModel -{ - private OnnxModel(int opset, OnnxGraphData graph, Dictionary initializers) - { - Opset = opset; - Graph = graph; - Initializers = initializers; - } - - /// The model's default-domain (ai.onnx) opset version. - public int Opset { get; } - internal OnnxGraphData Graph { get; } - internal Dictionary Initializers { get; } - - /// Loads an ONNX model from the file at . - public static OnnxModel Load(string path) - { - using FileStream fs = File.OpenRead(path); - return Load(fs); - } - - /// Loads an ONNX model by reading to its end. - public static OnnxModel Load(Stream source) => Build(OnnxProtoReader.Parse(source)); - - /// Parses the model straight from (pinned for the duration - /// of the call, no intermediate copy); weights are decoded into model-owned arrays, so the - /// memory may be released afterwards. - public static unsafe OnnxModel Load(ReadOnlySpan source) - { - fixed (byte* p = source) - { - using var ums = new UnmanagedMemoryStream(p, source.Length); - return Load(ums); - } - } - - private static OnnxModel Build(OnnxModelData parsed) - { - OnnxGraphData g = parsed.Graph!; - int opset = checked((int)(parsed.Opsets.FirstOrDefault( - static x => x.Domain.Length == 0 || x.Domain == "ai.onnx")?.Version ?? 0)); - Dictionary init = new(g.Initializers.Count); - foreach (OnnxTensorData t in g.Initializers) init[t.Name] = t; - return new OnnxModel(opset, g, init); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Onnx/OnnxModelData.cs b/src/Sdcb.SimdGcdrNet/Onnx/OnnxModelData.cs deleted file mode 100644 index 92063f4..0000000 --- a/src/Sdcb.SimdGcdrNet/Onnx/OnnxModelData.cs +++ /dev/null @@ -1,195 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Onnx; - -/// Raw parsed ONNX structures (protobuf level, pre-validation). -internal sealed class OnnxModelData -{ - public long IrVersion; - public List Opsets { get; } = []; - public OnnxGraphData? Graph; -} - -internal sealed class OnnxOpsetData -{ - public string Domain = ""; - public long Version; -} - -internal sealed class OnnxGraphData -{ - public List Nodes { get; } = []; - public List Initializers { get; } = []; - public List Inputs { get; } = []; - public List Outputs { get; } = []; - public List ValueInfo { get; } = []; -} - -internal sealed class OnnxNodeData -{ - public List Inputs { get; } = []; - public List Outputs { get; } = []; - public string Name = ""; - public string OpType = ""; - public string Domain = ""; - public List Attributes { get; } = []; - - public OnnxAttributeData? Attr(string name) - { - foreach (OnnxAttributeData a in Attributes) - if (a.Name == name) return a; - return null; - } - - public long IntAttr(string name, long fallback = 0) => Attr(name)?.Int ?? fallback; - public float FloatAttr(string name, float fallback = 0f) => Attr(name)?.Float ?? fallback; - public string StringAttr(string name, string fallback = "") => - Attr(name)?.String is byte[] b ? System.Text.Encoding.UTF8.GetString(b) : fallback; - public long[] IntsAttr(string name) => Attr(name)?.Ints.ToArray() ?? []; -} - -internal sealed class OnnxAttributeData -{ - public string Name = ""; - public int Type; - public float Float; - public long Int; - public byte[]? String; - public OnnxTensorData? Tensor; - public List Floats { get; } = []; - public List Ints { get; } = []; - public List Strings { get; } = []; -} - -internal sealed class OnnxTensorData -{ - public List Dims { get; } = []; - public int DataType; - public List FloatData { get; } = []; - public List Int32Data { get; } = []; - public List StringData { get; } = []; - public List Int64Data { get; } = []; - public byte[]? RawData; - public string Name = ""; - public List DoubleData { get; } = []; - public List UInt64Data { get; } = []; - - public const int TypeFloat = 1, TypeUInt8 = 2, TypeInt8 = 3, TypeUInt16 = 4, TypeInt16 = 5, - TypeInt32 = 6, TypeInt64 = 7, TypeString = 8, TypeBool = 9, TypeFloat16 = 10, - TypeDouble = 11, TypeUInt32 = 12, TypeUInt64 = 13; - - public int[] DimsArray() => Dims.Select(d => checked((int)d)).ToArray(); - - /// Decoded element count (product of dims; scalar = 1). - public long ElementCount() - { - long n = 1; - foreach (long d in Dims) n *= d; - return n; - } - - private float[]? _floats; - - /// - /// This tensor as floats (float/double/int sources), decoded once and shared: every - /// per-size plan of a model references the same read-only array. Raw float bytes are - /// dropped after decoding so the weights live in memory only once. - /// - public float[] AsFloats() - { - lock (this) - { - if (_floats is { } cached) return cached; - _floats = DecodeFloats(); - if (DataType == TypeFloat) RawData = null; - return _floats; - } - } - - private float[] DecodeFloats() - { - int count = checked((int)ElementCount()); - if (RawData is byte[] raw) - { - switch (DataType) - { - case TypeFloat: - { - float[] f = new float[checked(raw.Length / 4)]; - Buffer.BlockCopy(raw, 0, f, 0, raw.Length); - return f; - } - case TypeDouble: - { - double[] d = new double[checked(raw.Length / 8)]; - Buffer.BlockCopy(raw, 0, d, 0, raw.Length); - return d.Select(v => (float)v).ToArray(); - } - case TypeInt32: - { - int[] v = new int[checked(raw.Length / 4)]; - Buffer.BlockCopy(raw, 0, v, 0, raw.Length); - return v.Select(x => (float)x).ToArray(); - } - case TypeInt64: - { - long[] v = new long[checked(raw.Length / 8)]; - Buffer.BlockCopy(raw, 0, v, 0, raw.Length); - return v.Select(x => (float)x).ToArray(); - } - default: - throw new NotSupportedException($"Tensor '{Name}': raw dtype {DataType} cannot be read as float."); - } - } - if (FloatData.Count > 0) return FloatData.ToArray(); - if (Int32Data.Count > 0) return Int32Data.Select(v => (float)v).ToArray(); - if (Int64Data.Count > 0) return Int64Data.Select(v => (float)v).ToArray(); - if (DoubleData.Count > 0) return DoubleData.Select(v => (float)v).ToArray(); - if (count == 0) return []; - throw new InvalidDataException($"Tensor '{Name}' has no float-readable data."); - } - - /// Decodes this tensor to an int64 array. - public long[] AsInt64s() - { - byte[]? rawData; float[]? fl; - lock (this) { rawData = RawData; fl = _floats; } - if (DataType == TypeFloat && fl is not null) return fl.Select(x => (long)x).ToArray(); - if (rawData is byte[] raw) - { - switch (DataType) - { - case TypeInt64: - { - long[] v = new long[checked(raw.Length / 8)]; - Buffer.BlockCopy(raw, 0, v, 0, raw.Length); - return v; - } - case TypeInt32: - { - int[] v = new int[checked(raw.Length / 4)]; - Buffer.BlockCopy(raw, 0, v, 0, raw.Length); - return v.Select(x => (long)x).ToArray(); - } - case TypeFloat: - { - float[] f = new float[checked(raw.Length / 4)]; - Buffer.BlockCopy(raw, 0, f, 0, raw.Length); - return f.Select(x => (long)x).ToArray(); - } - default: - throw new NotSupportedException($"Tensor '{Name}': raw dtype {DataType} cannot be read as int64."); - } - } - if (Int64Data.Count > 0) return Int64Data.ToArray(); - if (Int32Data.Count > 0) return Int32Data.Select(v => (long)v).ToArray(); - if (FloatData.Count > 0) return FloatData.Select(v => (long)v).ToArray(); - if (ElementCount() == 0) return []; - throw new InvalidDataException($"Tensor '{Name}' has no int-readable data."); - } -} - -internal sealed class OnnxValueInfoData -{ - public string Name = ""; - public int ElementType; - public int[]? Shape; // -1 marks a dynamic dimension -} diff --git a/src/Sdcb.SimdGcdrNet/Onnx/OnnxProtoReader.cs b/src/Sdcb.SimdGcdrNet/Onnx/OnnxProtoReader.cs deleted file mode 100644 index d255248..0000000 --- a/src/Sdcb.SimdGcdrNet/Onnx/OnnxProtoReader.cs +++ /dev/null @@ -1,514 +0,0 @@ -using System.Buffers.Binary; -using System.Text; - -namespace Sdcb.SimdGcdrNet.Onnx; - -// A deliberately small protobuf decoder for the ONNX protobuf schema. It is -// kept independent of Google.Protobuf so the core remains entirely managed -// and has no generated-code or native runtime dependency. -// -// The reader consumes the input stream in place. Length-delimited protobuf -// messages are represented by bounded child readers rather than materialized -// as byte arrays. Only values that the model needs to retain (for example -// tensor raw_data and strings) are copied into managed storage. -internal static class OnnxProtoReader -{ - public static OnnxModelData Parse(Stream source) - { - if (source is null) throw new ArgumentNullException(nameof(source)); - ProtoReader reader = new(source, -1); - OnnxModelData model = new(); - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 0: model.IrVersion = checked((long)reader.ReadVarint()); break; - case 8 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxOpsetData opset = new(); - ParseOpset(ref nested, opset); - nested.EnsureFullyConsumed(); - model.Opsets.Add(opset); - break; - } - case 7 when wire == 2: - { - var nested = reader.EnterMessage(); - model.Graph = ParseGraph(ref nested); - nested.EnsureFullyConsumed(); - break; - } - default: reader.Skip(wire); break; - } - } - if (model.Graph is null) throw new InvalidDataException("ONNX ModelProto has no graph."); - return model; - } - - private static void ParseOpset(ref ProtoReader reader, OnnxOpsetData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: result.Domain = reader.ReadString(); break; - case 2 when wire == 0: result.Version = checked((long)reader.ReadVarint()); break; - default: reader.Skip(wire); break; - } - } - } - - private static OnnxGraphData ParseGraph(ref ProtoReader reader) - { - OnnxGraphData result = new(); - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxNodeData node = new(); - ParseNode(ref nested, node); - nested.EnsureFullyConsumed(); - result.Nodes.Add(node); - break; - } - case 5 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxTensorData tensor = new(); - ParseTensor(ref nested, tensor); - nested.EnsureFullyConsumed(); - result.Initializers.Add(tensor); - break; - } - case 11 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxValueInfoData value = new(); - ParseValueInfo(ref nested, value); - nested.EnsureFullyConsumed(); - result.Inputs.Add(value); - break; - } - case 12 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxValueInfoData value = new(); - ParseValueInfo(ref nested, value); - nested.EnsureFullyConsumed(); - result.Outputs.Add(value); - break; - } - case 13 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxValueInfoData value = new(); - ParseValueInfo(ref nested, value); - nested.EnsureFullyConsumed(); - result.ValueInfo.Add(value); - break; - } - default: reader.Skip(wire); break; - } - } - return result; - } - - private static void ParseNode(ref ProtoReader reader, OnnxNodeData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: result.Inputs.Add(reader.ReadString()); break; - case 2 when wire == 2: result.Outputs.Add(reader.ReadString()); break; - case 3 when wire == 2: result.Name = reader.ReadString(); break; - case 4 when wire == 2: result.OpType = reader.ReadString(); break; - case 5 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxAttributeData attribute = new(); - ParseAttribute(ref nested, attribute); - nested.EnsureFullyConsumed(); - result.Attributes.Add(attribute); - break; - } - case 7 when wire == 2: result.Domain = reader.ReadString(); break; - default: reader.Skip(wire); break; - } - } - if (result.OpType.Length == 0) - throw new InvalidDataException( - $"ONNX NodeProto has no op_type (inputs={result.Inputs.Count}, " + - $"outputs={result.Outputs.Count}, name='{result.Name}')."); - } - - private static void ParseAttribute(ref ProtoReader reader, OnnxAttributeData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: result.Name = reader.ReadString(); break; - case 20 when wire == 0: result.Type = checked((int)reader.ReadVarint()); break; - case 2 when wire == 5: result.Float = BitConverter.Int32BitsToSingle(unchecked((int)reader.ReadFixed32())); break; - case 3 when wire == 0: result.Int = unchecked((long)reader.ReadVarint()); break; - case 4 when wire == 2: result.String = reader.ReadBytes(); break; - case 5 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxTensorData tensor = new(); - ParseTensor(ref nested, tensor); - nested.EnsureFullyConsumed(); - result.Tensor = tensor; - break; - } - case 7 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedFloat(ref nested, result.Floats); - nested.EnsureFullyConsumed(); - break; - } - case 7 when wire == 5: result.Floats.Add(BitConverter.Int32BitsToSingle(unchecked((int)reader.ReadFixed32()))); break; - case 8 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedInt64(ref nested, result.Ints); - nested.EnsureFullyConsumed(); - break; - } - case 8 when wire == 0: result.Ints.Add(unchecked((long)reader.ReadVarint())); break; - case 9 when wire == 2: result.Strings.Add(reader.ReadBytes()); break; - default: reader.Skip(wire); break; - } - } - } - - private static void ParseTensor(ref ProtoReader reader, OnnxTensorData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedInt64(ref nested, result.Dims); - nested.EnsureFullyConsumed(); - break; - } - case 1 when wire == 0: result.Dims.Add(unchecked((long)reader.ReadVarint())); break; - case 2 when wire == 0: result.DataType = checked((int)reader.ReadVarint()); break; - case 4 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedFloat(ref nested, result.FloatData); - nested.EnsureFullyConsumed(); - break; - } - case 4 when wire == 5: result.FloatData.Add(BitConverter.Int32BitsToSingle(unchecked((int)reader.ReadFixed32()))); break; - case 5 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedInt32(ref nested, result.Int32Data); - nested.EnsureFullyConsumed(); - break; - } - case 5 when wire == 0: result.Int32Data.Add(unchecked((int)reader.ReadVarint())); break; - case 6 when wire == 2: result.StringData.Add(reader.ReadBytes()); break; - case 7 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedInt64(ref nested, result.Int64Data); - nested.EnsureFullyConsumed(); - break; - } - case 7 when wire == 0: result.Int64Data.Add(unchecked((long)reader.ReadVarint())); break; - case 8 when wire == 2: result.Name = reader.ReadString(); break; - case 9 when wire == 2: result.RawData = reader.ReadBytes(); break; - case 10 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedDouble(ref nested, result.DoubleData); - nested.EnsureFullyConsumed(); - break; - } - case 10 when wire == 1: result.DoubleData.Add(BitConverter.Int64BitsToDouble(unchecked((long)reader.ReadFixed64()))); break; - case 11 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedUInt64(ref nested, result.UInt64Data); - nested.EnsureFullyConsumed(); - break; - } - case 11 when wire == 1: result.UInt64Data.Add(reader.ReadFixed64()); break; - default: reader.Skip(wire); break; - } - } - if (result.Name.Length == 0) throw new InvalidDataException("ONNX TensorProto has no name."); - } - - private static void ParseValueInfo(ref ProtoReader reader, OnnxValueInfoData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: result.Name = reader.ReadString(); break; - case 2 when wire == 2: - { - var nested = reader.EnterMessage(); - ParseType(ref nested, result); - nested.EnsureFullyConsumed(); - break; - } - default: reader.Skip(wire); break; - } - } - } - - private static void ParseType(ref ProtoReader reader, OnnxValueInfoData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - if (field == 1 && wire == 2) - { - var nested = reader.EnterMessage(); - ParseTensorType(ref nested, result); - nested.EnsureFullyConsumed(); - } - else reader.Skip(wire); - } - } - - private static void ParseTensorType(ref ProtoReader reader, OnnxValueInfoData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 0: result.ElementType = checked((int)reader.ReadVarint()); break; - case 2 when wire == 2: - { - var nested = reader.EnterMessage(); - result.Shape = ParseShape(ref nested); - nested.EnsureFullyConsumed(); - break; - } - default: reader.Skip(wire); break; - } - } - } - - private static int[] ParseShape(ref ProtoReader reader) - { - List dims = []; - while (reader.TryReadTag(out int field, out int wire)) - { - if (field == 1 && wire == 2) - { - var dimension = reader.EnterMessage(); - long value = -1; - while (dimension.TryReadTag(out int df, out int dw)) - { - if (df == 1 && dw == 0) value = unchecked((long)dimension.ReadVarint()); - else dimension.Skip(dw); - } - dimension.EnsureFullyConsumed(); - dims.Add(value > 0 && value <= int.MaxValue ? (int)value : -1); - } - else reader.Skip(wire); - } - return dims.ToArray(); - } - - private static void ReadPackedFloat(ref ProtoReader reader, List destination) - { - if ((reader.Remaining & 3) != 0) throw new InvalidDataException("Packed float field is truncated."); - while (reader.Remaining > 0) - destination.Add(BitConverter.Int32BitsToSingle(unchecked((int)reader.ReadFixed32()))); - } - - private static void ReadPackedDouble(ref ProtoReader reader, List destination) - { - if ((reader.Remaining & 7) != 0) throw new InvalidDataException("Packed double field is truncated."); - while (reader.Remaining > 0) - destination.Add(BitConverter.Int64BitsToDouble(unchecked((long)reader.ReadFixed64()))); - } - - private static void ReadPackedInt32(ref ProtoReader reader, List destination) - { - while (reader.Remaining > 0) destination.Add(unchecked((int)reader.ReadVarint())); - } - - private static void ReadPackedInt64(ref ProtoReader reader, List destination) - { - while (reader.Remaining > 0) destination.Add(unchecked((long)reader.ReadVarint())); - } - - private static void ReadPackedUInt64(ref ProtoReader reader, List destination) - { - while (reader.Remaining > 0) destination.Add(reader.ReadVarint()); - } - - private ref struct ProtoReader - { - private readonly Stream _source; - // -1 denotes the unbounded root reader. Nested readers have an exact - // number of bytes left and can never consume their parent message. - private long _remaining; - - public ProtoReader(Stream source, long remaining) - { - _source = source; - _remaining = remaining; - } - - public long Remaining => _remaining; - - public bool TryReadTag(out int field, out int wire) - { - int first = ReadByte(allowEof: true); - if (first < 0) { field = wire = 0; return false; } - ulong tag = ReadVarint(first); - field = checked((int)(tag >> 3)); - wire = checked((int)(tag & 7)); - if (field <= 0) throw new InvalidDataException("Invalid protobuf field number."); - return true; - } - - public ulong ReadVarint() - { - int first = ReadByte(allowEof: false); - return ReadVarint(first); - } - - private ulong ReadVarint(int first) - { - ulong value = 0; - for (int shift = 0; shift < 64; shift += 7) - { - int next = shift == 0 ? first : ReadByte(allowEof: false); - byte b = checked((byte)next); - if (shift == 63 && b > 1) throw new InvalidDataException("Protobuf varint overflows UInt64."); - value |= (ulong)(b & 0x7f) << shift; - if ((b & 0x80) == 0) return value; - } - throw new InvalidDataException("Protobuf varint is too long."); - } - - public uint ReadFixed32() - { - Span bytes = stackalloc byte[4]; - ReadExact(bytes); - return BinaryPrimitives.ReadUInt32LittleEndian(bytes); - } - - public ulong ReadFixed64() - { - Span bytes = stackalloc byte[8]; - ReadExact(bytes); - return BinaryPrimitives.ReadUInt64LittleEndian(bytes); - } - - public byte[] ReadBytes() - { - ulong length = ReadVarint(); - if (length > int.MaxValue) throw new InvalidDataException("Protobuf length-delimited field is too large."); - int count = checked((int)length); - byte[] value = new byte[count]; - ReadExact(value); - return value; - } - - public string ReadString() - { - try { return new UTF8Encoding(false, true).GetString(ReadBytes()); } - catch (DecoderFallbackException ex) { throw new InvalidDataException("ONNX protobuf string is not valid UTF-8.", ex); } - } - - public ProtoReader EnterMessage() - { - ulong length = ReadVarint(); - if (length > long.MaxValue || (_remaining >= 0 && length > (ulong)_remaining)) - throw new InvalidDataException("Protobuf length-delimited field is out of bounds."); - if (_remaining >= 0) _remaining -= checked((long)length); - return new ProtoReader(_source, checked((long)length)); - } - - public void Skip(int wire) - { - switch (wire) - { - case 0: ReadVarint(); break; - case 1: _ = ReadFixed64(); break; - case 2: SkipLengthDelimited(); break; - case 3: - while (TryReadTag(out _, out int nestedWire)) - { - if (nestedWire == 4) return; - Skip(nestedWire); - } - throw new InvalidDataException("Unterminated protobuf group."); - case 4: throw new InvalidDataException("Unexpected protobuf end-group."); - case 5: _ = ReadFixed32(); break; - default: throw new NotSupportedException($"Unsupported protobuf wire type {wire}."); - } - } - - public readonly void EnsureFullyConsumed() - { - if (_remaining != 0) throw new InvalidDataException("Truncated protobuf nested message."); - } - - private void SkipLengthDelimited() - { - ulong length = ReadVarint(); - if (length > long.MaxValue || (_remaining >= 0 && length > (ulong)_remaining)) - throw new InvalidDataException("Protobuf length-delimited field is out of bounds."); - long left = checked((long)length); - Span scratch = stackalloc byte[1024]; - while (left > 0) - { - int chunk = (int)Math.Min(left, scratch.Length); - ReadExact(scratch[..chunk]); - left -= chunk; - } - } - - private int ReadByte(bool allowEof) - { - if (_remaining == 0) - { - if (allowEof) return -1; - throw new InvalidDataException("Truncated protobuf stream."); - } - int value = _source.ReadByte(); - if (value < 0) - { - if (allowEof && _remaining < 0) return -1; - throw new InvalidDataException("Truncated protobuf stream."); - } - if (_remaining > 0) _remaining--; - return value; - } - - private void ReadExact(scoped Span destination) - { - if (_remaining >= 0 && destination.Length > _remaining) - throw new InvalidDataException("Protobuf field overruns its message."); - int read = 0; - while (read < destination.Length) - { - int n = _source.Read(destination[read..]); - if (n <= 0) throw new InvalidDataException("Truncated protobuf stream."); - read += n; - } - if (_remaining > 0) _remaining -= destination.Length; - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Sdcb.SimdGcdrNet.csproj b/src/Sdcb.SimdGcdrNet/Sdcb.SimdGcdrNet.csproj index b974788..fe10bd3 100644 --- a/src/Sdcb.SimdGcdrNet/Sdcb.SimdGcdrNet.csproj +++ b/src/Sdcb.SimdGcdrNet/Sdcb.SimdGcdrNet.csproj @@ -14,4 +14,10 @@ + + + + + + diff --git a/tests/Sdcb.SimdGcdrNet.Tests/Avx512KernelTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/Avx512KernelTests.cs index eda6389..490ec79 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/Avx512KernelTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/Avx512KernelTests.cs @@ -1,5 +1,7 @@ using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Kernels; using Sdcb.SimdGcdrNet.Kernels; namespace Sdcb.SimdGcdrNet.Tests; @@ -32,91 +34,6 @@ private static float[] Both(Action run, int n) return avx512; } - [SkippableTheory] - [InlineData(3, 16, 3, 1, 37, 23, 1, 0f, false, 0)] // gc stem: 16-wide tail kernel, reflect, row runs - [InlineData(6, 32, 3, 1, 29, 41, 1, 0f, false, 1)] // dr stem: one block pair - [InlineData(16, 48, 3, 1, 11, 13, 0, 0.25f, true, 1)] // odd block count, pad value, residual - [InlineData(20, 20, 3, 2, 17, 19, 0, 0f, true, 2)] // stride 2, oc lane tail, sigmoid - [InlineData(32, 64, 3, 2, 12, 31, 1, 0f, false, 3)] // stride 2, reflect, gelu - [InlineData(24, 3, 3, 1, 9, 10, 0, 0f, false, 0)] // oc=3 (single partial lane block) - [InlineData(40, 36, 1, 1, 7, 33, 0, 0f, true, 1)] // 1x1 conv - [InlineData(8, 32, 5, 1, 14, 9, 1, 0f, false, 1)] // 5x5 (direct path), narrow rows - public void Conv_Direct_MatchesAvx2(int ic, int oc, int k, int stride, int h, int w, int mode, - float padValue, bool residual, int act) - { - var rng = new Random(ic * 131 + oc * 7 + k); - int pad = k / 2; - int oh = (h + 2 * pad - k) / stride + 1, ow = (w + 2 * pad - k) / stride + 1; - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * k * k), b = rng.Array(oc); - float[] res = residual ? rng.Array(oh * ow * oc) : []; - var p = new ConvP - { - Pads = [pad, pad, pad, pad], Strides = [stride, stride], Dilations = [1, 1], Kernel = [k, k], - PadMode = (PadMode)mode, PadValue = padValue, Act = (ActivationKind)act, - }; - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, k, k], b, 1, ic); // no Winograd: direct kernels - Both(dst => Nhwc.Conv(x, h, w, ic, dst, oh, ow, oc, p, kp, res), oh * ow * oc); - } - - [SkippableTheory] - [InlineData(16, 16, 9, 11, 0, 0f, false, 0)] - [InlineData(24, 20, 13, 6, 1, 0f, true, 1)] // ic % 16 != 0: 8-lane input transform - [InlineData(32, 48, 22, 18, 0, 0.5f, false, 3)] - [InlineData(64, 16, 17, 23, 1, 0f, true, 0)] - [InlineData(128, 64, 30, 41, 1, 0f, false, 1)] // several tile chunks per image - public void Conv_Winograd_MatchesAvx2(int ic, int oc, int h, int w, int mode, float padValue, bool residual, int act) - { - var rng = new Random(ic * 17 + oc); - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * 9), b = rng.Array(oc); - float[] res = residual ? rng.Array(h * w * oc) : []; - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], - PadMode = (PadMode)mode, PadValue = padValue, Act = (ActivationKind)act, - }; - Skip.IfNot(Nhwc.WinogradEligible(p, [oc, ic, 3, 3], ic), "Winograd disabled"); - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - kp.WinoU = Nhwc.PackWinograd(wt, oc, ic); - Both(dst => Nhwc.Conv(x, h, w, ic, dst, h, w, oc, p, kp, res), h * w * oc); - } - - [SkippableTheory] - [InlineData(57, 128, 128, false, 0)] - [InlineData(23, 160, 48, true, 3)] // odd block count, transB, gelu - [InlineData(5, 64, 20, false, 1)] // fewer rows than a tile, lane tail - public void Gemm_MatchesAvx2(int m, int k, int n, bool transB, int act) - { - var rng = new Random(m + k + n); - float[] a = rng.Array(m * k), bm = rng.Array(k * n), bias = rng.Array(n); - NhwcConvPlan kp = Nhwc.PackGemm(bm, k, n, transB, bias); - Both(dst => Nhwc.Gemm(a, m, k, dst, n, kp, (ActivationKind)act), m * n); - } - - [SkippableTheory] - [InlineData(32, 2, 2, 0, false)] - [InlineData(16, 3, 2, 1, true)] // padded taps, fused relu - public void MaxPool_MatchesAvx2(int c, int k, int s, int pad, bool relu) - { - var rng = new Random(c + k); - int h = 19, w = 26, oh = (h + 2 * pad - k) / s + 1, ow = (w + 2 * pad - k) / s + 1; - float[] x = rng.Array(h * w * c); - var p = new PoolP { Kernel = [k, k], Strides = [s, s], Pads = [pad, pad, pad, pad], FuseRelu = relu }; - Both(dst => Nhwc.MaxPool(x, h, w, c, dst, oh, ow, p), oh * ow * c); - } - - [SkippableTheory] - [InlineData(32, true, true, true)] - [InlineData(16, false, false, false)] - [InlineData(24, true, false, true)] // c % 16 != 0: generic resize - public void Resize_MatchesAvx2(int c, bool relu, bool addend, bool addRelu) - { - var rng = new Random(c); - int h = 13, w = 9, oh = 26, ow = 18; - float[] x = rng.Array(h * w * c), add = addend ? rng.Array(oh * ow * c) : []; - Both(dst => Nhwc.ResizeBilinear(x, h, w, c, dst, oh, ow, 2, 2, - relu ? ActivationKind.Relu : ActivationKind.None, add, addRelu), oh * ow * c); - } - [SkippableTheory] [InlineData(32, 3, true, true, 2)] [InlineData(16, 3, false, true, 0)] @@ -126,46 +43,9 @@ public void UpHead_MatchesAvx2(int c, int oc, bool resizeRelu, bool skipRelu, in var rng = new Random(c * oc); int h = 11, w = 15, oh = 22, ow = 30; // ow % 4 != 0: partial pixel group float[] x = rng.Array(h * w * c), skip = rng.Array(oh * ow * c), wt = rng.Array(oc * c), b = rng.Array(oc); - UpHeadPlan kp = Nhwc.PackUpHead(wt, oc, c, b); + UpHeadPlan kp = GcdrKernels.PackUpHead(wt, oc, c, b); var p = new UpHeadP { Scales = [1, 1, 2, 2], ResizeRelu = resizeRelu, SkipRelu = skipRelu, Act = (ActivationKind)act }; - Both(dst => Nhwc.UpHead(x, h, w, c, skip, dst, oh, ow, 2, 2, p, kp), oh * ow * oc); - } - - [SkippableTheory] - [InlineData(3, 16, false, 1, 0, 1)] // gc stem: 16-wide tail kernel, reflect, row runs - [InlineData(6, 32, false, 1, 0, 1)] // dr stem: one block pair - [InlineData(16, 48, false, 0, 0.25f, 0)] // odd block count, pad value - [InlineData(32, 64, true, 1, 0, 1)] // Winograd bands - [InlineData(24, 20, true, 0, 0, 3)] // Winograd, ic % 16 != 0, lane tail, gelu - public void ConvPool_MatchesAvx2(int ic, int oc, bool wino, int mode, float padValue, int act) - { - var rng = new Random(ic * 13 + oc); - int h = 22, w = 30; // pooled 11 x 15: odd tile counts - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * 9), b = rng.Array(oc); - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], - PadMode = (PadMode)mode, PadValue = padValue, Act = (ActivationKind)act, - Pool = new PoolP { Kernel = [2, 2], Strides = [2, 2], Pads = [0, 0, 0, 0], FuseRelu = act == 0 }, - }; - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - if (wino) - { - Skip.IfNot(Nhwc.WinogradEligible(p, [oc, ic, 3, 3], ic), "Winograd disabled"); - kp.WinoU = Nhwc.PackWinograd(wt, oc, ic); - } - float[] fused = Both(dst => Nhwc.ConvPool(x, h, w, ic, dst, h / 2, w / 2, oc, p, kp), h / 2 * (w / 2) * oc); - - // ... and equal to the separate conv -> maxpool on the AVX-512 tier - var conv = new ConvP - { - Pads = p.Pads, Strides = p.Strides, Dilations = p.Dilations, Kernel = p.Kernel, - PadMode = p.PadMode, PadValue = p.PadValue, Act = p.Act, - }; - float[] full = new float[h * w * oc], pooled = new float[h / 2 * (w / 2) * oc]; - Nhwc.Conv(x, h, w, ic, full, h, w, oc, conv, kp, default); - Nhwc.MaxPool(full, h, w, oc, pooled, h / 2, w / 2, p.Pool); - Assert.True(pooled.AsSpan().SequenceEqual(fused), "fused conv+pool differs from conv -> maxpool"); + Both(dst => GcdrKernels.UpHead(x, h, w, c, skip, dst, oh, ow, 2, 2, p, kp), oh * ow * oc); } [SkippableTheory] @@ -184,43 +64,16 @@ public void UpHead_SkipConv_MatchesAvx2(int skipC, int c, int skipAct, int act) PadMode = PadMode.Reflect, Act = (ActivationKind)skipAct, KernelPlan = Nhwc.Pack(cw, [c, skipC, 3, 3], cb, 1, skipC), }; - UpHeadPlan kp = Nhwc.PackUpHead(wt, oc, c, b); + UpHeadPlan kp = GcdrKernels.PackUpHead(wt, oc, c, b); var p = new UpHeadP { Scales = [1, 1, 2, 2], ResizeRelu = true, SkipRelu = true, Act = (ActivationKind)act, SkipConv = sc }; - float[] fused = Both(dst => Nhwc.UpHead(x, h, w, c, sx, dst, oh, ow, 2, 2, p, kp, oh, ow, skipC), oh * ow * oc); + float[] fused = Both(dst => GcdrKernels.UpHead(x, h, w, c, sx, dst, oh, ow, 2, 2, p, kp, oh, ow, skipC), oh * ow * oc); // ... and equal to UpHead over the stored skip tensor on the AVX-512 tier float[] skip = new float[oh * ow * c], want = new float[oh * ow * oc]; Nhwc.Conv(sx, oh, ow, skipC, skip, oh, ow, c, sc, (NhwcConvPlan)sc.KernelPlan!, default); var plain = new UpHeadP { Scales = p.Scales, ResizeRelu = true, SkipRelu = true, Act = p.Act }; - Nhwc.UpHead(x, h, w, c, skip, want, oh, ow, 2, 2, plain, kp); + GcdrKernels.UpHead(x, h, w, c, skip, want, oh, ow, 2, 2, plain, kp); Assert.True(want.AsSpan().SequenceEqual(fused), "skip recompute differs from the stored skip"); } - [SkippableTheory] - [InlineData("gc", 3)] - [InlineData("dr", 6)] - public void MemFusions_Avx512_MatchUnfusedAvx2(string net, int cin) - { - // whole net: memory-fused plan on the AVX-512 tier vs unfused plan on the AVX2 tier, bit for bit - Skip.IfNot(s_tier && Avx2.IsSupported && Fma.IsSupported, "AVX-512 tier not active"); - var m = Sdcb.SimdGcdrNet.Onnx.OnnxModel.Load(Path.Combine(AppContext.BaseDirectory, "testdata", "models", - net == "gc" ? "gcdrnet_gcnet_sim.onnx" : "gcdrnet_drnet_sim.onnx")); - int[] shape = [1, cin, 96, 128]; - var rnd = new Random(7); - float[] input = Enumerable.Range(0, cin * 96 * 128).Select(_ => (float)rnd.NextDouble()).ToArray(); - float[] Run(bool fuse, bool avx512) - { - ExecPlan p = GraphCompiler.Compile(m, shape, new CompileOptions(Nhwc: true, PhysicalIo: true, MemFuse: fuse)); - try - { - Nhwc.UseAvx512 = avx512; - using var s = new InferenceSession(p); - input.CopyTo(s.Input); s.Run(); - return s.Output.ToArray(); - } - finally { Nhwc.UseAvx512 = s_tier; } - } - float[] fused512 = Run(true, true), unfused2 = Run(false, false); - Assert.True(fused512.AsSpan().SequenceEqual(unfused2), $"{net}: fused AVX-512 differs from unfused AVX2"); - } } diff --git a/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs index 7cdcb99..f92bf5a 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs @@ -1,6 +1,8 @@ -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Kernels; using Sdcb.SimdGcdrNet.Kernels; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Onnx; using Xunit; namespace Sdcb.SimdGcdrNet.Tests; @@ -16,11 +18,11 @@ public void Fusion_ChannelShift_And_ResidualAdd() { // fused plan must contain the ChannelShift nodes and fused conv residual adds — // if a model export change breaks the FuseShift pattern, this fails loudly - ExecPlan dr = GraphCompiler.Compile(OnnxModel.Load(DrPath), [1, 6, 96, 128]); - Assert.Equal(8, dr.Nodes.Count(n => n.Op == ExecOp.ChannelShift)); + ExecPlan dr = GraphCompiler.Compile(OnnxModel.Load(DrPath), [1, 6, 96, 128], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }); + Assert.Equal(8, dr.Nodes.Count(n => n.Ext == ChannelShiftOp.Instance)); - ExecPlan gc = GraphCompiler.Compile(OnnxModel.Load(GcPath), [1, 3, 96, 128]); - Assert.True(gc.Nodes.Count(n => n.Op == ExecOp.ChannelShift) >= 4, + ExecPlan gc = GraphCompiler.Compile(OnnxModel.Load(GcPath), [1, 3, 96, 128], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }); + Assert.True(gc.Nodes.Count(n => n.Ext == ChannelShiftOp.Instance) >= 4, "gc plan lost its ChannelShift fusion"); } @@ -30,7 +32,7 @@ public void Fusion_Conv_Gelu_ThroughAlias() // the gelu decomposition commutes before its producer transpose, then // fuses into the 1x1 conv via the reshape Buf-alias — the plan must // contain no Gelu node and the conv must carry the epilogue - ExecPlan gc = GraphCompiler.Compile(OnnxModel.Load(GcPath), [1, 3, 96, 128]); + ExecPlan gc = GraphCompiler.Compile(OnnxModel.Load(GcPath), [1, 3, 96, 128], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }); Assert.DoesNotContain(gc.Nodes, n => n.Op == ExecOp.Gelu); Assert.True(gc.Nodes.Count(n => n.Op == ExecOp.Conv && ((ConvP)n.P!).Act == ActivationKind.Gelu) >= 2, @@ -47,7 +49,7 @@ public void Nhwc_TransposesDissolve(bool physicalIo, int expected) foreach (var (path, c) in new[] { (GcPath, 3), (DrPath, 6) }) { ExecPlan p = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, 96, 128], - new CompileOptions(Nhwc: true, PhysicalIo: physicalIo)); + new CompileOptions(Nhwc: true, PhysicalIo: physicalIo, Extensions: GcdrExtensions.Instance)); int n = p.Nodes.Count(m => m.Op == ExecOp.Transpose); Assert.True(n <= 8, $"{Path.GetFileName(path)}: {n} transposes"); Assert.Equal(expected, n); @@ -65,7 +67,7 @@ public void Nhwc_MatchesNchw() OnnxModel m = OnnxModel.Load(path); float[] Run(bool nhwc) { - using var s = new InferenceSession(GraphCompiler.Compile(m, shape, new CompileOptions(nhwc))); + using var s = new InferenceSession(GraphCompiler.Compile(m, shape, new CompileOptions(nhwc, Extensions: GcdrExtensions.Instance))); input.CopyTo(s.Input); s.Run(); return s.Output.ToArray(); } @@ -87,7 +89,7 @@ public void MemFusions_BitIdentical_AndShrinkArena() float[] input = Enumerable.Range(0, c * 96 * 128).Select(_ => (float)rnd.NextDouble()).ToArray(); (float[] y, ExecPlan p) Run(bool fuse) { - ExecPlan p = GraphCompiler.Compile(m, shape, new CompileOptions(Nhwc: true, PhysicalIo: true, MemFuse: fuse)); + ExecPlan p = GraphCompiler.Compile(m, shape, new CompileOptions(Nhwc: true, PhysicalIo: true, MemFuse: fuse, Extensions: GcdrExtensions.Instance)); using var s = new InferenceSession(p); input.CopyTo(s.Input); s.Run(); return (s.Output.ToArray(), p); @@ -110,7 +112,7 @@ public void MemFusions_GpuPlans_StemOnly() foreach (var (path, c) in new[] { (GcPath, 3), (DrPath, 6) }) { ExecPlan p = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, 96, 128], - new CompileOptions(Nhwc: true, PhysicalIo: true, Gpu: true)); + new CompileOptions(Nhwc: true, PhysicalIo: true, Target: PlanTarget.Vulkan, Extensions: GcdrExtensions.Instance)); Assert.Single(p.Nodes, n => n.P is ConvP { Pool: not null }); Assert.Equal(2, p.Nodes.Count(n => n.Op == ExecOp.MaxPool)); Assert.Contains(p.Nodes, n => n.P is UpHeadP { SkipConv: not null, SkipW: >= 0 }); @@ -124,26 +126,17 @@ public void NoFuse_MatchesFused() int[] shape = Golden.ShapeOf("small", "input"); OnnxModel m = OnnxModel.Load(GcPath); - float[] Fused() + // per-compile option, not GCDRNET_NOFUSE: the env var would also unfuse graphs that tests + // running in parallel compile meanwhile (byte-exact Vulkan comparisons then fail) + float[] Run(bool fuse) { - Environment.SetEnvironmentVariable("GCDRNET_NOFUSE", null); - using var s = new InferenceSession(GraphCompiler.Compile(m, shape)); + using var s = new InferenceSession(GraphCompiler.Compile(m, shape, + CompileOptions.Default with { Extensions = GcdrExtensions.Instance, Fuse = fuse })); input.CopyTo(s.Input); s.Run(); return s.Output.ToArray(); } - float[] Unfused() - { - Environment.SetEnvironmentVariable("GCDRNET_NOFUSE", "1"); - try - { - using var s = new InferenceSession(GraphCompiler.Compile(m, shape)); - input.CopyTo(s.Input); s.Run(); - return s.Output.ToArray(); - } - finally { Environment.SetEnvironmentVariable("GCDRNET_NOFUSE", null); } - } - var (max, rmse, _) = Golden.Diff(Fused(), Unfused()); + var (max, rmse, _) = Golden.Diff(Run(fuse: true), Run(fuse: false)); Assert.True(max < 1e-5f, $"fused vs unfused max diff {max} rmse {rmse}"); } @@ -175,25 +168,6 @@ public void DivConcat_MatchesOracle_Float(string tag) Assert.True(max < 1e-3f, $"dr_in[{tag}] max diff {max} rmse {rmse}"); } - [Fact] - public void Transpose_NonIdentityLeadingPerm_Correct() - { - // perm [1,0,3,2] must NOT take the last-two-dim fast path (would be wrong); - // it should fall through to the generic permute and still be correct - var rng = new Random(5); - int d0 = 2, d1 = 3, d2 = 4, d3 = 5; - float[] src = rng.Array(d0 * d1 * d2 * d3); - float[] got = new float[src.Length]; - Ops.Transpose(src, [d0, d1, d2, d3], got, [d1, d0, d3, d2], [1, 0, 3, 2]); - for (int i0 = 0; i0 < d0; i0++) - for (int i1 = 0; i1 < d1; i1++) - for (int i2 = 0; i2 < d2; i2++) - for (int i3 = 0; i3 < d3; i3++) - Assert.Equal( - src[((i0 * d1 + i1) * d2 + i2) * d3 + i3], - got[((i1 * d0 + i0) * d3 + i3) * d2 + i2]); - } - [Fact] public void ChannelShift_OffsetBeyondWidth_Zeros() { @@ -201,93 +175,10 @@ public void ChannelShift_OffsetBeyondWidth_Zeros() float[] src = Enumerable.Repeat(1f, 2 * 4 * 5).ToArray(); // c=2,h=4,w=5 float[] dst = Enumerable.Repeat(-1f, src.Length).ToArray(); var p = new ShiftP { Axis = 3, Offsets = [5], GroupEnds = [2] }; - Ops.ChannelShift(src, [1, 2, 4, 5], dst, [1, 2, 4, 5], p); + GcdrKernels.ChannelShiftNchw(src, [1, 2, 4, 5], dst, [1, 2, 4, 5], p); Assert.All(dst, v => Assert.Equal(0f, v)); } - [Fact] - public void Conv_GroupedNonDepthwise_MatchesReference() - { - // group=2 with ic=4 (not depthwise) routes to the scalar path — check vs manual - var rng = new Random(11); - int ic = 4, oc = 4, ih = 5, iw = 6, g = 2; - float[] x = rng.Array(ic * ih * iw); - float[] w = rng.Array(oc * (ic / g) * 9); - float[] dst = new float[oc * ih * iw]; - var p = new ConvP { Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], Group = g }; - Ops.Conv(x, [1, ic, ih, iw], w, [oc, ic / g, 3, 3], ReadOnlySpan.Empty, - dst, [1, oc, ih, iw], p); - - for (int o = 0; o < oc; o++) - for (int y = 0; y < ih; y++) - for (int xx = 0; xx < iw; xx++) - { - float acc = 0; - int gi = o / (oc / g) * (ic / g); - for (int c = 0; c < ic / g; c++) - for (int ky = 0; ky < 3; ky++) - for (int kx = 0; kx < 3; kx++) - { - int sy = y + ky - 1, sx = xx + kx - 1; - if ((uint)sy < (uint)ih && (uint)sx < (uint)iw) - acc += x[(gi + c) * ih * iw + sy * iw + sx] * - w[((o * (ic / g) + c) * 3 + ky) * 3 + kx]; - } - Assert.Equal(acc, dst[o * ih * iw + y * iw + xx], 5); - } - } - - [Fact] - public void Conv_ResidualAdd_MatchesManual() - { - // unit-level check of the Conv+Add epilogue (the fused path has no model coverage) - var rng = new Random(7); - int ic = 4, oc = 8, ih = 9, iw = 11; - float[] x = rng.Array(ic * ih * iw); - float[] w = rng.Array(oc * ic * 9); - float[] b = rng.Array(oc); - float[] res = rng.Array(oc * ih * iw); - float[] want = new float[oc * ih * iw], got = new float[oc * ih * iw]; - - var p = new ConvP { Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3] }; - Ops.Conv(x, [1, ic, ih, iw], w, [oc, ic, 3, 3], b, want, [1, oc, ih, iw], p); - for (int i = 0; i < want.Length; i++) want[i] += res[i]; - - Ops.Conv(x, [1, ic, ih, iw], w, [oc, ic, 3, 3], b, got, [1, oc, ih, iw], p, res); - var (max, _, _) = Golden.Diff(want, got); - Assert.True(max < 1e-4f, $"conv+add max diff {max}"); - } - - [SkippableTheory] - [InlineData(16, 16, 9, 11, 0, 0f, false, 0)] // constant pad - [InlineData(24, 20, 13, 6, 1, 0f, true, 1)] // reflect, residual, relu, oc lane tail, odd tiles - [InlineData(32, 48, 4, 4, 0, 0.5f, false, 3)] // nonzero pad value, gelu - [InlineData(64, 16, 17, 23, 1, 0f, true, 0)] - public void Conv_Winograd_MatchesDirect(int ic, int oc, int h, int w, int mode, float padValue, - bool residual, int act) - { - var rng = new Random(ic * 31 + oc); - float[] x = rng.Array(h * w * ic); // NHWC - float[] wt = rng.Array(oc * ic * 9); - float[] b = rng.Array(oc); - float[] res = residual ? rng.Array(h * w * oc) : []; - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], - PadMode = (PadMode)mode, PadValue = padValue, Act = (ActivationKind)act, - }; - Skip.IfNot(Nhwc.WinogradEligible(p, [oc, ic, 3, 3], ic), "Winograd disabled or ISA tier without it"); - NhwcConvPlan direct = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - NhwcConvPlan wino = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - wino.WinoU = Nhwc.PackWinograd(wt, oc, ic); - float[] want = new float[h * w * oc], got = new float[h * w * oc]; - Nhwc.Conv(x, h, w, ic, want, h, w, oc, p, direct, res); - Nhwc.Conv(x, h, w, ic, got, h, w, oc, p, wino, res); - var (max, _, _) = Golden.Diff(want, got); - float scale = want.Max(MathF.Abs); - Assert.True(max < 1e-4f * scale, $"winograd max diff {max} (|out| max {scale})"); - } - [Fact] public void Pipeline_SizeSwitching_MatchesFresh_Cpu() { diff --git a/tests/Sdcb.SimdGcdrNet.Tests/GoldenTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/GoldenTests.cs index 38b7a1e..e543ce5 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/GoldenTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/GoldenTests.cs @@ -1,5 +1,6 @@ -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Onnx; using Xunit; namespace Sdcb.SimdGcdrNet.Tests; diff --git a/tests/Sdcb.SimdGcdrNet.Tests/MetalTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/MetalTests.cs index c2ab8f4..37fd3d4 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/MetalTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/MetalTests.cs @@ -1,6 +1,8 @@ +using Sdcb.SimdOnnx.Backends.Metal; using Sdcb.SimdGcdrNet.Backends.Metal; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Onnx; using Xunit; namespace Sdcb.SimdGcdrNet.Tests; @@ -22,7 +24,7 @@ public void GcNet_MatchesGolden_Metal(string tag) float[] expect = Golden.F32(tag, "shadow"); OnnxModel m = OnnxModel.Load(GcPath); - using var sess = MetalBackend.Default.NewSession(m, shape); + using var sess = MetalBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); input.CopyTo(sess.Input); sess.Run(); var (max, rmse, _) = Golden.Diff(sess.Output, expect); @@ -42,7 +44,7 @@ public void DrNet_MatchesGolden_Metal(string tag) float[] expect = Golden.F32(tag, "output"); OnnxModel m = OnnxModel.Load(DrPath); - using var sess = MetalBackend.Default.NewSession(m, shape); + using var sess = MetalBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); input.CopyTo(sess.Input); sess.Run(); var (max, rmse, _) = Golden.Diff(sess.Output, expect); diff --git a/tests/Sdcb.SimdGcdrNet.Tests/MtlKernelTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/MtlKernelTests.cs deleted file mode 100644 index 45659eb..0000000 --- a/tests/Sdcb.SimdGcdrNet.Tests/MtlKernelTests.cs +++ /dev/null @@ -1,221 +0,0 @@ -using Sdcb.SimdGcdrNet.Backends.Metal; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Kernels; -using Xunit; - -namespace Sdcb.SimdGcdrNet.Tests; - -/// Standalone Metal kernel vs CPU kernel checks. -public unsafe class MtlKernelTests -{ - private static float[] RunGpu(MtlDevice dev, string shader, float[][] ins, int outLen, byte[] pc, uint groups, int[]? pcs = null) - { - using var pool = AutoReleasePool.Create(); - var bufs = new List(); - foreach (var f in ins) - { - var h = new Half[f.Length]; - for (int i = 0; i < f.Length; i++) h[i] = (Half)f[i]; - fixed (Half* hp = h) bufs.Add(dev.NewBuffer(hp, (nuint)(h.Length * 2))); - } - // extra inputs that must stay fp32 (bias, layernorm params) are passed via pcs tag - var bufsF = new List(); - var ob = dev.NewBuffer((nuint)(outLen * 2 + 64)); - var enc = dev.BeginCommands(); - enc.SetPipeline(dev.Pipe(shader)); - int bi = 0; - foreach (var b in bufs) { enc.SetBuffer(b, 0, bi++); } - enc.SetBuffer(ob, 0, bi++); - if (pcs != null) { /* placeholder */ } - fixed (byte* p = pc) enc.SetPc(p, (nuint)pc.Length, bi); - enc.Dispatch(groups, 1, 1, 256, 1, 1); - enc.End(); - enc.CommitAndWait(); - var dst = new float[outLen]; - var sp = new ReadOnlySpan(ob.Contents.ToPointer(), outLen); - for (int i = 0; i < outLen; i++) dst[i] = (float)sp[i]; - foreach (var b in bufs) b.Dispose(); - ob.Dispose(); - return dst; - } - - // generic: ins as fp16 tensors plus an fp32 tensor slot for bias - private static float[] RunGpuMixed(MtlDevice dev, string shader, - (float[] data, bool f32)[] ins, int outLen, byte[] pc, uint gx, uint gy = 1, uint gz = 1) - { - using var pool = AutoReleasePool.Create(); - var bufs = new List(); - foreach (var (f, f32) in ins) - { - if (f32) - fixed (float* fp = f) bufs.Add(dev.NewBuffer(fp, (nuint)(f.Length * 4))); - else - { - var h = new Half[f.Length]; - for (int i = 0; i < f.Length; i++) h[i] = (Half)f[i]; - fixed (Half* hp = h) bufs.Add(dev.NewBuffer(hp, (nuint)(h.Length * 2))); - } - } - var ob = dev.NewBuffer((nuint)(outLen * 2 + 64)); - var enc = dev.BeginCommands(); - enc.SetPipeline(dev.Pipe(shader)); - int bi = 0; - foreach (var b in bufs) enc.SetBuffer(b, 0, bi++); - enc.SetBuffer(ob, 0, bi++); - fixed (byte* p = pc) enc.SetPc(p, (nuint)pc.Length, bi); - enc.Dispatch(gx, gy, gz, 256, 1, 1); - enc.End(); - enc.CommitAndWait(); - var dst = new float[outLen]; - var sp = new ReadOnlySpan(ob.Contents.ToPointer(), outLen); - for (int i = 0; i < outLen; i++) dst[i] = (float)sp[i]; - foreach (var b in bufs) b.Dispose(); - ob.Dispose(); - return dst; - } - - private static float[] Rand(Random r, int n) - { - var a = new float[n]; - for (int i = 0; i < n; i++) a[i] = r.NextSingle() * 2 - 1; - return a; - } - - [SkippableTheory] - [InlineData("convpool", 3, 16, true, 0f, 1)] - [InlineData("convpool_k3", 3, 16, true, 0f, 1)] - [InlineData("convpool_k3", 6, 32, false, 0f, 0)] - [InlineData("convpool_k3", 7, 8, false, 0.25f, 1)] - [InlineData("convpool_k3", 1, 24, true, 0f, 0)] - public void ConvPool_MatchesCpu(string shader, int ic, int oc, bool reflect, float padValue, int act) - { - Skip.IfNot(MetalOk(), "no Metal device"); - var dev = MetalBackend.Default.Dev; - var rng = new Random(ic * 13 + oc); - int h = 22, w = 30, ph = h / 2, pw = w / 2; - float[] x = Rand(rng, h * w * ic), wt = Rand(rng, oc * ic * 9), b = Rand(rng, oc); - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], - PadMode = reflect ? PadMode.Reflect : PadMode.Constant, PadValue = padValue, - Act = act == 1 ? ActivationKind.Relu : ActivationKind.None, - Pool = new PoolP { Kernel = [2, 2], Strides = [2, 2], Pads = [0, 0, 0, 0], FuseRelu = act == 0 }, - }; - float[] cpu = new float[ph * pw * oc]; - Nhwc.ConvPool(x, h, w, ic, cpu, ph, pw, oc, p, Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic)); - float[] wk = new float[9 * ic * oc]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < 9; t++) wk[(t * ic + c) * oc + o] = wt[(o * ic + c) * 9 + t]; - uint flags = 1u | (reflect ? 2u : 0u) | (act == 1 ? 4u : 0u) | (act == 0 ? 8u : 0u); - byte[] pc = new byte[48]; - Buffer.BlockCopy(new uint[] { (uint)ic, (uint)h, (uint)w, (uint)oc, (uint)ph, (uint)pw, 3u, 3u, 1u, 1u, flags }, 0, pc, 0, 44); - Buffer.BlockCopy(new float[] { reflect ? 0f : padValue }, 0, pc, 44, 4); - float[] got = RunGpuMixed(dev, shader, [(x, false), (wk, false), (b, true)], cpu.Length, pc, - (uint)(ph * pw * (oc / 8) + 255) / 256); - AssertClose(got, cpu, 2e-2f, shader); - } - - - [SkippableTheory] - [InlineData(6, 32, 1, false, "conv_mm_32_1", false, 1)] // dr stem (scalar k gather) - [InlineData(3, 16, 1, true, "conv_mm_32_1", false, 1)] // gc stem, reflect - [InlineData(32, 64, 1, true, "conv_mm_64_0", false, 1)] - [InlineData(64, 128, 2, false, "conv_mm_128_0", false, 1)] - [InlineData(16, 32, 1, false, "conv_mm_32_0", false, 1)] - [InlineData(32, 64, 1, true, "conv_mm_64_0", true, 2)] // residual, gelu - [InlineData(64, 128, 2, false, "conv_mm_128_0", true, 0)] - [InlineData(16, 160, 1, false, "conv_mm_32_0", false, 1)] - [InlineData(32, 256, 1, false, "conv_mm_128_0", true, 1)] - [InlineData(16, 16, 1, true, "conv_mm_32_0", false, 1)] - [InlineData(16, 48, 1, false, "conv_mm_32_0", true, 2)] - [InlineData(8, 64, 2, true, "conv_mm_64_0", true, 1)] - [InlineData(3, 32, 1, false, "conv_mm_32_1", true, 2)] - [InlineData(128, 128, 1, true, "conv_mm_128_0", true, 2)] - public void ConvMm_MatchesCpu(int ic, int oc, int stride, bool reflect, string shader, bool residual, int act) - { - Skip.IfNot(MetalOk(), "no Metal device"); - var dev = MetalBackend.Default.Dev; - var rng = new Random(ic * 7 + oc); - int ih = 29, iw = 37, oh = (ih + 2 - 3) / stride + 1, ow = (iw + 2 - 3) / stride + 1; - float[] x = Rand(rng, ih * iw * ic).Select(v => (float)(Half)v).ToArray(); - float[] w = Rand(rng, oc * ic * 9).Select(v => (float)(Half)(v * 0.25f)).ToArray(); - float[] b = Rand(rng, oc); - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [stride, stride], Dilations = [1, 1], Kernel = [3, 3], - PadMode = reflect ? PadMode.Reflect : PadMode.Constant, - Act = act == 1 ? ActivationKind.Relu : act == 2 ? ActivationKind.Gelu : ActivationKind.None, - }; - float[] res = residual ? Rand(rng, oh * ow * oc).Select(v => (float)(Half)v).ToArray() : []; - float[] cpu = new float[oh * ow * oc]; - NhwcConvPlan kp = Nhwc.Pack(w, [oc, ic, 3, 3], b, 1, ic); - Nhwc.Conv(x, ih, iw, ic, cpu, oh, ow, oc, p, kp, res); - - int kr = 9 * ic, kd = (kr + 7) & ~7; - float[] wt = new float[oc * kd]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < 9; t++) wt[o * kd + t * ic + c] = w[(o * ic + c) * 9 + t]; - uint flags = 1u | (residual ? 2u : 0u) | (reflect ? 4u : 0u) | ((uint)act << 4); - uint m = (uint)(oh * ow); - byte[] pc = new byte[56]; - Buffer.BlockCopy(new uint[] { m, (uint)oc, (uint)kd, flags, (uint)ow, (uint)iw, (uint)ih, (uint)ic, 3u, - (uint)stride, (uint)stride, 1u, 1u, (uint)kr }, 0, pc, 0, 56); - uint bn = shader.StartsWith("conv_mm_128") ? 128u : shader.StartsWith("conv_mm_64") ? 64u : 32u; - float[] got = RunGpuMixed(dev, shader, - [(x, false), (wt, false), (b, true), (residual ? res : new float[8], false)], - cpu.Length, pc, (m + 63) / 64, ((uint)oc + bn - 1) / bn); - float scale = Math.Max(1f, cpu.Max(MathF.Abs)); - AssertClose(got, cpu, 2e-3f * scale, shader); - } - - - [SkippableTheory] - [InlineData(160)] - [InlineData(256)] - [InlineData(64)] - [InlineData(96)] - public void LayerNorm_MatchesCpu(int len) - { - Skip.IfNot(MetalOk(), "no Metal device"); - var dev = MetalBackend.Default.Dev; - var rng = new Random(len); - int rows = 97; - float[] x = Rand(rng, rows * len).Select(v => (float)(Half)v).ToArray(); - float[] w = Rand(rng, len), b = Rand(rng, len); - float eps = 1e-5f; - float[] want = new float[rows * len]; - for (int r = 0; r < rows; r++) - { - double m = 0, q = 0; - for (int i = 0; i < len; i++) { m += x[r * len + i]; q += x[r * len + i] * x[r * len + i]; } - m /= len; q = Math.Sqrt(q / len - m * m + eps); - for (int i = 0; i < len; i++) want[r * len + i] = (float)((x[r * len + i] - m) / q * w[i] + b[i]); - } - byte[] pc = new byte[12]; - Buffer.BlockCopy(new uint[] { (uint)len, (uint)rows }, 0, pc, 0, 8); - Buffer.BlockCopy(new float[] { eps }, 0, pc, 8, 4); - float[] got = RunGpuMixed(dev, "layernorm", [(x, false), (w, true), (b, true)], want.Length, pc, (uint)((rows + 7) / 8)); - AssertClose(got, want, 1e-2f, "layernorm"); - } - - private static void AssertClose(float[] got, float[] want, float tol, string tag) - { - double mx = 0, rmse = 0; - for (int i = 0; i < want.Length; i++) - { - double d = Math.Abs(got[i] - want[i]); - if (d > mx) mx = d; - rmse += d * d; - } - rmse = Math.Sqrt(rmse / want.Length); - Assert.True(mx < tol, $"{tag}: max {mx} rmse {rmse} (tol {tol})"); - } - - private static bool MetalOk() - { - try { return GcdrEnhancer.MetalUsable; } - catch { return false; } - } -} diff --git a/tests/Sdcb.SimdGcdrNet.Tests/NeonKernelTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/NeonKernelTests.cs index de23894..dc541b7 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/NeonKernelTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/NeonKernelTests.cs @@ -1,5 +1,7 @@ using System.Runtime.Intrinsics.Arm; -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Kernels; using Sdcb.SimdGcdrNet.Kernels; namespace Sdcb.SimdGcdrNet.Tests; @@ -46,75 +48,6 @@ private static float[] Both(Action run, int n, float tol = 2e-5f) PadMode = (PadMode)mode, PadValue = padValue, Act = (ActivationKind)act, }; - [SkippableTheory] - [InlineData(3, 16, 3, 1, 37, 23, 1, 0f, false, 0)] // gc stem: run of 9 (2 by-element groups + 1 broadcast) - [InlineData(6, 32, 3, 1, 29, 41, 1, 0f, false, 1)] // dr stem: run of 18 - [InlineData(16, 48, 3, 1, 11, 13, 0, 0.25f, true, 1)] // odd block count, pad value, residual - [InlineData(20, 20, 3, 2, 17, 19, 0, 0f, true, 2)] // stride 2, oc lane tail, sigmoid - [InlineData(32, 64, 3, 2, 12, 31, 1, 0f, false, 3)] // stride 2, reflect, gelu - [InlineData(24, 3, 3, 1, 9, 10, 0, 0f, false, 0)] // oc=3 (single partial lane block) - [InlineData(40, 36, 1, 1, 7, 33, 0, 0f, true, 1)] // 1x1 conv - [InlineData(5, 32, 5, 1, 14, 9, 1, 0f, false, 3)] // 5x5, reflect: border runs gathered, gelu - [InlineData(7, 16, 3, 1, 4, 3, 0, -0.5f, true, 2)] // tiny image: every run crosses a border - public void Conv_Direct_MatchesVector(int ic, int oc, int k, int stride, int h, int w, int mode, - float padValue, bool residual, int act) - { - var rng = new Random(ic * 131 + oc * 7 + k); - int pad = k / 2; - int oh = (h + 2 * pad - k) / stride + 1, ow = (w + 2 * pad - k) / stride + 1; - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * k * k), b = rng.Array(oc); - float[] res = residual ? rng.Array(oh * ow * oc) : []; - ConvP p = Conv3(k, stride, mode, padValue, act); - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, k, k], b, 1, ic); // no Winograd: direct kernels - Both(dst => Nhwc.Conv(x, h, w, ic, dst, oh, ow, oc, p, kp, res), oh * ow * oc); - } - - [SkippableTheory] - [InlineData(16, 16, 9, 11, 0, 0f, false, 0)] - [InlineData(24, 20, 13, 6, 1, 0f, true, 1)] - [InlineData(32, 48, 22, 18, 0, 0.5f, false, 3)] - [InlineData(64, 16, 17, 23, 1, 0f, true, 2)] - [InlineData(128, 64, 30, 41, 1, 0f, false, 1)] // several tile chunks per image - public void Conv_Winograd_MatchesVector(int ic, int oc, int h, int w, int mode, float padValue, bool residual, int act) - { - var rng = new Random(ic * 17 + oc); - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * 9), b = rng.Array(oc); - float[] res = residual ? rng.Array(h * w * oc) : []; - ConvP p = Conv3(3, 1, mode, padValue, act); - Skip.IfNot(Nhwc.WinogradEligible(p, [oc, ic, 3, 3], ic), "Winograd disabled"); - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - kp.WinoU = Nhwc.PackWinograd(wt, oc, ic); - Both(dst => Nhwc.Conv(x, h, w, ic, dst, h, w, oc, p, kp, res), h * w * oc, 2e-4f); - } - - [SkippableTheory] - [InlineData(16, 0, 0f, false, 0)] - [InlineData(20, 1, 0f, true, 3)] // c % 16 != 0, residual, gelu - [InlineData(22, 0, 0.5f, false, 2)] // c % 4 != 0: scalar channel tail, sigmoid - public void Depthwise_MatchesVector(int c, int mode, float padValue, bool residual, int act) - { - var rng = new Random(c * 5 + act); - int h = 13, w = 17; - float[] x = rng.Array(h * w * c), wt = rng.Array(c * 9), b = rng.Array(c); - float[] res = residual ? rng.Array(h * w * c) : []; - ConvP p = Conv3(3, 1, mode, padValue, act); - p.Group = c; - NhwcConvPlan kp = Nhwc.Pack(wt, [c, 1, 3, 3], b, c, c); - Both(dst => Nhwc.Conv(x, h, w, c, dst, h, w, c, p, kp, res), h * w * c); - } - - [SkippableTheory] - [InlineData(57, 128, 128, false, 0)] - [InlineData(23, 160, 48, true, 3)] // odd block count, transB, gelu - [InlineData(5, 66, 20, false, 1)] // fewer rows than a tile, k % 4 != 0, lane tail - public void Gemm_MatchesVector(int m, int k, int n, bool transB, int act) - { - var rng = new Random(m + k + n); - float[] a = rng.Array(m * k), bm = rng.Array(k * n), bias = rng.Array(n); - NhwcConvPlan kp = Nhwc.PackGemm(bm, k, n, transB, bias); - Both(dst => Nhwc.Gemm(a, m, k, dst, n, kp, (ActivationKind)act), m * n); - } - [SkippableTheory] [InlineData(32, 3, true, true, 2)] [InlineData(16, 3, false, true, 0)] @@ -125,48 +58,9 @@ public void UpHead_MatchesVector(int c, int oc, bool resizeRelu, bool skipRelu, var rng = new Random(c * oc); int h = 11, w = 15, oh = 22, ow = 30; float[] x = rng.Array(h * w * c), skip = rng.Array(oh * ow * c), wt = rng.Array(oc * c), b = rng.Array(oc); - UpHeadPlan kp = Nhwc.PackUpHead(wt, oc, c, b); + UpHeadPlan kp = GcdrKernels.PackUpHead(wt, oc, c, b); var p = new UpHeadP { Scales = [1, 1, 2, 2], ResizeRelu = resizeRelu, SkipRelu = skipRelu, Act = (ActivationKind)act }; - Both(dst => Nhwc.UpHead(x, h, w, c, skip, dst, oh, ow, 2, 2, p, kp), oh * ow * oc); - } - - [SkippableTheory] - [InlineData(128, 64)] - [InlineData(37, 18)] // c % 4 != 0 - public void LayerNorm_MatchesVector(int c, int rows) - { - var rng = new Random(c + rows); - float[] x = rng.Array(rows * c), wt = rng.Array(c), b = rng.Array(c); - Both(dst => Ops.LayerNorm(x, [rows, c], wt, b, dst, 1e-5f), rows * c); - } - - [SkippableTheory] - [InlineData(3, 16, false, 1, 0f, 1)] // gc stem - [InlineData(6, 32, false, 1, 0f, 1)] // dr stem - [InlineData(16, 48, false, 0, 0.25f, 0)] // odd block count, pad value - [InlineData(32, 64, true, 1, 0f, 1)] // Winograd bands - [InlineData(24, 20, true, 0, 0f, 3)] // Winograd, lane tail, gelu - public void ConvPool_MatchesVector_AndUnfused(int ic, int oc, bool wino, int mode, float padValue, int act) - { - var rng = new Random(ic * 13 + oc); - int h = 22, w = 30; - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * 9), b = rng.Array(oc); - ConvP p = Conv3(3, 1, mode, padValue, act); - p.Pool = new PoolP { Kernel = [2, 2], Strides = [2, 2], Pads = [0, 0, 0, 0], FuseRelu = act == 0 }; - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - if (wino) - { - Skip.IfNot(Nhwc.WinogradEligible(p, [oc, ic, 3, 3], ic), "Winograd disabled"); - kp.WinoU = Nhwc.PackWinograd(wt, oc, ic); - } - float[] fused = Both(dst => Nhwc.ConvPool(x, h, w, ic, dst, h / 2, w / 2, oc, p, kp), h / 2 * (w / 2) * oc, wino ? 2e-4f : 2e-5f); - - // ... and bit-identical to the separate conv -> maxpool on the NEON tier - ConvP conv = Conv3(3, 1, mode, padValue, act); - float[] full = new float[h * w * oc], pooled = new float[h / 2 * (w / 2) * oc]; - Nhwc.Conv(x, h, w, ic, full, h, w, oc, conv, kp, default); - Nhwc.MaxPool(full, h, w, oc, pooled, h / 2, w / 2, p.Pool); - Assert.True(pooled.AsSpan().SequenceEqual(fused), "fused conv+pool differs from conv -> maxpool"); + Both(dst => GcdrKernels.UpHead(x, h, w, c, skip, dst, oh, ow, 2, 2, p, kp), oh * ow * oc); } [SkippableTheory] @@ -181,39 +75,18 @@ public void UpHead_SkipConv_MatchesVector_AndStoredSkip(int skipC, int c, int sk float[] cw = rng.Array(c * skipC * 9), cb = rng.Array(c), wt = rng.Array(oc * c), b = rng.Array(oc); ConvP sc = Conv3(3, 1, (int)PadMode.Reflect, 0f, skipAct); sc.KernelPlan = Nhwc.Pack(cw, [c, skipC, 3, 3], cb, 1, skipC); - UpHeadPlan kp = Nhwc.PackUpHead(wt, oc, c, b); + UpHeadPlan kp = GcdrKernels.PackUpHead(wt, oc, c, b); var p = new UpHeadP { Scales = [1, 1, 2, 2], ResizeRelu = true, SkipRelu = true, Act = (ActivationKind)act, SkipConv = sc }; - float[] fused = Both(dst => Nhwc.UpHead(x, h, w, c, sx, dst, oh, ow, 2, 2, p, kp, oh, ow, skipC), oh * ow * oc); + float[] fused = Both(dst => GcdrKernels.UpHead(x, h, w, c, sx, dst, oh, ow, 2, 2, p, kp, oh, ow, skipC), oh * ow * oc); // ... and bit-identical to UpHead over the stored skip tensor on the NEON tier float[] skip = new float[oh * ow * c], want = new float[oh * ow * oc]; Nhwc.Conv(sx, oh, ow, skipC, skip, oh, ow, c, sc, (NhwcConvPlan)sc.KernelPlan!, default); var plain = new UpHeadP { Scales = p.Scales, ResizeRelu = true, SkipRelu = true, Act = p.Act }; - Nhwc.UpHead(x, h, w, c, skip, want, oh, ow, 2, 2, plain, kp); + GcdrKernels.UpHead(x, h, w, c, skip, want, oh, ow, 2, 2, plain, kp); Assert.True(want.AsSpan().SequenceEqual(fused), "skip recompute differs from the stored skip"); } - [SkippableTheory] - [InlineData("gc", 3)] - [InlineData("dr", 6)] - public void MemFusions_Neon_MatchUnfused(string net, int cin) - { - // whole net on the NEON tier: memory-fused plan vs unfused plan, bit for bit - SkipUnlessNeon(); - var m = Sdcb.SimdGcdrNet.Onnx.OnnxModel.Load(Path.Combine(AppContext.BaseDirectory, "testdata", "models", - net == "gc" ? "gcdrnet_gcnet_sim.onnx" : "gcdrnet_drnet_sim.onnx")); - int[] shape = [1, cin, 96, 128]; - var rnd = new Random(7); - float[] input = Enumerable.Range(0, cin * 96 * 128).Select(_ => (float)rnd.NextDouble()).ToArray(); - float[] Run(bool fuse) - { - ExecPlan p = GraphCompiler.Compile(m, shape, new CompileOptions(Nhwc: true, PhysicalIo: true, MemFuse: fuse)); - using var s = new InferenceSession(p); - input.CopyTo(s.Input); s.Run(); - return s.Output.ToArray(); - } - Assert.True(Run(true).AsSpan().SequenceEqual(Run(false)), $"{net}: fused NEON differs from unfused NEON"); - } } /// Tests that flip a process-wide kernel tier switch must not overlap any other test. diff --git a/tests/Sdcb.SimdGcdrNet.Tests/PixelFormatTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/PixelFormatTests.cs index 7368aa8..99d5e7d 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/PixelFormatTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/PixelFormatTests.cs @@ -1,4 +1,6 @@ +using Sdcb.SimdOnnx.Backends.Metal; using Sdcb.SimdGcdrNet.Backends.Metal; +using Sdcb.SimdOnnx.Backends.Vulkan; using Sdcb.SimdGcdrNet.Backends.Vulkan; using Xunit; diff --git a/tests/Sdcb.SimdGcdrNet.Tests/VkDebugTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/VkDebugTests.cs deleted file mode 100644 index 17737f5..0000000 --- a/tests/Sdcb.SimdGcdrNet.Tests/VkDebugTests.cs +++ /dev/null @@ -1,86 +0,0 @@ -using Sdcb.SimdGcdrNet.Backends.Vulkan; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; -using Xunit; - -namespace Sdcb.SimdGcdrNet.Tests; - -/// Node-by-node GPU vs CPU comparison. Enable with GCDRNET_VKDBG=1. -public class VkDebugTests -{ - private static string GcPath => Path.Combine(Golden.Dir, "models", "gcdrnet_gcnet_sim.onnx"); - private static string DrPath => Path.Combine(Golden.Dir, "models", "gcdrnet_drnet_sim.onnx"); - - [Theory] - [InlineData("gc", "small", "input")] - [InlineData("dr", "small", "dr_in")] - public void Nodes_Match(string which, string tag, string inputName) - { - if (Environment.GetEnvironmentVariable("GCDRNET_VKDBG") != "1") return; - try { _ = VkBackend.Default.Dev.DeviceName; } - catch { return; } // no Vulkan device — env-gated debug test, skip quietly - - OnnxModel m = OnnxModel.Load(which == "gc" ? GcPath : DrPath); - float[] input = Golden.F32(tag, inputName); - int[] shape = Golden.ShapeOf(tag, inputName); - - ExecPlan plan = GraphCompiler.Compile(m, shape, VkBackend.PlanOptions); - using var cpu = new InferenceSession(plan); - using var gpu = VkBackend.Default.NewSession(m, shape); - - input.CopyTo(cpu.Input); - input.CopyTo(gpu.Input); - gpu.DebugUploadInput(); - - var log = new StreamWriter(Path.Combine(Path.GetTempPath(), $"vkdbg_{which}_{tag}.txt")); - int bad = 0; - // fp16 storage: per-node error measured up to ~4e-2 on deep convs (relative ~1e-3 of activations) - float thr = VkBackend.Default.F16 ? 6e-2f : 1e-3f; - for (int i = 0; i < plan.Nodes.Length; i++) - { - ExecNode n = plan.Nodes[i]; - cpu.ExecNode(i); - var (ds, dc) = VkBackend.Default.GraphFor(m, shape).NodeRanges[i]; - gpu.DebugRunRange(ds, dc); - int ov = n.Out[0]; - CValue v = plan.Values[ov]; - float[] got = new float[v.ElementCount]; - gpu.DebugDownload(ov, got); - float[] want = cpu.ValueSpan(ov).ToArray(); - float maxd = 0; int at = -1; - for (int j = 0; j < want.Length; j++) - { - float d = Math.Abs(got[j] - want[j]); - if (d > maxd) { maxd = d; at = j; } - } - string status = maxd > thr ? " <-- BAD" : ""; - if (maxd > thr) - { - bad++; - if (bad <= 2) - { - int c = (int)v.ElementCount; - int[] sh = v.Shape; - int cw = sh[^1], chh = sh.Length >= 2 ? sh[^2] : 1; - for (int j = 0; j < c; j++) - if (Math.Abs(got[j] - want[j]) > thr) - { - int y = chh > 1 && cw > 0 ? (j / cw) % chh : j / cw; - int x = cw > 0 ? j % cw : 0; - int ch = cw > 0 && chh > 0 ? j / (cw * chh) : 0; - log.WriteLine($" [{j}] (c{ch},y{y},x{x}) gpu={got[j],12:g6} cpu={want[j],12:g6}"); - } - int badCnt = 0; - for (int j = 0; j < c; j++) - if (Math.Abs(got[j] - want[j]) > thr) badCnt++; - log.WriteLine($" badElems={badCnt}/{c}"); - for (int j = Math.Max(0, at - 2); j < Math.Min(c, at + 3); j++) - log.WriteLine($" [{j}] gpu={got[j],12:g6} cpu={want[j],12:g6}"); - } - } - log.WriteLine($"node[{i,3}] {n.Op,-14} {n.Name,-24} out={string.Join('x', v.Shape)} maxd={maxd:e3} @{at}{status}"); - } - log.Close(); - Assert.True(bad == 0, $"{bad} nodes diverged — see %TEMP%/vkdbg_{which}_{tag}.txt"); - } -} diff --git a/tests/Sdcb.SimdGcdrNet.Tests/VkKernelTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/VkKernelTests.cs index b6d6c33..5e4726e 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/VkKernelTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/VkKernelTests.cs @@ -1,5 +1,8 @@ +using Sdcb.SimdOnnx.Backends.Vulkan; using Sdcb.SimdGcdrNet.Backends.Vulkan; -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Kernels; using Sdcb.SimdGcdrNet.Kernels; using Xunit; @@ -8,56 +11,6 @@ namespace Sdcb.SimdGcdrNet.Tests; /// Standalone GPU kernel vs CPU kernel checks (bilinear, fused upsample tail). public unsafe class VkKernelTests { - [SkippableTheory] - [InlineData(0u, "bilinear")] // plain - [InlineData(7u, "bilinear")] // relu(resize) + relu(addend) - [InlineData(2u, "bilinear")] // resize + addend - [InlineData(0u, "bilinear4")] // 4-channel variant (integrated GPUs) - [InlineData(7u, "bilinear4")] - public void Bilinear_MatchesCpu(uint flags, string shader) - { - Skip.IfNot(VkOk(), "no Vulkan device"); - var dev = VkBackend.Default.Dev; - var rng = new Random(42); - int c = 4, ih = 3, iw = 4, oh = 6, ow = 8; - float[] src = Rand(rng, c * ih * iw); - float[] add = Rand(rng, c * oh * ow); - float[] dstCpu = new float[c * oh * ow]; - bool hasAdd = (flags & 2u) != 0; - // shaders address NHWC storage: src is [ih][iw][c] - Nhwc.ResizeBilinear(src, ih, iw, c, dstCpu, oh, ow, 2f, 2f, - (flags & 1u) != 0 ? ActivationKind.Relu : ActivationKind.None, - hasAdd ? add : default, (flags & 4u) != 0); - - // PC layout: 5 uints + flags uint + 2 floats - byte[] pcb = new byte[32]; - Buffer.BlockCopy(new uint[] { (uint)c, (uint)ih, (uint)iw, (uint)oh, (uint)ow, flags }, 0, pcb, 0, 24); - Buffer.BlockCopy(new float[] { 2f, 2f }, 0, pcb, 24, 8); - uint lanes = shader == "bilinear4" ? 4u : 1u; - float[] got = RunGpu(dev, shader, [src, add], dstCpu.Length, pcb, ((uint)dstCpu.Length / lanes + 255) / 256); - AssertClose(got, dstCpu, 1e-4f, shader); - } - - [SkippableTheory] - [InlineData("maxpool", 2, 2, 0, 0u)] - [InlineData("maxpool4", 2, 2, 0, 0u)] - [InlineData("maxpool4", 3, 2, 1, 1u)] // padded taps, fused relu - public void MaxPool_MatchesCpu(string shader, int k, int st, int pad, uint relu) - { - Skip.IfNot(VkOk(), "no Vulkan device"); - var dev = VkBackend.Default.Dev; - var rng = new Random(5); - int c = 8, ih = 9, iw = 11, oh = (ih + 2 * pad - k) / st + 1, ow = (iw + 2 * pad - k) / st + 1; - float[] x = Rand(rng, ih * iw * c), cpu = new float[oh * ow * c]; - Nhwc.MaxPool(x, ih, iw, c, cpu, oh, ow, - new PoolP { Kernel = [k, k], Strides = [st, st], Pads = [pad, pad, pad, pad], FuseRelu = relu != 0 }); - byte[] pc = new byte[48]; - Buffer.BlockCopy(new uint[] { (uint)c, (uint)ih, (uint)iw, (uint)oh, (uint)ow, (uint)k, (uint)k, (uint)st, (uint)st, - (uint)pad, (uint)pad, relu }, 0, pc, 0, 48); - uint lanes = shader == "maxpool4" ? 4u : 1u; - float[] got = RunGpu(dev, shader, [x], cpu.Length, pc, ((uint)cpu.Length / lanes + 255) / 256); - AssertClose(got, cpu, 1e-6f, shader); - } [SkippableTheory] [InlineData("shift", 2)] @@ -71,7 +24,7 @@ public void Shift_MatchesCpu(string shader, int axis) int c = 16, h = 7, w = 9; int[] ge = [4, 8, 12, 16], of = [-2, -1, 1, 2]; float[] x = Rand(rng, h * w * c), cpu = new float[h * w * c]; - Nhwc.ChannelShift(x, h, w, c, cpu, new ShiftP { Axis = axis, Offsets = of, GroupEnds = ge }); + GcdrKernels.ChannelShift(x, h, w, c, cpu, new ShiftP { Axis = axis, Offsets = of, GroupEnds = ge }); var pcu = new List { (uint)c, (uint)h, (uint)w, (uint)axis }; for (int i = 0; i < 5; i++) pcu.Add((uint)(i < ge.Length ? ge[i] : c)); for (int i = 0; i < 5; i++) pcu.Add((uint)(i < of.Length ? of[i] : 0)); @@ -82,35 +35,6 @@ public void Shift_MatchesCpu(string shader, int axis) AssertClose(got, cpu, 1e-6f, shader); } - [SkippableTheory] - [InlineData("depthwise", true, 1)] - [InlineData("depthwise4", true, 1)] - [InlineData("depthwise4", false, 3)] // constant pad, gelu - public void Depthwise_MatchesCpu(string shader, bool reflect, int act) - { - Skip.IfNot(VkOk(), "no Vulkan device"); - var dev = VkBackend.Default.Dev; - var rng = new Random(8); - int c = 12, h = 10, w = 13; - float[] x = Rand(rng, h * w * c), wt = Rand(rng, c * 9), b = Rand(rng, c), cpu = new float[h * w * c]; - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], Group = c, - PadMode = reflect ? PadMode.Reflect : PadMode.Constant, Act = (ActivationKind)act, - }; - Nhwc.Conv(x, h, w, c, cpu, h, w, c, p, Nhwc.Pack(wt, [c, 1, 3, 3], b, c, c), default); - float[] wp = new float[9 * c]; // [tap][C] - for (int ch = 0; ch < c; ch++) - for (int t = 0; t < 9; t++) wp[t * c + ch] = wt[ch * 9 + t]; - uint flags = 1u | (reflect ? 2u : 0u) | (act == 1 ? 4u : 0u) | (act == 3 ? 16u : 0u); - byte[] pc = new byte[52]; - Buffer.BlockCopy(new uint[] { (uint)c, (uint)h, (uint)w, (uint)h, (uint)w, 3u, 3u, 1u, 1u, 1u, 1u, flags }, 0, pc, 0, 48); - Buffer.BlockCopy(new float[] { 0f }, 0, pc, 48, 4); - uint lanes = shader == "depthwise4" ? 4u : 1u; - float[] got = RunGpu(dev, shader, [x, wp, b], cpu.Length, pc, ((uint)cpu.Length / lanes + 255) / 256); - AssertClose(got, cpu, 2e-3f, shader); // GPU erf polynomial vs CPU erf for gelu - } - [SkippableTheory] [InlineData(3, 16, 2u, 2u)] // gc tail: relu(skip), bias, sigmoid [InlineData(3, 32, 3u, 2u)] // + relu(resize) @@ -130,7 +54,7 @@ public void UpHead_MatchesCpu(int oc, int c, uint reluFlags, uint act) Act = act == 2u ? ActivationKind.Sigmoid : act == 1u ? ActivationKind.Relu : ActivationKind.None, }; float[] cpu = new float[oh * ow * oc]; - Nhwc.UpHead(x, ih, iw, c, skip, cpu, oh, ow, 2f, 2f, p, Nhwc.PackUpHead(w, oc, c, b)); + GcdrKernels.UpHead(x, ih, iw, c, skip, cpu, oh, ow, 2f, 2f, p, GcdrKernels.PackUpHead(w, oc, c, b)); uint flags = reluFlags | 4u | (act << 3); byte[] pcb = new byte[36]; @@ -140,132 +64,6 @@ public void UpHead_MatchesCpu(int oc, int c, uint reluFlags, uint act) AssertClose(got, cpu, 1e-4f, "uphead"); } - // stem conv + 2x2/2 max pool; pooled 11 x 15 leaves a partial last workgroup - [SkippableTheory] - [InlineData("convpool", 3, 16, true, 0f, 1)] - [InlineData("convpool_k3", 3, 16, true, 0f, 1)] // gc stem: reflect, conv relu - [InlineData("convpool_k3", 6, 32, false, 0f, 0)] // dr stem: zero pad, pool relu - [InlineData("convpool_k3", 7, 8, false, 0.25f, 1)] // ic 7, one channel group, pad value - [InlineData("convpool_k3", 1, 24, true, 0f, 0)] // ic 1, three channel groups - public void ConvPool_MatchesCpu(string shader, int ic, int oc, bool reflect, float padValue, int act) - { - Skip.IfNot(VkOk(), "no Vulkan device"); - var dev = VkBackend.Default.Dev; - var rng = new Random(ic * 13 + oc); - int h = 22, w = 30, ph = h / 2, pw = w / 2; - float[] x = Rand(rng, h * w * ic), wt = Rand(rng, oc * ic * 9), b = Rand(rng, oc); - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], - PadMode = reflect ? PadMode.Reflect : PadMode.Constant, PadValue = padValue, - Act = act == 1 ? ActivationKind.Relu : ActivationKind.None, - Pool = new PoolP { Kernel = [2, 2], Strides = [2, 2], Pads = [0, 0, 0, 0], FuseRelu = act == 0 }, - }; - float[] cpu = new float[ph * pw * oc]; - Nhwc.ConvPool(x, h, w, ic, cpu, ph, pw, oc, p, Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic)); - float[] wk = new float[9 * ic * oc]; // [K][oc], k = tap * ic + ci - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < 9; t++) wk[(t * ic + c) * oc + o] = wt[(o * ic + c) * 9 + t]; - uint flags = 1u | (reflect ? 2u : 0u) | (act == 1 ? 4u : 0u) | (act == 0 ? 8u : 0u); - byte[] pc = new byte[48]; - Buffer.BlockCopy(new uint[] { (uint)ic, (uint)h, (uint)w, (uint)oc, (uint)ph, (uint)pw, 3u, 3u, 1u, 1u, flags }, 0, pc, 0, 44); - Buffer.BlockCopy(new float[] { reflect ? 0f : padValue }, 0, pc, 44, 4); - float[] got = RunGpu(dev, shader, [x, wk, b], cpu.Length, pc, (uint)(ph * pw * (oc / 8) + 255) / 256); - AssertClose(got, cpu, 1e-4f, shader); - } - - // lite: false = the device's variant (LITE on wave64 / 32 KB parts), true = force the LITE build - [SkippableTheory] - [InlineData(6, 32, 1, false, "conv_cm_n32s", false, false, 1)] // dr stem (scalar k gather) - [InlineData(3, 16, 1, true, "conv_cm_n32s", false, false, 1)] // gc stem, reflect - [InlineData(32, 64, 1, true, "conv_cm_n64", false, false, 1)] - [InlineData(64, 128, 2, false, "conv_cm", false, false, 1)] // stride 2 - [InlineData(16, 32, 1, false, "conv_cm_n32", false, false, 1)] - [InlineData(6, 32, 1, false, "conv_cm_n32s", true, false, 1)] - [InlineData(32, 64, 1, true, "conv_cm_n64", true, true, 2)] // residual, gelu - [InlineData(64, 128, 2, false, "conv_cm", true, true, 0)] - [InlineData(16, 160, 1, false, "conv_cm_n32", true, false, 1)] // 5 N tiles: partial raster group - [InlineData(32, 256, 1, false, "conv_cm", false, true, 1)] - // shapes aimed at the SG16 build (8-row fragments, 16 lanes): half-used N tile (oc 16), - // partial N tile, minimal quad cin, any-cin stem with the epilogue, deep K (36 k-slices) - [InlineData(16, 16, 1, true, "conv_cm_n32", false, false, 1)] - [InlineData(16, 48, 1, false, "conv_cm_n32", false, true, 2)] - [InlineData(8, 64, 2, true, "conv_cm_n64", false, true, 1)] - [InlineData(3, 32, 1, false, "conv_cm_n32s", false, true, 2)] - [InlineData(128, 128, 1, true, "conv_cm", false, true, 2)] - public void ConvCm_MatchesCpu(int ic, int oc, int stride, bool reflect, string shader, bool lite, bool residual, int act) - { - Skip.IfNot(VkOk() && VkBackend.Default.Dev.CoopMat16, "no coopmat device"); - var dev = VkBackend.Default.Dev; - var rng = new Random(ic * 7 + oc); - int ih = 29, iw = 37, oh = (ih + 2 - 3) / stride + 1, ow = (iw + 2 - 3) / stride + 1; - // fp16-representable inputs so CPU and GPU see identical operands - float[] x = Rand(rng, ih * iw * ic).Select(v => (float)(Half)v).ToArray(); - float[] w = Rand(rng, oc * ic * 9).Select(v => (float)(Half)(v * 0.25f)).ToArray(); - float[] b = Rand(rng, oc); - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [stride, stride], Dilations = [1, 1], Kernel = [3, 3], - PadMode = reflect ? PadMode.Reflect : PadMode.Constant, - Act = act == 1 ? ActivationKind.Relu : act == 2 ? ActivationKind.Gelu : ActivationKind.None, - }; - float[] res = residual ? Rand(rng, oh * ow * oc).Select(v => (float)(Half)v).ToArray() : []; - float[] cpu = new float[oh * ow * oc]; - NhwcConvPlan kp = Nhwc.Pack(w, [oc, ic, 3, 3], b, 1, ic); - Nhwc.Conv(x, ih, iw, ic, cpu, oh, ow, oc, p, kp, res); - - int kr = 9 * ic, kd = (kr + 7) & ~7; - float[] wt = new float[oc * kd]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < 9; t++) wt[o * kd + t * ic + c] = w[(o * ic + c) * 9 + t]; - uint flags = 1u | (residual ? 2u : 0u) | (reflect ? 4u : 0u) | ((uint)act << 4); - uint m = (uint)(oh * ow); - byte[] pc = new byte[56]; - Buffer.BlockCopy(new uint[] { m, (uint)oc, (uint)kd, flags, (uint)ow, (uint)iw, (uint)ih, (uint)ic, 3u, - (uint)stride, (uint)stride, 1u, 1u, (uint)kr }, 0, pc, 0, 56); - uint bn = shader == "conv_cm" ? 128u : shader == "conv_cm_n64" ? 64u : 32u; - var bufs = new List { Half16(dev, x), Half16(dev, wt), F32(dev, b), Half16(dev, residual ? res : new float[8]) }; - VkBuffer ob = dev.NewStorageBuffer((ulong)cpu.Length * 2 + 16, true, preferHost: true); - bufs.Add(ob); - VkPipeline pipe = dev.GetCoopPipeline(shader, 5, 56, lite ? true : null); - IntPtr cmd = dev.NewCommandBuffer(); - IntPtr fence = dev.NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "begin"); - VkInferenceSession.RecordOne(dev, cmd, pipe, bufs.Select(bb => (bb, 0UL, bb.Size)).ToArray(), pc, - (m + 127) / 128, ((uint)oc + bn - 1) / bn); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "end"); - dev.Submit(cmd, fence); dev.WaitFence(fence); - dev.DestroyFence(fence); dev.FreeCommandBuffer(cmd); - Half[] gh = new Half[cpu.Length]; - fixed (Half* gp = gh) dev.Download(ob, gp, (ulong)gh.Length * 2); - foreach (VkBuffer bb in bufs) bb.Free(); - float[] got = gh.Select(h => (float)h).ToArray(); - float scale = Math.Max(1f, cpu.Max(MathF.Abs)); - AssertClose(got, cpu, 2e-3f * scale, shader); - } - - /// The SG16 build (8x16x16, 16 lanes) is only ever chosen with its pin in place and never - /// next to the 32-lane builds; devices that cannot run 16-lane subgroups never select it. - [SkippableFact] - public void CoopGate_Consistent() - { - Skip.IfNot(VkOk(), "no Vulkan device"); - var dev = VkBackend.Default.Dev; - if (dev.CoopM8) - { - Assert.True(dev.CoopMat16); - Assert.Equal(16u, dev.CoopSubgroupSize); - Assert.False(dev.CoopLite); - Assert.True(dev.SubgroupMin <= 16 && dev.SubgroupMax >= 16); - } - else if (dev.CoopMat16) - Assert.True(dev.CoopSubgroupSize is 0u or 32u); - if (dev.SubgroupMin > 16) Assert.False(dev.CoopM8); - } - private static VkBuffer Half16(VkDevice dev, float[] a) { Half[] h = a.Select(v => (Half)v).ToArray(); @@ -303,7 +101,7 @@ private static float[] RunGpu(VkDevice dev, string shader, float[][] inputs, int } VkBuffer ob = dev.NewStorageBuffer((ulong)outLen * 4, true, preferHost: true); bufs.Add(ob); - VkPipeline pipe = dev.GetPipeline(shader, bufs.Count, pc.Length); + VkPipeline pipe = dev.GetPipeline(shader, bufs.Count, pc.Length, owner: GcdrExtensions.Owner); IntPtr cmd = dev.NewCommandBuffer(); IntPtr fence = dev.NewFence(); var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; diff --git a/tests/Sdcb.SimdGcdrNet.Tests/VulkanDeviceLossTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/VulkanDeviceLossTests.cs new file mode 100644 index 0000000..49a0a87 --- /dev/null +++ b/tests/Sdcb.SimdGcdrNet.Tests/VulkanDeviceLossTests.cs @@ -0,0 +1,34 @@ +using Sdcb.SimdOnnx.Backends.Vulkan; + +namespace Sdcb.SimdGcdrNet.Tests; + +/// +/// Marks the process-wide Vulkan device lost, which disturbs every other Vulkan test running in the +/// process: opt-in (SIMDONNX_TEST_DEVICELOSS=1), run it on its own (--filter DeviceLoss). +/// +public class VulkanDeviceLossTests +{ + private static string GcPath => Path.Combine(Golden.Dir, "models", "gcdrnet_gcnet_sim.onnx"); + private static string DrPath => Path.Combine(Golden.Dir, "models", "gcdrnet_drnet_sim.onnx"); + + [SkippableFact] + public void Enhance_RecoversFromDeviceLoss_Vulkan() + { + Skip.IfNot(Environment.GetEnvironmentVariable("SIMDONNX_TEST_DEVICELOSS") == "1", "opt-in: SIMDONNX_TEST_DEVICELOSS=1"); + VkBackend before; + try { before = VkBackend.Default; } + catch { throw new SkipException("no Vulkan device"); } + var (_, _, _, _, oh, ow) = Golden.Dims("small"); + byte[] src = Golden.Bgr("small"); + + using var enh = GcdrEnhancer.FromFiles(GcPath, DrPath, GcdrBackend.Vulkan); + byte[] first = new byte[ow * oh * 3], second = new byte[ow * oh * 3]; + enh.Enhance(src, ow, oh, ow * 3, first, ow * 3); + + before.Dev.Lost = true; + enh.Enhance(src, ow, oh, ow * 3, second, ow * 3); + + Assert.NotSame(before, VkBackend.Default); + Assert.Equal(first, second); + } +} diff --git a/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs index 24e5f93..fbd11b7 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs @@ -1,6 +1,8 @@ +using Sdcb.SimdOnnx.Backends.Vulkan; using Sdcb.SimdGcdrNet.Backends.Vulkan; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Onnx; using Xunit; namespace Sdcb.SimdGcdrNet.Tests; @@ -22,7 +24,7 @@ public void GcNet_MatchesGolden_Vulkan(string tag) float[] expect = Golden.F32(tag, "shadow"); OnnxModel m = OnnxModel.Load(GcPath); - using var sess = VkBackend.Default.NewSession(m, shape); + using var sess = VkBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); input.CopyTo(sess.Input); sess.Run(); var (max, rmse, _) = Golden.Diff(sess.Output, expect); @@ -42,7 +44,7 @@ public void DrNet_MatchesGolden_Vulkan(string tag) float[] expect = Golden.F32(tag, "output"); OnnxModel m = OnnxModel.Load(DrPath); - using var sess = VkBackend.Default.NewSession(m, shape); + using var sess = VkBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); input.CopyTo(sess.Input); sess.Run(); var (max, rmse, _) = Golden.Diff(sess.Output, expect); @@ -153,27 +155,31 @@ public class VulkanConvDotTests [InlineData("dr", "mid", "dr_in", "output")] public void ConvDotRoute_MatchesGolden(string which, string tag, string inName, string outName) { - VkDevice dev; - try { dev = VkBackend.Default.Dev; } - catch { dev = null!; } - Skip.If(dev is null || !VkBackend.Default.F16, "needs a Vulkan device with fp16 storage"); + VkBackend? be = s_dotBackend.Value; + Skip.If(be is null || !be.F16, "needs a Vulkan device with fp16 storage"); float[] input = Golden.F32(tag, inName); int[] shape = Golden.ShapeOf(tag, inName); float[] expect = Golden.F32(tag, outName); - bool saved = dev.ConvDot; - dev.ConvDot = true; + OnnxModel m = OnnxModel.Load(which == "gc" ? GcPath : DrPath); + using var sess = be.NewSession(m, shape, GcdrExtensions.Instance); + Assert.Contains(be.GraphFor(m, shape, ext: GcdrExtensions.Instance).Dispatches, d => d.Name!.StartsWith("conv_dot", StringComparison.Ordinal)); + input.CopyTo(sess.Input); + sess.Run(); + var (max, rmse, _) = Golden.Diff(sess.Output, expect); + Assert.True(max < 2e-2f, $"{which}-convdot[{tag}] max diff {max} rmse {rmse}"); + } + + // a private device with the conv_dot route forced on: flipping it on the shared default device + // would re-route graphs other tests compile concurrently (byte-exact comparisons then fail) + private static readonly Lazy s_dotBackend = new(() => + { try { - // a fresh model instance: graphs are cached per model, so this compiles a new one - OnnxModel m = OnnxModel.Load(which == "gc" ? GcPath : DrPath); - using var sess = VkBackend.Default.NewSession(m, shape); - Assert.Contains(VkBackend.Default.GraphFor(m, shape).Dispatches, d => d.Name!.StartsWith("conv_dot", StringComparison.Ordinal)); - input.CopyTo(sess.Input); - sess.Run(); - var (max, rmse, _) = Golden.Diff(sess.Output, expect); - Assert.True(max < 2e-2f, $"{which}-convdot[{tag}] max diff {max} rmse {rmse}"); + var be = new VkBackend(); + be.Dev.ConvDot = true; + return be; } - finally { dev.ConvDot = saved; } - } + catch { return null; } + }); }