diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1bb3856..682e7d9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -4,7 +4,7 @@ name: test
# smoke - correctness: dotnet test + corpus smoke on every platform / ISA tier (+ Metal)
# bench - performance: corpus benchmark (.github/scripts/bench.ps1), several replicas per platform
# report - merges every result into the run summary (tests/Sdcb.SimdGcdrNet.TestReport)
-# Job budget: 35 jobs (40 concurrent); macOS: 2 smoke + 3 bench (5 concurrent).
+# Job budget: 36 jobs (40 concurrent); macOS: 2 smoke + 3 bench (5 concurrent).
on:
push:
@@ -33,6 +33,9 @@ concurrency:
env:
DOTNET_NOLOGO: 1
+ # jobs build against the pinned Sdcb.SimdOnnx package (what ships); the engine-head smoke builds
+ # sdcb/SimdOnnx main from source next to this checkout (MSBuild reads env vars as properties)
+ SimdOnnxProject: ${{ github.workspace }}/_simdonnx/src/Sdcb.SimdOnnx/Sdcb.SimdOnnx.csproj
DOTNET_CLI_TELEMETRY_OPTOUT: 1
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
@@ -57,11 +60,15 @@ jobs:
name: smoke ${{ matrix.name }}
runs-on: ${{ matrix.os }}
timeout-minutes: 30
+ env:
+ UseLocalSimdOnnx: ${{ matrix.enginehead && 'true' || 'false' }}
strategy:
fail-fast: false
matrix:
include:
- { name: win-x64, os: windows-2025, count: 5 }
+ # the same tests against sdcb/SimdOnnx main from source: engine changes that break this library
+ - { name: linux-x64-engine-head, os: ubuntu-24.04, count: 2, enginehead: true }
- { name: win-arm64, os: windows-11-arm, count: 5 }
- { name: linux-x64, os: ubuntu-24.04, count: 5, pack: true }
- { name: linux-arm64, os: ubuntu-24.04-arm, count: 5 }
@@ -77,6 +84,9 @@ jobs:
- { name: linux-arm64-scalar, os: ubuntu-24.04-arm, count: 2, maxmp: 1, isa: DOTNET_EnableHWIntrinsic=0 }
steps:
- uses: actions/checkout@v7
+ - uses: actions/checkout@v7
+ if: matrix.enginehead
+ with: { repository: sdcb/SimdOnnx, path: _simdonnx }
- uses: actions/download-artifact@v8
with: { name: realdae, path: tests/testdata/realdae }
- name: Build
diff --git a/Directory.Build.props b/Directory.Build.props
index 4d17f13..914a541 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -4,8 +4,15 @@
never force a models release). Edit here and keep build/00-common.linq in sync.
-->
- 1.0.0
+ 2.0.0
1.0.0
+
+ 0.1.0-preview.4
+ $(MSBuildThisFileDirectory)..\Sdcb.SimdOnnx\src\Sdcb.SimdOnnx\Sdcb.SimdOnnx.csproj
+
+ $(NoWarn);NU5104
@@ -39,7 +46,7 @@
Sdcb.SimdGcdrNet
$(SdcbSimdGcdrNetVersion)
true
- Pure C# GCDRNet inference: document image shadow removal / appearance enhancement. Self-contained ONNX graph engine with CPU SIMD (AVX-512/AVX2/NEON/Vector) and GPU (Vulkan, Metal) backends; no onnxruntime dependency. Accepts interleaved BGR24 / RGB24 / BGRA32 / RGBA32. Models ship separately in Sdcb.SimdGcdrNet.Models.
+ Pure C# GCDRNet inference: document image shadow removal / appearance enhancement. Runs on the pure C# Sdcb.SimdOnnx engine (CPU SIMD AVX-512/AVX2/NEON/Vector, GPU Vulkan/Metal) with GCDRNet-specific fused kernels; no onnxruntime dependency. Accepts interleaved BGR24 / RGB24 / BGRA32 / RGBA32. Models ship separately in Sdcb.SimdGcdrNet.Models.
Sdcb;GCDRNet;document;document-enhancement;shadow-removal;image-enhancement;onnx;inference;deep-learning;SIMD;AVX512;NEON;wasm;AOT;Vulkan;Metal;linqpad-samples
diff --git a/README.md b/README.md
index 48d63a2..883ba42 100644
--- a/README.md
+++ b/README.md
@@ -9,8 +9,8 @@
**Sdcb.SimdGcdrNet** is a pure C# implementation of [GCDRNet](https://github.com/ZZZHANG-jx/GCDRNet)
(*Appearance Enhancement for Camera-captured Document Images in the Wild*, IEEE TAI 2023) —
document-image shadow removal and appearance enhancement **without onnxruntime or any other
-third-party inference engine**. It ships its own compiled-model inference engine, inspired by
-`Sdcb.SimdPaddleOCR`.
+third-party inference engine**. Inference runs on [Sdcb.SimdOnnx](https://github.com/sdcb/SimdOnnx),
+a pure C# engine shared with `Sdcb.SimdPaddleOCR` (it grew out of this repository's own engine).
The model is a two-stage pipeline:
@@ -110,19 +110,20 @@ dotnet run --project examples/PaddleOCR.Console -- [image] [output.jpg]
### Layout
-- `src/Sdcb.SimdGcdrNet/Onnx` — minimal protobuf reader + ONNX model/graph objects
-- `src/Sdcb.SimdGcdrNet/Engine` — `GraphCompiler` (specialize → fold → fuse → arena-plan),
- `InferenceSession`
-- `src/Sdcb.SimdGcdrNet/Kernels` — Scalar / `Vector` / AVX2 / AVX-512 kernels (`Ops`, `Avx2Kernels`, `Nhwc`,
- `PixelOps`, `SimdMath`)
-- `src/Sdcb.SimdGcdrNet/Backends/Vulkan` — `Vk` P/Invoke layer, `VkDevice`, `VkShaders` (embedded
- SPIR-V), `VkGraph` dispatch lowering, `VkSession`, `VkPipeline2` (fused pipeline), `VkBackend`
-- `src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders` — GLSL sources; `tools/build-shaders.bat`
- recompiles `.comp` → `.spv` (committed; dev-time only, not part of `dotnet build`)
-- `src/Sdcb.SimdGcdrNet/Backends/Metal` — `ObjC`/`Metal` P/Invoke layer, `MtlGraph` dispatch
- lowering, `MtlSession`, `MtlPipeline` (fused pipeline), `MetalBackend`
-- `src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders` — MSL sources (`.metal`, embedded resources,
- compiled to PSOs at run time — no pre-build step)
+The ONNX engine (protobuf reader, graph compiler, CPU SIMD kernels, Vulkan / Metal backends) is
+[Sdcb.SimdOnnx](https://github.com/sdcb/SimdOnnx); this library holds the GCDRNet-specific parts:
+
+- `src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs` — public API: backend selection, per-size session pools,
+ gc → dr stitching, arena stacking
+- `src/Sdcb.SimdGcdrNet/Extensions` — engine extensions (`GcdrExtensions`): fused operators
+ (`UpHeadOp`, `ChannelShiftOp`) with their CPU kernels (Vector / AVX-512 / NEON / WASM SIMD) and
+ graph passes
+- `src/Sdcb.SimdGcdrNet/Kernels` — `PixelOps` (interleaved pixels ↔ tensors, padding)
+- `src/Sdcb.SimdGcdrNet/Backends/Vulkan` — `VkPipeline2` (fused gc + dr pipeline on SimdOnnx's
+ Vulkan backend); `Shaders` holds the GCDRNet GLSL kernels, `tools/build-shaders.bat` recompiles
+ `.comp` → `.spv` (committed; dev-time only, not part of `dotnet build`)
+- `src/Sdcb.SimdGcdrNet/Backends/Metal` — `MtlPipeline` (fused pipeline on SimdOnnx's Metal
+ backend); `Shaders` holds MSL sources (embedded, compiled to PSOs at run time)
- `bench/Sdcb.SimdGcdrNet.Benchmark` — `--check`, `--bench` (CPU), `--benchvk` (Vulkan e2e),
`--benchmtl` / `--checkgpu` / `--mtlprof` / `--mtrace` (Metal e2e, diff check, per-shader
profile, CPU-vs-Metal per-node trace), `--vkprof` (per-shader/per-dispatch GPU timing), `--vkcheck` / `--vknodes`
@@ -173,8 +174,10 @@ runs two waves in parallel, then merges them into one report (the run's summary
- Metal path requires macOS on Apple Silicon with Apple7+ (simdgroup-MMA, i.e. M1+; an x64 process
under Rosetta also qualifies); `Auto` checks `SupportsApple7` before choosing it,
`GcdrBackend.Metal` selects it unconditionally. Set `GCDRNET_MTL_VERBOSE=1` to print the chosen device.
-- Thread-safety: `GcdrEnhancer` is thread-safe (CPU: per-size session pools; Vulkan: size-agnostic worker pool); individual
- `InferenceSession`/`VkInferenceSession` objects are not.
+- Thread-safety: `GcdrEnhancer` is thread-safe (CPU: per-size session pools; Vulkan: size-agnostic worker pool).
+- 2.0 moved the engine out: `InferenceSession`, `CompiledModel`, `OnnxModel`, `VkInferenceSession` and
+ `MtlInferenceSession` are no longer in `Sdcb.SimdGcdrNet.*`; use the same types from
+ `Sdcb.SimdOnnx.*` (a dependency of this package). `GcdrEnhancer`'s API is unchanged.
- Not covered: batched inference (N>1), fp16/int8.
- Native AOT works (`IsAotCompatible`, no trim/AOT warnings; CPU and Vulkan output verified identical to JIT). On **x64** the executable
project must set the instruction set, otherwise ILC compiles for the SSE baseline, folds
@@ -202,8 +205,8 @@ are from the RealDAE dataset of the same paper.
## 中文
**Sdcb.SimdGcdrNet** 是 [GCDRNet](https://github.com/ZZZHANG-jx/GCDRNet)(相机拍摄文档去阴影/外观增强,
-IEEE TAI 2023)的**纯 C# 推理库**——不依赖 onnxruntime 等第三方推理引擎,像 `Sdcb.SimdPaddleOCR`
-一样自带 CompiledModel 推理引擎。
+IEEE TAI 2023)的**纯 C# 推理库**——不依赖 onnxruntime 等第三方推理引擎,与 `Sdcb.SimdPaddleOCR`
+共用纯 C# 推理引擎 [Sdcb.SimdOnnx](https://github.com/sdcb/SimdOnnx)(2.0 起引擎类型移到 `Sdcb.SimdOnnx.*`)。
模型为两级串联:**GC-Net** 估计逐像素光照/阴影图 `I_sm` → `I_GC = clamp(I_s/I_sm)` 除法校正 →
**DR-Net** 消费 6 通道 `[I_s; I_GC]` 输出增强图。
diff --git a/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs b/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs
index ad03c02..6aad87e 100644
--- a/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs
+++ b/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs
@@ -195,7 +195,7 @@ private static JsonObject HostInfo()
string isa = RuntimeInformation.ProcessArchitecture switch
{
Architecture.X64 or Architecture.X86 =>
- Kernels.Nhwc.UseAvx512 ? "avx512" : Avx2.IsSupported ? "avx2" : Avx.IsSupported ? "avx" :
+ Sdcb.SimdOnnx.Kernels.Nhwc.UseAvx512 ? "avx512" : Avx2.IsSupported ? "avx2" : Avx.IsSupported ? "avx" :
System.Numerics.Vector.IsHardwareAccelerated ? "sse" : "scalar",
Architecture.Arm64 => AdvSimd.IsSupported ? "neon" : "scalar",
_ => System.Numerics.Vector.IsHardwareAccelerated ? "vector" : "scalar",
diff --git a/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs b/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs
index 100c298..1f3caa5 100644
--- a/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs
+++ b/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs
@@ -1,7 +1,8 @@
using System.Diagnostics;
using System.Runtime.InteropServices;
-using Sdcb.SimdGcdrNet.Engine;
-using Sdcb.SimdGcdrNet.Onnx;
+using Sdcb.SimdOnnx.Engine;
+using Sdcb.SimdGcdrNet.Extensions;
+using Sdcb.SimdOnnx.Onnx;
namespace Sdcb.SimdGcdrNet.Benchmark;
@@ -44,7 +45,7 @@ public static int Main(string[] args)
var m = OnnxModel.Load(mpath);
int c = which == "gc" ? 3 : 6;
// run the SAME (gpu-layout) plan on CPU and Metal so node indices align
- var cpuPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdGcdrNet.Backends.Metal.MetalBackend.PlanOptions);
+ var cpuPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdOnnx.Backends.Metal.MetalBackend.PlanOptions with { Extensions = GcdrExtensions.Instance });
using var cpu = new InferenceSession(cpuPlan);
var rng = new Random(0);
if (args.Length > 4)
@@ -56,9 +57,9 @@ public static int Main(string[] args)
}
else
for (int i = 0; i < cpu.Input.Length; i++) cpu.Input[i] = rng.NextSingle();
- var mbe = Sdcb.SimdGcdrNet.Backends.Metal.MetalBackend.Default;
- var mPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdGcdrNet.Backends.Metal.MetalBackend.PlanOptions);
- using var mtl = mbe.NewSession(m, [1, c, h, w]);
+ var mbe = Sdcb.SimdOnnx.Backends.Metal.MetalBackend.Default;
+ var mPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdOnnx.Backends.Metal.MetalBackend.PlanOptions with { Extensions = GcdrExtensions.Instance });
+ using var mtl = mbe.NewSession(m, [1, c, h, w], GcdrExtensions.Instance);
cpu.Input.CopyTo(mtl.Input);
mtl.DebugUploadInput();
if (Environment.GetEnvironmentVariable("MTRACE_NODES") == "1")
@@ -106,9 +107,9 @@ public static int Main(string[] args)
var exp = new float[wsrc.Length];
for (int o2 = 0; o2 < oc_; o2++) for (int c2 = 0; c2 < ic_; c2++) for (int t = 0; t < taps; t++)
exp[(t * ic_ + c2) * oc_ + o2] = wsrc[(o2 * ic_ + c2) * taps + t];
- var mw = mtl.DebugConst(nd_.In[1], 2, wsrc.Length);
+ var mw = mtl.DebugConst(nd_.In[1], ConstKind.ConvW, wsrc.Length);
int wb = -1; for (int k = 0; k < exp.Length; k++) if (Math.Abs(mw[k] - exp[k]) > 1e-3) { wb = k; break; }
- Console.WriteLine($" w-check: pack len {exp.Length} off {mtl.DebugConstOff(nd_.In[1], 2)} first-diff {wb} head {string.Join(' ', mw.Take(8).Select(x => $"{x:F3}"))} vs exp {string.Join(' ', exp.Take(8).Select(x => $"{x:F3}"))}");
+ Console.WriteLine($" w-check: pack len {exp.Length} off {mtl.DebugConstOff(nd_.In[1], ConstKind.ConvW)} first-diff {wb} head {string.Join(' ', mw.Take(8).Select(x => $"{x:F3}"))} vs exp {string.Join(' ', exp.Take(8).Select(x => $"{x:F3}"))}");
if (wb >= 0) Console.WriteLine($" @{wb}: got {mw[wb]:F4} exp {exp[wb]:F4}; around got {string.Join(' ', mw.Skip(wb).Take(6).Select(x => $"{x:F3}"))} exp {string.Join(' ', exp.Skip(wb).Take(6).Select(x => $"{x:F3}"))}");
}
if (bad < 0 && (rmse > 1e-2 || nan > 0 || nanw > 0))
@@ -177,7 +178,7 @@ public static int Main(string[] args)
// --node gc|dr h w idx reps : time one node in isolation (after a full warmup run)
bool isDr = args[1] == "dr";
int nh = int.Parse(args[2]), nw = int.Parse(args[3]), idx = int.Parse(args[4]), nreps = int.Parse(args[5]);
- var cm = CompiledModel.Compile(OnnxModel.Load(isDr ? drPath : gcPath), [1, isDr ? 6 : 3, nh, nw]);
+ var cm = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(isDr ? drPath : gcPath), [1, isDr ? 6 : 3, nh, nw], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }));
var ss = cm.CreateSession();
new Random(0).NextBytes(MemoryMarshal.AsBytes(ss.Input));
if (args.Length > 6 && args[6] == "u") { var rr = new Random(0); var inp = ss.Input; for (int i = 0; i < inp.Length; i++) inp[i] = rr.NextSingle(); }
@@ -202,8 +203,8 @@ public static int Main(string[] args)
foreach ((string tag, string mp, int cin) in new[] { ("gc", gcPath, 3), ("dr", drPath, 6) })
{
var m = OnnxModel.Load(mp);
- var cpu = CompiledModel.Compile(m, [1, cin, ch, cw]).CreateSession();
- using var vk = Backends.Vulkan.VkBackend.Default.NewSession(m, [1, cin, ch, cw]);
+ var cpu = new CompiledModel(GraphCompiler.Compile(m, [1, cin, ch, cw], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })).CreateSession();
+ using var vk = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.NewSession(m, [1, cin, ch, cw], GcdrExtensions.Instance);
var rng = new Random(7);
for (int i = 0; i < cpu.Input.Length; i++) cpu.Input[i] = vk.Input[i] = rng.NextSingle();
cpu.Run(); vk.Run();
@@ -227,14 +228,14 @@ public static int Main(string[] args)
int nh = args.Length > 2 ? int.Parse(args[2]) : 128, nw = args.Length > 3 ? int.Parse(args[3]) : 192;
var m = OnnxModel.Load(isDr ? drPath : gcPath);
int[] shape = [1, isDr ? 6 : 3, nh, nw];
- var plan = GraphCompiler.Compile(m, shape, Backends.Vulkan.VkBackend.PlanOptions);
+ var plan = GraphCompiler.Compile(m, shape, Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.PlanOptions with { Extensions = GcdrExtensions.Instance });
using var cpu = new InferenceSession(plan);
- using var gpu = Backends.Vulkan.VkBackend.Default.NewSession(m, shape);
+ using var gpu = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.NewSession(m, shape, GcdrExtensions.Instance);
var rng = new Random(7);
for (int i = 0; i < cpu.Input.Length; i++) cpu.Input[i] = gpu.Input[i] = rng.NextSingle();
gpu.DebugUploadInput();
- var ranges = Backends.Vulkan.VkBackend.Default.GraphFor(m, shape).NodeRanges;
- var gd = Backends.Vulkan.VkBackend.Default.GraphFor(m, shape).Dispatches;
+ var ranges = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.GraphFor(m, shape, ext: GcdrExtensions.Instance).NodeRanges;
+ var gd = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.GraphFor(m, shape, ext: GcdrExtensions.Instance).Dispatches;
for (int i = 0; i < plan.Nodes.Length; i++)
{
var nd = plan.Nodes[i];
@@ -304,11 +305,11 @@ public static int Main(string[] args)
{
int h = args.Length > 1 ? int.Parse(args[1]) : 1152;
int w = args.Length > 2 ? int.Parse(args[2]) : 1600;
- var be = Backends.Metal.MetalBackend.Default;
+ var be = Sdcb.SimdOnnx.Backends.Metal.MetalBackend.Default;
foreach ((string tag, string mp, int cin) in new[] { ("gc", gcPath, 3), ("dr", drPath, 6) })
{
var m = OnnxModel.Load(mp);
- using var s = be.NewSession(m, [1, cin, h, w]);
+ using var s = be.NewSession(m, [1, cin, h, w], GcdrExtensions.Instance);
s.Run(); // warm
var agg = s.DebugProfile();
Console.WriteLine($"--- {tag} metal per-shader (total {agg.Values.Sum():F1} ms) ---");
@@ -387,7 +388,7 @@ public static int Main(string[] args)
// destroy, recreate — any hash change = stale-read/nondeterminism at session level.
private static int VkDup(string gcPath, string drPath, int h, int w, bool useDr)
{
- var be = Backends.Vulkan.VkBackend.Default;
+ var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default;
string path = useDr ? drPath : gcPath;
int c = useDr ? 6 : 3;
var m = OnnxModel.Load(path);
@@ -399,7 +400,7 @@ static ulong Fhash(ReadOnlySpan d)
}
for (int round = 0; round < 4; round++)
{
- using var s = be.NewSession(m, [1, c, h, w]);
+ using var s = be.NewSession(m, [1, c, h, w], GcdrExtensions.Instance);
var rng = new Random(42);
for (int i = 0; i < s.Input.Length; i++) s.Input[i] = rng.NextSingle();
s.Run();
@@ -409,7 +410,7 @@ static ulong Fhash(ReadOnlySpan d)
// interleave: run smaller sessions to churn memory
foreach (int cs in new[] { 1024, 1312, 1152 })
{
- using var s2 = be.NewSession(m, [1, c, cs, cs]);
+ using var s2 = be.NewSession(m, [1, c, cs, cs], GcdrExtensions.Instance);
new Random(0).NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(s2.Input));
s2.Run();
}
@@ -423,7 +424,7 @@ private static void Analyze(string gcPath, string drPath, int h, int w)
{
foreach (var (path, c) in new[] { (gcPath, 3), (drPath, 6) })
{
- var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w]);
+ var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance });
var written = new List<(long s, long e)>();
bool Covered(long s, long e) =>
written.Any(r => r.s <= s && r.e >= e);
@@ -463,7 +464,7 @@ bool Covered(long s, long e) =>
// stage first diverges.
private static int Probe(string gcPath, string drPath, string imgPath, int[] churnPx)
{
- var be = Backends.Vulkan.VkBackend.Default;
+ var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default;
var gc = OnnxModel.Load(gcPath); var dr = OnnxModel.Load(drPath);
int iw, ih; byte[] bgr;
(iw, ih) = ImageSize(imgPath);
@@ -488,10 +489,10 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag)
{
p.Enhance(bgr, iw * 3, dst, iw * 3);
p.DebugIntermediates(gcI, gcO, drI);
- Console.WriteLine($"{tag}: dst={Fb(dst):x16} gcIn={Ff(gcI):x16} gcOut={Ff(gcO):x16} drIn={Ff(drI):x16} live={Backends.Vulkan.VkDevice.LiveDeviceBytes >> 20}MB");
+ Console.WriteLine($"{tag}: dst={Fb(dst):x16} gcIn={Ff(gcI):x16} gcOut={Ff(gcO):x16} drIn={Ff(drI):x16} live={Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.LiveDeviceBytes >> 20}MB");
}
- var p1 = be.NewPipe(gc, dr, ih, iw);
+ var p1 = NewVkPipe(be, gc, dr, ih, iw);
Run(p1, "p1-run1");
Run(p1, "p1-run2");
var churn = new List();
@@ -499,7 +500,7 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag)
{
int cw = Math.Max(64, (int)Math.Sqrt(px * 16.0 / 9) / 32 * 32);
int chh = Math.Max(64, px / Math.Max(cw, 1) / 32 * 32);
- var cp = be.NewPipe(gc, dr, chh, cw);
+ var cp = NewVkPipe(be, gc, dr, chh, cw);
byte[] cb = new byte[chh * cw * 3]; new Random(px).NextBytes(cb);
byte[] cd = new byte[chh * cw * 3];
cp.Enhance(cb, cw * 3, cd, cw * 3);
@@ -507,7 +508,7 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag)
if (churn.Count > 1) { churn[0].Dispose(); churn.RemoveAt(0); }
}
foreach (var c in churn) c.Dispose();
- var p2 = be.NewPipe(gc, dr, ih, iw);
+ var p2 = NewVkPipe(be, gc, dr, ih, iw);
Run(p2, "p2-after-churn");
p2.Dispose(); p1.Dispose();
return 0;
@@ -516,14 +517,14 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag)
// bisect which plan value first diverges between two pipeline instances.
private static int Bisect(string gcPath, string drPath, string imgPath, int[] churnPx)
{
- var be = Backends.Vulkan.VkBackend.Default;
+ var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default;
var gc = OnnxModel.Load(gcPath); var dr = OnnxModel.Load(drPath);
int iw, ih;
(iw, ih) = ImageSize(imgPath);
byte[] bgr = DecodeBgr(imgPath);
byte[] dst = new byte[bgr.Length];
- var p1 = be.NewPipe(gc, dr, ih, iw);
+ var p1 = NewVkPipe(be, gc, dr, ih, iw);
p1.Enhance(bgr, iw * 3, dst, iw * 3);
// replicate corpus allocation history: run every corpus image sorted by px
// (up to `churnPx[0]` predecessors, or all when empty) with real (h,w) sizes
@@ -537,7 +538,7 @@ private static int Bisect(string gcPath, string drPath, string imgPath, int[] ch
var alive = new List();
foreach (var e in seq.Take(take))
{
- var cp = be.NewPipe(gc, dr, e.h, e.w);
+ var cp = NewVkPipe(be, gc, dr, e.h, e.w);
byte[] cb = new byte[e.h * e.w * 3]; new Random(e.w * e.h).NextBytes(cb);
byte[] cd = new byte[e.h * e.w * 3];
cp.Enhance(cb, e.w * 3, cd, e.w * 3);
@@ -545,7 +546,7 @@ private static int Bisect(string gcPath, string drPath, string imgPath, int[] ch
if (alive.Count > 1) { alive[0].Dispose(); alive.RemoveAt(0); }
}
foreach (var c in alive) c.Dispose();
- var p2 = be.NewPipe(gc, dr, ih, iw);
+ var p2 = NewVkPipe(be, gc, dr, ih, iw);
p2.Enhance(bgr, iw * 3, dst, iw * 3);
// dump BOTH whole gc arenas in one shot each, diff per plan value on host
@@ -749,7 +750,7 @@ static ulong Ff(Span d)
};
}
- Backends.Vulkan.VkDevice.ResetPeakDeviceBytes();
+ Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.ResetPeakDeviceBytes();
long gc0 = GC.GetTotalMemory(true);
double wsLast = wsLoaded;
var hashes = new Dictionary();
@@ -824,8 +825,8 @@ static ulong Ff(Span d)
Console.WriteLine($" ws: init {wsInit0:F0} -> loaded {wsLoaded:F0} MB ({wsLoaded - wsInit0:+0;-0}), " +
$"loaded -> last {wsLast:F0} MB (delta {wsLast - wsLoaded:+0;-0} MB), peak {wsPeak:F0} MB");
Console.WriteLine($" mem: managed {gcD / 1048576.0:+0;-0} MB (after forced GC), " +
- $"vkDevice live {Backends.Vulkan.VkDevice.LiveDeviceBytes / 1048576.0:F0} MB " +
- $"(peak {Backends.Vulkan.VkDevice.PeakDeviceBytes / 1048576.0:F0} MB)");
+ $"vkDevice live {Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.LiveDeviceBytes / 1048576.0:F0} MB " +
+ $"(peak {Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.PeakDeviceBytes / 1048576.0:F0} MB)");
return 0;
}
@@ -948,7 +949,7 @@ private static int Trace(string gcPath, string drPath, int h, int w)
{
foreach (var (path, c) in new[] { (gcPath, 3), (drPath, 6) })
{
- var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w]);
+ var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance });
long maxT = plan.Values.Max(v => v.Shape.Aggregate(1L, (a, x) => a * x));
Console.WriteLine($"max tensor {maxT * 4 / 1048576.0:F0} MB, arena {plan.ArenaFloats * 4L / 1048576.0:F0} MB");
using var s = new InferenceSession(plan);
@@ -972,8 +973,8 @@ private static int Trace(string gcPath, string drPath, int h, int w)
private static int Bench(string gcPath, string drPath, int h, int w, int reps)
{
- var gc = CompiledModel.Compile(OnnxModel.Load(gcPath), [1, 3, h, w]);
- var dr = CompiledModel.Compile(OnnxModel.Load(drPath), [1, 6, h, w]);
+ var gc = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(gcPath), [1, 3, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }));
+ var dr = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(drPath), [1, 6, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }));
var s1 = gc.CreateSession(); var s2 = dr.CreateSession();
new Random(0).NextBytes(MemoryMarshal.AsBytes(s1.Input));
new Random(1).NextBytes(MemoryMarshal.AsBytes(s2.Input));
@@ -1033,11 +1034,11 @@ private static int BenchE2e(string gcPath, string drPath, int h, int w, int reps
private static int VkProf(string gcPath, string drPath, int h, int w)
{
- var be = Backends.Vulkan.VkBackend.Default;
+ var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default;
foreach ((string tag, string mp, int cin) in new[] { ("gc", gcPath, 3), ("dr", drPath, 6) })
{
var m = OnnxModel.Load(mp);
- using var s = be.NewSession(m, [1, cin, h, w]);
+ using var s = be.NewSession(m, [1, cin, h, w], GcdrExtensions.Instance);
s.Run(); // warm
var agg = s.DebugProfile();
double tot = agg.Values.Sum();
@@ -1078,4 +1079,11 @@ private static string FindTestdata()
}
throw new DirectoryNotFoundException("testdata not found");
}
-}
+
+ private static Backends.Vulkan.VkGcdrPipeline NewVkPipe(Sdcb.SimdOnnx.Backends.Vulkan.VkBackend be, OnnxModel gc, OnnxModel dr, int ih, int iw)
+ {
+ var p = new Backends.Vulkan.VkGcdrPipeline(be, gc, dr);
+ try { p.Bind(ih, iw); return p; }
+ catch { p.Dispose(); throw; }
+ }
+}
\ No newline at end of file
diff --git a/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs b/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs
index 0362e48..af12d38 100644
--- a/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs
+++ b/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs
@@ -5,9 +5,11 @@
using System.Runtime.Intrinsics.Wasm;
using System.Text.Json;
using System.Text.Json.Nodes;
-using Sdcb.SimdGcdrNet.Engine;
+using Sdcb.SimdOnnx.Engine;
+using Sdcb.SimdGcdrNet.Extensions;
+using Sdcb.SimdOnnx.Kernels;
using Sdcb.SimdGcdrNet.Kernels;
-using Sdcb.SimdGcdrNet.Onnx;
+using Sdcb.SimdOnnx.Onnx;
using Sdcb.SimdGcdrNet.Testing;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
@@ -274,8 +276,8 @@ private static async Task Corpus(string backend, int count, JsonObject doc)
private static async Task Bench(int h, int w, int reps, JsonObject doc)
{
var (gcBytes, drBytes) = await Models();
- var gc = CompiledModel.Compile(OnnxModel.Load(gcBytes), [1, 3, h, w]);
- var dr = CompiledModel.Compile(OnnxModel.Load(drBytes), [1, 6, h, w]);
+ var gc = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(gcBytes), [1, 3, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }));
+ var dr = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(drBytes), [1, 6, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }));
using var s1 = gc.CreateSession();
using var s2 = dr.CreateSession();
new Random(0).NextBytes(MemoryMarshal.AsBytes(s1.Input));
@@ -381,7 +383,7 @@ private static async Task TiersCore(int h, int w, JsonObject doc)
bool ok = true;
foreach (var (tag, model, c) in new[] { ("gc", gcBytes, 3), ("dr", drBytes, 6) })
{
- var cm = CompiledModel.Compile(OnnxModel.Load(model), [1, c, h, w]);
+ var cm = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(model), [1, c, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }));
var outs = new float[2][];
for (int pass = 0; pass < 2; pass++)
{
diff --git a/build/00-common.linq b/build/00-common.linq
index 49e400d..d6593b2 100644
--- a/build/00-common.linq
+++ b/build/00-common.linq
@@ -25,7 +25,7 @@ static void Run(string exe, string args, Encoding encoding) => Util.Cmd(exe, arg
// dependencies) only when the models change. Keep the Directory.Build.props defaults in sync.
static ProjectVersion[] Projects =
{
- new("Sdcb.SimdGcdrNet", "1.0.0"),
+ new("Sdcb.SimdGcdrNet", "2.0.0"),
new("Sdcb.SimdGcdrNet.Models", "1.0.0"),
};
diff --git a/docs/rockchip-rk3399.md b/docs/rockchip-rk3399.md
index ecf8a9a..1b05efb 100644
--- a/docs/rockchip-rk3399.md
+++ b/docs/rockchip-rk3399.md
@@ -84,7 +84,7 @@ CPU 档位(`--bench 512 512`,gc+dr 推理,锁 1008 MHz):
NEON 档位(`Nhwc.UseNeon = AdvSimd.Arm64.IsSupported`)沿用 x86 各档的算子结构,以下是只在 NEON 上走的部分:
-1. **6×16 micro-kernel `TileNeonK`**(`Kernels/Nhwc.Neon.cs`)
+1. **6×16 micro-kernel `TileNeonK`**(Sdcb.SimdOnnx `src/Sdcb.SimdOnnx/Kernels/Nhwc.Neon.cs`)
- 每像素一次 128 位读入 4 个输入通道,`fmla` by element 累加到 24 个寄存器累加器(共 32 个向量寄存器)。
- 下一组权重向量提前一组读入,照顾按序执行的 A53。`ic % 4` 的余数用 `ld1r` 广播。
- 单核微基准(`taps × ic` 次累加,L1 内):A72 6.3–7.1 GFLOP/s,约为峰值 7.5 的 85–95%;A53 3.5–3.8 GFLOP/s,约为峰值的 55%。
@@ -97,7 +97,7 @@ NEON 档位(`Nhwc.UseNeon = AdvSimd.Arm64.IsSupported`)沿用 x86 各档的
3. **Winograd**
- GEMM 走 `TileNeonK`,输出 epilogue 改为向量化。
- V 块预算降到 16 KB(ic ≥ 32 时 T = 6)。4 个 A53 共享 512 KB L2,块大了会互相挤出。
- - U(36·oc·ic)超过 2 MB 的深层改走三阶段分组调度 `WinogradGrouped`(`Kernels/NhwcWinograd.Neon.cs`):输入变换 → 每个 (xi, oc 块) 一个 GEMM 任务,每块 U 每组只读一次 → 输出变换。原来是每 6 个 tile 就把整块 U 从 DRAM 流一遍。
+ - U(36·oc·ic)超过 2 MB 的深层改走三阶段分组调度 `WinogradGrouped`(Sdcb.SimdOnnx `src/Sdcb.SimdOnnx/Kernels/NhwcWinograd.Neon.cs`):输入变换 → 每个 (xi, oc 块) 一个 GEMM 任务,每块 U 每组只读一次 → 输出变换。原来是每 6 个 tile 就把整块 U 从 DRAM 流一遍。
- 分组调度与原调度逐位相同:变换、k 序都一样。512→256 层(1600×1152,锁频)从 105 ms 降到 83 ms,256→128 从 120 ms 降到 93 ms。
4. **向量化激活**:`SimdMath` 增加 Vector128 版 Exp / Sigmoid / Erf / GELU,公式与 AVX2 版一致,用 FMA。
- 深度卷积 + GELU 原来在非 AVX 档逐元素走标量 `ErfScalar`,现在向量化:gc 128 通道那一层从 21.6 ms 降到 8.5 ms(512²,锁频)。
diff --git a/docs/wasm.md b/docs/wasm.md
index e64fa47..606c68e 100644
--- a/docs/wasm.md
+++ b/docs/wasm.md
@@ -89,7 +89,7 @@ Transpose 现在是 gc / dr 里仅次于 Conv 的算子(未优化,两档相
各平台的 CPU 档位:x64 上 AVX-512 / AVX2,arm64 上 NEON,browser-wasm 上 **PackedSimd**;没有专门内核的算子(Transpose、Resize、MaxPool 等)仍走 `Vector`,再往下是标量。PackedSimd 档(`Nhwc.UsePacked = PackedSimd.IsSupported`)参照 SimdPaddleOCR 的 WASM 优化(`Nhwc.Conv.PackedSimd`),结构上对应本仓库 NEON 档已有的特化点:
-1. **4 像素 × 8 输出通道 micro-kernel `TilePkK`**(`Kernels/Nhwc.PackedSimd.cs`)
+1. **4 像素 × 8 输出通道 micro-kernel `TilePkK`**(Sdcb.SimdOnnx `src/Sdcb.SimdOnnx/Kernels/Nhwc.PackedSimd.cs`)
- V8 在 x64 上最多给 v128 分 15 个 XMM 寄存器,6 行 tile 会在内层循环溢出。改为 8 个累加器 + 2 个权重 + 1 个 splat,每个 16 通道打包块跑两遍。
- 输入直接 `v128.load32_splat` 从内存广播,乘、加分开(WASM SIMD 无 FMA),按 k 升序累加:与 `Vector` 档逐 lane 运算顺序相同。
- 用于 1×1 / Gemm(`GemmPk`)、直接卷积(`DenseRowPk`)和 Winograd 的 36 个小 GEMM(tile 行数随之改为 4)。
diff --git a/nuget.config b/nuget.config
new file mode 100644
index 0000000..9085df2
--- /dev/null
+++ b/nuget.config
@@ -0,0 +1,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Metal.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/Metal.cs
deleted file mode 100644
index 0263c36..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Metal.cs
+++ /dev/null
@@ -1,337 +0,0 @@
-using System.Runtime.InteropServices;
-
-namespace Sdcb.SimdGcdrNet.Backends.Metal;
-
-// Thin Metal wrappers over ObjC msgSend. +1-owned objects are released on
-// Dispose; autoreleased objects (commandBuffer, computeCommandEncoder, …) are
-// only valid inside the current NSAutoreleasePool — all public entry points
-// that create them take their own pool.
-
-// MTLResourceOptions: StorageModeShared = 0 << 4 = 0 (the zero value), which
-// is what the whole backend uses — unified memory means the same allocation
-// is CPU- and GPU-visible without didModifyRange/coherence calls.
-internal static class MtlStorageMode { public const nuint Shared = 0; }
-
-// MTLDataType values (MTLDataType.h) — only what NewPipeline's intConstants
-// path uses.
-internal static class MtlDataType
-{
- public const nuint Int = 29;
-}
-
-internal sealed class MtlDevice : IDisposable
-{
- public readonly IntPtr H;
- private readonly IntPtr _queue;
- // shader library + PSO cache, shared by every MetalGraphModel on this
- // device (each MtlGcdrPipeline carries gc+dr models — per-model
- // libraries would recompile ~2.5k lines of MSL three times and leak the
- // whole set on every release). Disposed with the device.
- private MtlLibrary? _lib;
- private readonly Dictionary _pipes = new();
-
- private MtlDevice(IntPtr h)
- {
- H = h;
- // newCommandQueue is +1 — owned, released in Dispose.
- _queue = ObjC.Send0(h, ObjC.S.NewCommandQueue);
- if (_queue == 0) throw new InvalidOperationException("Metal: newCommandQueue returned nil");
- }
-
- public IntPtr Queue => _queue;
-
- public string Name => ObjC.NsToString(ObjC.Send0(H, ObjC.S.Name)) ?? "?";
-
- // MTLGPUFamily enum values (MTLGPUFamily.h)
- public bool SupportsFamily(int family) => ObjC.SendBool1L(H, ObjC.S.SupportsFamily, family) != 0;
- public bool SupportsApple1 => SupportsFamily(1001);
- public bool SupportsApple2 => SupportsFamily(1002);
- public bool SupportsApple3 => SupportsFamily(1003);
- public bool SupportsApple4 => SupportsFamily(1004);
- public bool SupportsApple5 => SupportsFamily(1005);
- public bool SupportsApple6 => SupportsFamily(1006);
- public bool SupportsApple7 => SupportsFamily(1007);
- public bool SupportsApple8 => SupportsFamily(1008);
- public bool SupportsApple9 => SupportsFamily(1009);
- public bool SupportsMetal3 => SupportsFamily(5001);
- public bool SupportsMetal4 => SupportsFamily(5002);
-
- public nuint MaxThreadgroupMemoryLength => ObjC.SendNuint(H, ObjC.S.MaxThreadgroupMemoryLength);
-
- /// Defensive check for optional selectors — an unrecognized
- /// selector on this paravirt driver raises an ObjC exception that kills
- /// the process, so never call a maybe-absent method unconditionally.
- public bool RespondsTo(IntPtr sel) => ObjC.SendBool1P(H, ObjC.S.RespondsToSelector, sel) != 0;
- public bool HasUnifiedMemory => ObjC.SendBool(H, ObjC.S.HasUnifiedMemory) != 0;
- public ulong RecommendedMaxWorkingSetSize => (ulong)ObjC.SendNuint(H, ObjC.S.RecommendedMaxWorkingSetSize);
- public nuint CurrentAllocatedSize => ObjC.SendNuint(H, ObjC.S.CurrentAllocatedSize);
- public ulong RegistryID => (ulong)ObjC.SendNuint(H, ObjC.S.RegistryID);
-
- public static MtlDevice? Probe()
- {
- if (!OperatingSystem.IsMacOS()) return null;
- using var pool = AutoReleasePool.Create();
- // MTLCreateSystemDefaultDevice follows the Create rule: already +1.
- IntPtr h = ObjC.CreateSystemDefaultDevice();
- return h == 0 ? null : new MtlDevice(h);
- }
-
- /// Pipeline for a kernel in the embedded shader library,
- /// compiled once and cached per device.
- internal MtlPipeline Pipe(string name)
- {
- lock (_pipes)
- {
- if (!_pipes.TryGetValue(name, out MtlPipeline? p))
- {
- p = NewPipeline(ShaderLibrary(), name);
- // every kernel is dispatched at 256 threads/threadgroup
- if (p.MaxTotalThreadsPerThreadgroup < 256)
- {
- p.Dispose();
- throw new NotSupportedException(
- $"Metal: {name} pso supports {p.MaxTotalThreadsPerThreadgroup} threads/group");
- }
- _pipes[name] = p;
- }
- return p;
- }
- }
-
- private MtlLibrary ShaderLibrary()
- {
- if (_lib is not null) return _lib;
- var asm = typeof(MtlDevice).Assembly;
- var names = asm.GetManifestResourceNames()
- .Where(n => n.StartsWith("Sdcb.SimdGcdrNet.Backends.Metal.Shaders.")
- && n.EndsWith(".metal"))
- .OrderBy(n => n).ToArray();
- if (names.Length == 0)
- throw new FileNotFoundException("Metal: no embedded .metal shaders");
- var sb = new System.Text.StringBuilder();
- foreach (var n in names)
- {
- using Stream s = asm.GetManifestResourceStream(n)!;
- using var r = new StreamReader(s);
- sb.AppendLine(r.ReadToEnd());
- }
- return _lib = NewLibrary(sb.ToString());
- }
-
- public MtlBuffer NewBuffer(nuint bytes)
- {
- // MTLResourceStorageModeShared = 0 options.
- IntPtr b = ObjC.Send1N1N(H, ObjC.S.NewBufferWithLengthOptions, bytes, 0);
- if (b == 0) throw new InvalidOperationException($"Metal: newBufferWithLength({bytes}) failed");
- return new MtlBuffer(b, bytes);
- }
-
- public unsafe MtlBuffer NewBuffer(void* src, nuint bytes)
- {
- IntPtr b = ObjC.Send1P2N(H, ObjC.S.NewBufferWithBytesLengthOptions, (IntPtr)src, bytes, 0);
- if (b == 0) throw new InvalidOperationException($"Metal: newBufferWithBytes({bytes}) failed");
- return new MtlBuffer(b, bytes);
- }
-
- /// Compile MSL source into a library. Throws with the NSError
- /// localizedDescription on failure — MSL compile errors must never be
- /// silently swallowed.
- public unsafe MtlLibrary NewLibrary(string source, bool fastMath = false)
- {
- using var pool = AutoReleasePool.Create();
- IntPtr src = ObjC.NsStr(source);
- IntPtr opts = MakeCompileOptions(fastMath);
- IntPtr err = 0;
- IntPtr lib;
- lib = ObjC.Send3P(H, ObjC.S.NewLibraryWithSourceOptionsError, src, opts, (IntPtr)(&err));
- ObjC.Release(src);
- ObjC.Release(opts);
- if (lib == 0) throw new InvalidOperationException("MSL compile failed: " + DescribeError(err));
- return new MtlLibrary(lib);
- }
-
- private static IntPtr MakeCompileOptions(bool fastMath)
- {
- // MTLCompileOptions: mathMode is a private-ish key; the public knob is
- // fastMathEnabled (deprecated in MSL 3.x but still honored) plus
- // languageVersion. MSL 3.1 = 0x30001 << 16? No — MTLLanguageVersion is
- // (major<<16)+minor: 2.4=0x20004, 3.1=0x30001. We request 3.1 when
- // supported for simdgroup_matrix; harmless if clamped.
- IntPtr o = ObjC.Send0(ObjC.Send0(ObjC.ClassMTLCompileOptions, ObjC.S.Alloc), ObjC.S.Init);
- ObjC.SendV1N(o, ObjC.S.SetLanguageVersion, (nuint)0x00030001); // MSL 3.1
- ObjC.SendV1B(o, ObjC.S.SetFastMathEnabled, (byte)(fastMath ? 1 : 0));
- return o;
- }
-
- public unsafe MtlPipeline NewPipeline(MtlLibrary lib, string fnName, int[]? intConstants = null)
- {
- using var pool = AutoReleasePool.Create();
- IntPtr name = ObjC.NsStr(fnName);
- IntPtr fn;
- if (intConstants is { Length: > 0 })
- {
- IntPtr fc = ObjC.Send0(ObjC.Send0(ObjC.ClassMTLFunctionConstantValues, ObjC.S.Alloc), ObjC.S.Init);
- for (int i = 0; i < intConstants.Length; ++i)
- {
- int v = intConstants[i];
- // setConstantValue:type:atIndex: — (const void*, MTLDataType, NSUInteger).
- ObjC.Send3P(fc, ObjC.S.SetConstantValueTypeAtIndex, (IntPtr)(&v), (IntPtr)(nint)(long)MtlDataType.Int, (IntPtr)(nint)i);
- }
- IntPtr err = 0;
- fn = ObjC.Send3P(lib.H, ObjC.S.NewFunctionWithNameConstantValuesError, name, fc, (IntPtr)(&err));
- ObjC.Release(fc);
- if (fn == 0) { ObjC.Release(name); throw new InvalidOperationException($"Metal: function '{fnName}' failed: {DescribeError(err)}"); }
- }
- else
- {
- fn = ObjC.Send1P(lib.H, ObjC.S.NewFunctionWithName, name);
- }
- ObjC.Release(name);
- if (fn == 0) throw new InvalidOperationException($"Metal: no function '{fnName}' in library");
-
- IntPtr err2 = 0;
- IntPtr pso = ObjC.Send2P(H, ObjC.S.NewComputePipelineStateWithFunctionError, fn, (IntPtr)(&err2));
- ObjC.Release(fn);
- if (pso == 0) throw new InvalidOperationException($"Metal: PSO '{fnName}' failed: {DescribeError(err2)}");
- return new MtlPipeline(pso, fnName);
- }
-
- internal static unsafe string DescribeError(IntPtr err)
- {
- if (err == 0) return "(no NSError)";
- IntPtr desc = ObjC.Send0(err, ObjC.S.LocalizedDescription);
- return ObjC.NsToString(desc) ?? "(unreadable NSError)";
- }
-
- /// Begin a command buffer + compute encoder pair. Both are
- /// autoreleased — caller must hold an AutoReleasePool open.
- public MtlEncoder BeginCommands()
- {
- IntPtr cmd = ObjC.Send0(_queue, ObjC.S.CommandBuffer);
- IntPtr enc = ObjC.Send0(cmd, ObjC.S.ComputeCommandEncoder);
- return new MtlEncoder(cmd, enc);
- }
-
- /// Begin encoder with a specific dispatch type
- /// (MTLDispatchType: 0=Serial, 1=Concurrent — concurrent allows the GPU to
- /// overlap dispatches that don't depend on each other; serial inserts an
- /// implicit barrier between dispatches which is what the schedule assumes).
- public MtlEncoder BeginCommands(bool concurrent)
- {
- IntPtr cmd = ObjC.Send0(_queue, ObjC.S.CommandBuffer);
- IntPtr enc = concurrent
- ? ObjC.Send1N(cmd, ObjC.S.ComputeCommandEncoderWithDispatchType, 1)
- : ObjC.Send0(cmd, ObjC.S.ComputeCommandEncoder);
- return new MtlEncoder(cmd, enc);
- }
-
- public void Dispose()
- {
- lock (_pipes)
- {
- foreach (MtlPipeline p in _pipes.Values) p.Dispose();
- _pipes.Clear();
- }
- _lib?.Dispose();
- ObjC.Release(_queue);
- ObjC.Release(H);
- }
-}
-
-internal sealed class MtlBuffer : IDisposable
-{
- public IntPtr H;
- public readonly nuint Length;
- public MtlBuffer(IntPtr h, nuint length) { H = h; Length = length; }
- public IntPtr Contents => ObjC.Send0(H, ObjC.S.Contents);
- public void Dispose() { ObjC.Release(H); H = 0; }
-}
-
-internal sealed class MtlLibrary : IDisposable
-{
- public readonly IntPtr H;
- public MtlLibrary(IntPtr h) => H = h;
- public void Dispose() => ObjC.Release(H);
-}
-
-internal sealed class MtlPipeline : IDisposable
-{
- public readonly IntPtr Pso;
- public readonly string Name;
- public readonly nuint MaxTotalThreadsPerThreadgroup;
- public readonly nuint ThreadExecutionWidth;
- public MtlPipeline(IntPtr pso, string name)
- {
- Pso = pso;
- Name = name;
- MaxTotalThreadsPerThreadgroup = ObjC.SendNuint(pso, ObjC.S.MaxTotalThreadsPerThreadgroup);
- ThreadExecutionWidth = ObjC.SendNuint(pso, ObjC.S.ThreadExecutionWidth);
- }
- public void Dispose() => ObjC.Release(Pso);
-}
-
-/// One-shot command-buffer + compute-encoder pair. The objects are
-/// autoreleased; keep the enclosing AutoReleasePool alive until Commit has
-/// been issued (and waitUntilCompleted returned if you read results).
-internal readonly struct MtlEncoder
-{
- public readonly IntPtr Cmd;
- public readonly IntPtr Enc;
-
- public MtlEncoder(IntPtr cmd, IntPtr enc) { Cmd = cmd; Enc = enc; }
-
- public void SetPipeline(MtlPipeline p) => ObjC.Send1P(Enc, ObjC.S.SetComputePipelineState, p.Pso);
-
- public void SetBuffer(MtlBuffer buf, ulong byteOffset, int index) =>
- ObjC.SendV1P2N(Enc, ObjC.S.SetBufferOffsetAtIndex, buf.H, (nuint)byteOffset, (nuint)index);
-
- public unsafe void SetPc(in T value, int index) where T : unmanaged
- {
- fixed (T* p = &value)
- ObjC.SendV1P2N(Enc, ObjC.S.SetBytesLengthAtIndex, (IntPtr)p, (nuint)sizeof(T), (nuint)index);
- }
-
- public unsafe void SetPc(void* data, nuint bytes, int index) =>
- ObjC.SendV1P2N(Enc, ObjC.S.SetBytesLengthAtIndex, (IntPtr)data, bytes, (nuint)index);
-
- public void SetThreadgroupMemory(nuint bytes, int index) =>
- ObjC.SendV1N1N(Enc, ObjC.S.SetThreadgroupMemoryLengthAtIndex, bytes, (nuint)index);
-
- public void Dispatch(nuint gx, nuint gy, nuint gz, nuint tx, nuint ty, nuint tz) =>
- ObjC.SendV2Size(Enc, ObjC.S.DispatchThreadgroupsThreadsPerThreadgroup,
- new ObjC.MTLSize(gx, gy, gz), new ObjC.MTLSize(tx, ty, tz));
-
- /// MTLBarrierScope buffers(=1): visibility barrier between
- /// dispatches — needed when a later dispatch consumes a buffer written by
- /// an earlier one and we used a Concurrent encoder. Serial encoders imply
- /// this already.
- public void BarrierBuffers() => ObjC.SendV1N(Enc, ObjC.S.MemoryBarrierWithScope, 1);
-
- public void End() => ObjC.Send0(Enc, ObjC.S.EndEncoding);
-
- public void Commit() => ObjC.Send0(Cmd, ObjC.S.Commit);
-
- /// Wait for completion and check status; throws with NSError on
- /// any non-completed terminal state (incl. GPU watchdog kills).
- public void CommitAndWait()
- {
- Commit();
- ObjC.Send0(Cmd, ObjC.S.WaitUntilCompleted);
- long status = (long)ObjC.SendLong(Cmd, ObjC.S.Status);
- // MTLCommandBufferStatus: 4=Completed; 5=Error; others shouldn't occur post-wait.
- if (status != 4)
- {
- IntPtr err = ObjC.Send0(Cmd, ObjC.S.Error);
- throw new InvalidOperationException($"Metal command buffer failed (status={status}): {MtlDevice.DescribeError(err)}");
- }
- }
-
- /// GPU-side wall time of the buffer (seconds). 0 when timestamps
- /// are unavailable (paravirt drivers often return 0).
- public double GpuDurationSeconds()
- {
- double s = ObjC.SendDouble(Cmd, ObjC.S.GpuStartTime);
- double e = ObjC.SendDouble(Cmd, ObjC.S.GpuEndTime);
- return e > s ? e - s : 0;
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MetalBackend.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MetalBackend.cs
deleted file mode 100644
index 735e478..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MetalBackend.cs
+++ /dev/null
@@ -1,86 +0,0 @@
-using Sdcb.SimdGcdrNet.Engine;
-using Sdcb.SimdGcdrNet.Onnx;
-
-namespace Sdcb.SimdGcdrNet.Backends.Metal;
-
-///
-/// Metal backend root: the device, the per-(model, shape) graph cache, and
-/// factories for inference sessions and fused enhancer pipelines. Thread-safe;
-/// dispose with the process or when the backend is torn down.
-///
-internal sealed class MetalBackend : IDisposable
-{
- internal readonly MtlDevice Dev;
- private readonly Dictionary<(OnnxModel m, string shape, bool physIo), MtlGraphModel> _graphs = new();
- private readonly object _gate = new();
-
- private static readonly Lazy s_default = new(() => new MetalBackend());
- internal static MetalBackend Default => s_default.Value;
-
- internal static CompileOptions PlanOptions => new(Nhwc: true, PhysicalIo: false, Gpu: true);
- internal static CompileOptions PipeOptions => new(Nhwc: true, PhysicalIo: true, Gpu: true);
-
- internal MetalBackend()
- {
- Dev = MtlDevice.Probe() ?? throw new PlatformNotSupportedException("Metal: no device");
- if (Environment.GetEnvironmentVariable("GCDRNET_MTL_VERBOSE") == "1")
- Console.Error.WriteLine($"[metal] device: {Dev.Name}");
- }
-
- /// New inference session for a model and input shape (non-physical I/O plan).
- public MtlInferenceSession NewSession(OnnxModel m, int[] shape)
- {
- MtlGraphModel g = AcquireGraph(m, shape, physIo: false); // caller ref: later evictions can't free it under us
- try { return new MtlInferenceSession(Dev, g.Plan, g); }
- finally { g.Release(); }
- }
-
- /// The cached graph for (model, shape); acquires a reference for the caller.
- internal MtlGraphModel GraphFor(OnnxModel m, int[] shape, bool physIo = false)
- {
- var key = (m, string.Join('x', shape), physIo);
- lock (_gate)
- {
- if (_graphs.TryGetValue(key, out var gg))
- {
- gg.Acquire();
- return gg;
- }
- }
- ExecPlan plan = GraphCompiler.Compile(m, shape, physIo ? PipeOptions : PlanOptions);
- var g = new MtlGraphModel(Dev, plan, constKey: m);
- lock (_gate)
- {
- if (_graphs.TryGetValue(key, out var won)) { g.Release(); won.Acquire(); return won; }
- _graphs[key] = g;
- // bound the cache: graphs share the model's const store, so eviction only
- // frees the small per-size plan+dispatch list
- while (_graphs.Count > MaxGraphs)
- {
- var victim = _graphs.Keys.First(k => !k.Equals(key));
- _graphs.Remove(victim, out var old);
- old?.Release();
- }
- g.Acquire();
- return g;
- }
- }
- private const int MaxGraphs = 32;
-
- /// Alias: acquires the graph for the caller (caller must Release).
- internal MtlGraphModel AcquireGraph(OnnxModel m, int[] shape, bool physIo)
- => GraphFor(m, shape, physIo);
-
- /// New fused enhancer worker (size-agnostic; binds per frame).
- internal MtlGcdrPipeline NewWorker(OnnxModel gc, OnnxModel dr) => new(this, gc, dr);
-
- /// New fused enhancer worker pre-bound to a size.
- internal MtlGcdrPipeline NewPipe(OnnxModel gc, OnnxModel dr, int ih, int iw)
- {
- var p = new MtlGcdrPipeline(this, gc, dr);
- p.Bind(ih, iw);
- return p;
- }
-
- public void Dispose() => Dev.Dispose();
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlGraph.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlGraph.cs
deleted file mode 100644
index e06e77c..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlGraph.cs
+++ /dev/null
@@ -1,798 +0,0 @@
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using Sdcb.SimdGcdrNet.Engine;
-
-namespace Sdcb.SimdGcdrNet.Backends.Metal;
-
-/// Binding slots resolved per-session: which underlying buffer.
-internal enum MtlSlot : int { Arena = 0, Consts = 1, Dummy = 2, Direct = 3 }
-
-internal readonly struct MtlBind
-{
- public readonly MtlSlot Slot;
- public readonly MtlBuffer? Buf; // when Slot == Direct
- public readonly ulong Offset, Size;
- public MtlBind(MtlSlot s, ulong off, ulong size) { Slot = s; Offset = off; Size = size; }
- public MtlBind(MtlBuffer b, ulong off, ulong size) { Slot = MtlSlot.Direct; Buf = b; Offset = off; Size = size; }
-}
-
-/// One recorded compute dispatch.
-internal sealed class MtlDispatch
-{
- public required MtlPipeline Pipe;
- public required MtlBind[] Binds;
- public required byte[] Pc;
- public uint Gx, Gy = 1, Gz = 1;
- public string Name = "";
-}
-
-///
-/// Device-side const tensors (backend-packed weights) plus a 64B zero buffer,
-/// shared by every of one model: per-size plans copy
-/// the same initializers, so instead of re-packing and re-uploading the weights
-/// per input size, a new graph whose const list is element-for-element identical
-/// (same consumer packing, same contents) reuses the existing buffer. A graph
-/// whose consts differ gets a private, unshared store. Ref-counted; the
-/// buffers are freed when the last graph releases.
-///
-internal sealed class MtlConstStore
-{
- private static readonly Dictionary<(MtlDevice dev, object key), MtlConstStore> s_shared = new();
-
- private readonly (MtlDevice, object)? _key;
- private readonly float[][] _src;
- private readonly int[] _kinds;
- private int _refs = 1;
- internal readonly MtlBuffer Consts, Dummy;
- internal readonly ulong[] Offsets;
- internal readonly ulong Bytes;
-
- private MtlConstStore(MtlDevice dev, (MtlDevice, object)? key, float[][] src, int[] kinds,
- ulong[] offsets, byte[] host)
- {
- _key = key;
- _src = src;
- _kinds = kinds;
- Offsets = offsets;
- Bytes = (ulong)host.LongLength;
- unsafe
- {
- fixed (byte* hp = host) Consts = dev.NewBuffer(hp, (nuint)host.Length);
- byte* z = stackalloc byte[64];
- new Span(z, 64).Clear();
- Dummy = dev.NewBuffer(z, 64);
- }
- }
-
- ///
- /// Returns a store holding (kinds = consumer packing ids),
- /// shared under when an identical list was uploaded before.
- /// (offsets, host bytes) only runs when a new store is built.
- ///
- internal static MtlConstStore Acquire(MtlDevice dev, object? key, float[][] src, int[] kinds,
- Func<(ulong[] offsets, byte[] host)> pack)
- {
- if (key is null)
- {
- var (o, h) = pack();
- return new MtlConstStore(dev, null, src, kinds, o, h);
- }
- var k = (dev, key);
- lock (s_shared)
- {
- if (s_shared.TryGetValue(k, out MtlConstStore? s) && s.Matches(src, kinds))
- {
- s._refs++;
- return s;
- }
- var (offs, host) = pack();
- bool share = s is null; // keep the first registered store on a mismatch
- var ns = new MtlConstStore(dev, share ? k : null, src, kinds, offs, host);
- if (share) s_shared[k] = ns;
- return ns;
- }
- }
-
- private bool Matches(float[][] src, int[] kinds)
- {
- if (src.Length != _src.Length || !kinds.AsSpan().SequenceEqual(_kinds)) return false;
- for (int i = 0; i < src.Length; i++)
- if (!ReferenceEquals(src[i], _src[i]) && !src[i].AsSpan().SequenceEqual(_src[i])) return false;
- return true;
- }
-
- internal void Release()
- {
- lock (s_shared)
- {
- if (--_refs > 0) return;
- if (_key is { } k && s_shared.TryGetValue(k, out MtlConstStore? cur) && cur == this)
- s_shared.Remove(k);
- }
- Consts.Dispose();
- Dummy.Dispose();
- }
-}
-
-///
-/// Per-(device, ExecPlan) half of the Metal backend: backend-private constant
-/// packing/upload and the node list compiled to
-/// records. Sessions own the activation arena and resolve slot ids to live
-/// buffers. Activations and packed weights are stored as fp16; all shader math
-/// is fp32; conv bias and LayerNorm params stay fp32.
-///
-internal sealed class MtlGraphModel : IDisposable
-{
- private readonly MtlDevice _dev;
- private readonly ExecPlan _plan;
- private int _refs = 1; // the graph cache holds the first reference
- private readonly MtlConstStore _store;
- internal readonly MtlBuffer Consts; // all const tensors, backend-packed (shared store)
- private readonly Dictionary<(int v, CK k), ulong> _constOff = new();
- internal readonly MtlBuffer Dummy; // 64B zero buffer for unused bindings
- internal readonly MtlDispatch[] Dispatches;
- internal readonly ulong ConstBytes;
- /// Activations + weights stored as float16 (fp32 math in every shader).
- internal const bool F16 = true;
- /// Bytes per stored activation element.
- internal int Es => 2;
-
- /// How a const is laid out on the device (decided by its consumer).
- private enum CK { Raw, F32, ConvW, DwW, ConvWT, GemmNK }
-
- /// Metal device the graph is lowered for.
- /// Compiled execution plan to lower.
- /// Sharing key for the const store (the source model): graphs of the
- /// same model at other sizes reuse its uploaded weights. Null = private store.
- internal MtlGraphModel(MtlDevice dev, ExecPlan plan, object? constKey = null)
- {
- _dev = dev;
- _plan = plan;
- if (!plan.Nhwc) throw new NotSupportedException("Metal backend requires a layout-planned (NHWC) plan.");
-
- // ---- consts: per-consumer packing ----
- var need = new List<(int v, CK k)>();
- void Need(int v, CK k)
- {
- if (v < 0 || plan.Values[v].Floats is null) return;
- if (!need.Contains((v, k))) need.Add((v, k));
- }
- foreach (ExecNode n in plan.Nodes)
- {
- switch (n.P)
- {
- case ConvP cp:
- Need(n.In[0], CK.Raw);
- Need(n.In[1], cp.Group > 1 ? CK.DwW : UseMmConv(plan, n, cp) ? CK.ConvWT : CK.ConvW);
- Need(cp.BiasValue, CK.F32);
- Need(cp.AddValue, CK.Raw);
- break;
- case GemmP gp:
- Need(n.In[0], CK.Raw); Need(n.In[1], UseMmGemm(plan, n, gp) ? CK.GemmNK : CK.Raw);
- Need(gp.BiasValue, CK.F32);
- break;
- case LnP:
- Need(n.In[0], CK.Raw); Need(n.In[1], CK.F32); Need(n.In[2], CK.F32);
- break;
- case UpHeadP up:
- Need(n.In[2], CK.Raw);
- if (n.In.Length > 3) Need(n.In[3], CK.F32);
- if (up.SkipConv is not null) { Need(up.SkipW, CK.ConvW); Need(up.SkipB, CK.F32); }
- break;
- default:
- foreach (int iv in n.In) Need(iv, CK.Raw);
- break;
- }
- }
- foreach (var (v, k) in need)
- {
- CValue cv = plan.Values[v];
- if (k == CK.Raw && cv.Layout is { } l && !Engine.Layout.IsIdentity(l))
- throw new NotSupportedException($"Metal: const '{cv.Name}' with non-identity layout");
- }
- float[][] srcs = need.Select(x => plan.Values[x.v].Floats!).ToArray();
- int[] kinds = need.Select(x => (int)x.k).ToArray();
- _store = MtlConstStore.Acquire(dev, constKey, srcs, kinds, () =>
- {
- ulong total = 0;
- var offs = new ulong[need.Count];
- var items = new List<(ulong off, float[] data, bool half)>();
- for (int i = 0; i < need.Count; i++)
- {
- var (v, k) = need[i];
- CValue cv = plan.Values[v];
- float[] data = k switch
- {
- CK.ConvW => PackConv(cv),
- CK.ConvWT => PackConvT(cv),
- CK.GemmNK => PackGemmNK(cv, plan.Nodes.First(m => m.P is GemmP && m.In[1] == v)),
- CK.DwW => PackDw(cv),
- _ => cv.Floats!,
- };
- bool half = k != CK.F32;
- total = (total + 63) & ~63UL;
- offs[i] = total;
- items.Add((total, data, half));
- total += (ulong)data.Length * (half ? 2UL : 4UL);
- }
- byte[] host = new byte[Math.Max((total + 63) & ~63UL, 64)];
- foreach (var (off, data, half) in items)
- {
- if (half)
- {
- Span dst = MemoryMarshal.Cast(host.AsSpan(checked((int)off), data.Length * 2));
- for (int i = 0; i < data.Length; i++) dst[i] = (Half)data[i];
- }
- else
- MemoryMarshal.AsBytes(data.AsSpan()).CopyTo(host.AsSpan(checked((int)off)));
- }
- return (offs, host);
- });
- for (int i = 0; i < need.Count; i++) _constOff[need[i]] = _store.Offsets[i];
- Consts = _store.Consts;
- Dummy = _store.Dummy;
- ConstBytes = _store.Bytes;
-
- var disps = new List();
- NodeRanges = new (int start, int count)[plan.Nodes.Length];
- for (int i = 0; i < plan.Nodes.Length; i++)
- {
- int start = disps.Count;
- disps.AddRange(Emit(plan.Nodes[i]));
- NodeRanges[i] = (start, disps.Count - start);
- }
- Dispatches = disps.ToArray();
- }
-
- // ONNX [OC][IC][KH][KW] -> [K][OC], k = (ky*KW + kx)*IC + ic
- private static float[] PackConv(CValue w)
- {
- int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3];
- float[] src = w.Floats!, dst = new float[src.Length];
- for (int o = 0; o < oc; o++)
- for (int c = 0; c < ic; c++)
- for (int t = 0; t < taps; t++)
- dst[(t * ic + c) * oc + o] = src[(o * ic + c) * taps + t];
- return dst;
- }
-
- // ONNX [OC][IC][KH][KW] -> [OC][Kpad], k = (ky*KW + kx)*IC + ic, Kpad = K rounded up to 8
- private static float[] PackConvT(CValue w)
- {
- int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3], k = (taps * ic + 7) & ~7;
- float[] src = w.Floats!, dst = new float[oc * k];
- for (int o = 0; o < oc; o++)
- for (int c = 0; c < ic; c++)
- for (int t = 0; t < taps; t++)
- dst[o * k + t * ic + c] = src[(o * ic + c) * taps + t];
- return dst;
- }
-
- /// simdgroup-MMA conv eligibility (conv_mm): dense, output channels in
- /// 16 B quads, an epilogue it implements, M within the grid limit. cin%4!=0 uses
- /// the scalar-gather variant; constant pad must be zero.
- private static bool UseMmConv(ExecPlan plan, ExecNode n, ConvP cp)
- {
- if (cp.Group != 1 || cp.Pool is not null || cp.Dilations.Any(d => d != 1)) return false;
- if (cp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false;
- if (cp.PadMode == PadMode.Constant && cp.PadValue != 0f) return false;
- int[] so = plan.Values[n.Out[0]].PhysShape; // [1,h,w,c]
- long m = (long)so[1] * so[2];
- return so[3] % 8 == 0 && m / 64 <= int.MaxValue;
- }
-
- // Gemm B -> [N][K] (k fastest), the conv_mm weight layout
- private static float[] PackGemmNK(CValue b, ExecNode n)
- {
- var gp = (GemmP)n.P!;
- if (gp.TransB) return b.Floats!; // already [N][K]
- int k = b.Shape[0], nn = b.Shape[1];
- float[] src = b.Floats!, dst = new float[src.Length];
- for (int i = 0; i < k; i++)
- for (int j = 0; j < nn; j++) dst[j * k + i] = src[i * nn + j];
- return dst;
- }
-
- /// MMA Gemm (a 1x1 conv over M "pixels" through conv_mm): const B,
- /// K/N in 16 B quads, relu/gelu/none epilogue.
- private static bool UseMmGemm(ExecPlan plan, ExecNode n, GemmP gp)
- {
- if (plan.Values[n.In[1]].Floats is null || plan.Values[n.In[1]].Shape.Length != 2) return false;
- // B packed per value: only when no other node reads it (its transB could differ)
- if (plan.Nodes.Count(m => m.In.Contains(n.In[1])) != 1) return false;
- if (gp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false;
- CValue a = plan.Values[n.In[0]];
- int k = a.Shape[^1], nn = plan.Values[n.Out[0]].Shape[^1];
- if (a.PhysShape[^1] != k) return false;
- long m = a.ElementCount / k;
- return k % 8 == 0 && nn % 8 == 0 && m / 64 <= int.MaxValue;
- }
-
- // ONNX [C][1][KH][KW] -> [tap][C]
- private static float[] PackDw(CValue w)
- {
- int c = w.Shape[0], taps = w.Shape[2] * w.Shape[3];
- float[] src = w.Floats!, dst = new float[src.Length];
- for (int ch = 0; ch < c; ch++)
- for (int t = 0; t < taps; t++) dst[t * c + ch] = src[ch * taps + t];
- return dst;
- }
-
- /// Dispatch index range per source ExecNode (for per-node debug runs).
- internal readonly (int start, int count)[] NodeRanges;
-
- // ---- value → bind resolution (byte offsets/sizes: all long math) ----
- private MtlBind Bind(int valueId, CK kind = CK.Raw)
- {
- CValue v = _plan.Values[valueId];
- if (v.Buf >= 0)
- return new MtlBind(MtlSlot.Arena, (ulong)_plan.BufOffset[v.Buf] * (ulong)Es,
- (ulong)Math.Max(v.ElementCount, 1) * (ulong)Es);
- if (_constOff.TryGetValue((valueId, kind), out ulong off))
- {
- ulong es = kind != CK.F32 ? 2UL : 4UL;
- // ConvWT rows are K-padded to 8: bind the packed length, not the logical one
- long elems = kind == CK.ConvWT
- ? (long)v.Shape[0] * ((v.Shape[1] * v.Shape[2] * v.Shape[3] + 7) & ~7)
- : Math.Max(v.ElementCount, 1);
- return new MtlBind(MtlSlot.Consts, off, (ulong)elems * es);
- }
- if (v.Floats != null)
- throw new InvalidOperationException($"const value '{v.Name}' not packed as {kind}");
- throw new InvalidOperationException($"value '{v.Name}' has no GPU storage (ints must be folded)");
- }
-
- private static byte[] PcOf(params uint[] ints)
- {
- byte[] b = new byte[ints.Length * 4];
- Buffer.BlockCopy(ints, 0, b, 0, b.Length);
- return b;
- }
-
- private static byte[] PcOfF(uint[] ints, params float[] floats)
- {
- byte[] b = new byte[(ints.Length + floats.Length) * 4];
- Buffer.BlockCopy(ints, 0, b, 0, ints.Length * 4);
- Buffer.BlockCopy(floats, 0, b, ints.Length * 4, floats.Length * 4);
- return b;
- }
-
- private static uint[] U(params int[] v) { uint[] u = new uint[v.Length]; for (int i = 0; i < v.Length; i++) u[i] = (uint)v[i]; return u; }
-
- private static uint G(uint n) => (n + 255) / 256;
-
- ///
- /// Broadcast strides (padded rank-4, storage order of the output) of operand
- /// onto output . Output physical dim i
- /// holds logical dim o.Layout[i]; the operand's logical dim is that minus the
- /// numpy right-alignment offset.
- ///
- private static int[] PhysBroadcastStrides(CValue a, CValue o)
- {
- int rank = o.Shape.Length, off = rank - a.Shape.Length;
- if (rank > 4) throw new NotSupportedException("Metal elementwise: rank > 4");
- int[] lo = o.Layout ?? Engine.Layout.Identity(rank);
- int[] la = a.Layout ?? Engine.Layout.Identity(a.Shape.Length);
- int[] ap = a.PhysShape;
- int[] aStr = new int[ap.Length];
- for (int i = ap.Length - 1, acc = 1; i >= 0; i--) { aStr[i] = acc; acc *= ap[i]; }
- int[] r4 = new int[4];
- for (int i = 0; i < rank; i++)
- {
- int d = lo[i] - off;
- r4[4 - rank + i] = d < 0 || a.Shape[d] == 1 ? 0 : aStr[Array.IndexOf(la, d)];
- }
- return r4;
- }
-
- private IEnumerable Emit(ExecNode n)
- {
- CValue[] V = _plan.Values;
- // all addressing on storage-order shapes
- int[] shapeIn(int k) => V[n.In[k]].PhysShape;
- int[] shapeOut() => V[n.Out[0]].PhysShape;
- int nOut() => checked((int)V[n.Out[0]].ElementCount);
-
- switch (n.Op)
- {
- case ExecOp.Copy:
- yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf((uint)nOut(), 4), G((uint)nOut()));
- break;
-
- case ExecOp.Relu or ExecOp.Sigmoid or ExecOp.Erf or ExecOp.Gelu:
- {
- uint mode = n.Op switch
- {
- ExecOp.Relu => 0, ExecOp.Sigmoid => 1, ExecOp.Erf => 2, _ => 3,
- };
- yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf((uint)nOut(), mode), G((uint)nOut()));
- break;
- }
-
- case ExecOp.Elementwise:
- {
- var p = (EwP)n.P!;
- CValue a = V[n.In[0]], bv = V[n.In[1]], o = V[n.Out[0]];
- int[] os = Pad4(shapeOut());
- int[] ostr = Strides(os);
- int[] astr = PhysBroadcastStrides(a, o);
- int[] bstr = PhysBroadcastStrides(bv, o);
- uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, _ => 0u };
- // same-storage fast path: vec4 (n % 4 guard — no OOB tail writes)
- bool dense = astr.SequenceEqual(ostr) && bstr.SequenceEqual(ostr);
- if (nOut() % 4 == 0 && dense)
- {
- yield return ND("ew2v", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])],
- PcOf((uint)(nOut() / 4), (uint)p.Op, act), G((uint)(nOut() / 4)));
- break;
- }
- yield return ND("ew2", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])],
- PcOf(U(nOut(), ostr[0], ostr[1], ostr[2],
- astr[0], astr[1], astr[2], astr[3],
- bstr[0], bstr[1], bstr[2], bstr[3], (int)p.Op, (int)act)),
- G((uint)nOut()));
- break;
- }
-
- case ExecOp.Conv:
- {
- var p = (ConvP)n.P!;
- RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]);
- if (p.Pool is { } pool)
- {
- // small-ic stem conv + 2x2/2 max pool (GraphCompiler.GpuSmallConv)
- int[] ci = shapeIn(0), co = shapeOut(), cw = V[n.In[1]].Shape;
- if (!GraphCompiler.GpuSmallConv(p, cw, ci[3]) || p.AddValue >= 0 || ci[1] != co[1] * 2 || ci[2] != co[2] * 2)
- throw new NotSupportedException("Metal convpool: unsupported conv");
- uint pf = (p.BiasValue >= 0 ? 1u : 0u) | (p.PadMode == PadMode.Reflect ? 2u : 0u)
- | (p.Act == ActivationKind.Relu ? 4u : 0u) | (pool.FuseRelu ? 8u : 0u);
- bool k3 = p.Kernel[0] == 3 && p.Kernel[1] == 3;
- yield return ND(k3 ? "convpool_k3" : "convpool",
- [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), Bind(n.Out[0])],
- PcOfF(U(ci[3], ci[1], ci[2], cw[0], co[1], co[2], p.Kernel[0], p.Kernel[1], p.Pads[0], p.Pads[1], (int)pf),
- p.PadMode == PadMode.Constant ? p.PadValue : 0f),
- G((uint)co[1] * (uint)co[2] * (uint)(cw[0] / 8)), name: "convpool");
- break;
- }
- if (p.AddValue >= 0) RequireNhwc(p.AddValue);
- int[] si = shapeIn(0), sw_ = V[n.In[1]].Shape, so = shapeOut(); // [1,h,w,c]
- uint ih = (uint)si[1], iw = (uint)si[2], ic = (uint)si[3];
- uint oc = (uint)sw_[0], oh = (uint)so[1], ow = (uint)so[2];
- if (p.PadMode == PadMode.Constant && p.PadValue != 0f)
- throw new NotSupportedException("Metal conv: nonzero constant pad");
- if (p.Dilations.Any(d => d != 1))
- throw new NotSupportedException("Metal conv: dilation");
- if (p.Act == ActivationKind.Sigmoid)
- throw new NotSupportedException("Metal conv: sigmoid epilogue");
- uint flags = (p.BiasValue >= 0 ? 1u : 0u)
- | (p.PadMode == PadMode.Reflect ? 2u : 0u)
- | (p.Act == ActivationKind.Relu ? 4u : 0u)
- | (p.AddValue >= 0 ? 8u : 0u)
- | (p.Act == ActivationKind.Gelu ? 16u : 0u);
- if (p.Group > 1)
- {
- if (p.Group != ic || oc != ic) throw new NotSupportedException("grouped non-depthwise conv");
- if (p.AddValue >= 0) throw new NotSupportedException("depthwise conv + residual");
- bool v4 = ic % 4 == 0;
- yield return ND(v4 ? "depthwise4" : "depthwise",
- [Bind(n.In[0]), Bind(n.In[1], CK.DwW), B(p.BiasValue, CK.F32), Bind(n.Out[0])],
- PcOfF(U((int)ic, (int)ih, (int)iw, (int)oh, (int)ow, p.Kernel[0], p.Kernel[1],
- p.Strides[0], p.Strides[1], p.Pads[0], p.Pads[1], (int)(flags & ~8u)), p.PadValue),
- G(ic * oh * ow / (v4 ? 4u : 1u)), name: "depthwise");
- break;
- }
- if (UseMmConv(_plan, n, p))
- {
- // BM = 64 rows (pixels); BN by oc divisibility; SCK when cin%4!=0
- uint sck = ic % 4 != 0 ? 1u : 0u;
- (string cm, uint bn) = oc % 128 == 0 ? ("conv_mm_128", 128u)
- : oc % 64 == 0 ? ("conv_mm_64", 64u) : ("conv_mm_32", 32u);
- cm += "_" + sck;
- uint m = oh * ow, kr = (uint)(p.Kernel[0] * p.Kernel[1]) * ic, kd = (kr + 7u) & ~7u;
- uint cflags = (p.BiasValue >= 0 ? 1u : 0u) | (p.AddValue >= 0 ? 2u : 0u)
- | (p.PadMode == PadMode.Reflect ? 4u : 0u)
- | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4);
- yield return ND(cm,
- [Bind(n.In[0]), Bind(n.In[1], CK.ConvWT), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])],
- PcOf(m, oc, kd, cflags, ow, iw, ih, ic, (uint)p.Kernel[1],
- (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], kr),
- (m + 63) / 64, (oc + bn - 1) / bn, name: "conv_mm");
- break;
- }
- yield return ND("conv_tile",
- [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])],
- PcOf(ic, ih, iw, oc, oh, ow, (uint)p.Kernel[0], (uint)p.Kernel[1],
- (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1],
- flags, 0),
- (ow + 15) / 16, (oh + 7) / 8, (oc + 63) / 64, name: "conv");
- break;
- }
-
- case ExecOp.MaxPool:
- {
- var p = (PoolP)n.P!;
- RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]);
- int[] si = shapeIn(0), so = shapeOut();
- bool v4 = si[3] % 4 == 0;
- yield return ND(v4 ? "maxpool4" : "maxpool", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf((uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2],
- (uint)p.Kernel[0], (uint)p.Kernel[1], (uint)p.Strides[0], (uint)p.Strides[1],
- (uint)p.Pads[0], (uint)p.Pads[1], p.FuseRelu ? 1u : 0u),
- G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "maxpool");
- break;
- }
-
- case ExecOp.LayerNorm:
- {
- var p = (LnP)n.P!;
- uint len = (uint)V[n.In[0]].Shape[^1];
- if ((uint)shapeIn(0)[^1] != len) throw new NotSupportedException("Metal LayerNorm: last dim not innermost");
- uint rows = (uint)(V[n.In[0]].ElementCount / len);
- yield return ND("layernorm",
- [Bind(n.In[0]), Bind(n.In[1], CK.F32), Bind(n.In[2], CK.F32), Bind(n.Out[0])],
- PcOfF([len, rows], p.Eps), (rows + 7) / 8, name: "layernorm");
- break;
- }
-
- case ExecOp.Gemm:
- {
- var p = (GemmP)n.P!;
- uint k = (uint)V[n.In[0]].Shape[^1];
- if ((uint)shapeIn(0)[^1] != k) throw new NotSupportedException("Metal Gemm: K not innermost");
- uint m = (uint)(V[n.In[0]].ElementCount / k);
- uint nn = (uint)V[n.Out[0]].Shape[^1];
- if (UseMmGemm(_plan, n, p))
- {
- (string cm, uint bn) = nn % 128 == 0 ? ("conv_mm_128", 128u)
- : nn % 64 == 0 ? ("conv_mm_64", 64u) : ("conv_mm_32", 32u);
- cm += "_0";
- uint cflags = (p.BiasValue >= 0 ? 1u : 0u)
- | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4);
- // [M,K] x [N,K]^T as a 1x1 conv over a 1 x M image
- yield return ND(cm,
- [Bind(n.In[0]), Bind(n.In[1], CK.GemmNK), B(p.BiasValue, CK.F32), B(-1), Bind(n.Out[0])],
- PcOf(m, nn, k, cflags, m, m, 1u, k, 1u, 1u, 1u, 0u, 0u, k),
- (m + 63) / 64, (nn + bn - 1) / bn, name: "gemm_mm");
- break;
- }
- uint gflags = (p.TransB ? 1u : 0u) | (p.BiasValue >= 0 ? 2u : 0u)
- | (p.Act == ActivationKind.Relu ? 4u : 0u)
- | (p.Act == ActivationKind.Gelu ? 8u : 0u);
- yield return ND("gemm",
- [Bind(n.In[0]), Bind(n.In[1]), B(p.BiasValue, CK.F32), Bind(n.Out[0])],
- PcOf(m, nn, k, gflags),
- (nn + 63) / 64, (m + 63) / 64, name: "gemm");
- break;
- }
-
- case ExecOp.ResizeBilinear:
- {
- var p = (ResizeP)n.P!;
- RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]);
- bool hasAdd = n.In.Length > 1;
- if (hasAdd) RequireNhwc(n.In[1]);
- int[] si = shapeIn(0), so = shapeOut();
- uint flags = (p.FuseRelu ? 1u : 0u) | (hasAdd ? 2u : 0u) | (p.AddRelu ? 4u : 0u);
- bool v4 = si[3] % 4 == 0;
- yield return ND(v4 ? "bilinear4" : "bilinear", [Bind(n.In[0]), hasAdd ? Bind(n.In[1]) : B(-1), Bind(n.Out[0])],
- PcOfF([(uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], flags],
- p.Scales[2], p.Scales[3]),
- G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "bilinear");
- break;
- }
-
- case ExecOp.UpHead:
- {
- var p = (UpHeadP)n.P!;
- RequireNhwc(n.In[0]); RequireNhwc(n.In[1]); RequireNhwc(n.Out[0]);
- int[] si = shapeIn(0), so = shapeOut();
- uint c = (uint)si[3], oc = (uint)so[3];
- if (oc > 8 || c > 128 || c % 4 != 0) throw new NotSupportedException($"Metal UpHead: c={c} oc={oc}");
- bool hasBias = n.In.Length > 3;
- uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, ActivationKind.None => 0u,
- _ => throw new NotSupportedException($"Metal UpHead: {p.Act}") };
- uint flags = (p.ResizeRelu ? 1u : 0u) | (p.SkipRelu ? 2u : 0u) | (hasBias ? 4u : 0u) | (act << 3);
- if (p.SkipConv is { } sc)
- {
- // skip recomputed from the stem conv's input (GraphCompiler.GpuSmallConv, c <= 32)
- int[] xi = shapeIn(1), cw = V[p.SkipW].Shape;
- if (!GraphCompiler.GpuSmallConv(sc, cw, xi[3]) || cw[0] != c || c > 32 || xi[1] != so[1] || xi[2] != so[2])
- throw new NotSupportedException("Metal UpHead: unsupported skip conv");
- uint cf = (sc.BiasValue >= 0 ? 1u : 0u) | (sc.PadMode == PadMode.Reflect ? 2u : 0u)
- | (sc.Act == ActivationKind.Relu ? 4u : 0u);
- yield return ND("uphead_sc",
- [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0]),
- Bind(p.SkipW, CK.ConvW), B(p.SkipB, CK.F32)],
- PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags],
- p.Scales[2], p.Scales[3]).Concat(PcOfF(U(xi[3], sc.Kernel[0], sc.Kernel[1], sc.Pads[0], sc.Pads[1], (int)cf),
- sc.PadMode == PadMode.Constant ? sc.PadValue : 0f)).ToArray(),
- G((uint)so[1] * (uint)so[2]), name: "uphead_sc");
- break;
- }
- yield return ND("uphead",
- [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0])],
- PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags],
- p.Scales[2], p.Scales[3]),
- G((uint)so[1] * (uint)so[2]));
- break;
- }
-
- case ExecOp.Pad:
- {
- // generic rank-4 pad in storage order (pads permuted to physical dims)
- var p = (PadP)n.P!;
- CValue x = V[n.In[0]];
- if (x.Shape.Length != 4) throw new NotSupportedException("Metal pad: rank != 4");
- int[] l = x.Layout ?? Engine.Layout.Identity(4);
- int[] id4 = shapeIn(0), od4 = shapeOut();
- int[] pb4 = new int[4];
- for (int d = 0; d < 4; d++) pb4[d] = p.Pads[l[d]];
- yield return ND("pad", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOfF(U(id4[0], id4[1], id4[2], id4[3], od4[0], od4[1], od4[2], od4[3],
- pb4[0], pb4[1], pb4[2], pb4[3], p.Mode == PadMode.Reflect ? 1 : 0),
- p.Value),
- G((uint)V[n.Out[0]].ElementCount));
- break;
- }
-
- case ExecOp.Transpose:
- {
- // only planner relayouts reach here; the perm is physical
- var p = (TransposeP)n.P!;
- int[] is4 = Pad4(shapeIn(0));
- int[] os4 = Pad4(shapeOut());
- int[] ostr = Strides(os4), istr = Strides(is4);
- int[] perm4 = PermuteTo4(p.Perm, shapeIn(0).Length);
- yield return ND("transpose", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf(U(nOut(), ostr[0], ostr[1], ostr[2],
- istr[perm4[0]], istr[perm4[1]], istr[perm4[2]], istr[perm4[3]])),
- G((uint)nOut()));
- break;
- }
-
- case ExecOp.Slice:
- {
- // SliceP.Axes are physical after the layout pass
- var p = (SliceP)n.P!;
- int[] is4 = Pad4(shapeIn(0)), os4 = Pad4(shapeOut());
- int[] ostr = Strides(os4), istr = Strides(is4);
- uint[] st = [0, 0, 0, 0], sp = [1, 1, 1, 1];
- int shift = 4 - shapeIn(0).Length;
- for (int i = 0; i < p.Axes.Length; i++)
- {
- st[p.Axes[i] + shift] = (uint)p.Starts[i];
- sp[p.Axes[i] + shift] = (uint)p.Steps[i];
- }
- yield return ND("slice", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf(U(nOut(), ostr[0], ostr[1], ostr[2],
- (int)st[0], (int)st[1], (int)st[2], (int)st[3],
- (int)sp[0], (int)sp[1], (int)sp[2], (int)sp[3],
- istr[0], istr[1], istr[2], istr[3])),
- G((uint)nOut()));
- break;
- }
-
- case ExecOp.Concat:
- {
- // ConcatP.Axis is physical after the layout pass
- var p = (ConcatP)n.P!;
- int[] os = shapeOut();
- int axis = p.Axis;
- int inner = 1, dstAxis = os[axis];
- for (int d = axis + 1; d < os.Length; d++) inner *= os[d];
- int axisOff = 0;
- foreach (int iv in n.In)
- {
- int[] ishape = V[iv].PhysShape;
- int cnt = checked((int)V[iv].ElementCount);
- if (cnt > 0)
- {
- yield return ND("concat_ax", [Bind(iv), Bind(n.Out[0])],
- PcOf((uint)cnt, (uint)inner, (uint)ishape[axis], (uint)dstAxis, (uint)axisOff),
- G((uint)cnt));
- }
- axisOff += ishape[axis];
- }
- break;
- }
-
- case ExecOp.ChannelShift:
- {
- var p = (ShiftP)n.P!;
- RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]);
- int[] si = shapeIn(0); // [1,h,w,c]
- var ge = p.GroupEnds; var of = p.Offsets;
- if (ge.Length > 5) throw new NotSupportedException("Metal shift: > 5 groups");
- var pc = new List { (uint)si[3], (uint)si[1], (uint)si[2], (uint)p.Axis };
- for (int i = 0; i < 5; i++) pc.Add((uint)(i < ge.Length ? ge[i] : si[3]));
- for (int i = 0; i < 5; i++) pc.Add((uint)(i < of.Length ? of[i] : 0));
- bool v4 = si[3] % 4 == 0 && ge.All(e => e % 4 == 0);
- yield return ND(v4 ? "shift4" : "shift", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf(pc.ToArray()), G((uint)nOut() / (v4 ? 4u : 1u)), name: "shift");
- break;
- }
-
- default:
- throw new NotSupportedException($"Metal: ExecOp {n.Op} not implemented");
- }
-
- MtlDispatch ND(string shader, MtlBind[] binds, byte[] pc, uint gx, uint gy = 1, uint gz = 1, string? name = null)
- {
- return new MtlDispatch
- {
- Pipe = _dev.Pipe(shader),
- Pc = pc, Binds = binds, Gx = gx, Gy = gy, Gz = gz,
- Name = $"{shader}({n.Name})",
- };
- }
-
- MtlBind B(int valueId, CK kind = CK.Raw) => valueId >= 0 ? Bind(valueId, kind) : new MtlBind(MtlSlot.Dummy, 0, 64);
- }
-
- private void RequireNhwc(int v)
- {
- CValue cv = _plan.Values[v];
- if (cv.Shape.Length != 4 || !Engine.Layout.Equivalent(cv.Shape, cv.Layout ?? Engine.Layout.Identity(4), Engine.Layout.Nhwc))
- throw new NotSupportedException($"Metal: '{cv.Name}' expected NHWC storage, got {Engine.Layout.Str(cv.Layout)}");
- }
-
- // pad a shape to rank 4 with leading 1s
- private static int[] Pad4(int[] s)
- {
- if (s.Length == 4) return s;
- int[] r = [1, 1, 1, 1];
- s.CopyTo(r, 4 - s.Length);
- return r;
- }
-
- private static int[] Strides(int[] shape4)
- {
- int[] s = new int[4]; s[3] = 1;
- for (int i = 2; i >= 0; i--) s[i] = s[i + 1] * shape4[i + 1];
- return s;
- }
-
- // lift an original-rank perm into padded rank-4 (leading dims map to themselves)
- private static int[] PermuteTo4(int[] perm, int rank)
- {
- int shift = 4 - rank;
- int[] p4 = [0, 1, 2, 3];
- for (int i = 0; i < rank; i++) p4[i + shift] = perm[i] + shift;
- return p4;
- }
-
- /// Sessions hold a reference so cache eviction can't free live const/dummy buffers.
- public void Acquire()
- {
- if (Interlocked.Increment(ref _refs) <= 1)
- throw new ObjectDisposedException(nameof(MtlGraphModel), "const buffers already freed");
- }
- /// Debug/test: const buffer + byte offset of (value, packing-kind) for diagnostics.
- internal (MtlBuffer buf, ulong off, bool half) DebugConst(int valueId, int kind)
- {
- return _constOff.TryGetValue((valueId, (CK)kind), out ulong o)
- ? (Consts, o, (CK)kind != CK.F32)
- : (Dummy, ulong.MaxValue, false);
- }
-
- /// Acquire unless already freed (no resurrection; for racing cache readers).
- public bool TryAcquire()
- {
- while (true)
- {
- int r = Volatile.Read(ref _refs);
- if (r <= 0) return false;
- if (Interlocked.CompareExchange(ref _refs, r + 1, r) == r) return true;
- }
- }
- public void Release()
- {
- if (Interlocked.Decrement(ref _refs) == 0) _store.Release();
- }
-
- /// The plan this graph was compiled from (arena offsets, I/O values).
- internal ExecPlan Plan => _plan;
- public void Dispose() => Release();
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs
index b55f67c..6e49615 100644
--- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs
+++ b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs
@@ -1,5 +1,7 @@
-using Sdcb.SimdGcdrNet.Engine;
-using Sdcb.SimdGcdrNet.Onnx;
+using Sdcb.SimdGcdrNet.Extensions;
+using Sdcb.SimdOnnx.Backends.Metal;
+using Sdcb.SimdOnnx.Engine;
+using Sdcb.SimdOnnx.Onnx;
namespace Sdcb.SimdGcdrNet.Backends.Metal;
@@ -56,9 +58,9 @@ internal void Bind(int ih, int iw)
if (_gcG is null || h != _h || w != _w)
{
// a new size compiles two plans + graphs: build them side by side
- var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], physIo: true));
+ var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], physIo: true, GcdrExtensions.Instance));
MtlGraphModel gd;
- try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], physIo: true); }
+ try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], physIo: true, GcdrExtensions.Instance); }
catch { try { tg.GetAwaiter().GetResult().Release(); } catch { } throw; }
MtlGraphModel gg;
try { gg = tg.GetAwaiter().GetResult(); }
@@ -109,7 +111,7 @@ private void RecordAll(MtlEncoder enc, int srcStride)
uint h = (uint)_h, w = (uint)_w;
// bgr2nhwc -> gc arena input
- enc.SetPipeline(_dev.Pipe("bgr2nhwc"));
+ enc.SetPipeline(_dev.Pipe("bgr2nhwc", GcdrExtensions.Owner));
enc.SetBuffer(_bgrIn!, 0, 0);
enc.SetBuffer(_arena!, _gcBase + InOff(gcPlan), 1);
var pc = stackalloc uint[7] { h, w, (uint)_ih, (uint)_iw, (uint)_pt, (uint)_pl, (uint)srcStride };
@@ -121,7 +123,7 @@ private void RecordAll(MtlEncoder enc, int srcStride)
// div6: gc input + gc output -> dr arena input
uint hw = h * w;
- enc.SetPipeline(_dev.Pipe("div6"));
+ enc.SetPipeline(_dev.Pipe("div6", GcdrExtensions.Owner));
enc.SetBuffer(_arena!, _gcBase + InOff(gcPlan), 0);
enc.SetBuffer(_arena!, _gcBase + OutOff(gcPlan), 1);
enc.SetBuffer(_arena!, InOff(drPlan), 2);
@@ -134,7 +136,7 @@ private void RecordAll(MtlEncoder enc, int srcStride)
// nhwc2bgr -> bgrOut
uint nwords = ((uint)_ih * (uint)_iw * 3u + 3u) / 4u;
- enc.SetPipeline(_dev.Pipe("nhwc2bgr"));
+ enc.SetPipeline(_dev.Pipe("nhwc2bgr", GcdrExtensions.Owner));
enc.SetBuffer(_arena!, OutOff(drPlan), 0);
enc.SetBuffer(_bgrOut!, 0, 1);
var pc3 = stackalloc uint[7] { h, w, (uint)_ih, (uint)_iw, (uint)_pt, (uint)_pl, nwords };
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlSession.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlSession.cs
deleted file mode 100644
index ceaab1e..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlSession.cs
+++ /dev/null
@@ -1,250 +0,0 @@
-using Sdcb.SimdGcdrNet.Engine;
-
-namespace Sdcb.SimdGcdrNet.Backends.Metal;
-
-///
-/// GPU counterpart of : one activation arena in
-/// shared (CPU+GPU-visible) memory; Run() writes the input window straight
-/// into the arena, records the graph's dispatches into one command buffer and
-/// waits once — no staging copies (unified memory). Not thread-safe.
-///
-public sealed unsafe class MtlInferenceSession : IDisposable
-{
- private readonly MtlDevice _dev;
- private readonly MtlGraphModel _graph;
- private readonly ExecPlan _plan;
-
- internal readonly MtlBuffer Arena; // shared-memory activation arena
- private readonly float[] _inHost, _outHost;
- private readonly ulong _inOff, _outOff;
- private readonly int _inFloats, _outFloats;
- internal int Es => _graph.Es; // bytes per stored element (2 = fp16)
-
- internal MtlInferenceSession(MtlDevice dev, ExecPlan plan, MtlGraphModel graph)
- {
- _dev = dev;
- _plan = plan;
- _graph = graph;
- _graph.Acquire();
-
- Arena = dev.NewBuffer((nuint)((ulong)plan.ArenaFloats * (ulong)Es + 64));
- _inFloats = checked((int)plan.Values[plan.InputValue].ElementCount);
- _outFloats = checked((int)plan.Values[plan.OutputValue].ElementCount);
- _inOff = (ulong)plan.BufOffset[plan.Values[plan.InputValue].Buf] * (ulong)Es;
- _outOff = (ulong)plan.BufOffset[plan.Values[plan.OutputValue].Buf] * (ulong)Es;
- _inHost = new float[_inFloats];
- _outHost = new float[_outFloats];
- }
-
- internal MtlBuffer ResolveBuf(MtlBind b) => Resolve(b).buf;
- internal ulong ResolveOff(MtlBind b) => b.Offset;
- internal (MtlBuffer buf, ulong off, ulong size) Resolve(MtlBind b) => b.Slot switch
- {
- MtlSlot.Arena => (Arena, b.Offset, b.Size),
- MtlSlot.Consts => (_graph.Consts, b.Offset, b.Size),
- MtlSlot.Direct => (b.Buf!, b.Offset, b.Size),
- _ => (_graph.Dummy, b.Offset, b.Size),
- };
-
- /// Input tensor shape (e.g. [1,3,H,W]).
- public int[] InputShape => _plan.Values[_plan.InputValue].Shape;
- /// Output tensor shape (e.g. [1,3,H,W]).
- public int[] OutputShape => _plan.Values[_plan.OutputValue].Shape;
-
- /// The writable input window (staged into the arena on ).
- public Span Input => _inHost;
- /// The output tensor after .
- public ReadOnlySpan Output => _outHost;
-
- internal ulong InputOffset => _inOff;
- internal ulong OutputOffset => _outOff;
- internal ulong InputBytes => (ulong)_inFloats * (ulong)Es;
- internal ulong OutputBytes => (ulong)_outFloats * (ulong)Es;
-
- /// Host float -> stored elements (fp16 narrowing).
- internal static void ToStorage(ReadOnlySpan src, void* dst)
- {
- var d = new Span(dst, src.Length);
- for (int i = 0; i < src.Length; i++) d[i] = (Half)src[i];
- }
-
- /// Stored elements -> host float (fp16 widening).
- internal static void FromStorage(void* src, Span dst)
- {
- var sp = new ReadOnlySpan(src, dst.Length);
- for (int i = 0; i < dst.Length; i++) dst[i] = (float)sp[i];
- }
-
- /// Downloads dst.Length stored elements at byte offset .
- internal void DownloadElems(MtlBuffer buf, ulong off, Span dst)
- => FromStorage((byte*)buf.Contents + off, dst);
-
- /// Record one dispatch on an open encoder.
- internal static void RecordOne(MtlEncoder enc, MtlGraphModel g,
- MtlDispatch d, MtlBuffer arena, ulong arenaBase = 0)
- {
- enc.SetPipeline(d.Pipe);
- for (int b = 0; b < d.Binds.Length; b++)
- {
- MtlBind mb = d.Binds[b];
- (MtlBuffer buf, ulong off, _) = mb.Slot switch
- {
- MtlSlot.Arena => (arena, arenaBase + mb.Offset, mb.Size),
- MtlSlot.Consts => (g.Consts, mb.Offset, mb.Size),
- MtlSlot.Direct => (mb.Buf!, mb.Offset, mb.Size),
- _ => (g.Dummy, mb.Offset, mb.Size),
- };
- enc.SetBuffer(buf, off, b);
- }
- if (d.Pc.Length > 0)
- fixed (byte* pc = d.Pc)
- enc.SetPc(pc, (nuint)d.Pc.Length, d.Binds.Length);
- enc.Dispatch(d.Gx, d.Gy, d.Gz, 256, 1, 1);
- }
-
- /// Record all dispatches on an open encoder (serial dispatch order).
- internal void RecordCompute(MtlEncoder enc)
- {
- foreach (var d in _graph.Dispatches)
- RecordOne(enc, _graph, d, Arena);
- }
-
- /// Debug/test: run the dispatches of ExecNode i (sequential state persists).
- internal void DebugRunNode(int i)
- {
- var (start, count) = _graph.NodeRanges[i];
- if (count > 0) DebugRunRange(start, count);
- }
-
- /// Debug/test: a value's arena window as host floats.
- internal float[] DebugValue(int valueId)
- {
- var dst = new float[checked((int)_plan.Values[valueId].ElementCount)];
- DebugDownload(valueId, dst);
- return dst;
- }
-
- /// Debug/test: run a single dispatch-list slice on its own command buffer.
- internal void DebugRunRange(int dStart, int dCount)
- {
- using var pool = AutoReleasePool.Create();
- var enc = _dev.BeginCommands();
- for (int i = dStart; i < dStart + dCount; i++)
- RecordOne(enc, _graph, _graph.Dispatches[i], Arena);
- enc.End();
- enc.CommitAndWait();
- }
-
- ///
- /// Debug/test: time each dispatch by running it alone (commit + wait) and
- /// wall-timing; includes ~45us launch overhead per dispatch — rank only.
- /// Returns name→best-of-2 ms.
- ///
- internal Dictionary DebugProfile()
- {
- int n = _graph.Dispatches.Length;
- var best = new double[n];
- Array.Fill(best, double.MaxValue);
- // warm pass to keep state identical to production (input already in arena)
- using (var pool = AutoReleasePool.Create())
- {
- var enc = _dev.BeginCommands();
- foreach (var d in _graph.Dispatches) RecordOne(enc, _graph, d, Arena);
- enc.End();
- enc.CommitAndWait();
- }
- var sw = new System.Diagnostics.Stopwatch();
- for (int rep = 0; rep < 2; rep++)
- for (int i = 0; i < n; i++)
- {
- var d = _graph.Dispatches[i];
- using var pool = AutoReleasePool.Create();
- var enc = _dev.BeginCommands();
- RecordOne(enc, _graph, d, Arena);
- enc.End();
- sw.Restart();
- enc.CommitAndWait();
- double ms = sw.Elapsed.TotalMilliseconds;
- if (ms < best[i]) best[i] = ms;
- }
- var agg = new Dictionary();
- PerDispatch = new (string, double)[n];
- for (int i = 0; i < n; i++)
- {
- string name = _graph.Dispatches[i].Name ?? $"d{i}";
- PerDispatch[i] = (name, best[i]);
- name = name[..(name.IndexOf('(') is int cut && cut > 0 ? cut : name.Length)];
- agg[name] = agg.TryGetValue(name, out double v) ? v + best[i] : best[i];
- }
- return agg;
- }
-
- /// Per-dispatch timings from the last call.
- internal (string Name, double Ms)[]? PerDispatch;
-
- /// Debug: the compiled plan backing this session.
- internal ExecPlan DebugPlan => _plan;
-
- /// Debug/test: const offset (ulong.MaxValue when not packed).
- internal ulong DebugConstOff(int valueId, int kind) => _graph.DebugConst(valueId, kind).off;
-
- /// Debug/test: const storage of a value (kind 0 raw,1 f32,2 convw,3 dww,4 convwt,5 gemmnk).
- internal float[] DebugConst(int valueId, int kind, int len)
- {
- var (buf, off, half) = _graph.DebugConst(valueId, kind);
- var dst = new float[len];
- if (half)
- {
- var h = new ReadOnlySpan((byte*)buf.Contents + off, len);
- for (int i = 0; i < len; i++) dst[i] = (float)h[i];
- }
- else
- new ReadOnlySpan((byte*)buf.Contents + off, len).CopyTo(dst);
- return dst;
- }
-
- /// Debug/test: dispatch names for ExecNode i.
- internal string[] DebugNodeDispatches(int i)
- {
- var (st, cnt) = _graph.NodeRanges[i];
- return _graph.Dispatches.Skip(st).Take(cnt).Select(d => d.Name ?? "").ToArray();
- }
-
- /// Debug: read the whole arena (ArenaFloats floats) into dst.
- internal void DebugArena(Span dst)
- => FromStorage(Arena.Contents.ToPointer(), dst.Slice(0, _plan.ArenaFloats));
-
- /// Debug: graph consts buffer + the session's arena buffer.
- internal (MtlBuffer consts, MtlBuffer arena) DebugBuffers => (_graph.Consts, Arena);
-
- /// Debug/test: download an activation value's region into dst.
- internal void DebugDownload(int valueId, float[] dst)
- {
- CValue v = _plan.Values[valueId];
- FromStorage((byte*)Arena.Contents + (ulong)_plan.BufOffset[v.Buf] * (ulong)Es,
- dst.AsSpan(0, checked((int)v.ElementCount)));
- }
-
- /// Debug/test: copy host input into the arena's input region.
- internal void DebugUploadInput()
- => ToStorage(_inHost, (byte*)Arena.Contents + _inOff);
-
- /// Write input into the arena and run all dispatches.
- public void Run()
- {
- ToStorage(_inHost, (byte*)Arena.Contents + _inOff);
- using var pool = AutoReleasePool.Create();
- var enc = _dev.BeginCommands();
- RecordCompute(enc);
- enc.End();
- enc.CommitAndWait();
- FromStorage((byte*)Arena.Contents + _outOff, _outHost);
- }
-
- /// Releases the session's device arena and its reference to the shared graph.
- public void Dispose()
- {
- Arena.Dispose();
- _graph.Release();
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/ObjC.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/ObjC.cs
deleted file mode 100644
index d0852a9..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Metal/ObjC.cs
+++ /dev/null
@@ -1,229 +0,0 @@
-using System.Runtime.InteropServices;
-
-namespace Sdcb.SimdGcdrNet.Backends.Metal;
-
-// Minimal libobjc P/Invoke layer — pure net10.0 C#, no macOS workload.
-// Ownership follows Cocoa rules: alloc/init/newXxx/copy return +1 (caller
-// releases); everything else is autoreleased within the enclosing
-// NSAutoreleasePool. Selectors and class handles are cached in static
-// readonly fields — a dispatch already costs enough objc_msgSend calls that
-// re-registering selector strings per call is wasted marshaling.
-internal static unsafe partial class ObjC
-{
- private const string LibObjC = "libobjc.dylib";
- private const string LibMetal = "/System/Library/Frameworks/Metal.framework/Metal";
-
- [LibraryImport(LibObjC, EntryPoint = "objc_getClass")]
- private static partial IntPtr GetClassRaw(IntPtr name);
-
- [LibraryImport(LibObjC, EntryPoint = "sel_registerName")]
- private static partial IntPtr RegSelRaw(IntPtr name);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial IntPtr Send0(IntPtr recv, IntPtr sel);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial IntPtr Send1P(IntPtr recv, IntPtr sel, IntPtr a);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial IntPtr Send2P(IntPtr recv, IntPtr sel, IntPtr a, IntPtr b);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial IntPtr Send3P(IntPtr recv, IntPtr sel, IntPtr a, IntPtr b, IntPtr c);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial IntPtr Send1N(IntPtr recv, IntPtr sel, nuint a);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial IntPtr Send1P1N(IntPtr recv, IntPtr sel, IntPtr a, nuint b);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial IntPtr Send2P1N(IntPtr recv, IntPtr sel, IntPtr a, IntPtr b, nuint c);
-
- // newBufferWithBytes:length:options: / newBufferWithLength:options:
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial IntPtr Send1P2N(IntPtr recv, IntPtr sel, IntPtr a, nuint b, nuint c);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial IntPtr Send1N1N(IntPtr recv, IntPtr sel, nuint a, nuint b);
-
- // setBuffer:offset:atIndex: / setBytes:length:atIndex:
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial void SendV1P2N(IntPtr recv, IntPtr sel, IntPtr a, nuint b, nuint c);
-
- // dispatchThreadgroups:threadsPerThreadgroup: — MTLSize by value (24B)
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial void SendV2Size(IntPtr recv, IntPtr sel, MTLSize a, MTLSize b);
-
- // setThreadgroupMemoryLength:atIndex:
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial void SendV1N1N(IntPtr recv, IntPtr sel, nuint a, nuint b);
-
- // single-NSUInteger void calls (setLanguageVersion:, memoryBarrierWithScope:, …)
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial void SendV1N(IntPtr recv, IntPtr sel, nuint a);
-
- // single-BOOL void calls (setFastMathEnabled:…) — BOOL is 1 byte on arm64.
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial void SendV1B(IntPtr recv, IntPtr sel, byte a);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial byte SendBool(IntPtr recv, IntPtr sel);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial byte SendBool1P(IntPtr recv, IntPtr sel, IntPtr a);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial byte SendBool1N(IntPtr recv, IntPtr sel, nuint a);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial nuint SendNuint(IntPtr recv, IntPtr sel);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial double SendDouble(IntPtr recv, IntPtr sel);
-
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial long SendLong(IntPtr recv, IntPtr sel);
-
- // supportsFamily: — (id,SEL,NSInteger)
- [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")]
- public static partial byte SendBool1L(IntPtr recv, IntPtr sel, nint a);
-
- [LibraryImport(LibMetal, EntryPoint = "MTLCreateSystemDefaultDevice")]
- public static partial IntPtr CreateSystemDefaultDevice();
-
- [StructLayout(LayoutKind.Sequential)]
- public readonly struct MTLSize
- {
- public readonly nuint X, Y, Z;
- public MTLSize(nuint x, nuint y, nuint z) { X = x; Y = y; Z = z; }
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public readonly struct NSRange
- {
- public readonly nuint Location, Length;
- public NSRange(nuint location, nuint length) { Location = location; Length = length; }
- }
-
- // ---- cached class/selector tables ------------------------------------
- // sel_registerName returns the same SEL for the same string, so register
- // once. Classes likewise are process-wide stable.
-
- private static readonly IntPtr s_classNsString = Cls("NSString");
- private static readonly IntPtr s_classNsAutoreleasePool = Cls("NSAutoreleasePool");
- private static readonly IntPtr s_classMtlCompileOptions = Cls("MTLCompileOptions");
- private static readonly IntPtr s_classMtlFunctionConstantValues = Cls("MTLFunctionConstantValues");
-
- public static IntPtr ClassNSString => s_classNsString;
- public static IntPtr ClassNSAutoreleasePool => s_classNsAutoreleasePool;
- public static IntPtr ClassMTLCompileOptions => s_classMtlCompileOptions;
- public static IntPtr ClassMTLFunctionConstantValues => s_classMtlFunctionConstantValues;
-
- public static IntPtr Cls(string name)
- {
- IntPtr n = Marshal.StringToHGlobalAnsi(name);
- try { return GetClassRaw(n); } finally { Marshal.FreeHGlobal(n); }
- }
-
- public static IntPtr Sel(string name)
- {
- IntPtr n = Marshal.StringToHGlobalAnsi(name);
- try { return RegSelRaw(n); } finally { Marshal.FreeHGlobal(n); }
- }
-
- public static class S
- {
- public static readonly IntPtr Alloc = Sel("alloc");
- public static readonly IntPtr Init = Sel("init");
- public static readonly IntPtr Release = Sel("release");
- public static readonly IntPtr Retain = Sel("retain");
- public static readonly IntPtr Drain = Sel("drain");
- public static readonly IntPtr InitWithUTF8String = Sel("initWithUTF8String:");
- public static readonly IntPtr Utf8String = Sel("UTF8String");
- public static readonly IntPtr LocalizedDescription = Sel("localizedDescription");
- public static readonly IntPtr Length = Sel("length");
-
- public static readonly IntPtr Name = Sel("name");
- public static readonly IntPtr RespondsToSelector = Sel("respondsToSelector:");
- public static readonly IntPtr SupportsFamily = Sel("supportsFamily:");
- public static readonly IntPtr MaxThreadgroupMemoryLength = Sel("maxThreadgroupMemoryLength");
- public static readonly IntPtr MaxThreadsPerThreadgroup = Sel("maxThreadsPerThreadgroup");
- public static readonly IntPtr MaxTotalThreadsPerThreadgroup = Sel("maxTotalThreadsPerThreadgroup");
- public static readonly IntPtr ThreadExecutionWidth = Sel("threadExecutionWidth");
- public static readonly IntPtr HasUnifiedMemory = Sel("hasUnifiedMemory");
- public static readonly IntPtr RecommendedMaxWorkingSetSize = Sel("recommendedMaxWorkingSetSize");
- public static readonly IntPtr CurrentAllocatedSize = Sel("currentAllocatedSize");
- public static readonly IntPtr RegistryID = Sel("registryID");
-
- public static readonly IntPtr NewCommandQueue = Sel("newCommandQueue");
- public static readonly IntPtr NewBufferWithBytesLengthOptions = Sel("newBufferWithBytes:length:options:");
- public static readonly IntPtr NewBufferWithLengthOptions = Sel("newBufferWithLength:options:");
- public static readonly IntPtr Contents = Sel("contents");
- public static readonly IntPtr NewLibraryWithSourceOptionsError = Sel("newLibraryWithSource:options:error:");
- public static readonly IntPtr NewFunctionWithName = Sel("newFunctionWithName:");
- public static readonly IntPtr NewFunctionWithNameConstantValuesError = Sel("newFunctionWithName:constantValues:error:");
- public static readonly IntPtr NewComputePipelineStateWithFunctionError = Sel("newComputePipelineStateWithFunction:error:");
- public static readonly IntPtr FunctionName = Sel("functionName");
- public static readonly IntPtr FunctionConstantsDictionary = Sel("functionConstantsDictionary");
- public static readonly IntPtr SetConstantValueTypeAtIndex = Sel("setConstantValue:type:atIndex:");
-
- public static readonly IntPtr SetFastMathEnabled = Sel("setFastMathEnabled:");
- public static readonly IntPtr SetMathMode = Sel("setMathMode:");
- public static readonly IntPtr SetLanguageVersion = Sel("setLanguageVersion:");
-
- public static readonly IntPtr CommandBuffer = Sel("commandBuffer");
- public static readonly IntPtr ComputeCommandEncoder = Sel("computeCommandEncoder");
- public static readonly IntPtr ComputeCommandEncoderWithDispatchType = Sel("computeCommandEncoderWithDispatchType:");
- public static readonly IntPtr SetComputePipelineState = Sel("setComputePipelineState:");
- public static readonly IntPtr SetBufferOffsetAtIndex = Sel("setBuffer:offset:atIndex:");
- public static readonly IntPtr SetBytesLengthAtIndex = Sel("setBytes:length:atIndex:");
- public static readonly IntPtr DispatchThreadgroupsThreadsPerThreadgroup = Sel("dispatchThreadgroups:threadsPerThreadgroup:");
- public static readonly IntPtr DispatchThreadsThreadsPerThreadgroup = Sel("dispatchThreads:threadsPerThreadgroup:");
- public static readonly IntPtr SetThreadgroupMemoryLengthAtIndex = Sel("setThreadgroupMemoryLength:atIndex:");
- public static readonly IntPtr MemoryBarrierWithScope = Sel("memoryBarrierWithScope:");
- public static readonly IntPtr EndEncoding = Sel("endEncoding");
- public static readonly IntPtr Commit = Sel("commit");
- public static readonly IntPtr WaitUntilCompleted = Sel("waitUntilCompleted");
- public static readonly IntPtr Status = Sel("status");
- public static readonly IntPtr Error = Sel("error");
- public static readonly IntPtr GpuStartTime = Sel("GPUStartTime");
- public static readonly IntPtr GpuEndTime = Sel("GPUEndTime");
- public static readonly IntPtr Label = Sel("label");
- public static readonly IntPtr SetLabel = Sel("setLabel:");
- }
-
- // NSString* with UTF-8 — caller owns (+1 via alloc/init); pass owns:false
- // to consume within the current autorelease pool.
- public static IntPtr NsStr(string s)
- {
- IntPtr obj = Send0(s_classNsString, S.Alloc);
- IntPtr cstr = Marshal.StringToHGlobalAnsi(s);
- try { return Send1P(obj, S.InitWithUTF8String, cstr); }
- finally { Marshal.FreeHGlobal(cstr); }
- }
-
- public static string? NsToString(IntPtr nsString)
- {
- if (nsString == 0) return null;
- IntPtr utf8 = Send0(nsString, S.Utf8String);
- return utf8 == 0 ? null : Marshal.PtrToStringUTF8(utf8);
- }
-
- public static void Release(IntPtr obj) { if (obj != 0) Send0(obj, S.Release); }
- public static IntPtr Retain(IntPtr obj) => Send0(obj, S.Retain);
-}
-
-/// Scoped NSAutoreleasePool — .NET worker threads have none, so every
-/// public entry that allocates autoreleased ObjC objects wraps itself in one.
-/// Drain at Dispose releases the pool itself.
-internal readonly struct AutoReleasePool : IDisposable
-{
- private readonly IntPtr _pool;
- private AutoReleasePool(IntPtr pool) => _pool = pool;
-
- public static AutoReleasePool Create() =>
- new(ObjC.Send0(ObjC.Send0(ObjC.ClassNSAutoreleasePool, ObjC.S.Alloc), ObjC.S.Init));
-
- public void Dispose() => ObjC.Send0(_pool, ObjC.S.Drain);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/common.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/common.metal
deleted file mode 100644
index b8b9f95..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/common.metal
+++ /dev/null
@@ -1,43 +0,0 @@
-// Common helpers for the GCDRNet Metal kernels (fp16 storage, fp32 math —
-// same contract as the Vulkan *_h shaders: half is storage-only, every load
-// widens to float and all arithmetic runs in fp32).
-#include
-using namespace metal;
-
-static inline float ferf1(float v)
-{
- float s = v < 0.0f ? -1.0f : 1.0f;
- float ax = fabs(v);
- float t = 1.0f / (1.0f + 0.3275911f * ax);
- float poly = ((((1.061405429f * t - 1.453152027f) * t + 1.421413741f) * t
- - 0.284496736f) * t + 0.254829592f) * t;
- return (1.0f - poly * exp(-ax * ax)) * s;
-}
-
-static inline float fexp1(float v)
-{
- v = clamp(v, -87.3f, 88.7f);
- float fx = v * 1.4426950408889634f + 0.5f;
- float nf = floor(fx);
- float r = v - nf * 0.693359375f - nf * -0.00021219444005469058f;
- float poly = (((((1.9875691500e-4f * r + 1.3981999507e-3f) * r + 8.3334519073e-3f) * r
- + 4.1665795894e-2f) * r + 1.6666665459e-1f) * r + 5.0000001201e-1f) * r * r + r + 1.0f;
- return poly * as_type((uint(int(nf)) + 127u) << 23);
-}
-
-// Conv epilogue activation: bits 4-6 (relu=1, gelu=2, sigmoid=3).
-static inline float cact(float v, uint flags)
-{
- uint a = (flags >> 4) & 7u;
- if (a == 1u) return max(v, 0.0f);
- if (a == 2u) return 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f));
- if (a == 3u) return 1.0f / (1.0f + fexp1(-v));
- return v;
-}
-
-static inline int mirror1(int i, int d)
-{
- if (d <= 1) return 0;
- while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i;
- return i;
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv.metal
deleted file mode 100644
index 9cb02a7..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv.metal
+++ /dev/null
@@ -1,333 +0,0 @@
-// Conv kernels: conv_tile (implicit-GEMM tiled fallback, conv_gen/k33/k11
-// equivalent), depthwise(4), convpool(_k3) fused stem+pool. fp16 storage, fp32 math.
-
-struct PcConv {
- uint ic, ih, iw;
- uint oc, oh, ow;
- uint kh, kw, sh, sw;
- uint pt, pl;
- uint flags; // bit0 bias, bit1 reflectPad, bit2 relu, bit3 hasAdd, bit4 gelu
- uint pad0;
-};
-
-// conv_tile: O[px,oc] = sum_k X[px,k]*W[k,oc], k = (ky*kw+kx)*ic+ci (coalesced).
-// W backend-packed [K][OC] oc-fastest. 3D dispatch: gx=x-tile(16), gy=y-tile(8),
-// gz=oc-tile(64); 256 threads, each 4oc x 8px. K-chunk 32 staged.
-kernel void conv_tile(device const half* x [[buffer(0)]],
- device const half* w [[buffer(1)]],
- device const float* b [[buffer(2)]],
- device const half* r [[buffer(3)]],
- device half* o [[buffer(4)]],
- constant PcConv& p [[buffer(5)]],
- uint3 wg [[threadgroup_position_in_grid]],
- uint3 tp [[thread_position_in_threadgroup]])
-{
- const uint tid = tp.x;
- const uint BM = 64u, BH = 8u, BW = 16u, BN = BH * BW, BK = 32u, BKP = 36u;
- threadgroup float4 Asv[BK * BM / 4];
- threadgroup float4 Bsv[BN * BKP / 4];
- uint oc0 = wg.z * BM;
- uint oy0 = wg.y * BH;
- uint ox0 = wg.x * BW;
- uint K = p.kh * p.kw * p.ic;
- uint tm = tid & 15u, tn = tid >> 4;
- float4 acc[8] = { float4(0.0f), float4(0.0f), float4(0.0f), float4(0.0f),
- float4(0.0f), float4(0.0f), float4(0.0f), float4(0.0f) };
- bool reflect = (p.flags & 2u) != 0u;
- uint bk = tid & 31u;
-
- for (uint k0 = 0u; k0 < K; k0 += BK)
- {
- for (uint e = tid; e < BK * BM; e += 256u)
- {
- uint k = e >> 6, m = e & 63u;
- uint gm = oc0 + m, gk = k0 + k;
- ((threadgroup float*)Asv)[e] = (gm < p.oc && gk < K) ? float(w[gk * p.oc + gm]) : 0.0f;
- }
- uint gk = k0 + bk;
- bool kok = gk < K;
- uint tap = kok ? gk / p.ic : 0u;
- uint ch = gk - tap * p.ic;
- uint ky = tap / p.kw, kx = tap - ky * p.kw;
- for (uint n = tid >> 5; n < BN; n += 8u)
- {
- float v = 0.0f;
- uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u);
- if (kok && oy < p.oh && ox < p.ow)
- {
- int sy = int(oy * p.sh + ky) - int(p.pt);
- int sx = int(ox * p.sw + kx) - int(p.pl);
- if (reflect)
- {
- if (uint(sy) >= p.ih) sy = mirror1(sy, int(p.ih));
- if (uint(sx) >= p.iw) sx = mirror1(sx, int(p.iw));
- v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]);
- }
- else if (uint(sy) < p.ih && uint(sx) < p.iw)
- v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]);
- }
- uint si = n * BKP + bk;
- ((threadgroup float*)Bsv)[si] = v;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint k = 0u; k < BK; k += 4u)
- {
- float4 a0 = Asv[k * 16u + tm];
- float4 a1 = Asv[(k + 1u) * 16u + tm];
- float4 a2 = Asv[(k + 2u) * 16u + tm];
- float4 a3 = Asv[(k + 3u) * 16u + tm];
- for (int j = 0; j < 8; j++)
- {
- float4 bv = Bsv[(tn * 8u + uint(j)) * (BKP / 4u) + (k >> 2)];
- acc[j] += a0 * bv.x + a1 * bv.y + a2 * bv.z + a3 * bv.w;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- bool hasBias = (p.flags & 1u) != 0u;
- uint gm0 = oc0 + tm * 4u;
- float4 bias = float4(
- hasBias && gm0 < p.oc ? b[gm0] : 0.0f,
- hasBias && gm0 + 1u < p.oc ? b[gm0 + 1u] : 0.0f,
- hasBias && gm0 + 2u < p.oc ? b[gm0 + 2u] : 0.0f,
- hasBias && gm0 + 3u < p.oc ? b[gm0 + 3u] : 0.0f);
- for (int j = 0; j < 8; j++)
- {
- uint n = tn * 8u + uint(j);
- uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u);
- if (oy >= p.oh || ox >= p.ow || gm0 >= p.oc) continue;
- uint base = (oy * p.ow + ox) * p.oc + gm0;
- uint full = gm0 + 3u < p.oc ? 4u : p.oc - gm0;
- for (uint l = 0u; l < 4u && l < full; l++)
- {
- float v = acc[j][l] + bias[l];
- if ((p.flags & 8u) != 0u) v += float(r[base + l]);
- if ((p.flags & 4u) != 0u) v = max(v, 0.0f);
- if ((p.flags & 16u) != 0u) v = 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f));
- o[base + l] = half(v);
- }
- }
-}
-
-// ---- depthwise / depthwise4: out[oy,ox,c] = bias[c] + sum w[tap,c]*x[sy,sx,c] ----
-struct PcDw {
- uint c, ih, iw, oh, ow;
- uint kh, kw, sh, sw, pt, pl;
- uint flags; // bit0 hasBias, bit1 reflect, bit2 relu, bit4 gelu
- float padVal;
-};
-kernel void depthwise(device const half* x [[buffer(0)]],
- device const half* w [[buffer(1)]],
- device const float* b [[buffer(2)]],
- device half* o [[buffer(3)]],
- constant PcDw& p [[buffer(4)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= p.c * p.oh * p.ow) return;
- uint ch = gid % p.c, px = gid / p.c;
- uint ox = px % p.ow, oy = px / p.ow;
- bool refl = (p.flags & 2u) != 0u;
- float acc = ((p.flags & 1u) != 0u) ? b[ch] : 0.0f;
- for (uint ky = 0u; ky < p.kh; ky++)
- {
- int sy = int(oy * p.sh + ky) - int(p.pt);
- for (uint kx = 0u; kx < p.kw; kx++)
- {
- int sx = int(ox * p.sw + kx) - int(p.pl);
- float wv = float(w[(ky * p.kw + kx) * p.c + ch]);
- int yy = sy, xx = sx;
- if (uint(sy) >= p.ih || uint(sx) >= p.iw)
- {
- if (!refl) { acc += wv * p.padVal; continue; }
- if (uint(sy) >= p.ih) yy = mirror1(sy, int(p.ih));
- if (uint(sx) >= p.iw) xx = mirror1(sx, int(p.iw));
- }
- acc += wv * float(x[(uint(yy) * p.iw + uint(xx)) * p.c + ch]);
- }
- }
- if ((p.flags & 4u) != 0u) acc = max(acc, 0.0f);
- if ((p.flags & 16u) != 0u) acc = 0.5f * acc * (1.0f + ferf1(acc * 0.7071067811865475f));
- o[gid] = half(acc);
-}
-kernel void depthwise4(device const half4* x [[buffer(0)]],
- device const half4* w [[buffer(1)]],
- device const float4* b [[buffer(2)]],
- device half4* o [[buffer(3)]],
- constant PcDw& p [[buffer(4)]],
- uint gid [[thread_position_in_grid]])
-{
- uint c4 = p.c >> 2;
- if (gid >= c4 * p.oh * p.ow) return;
- uint ch = gid % c4, px = gid / c4;
- uint ox = px % p.ow, oy = px / p.ow;
- bool refl = (p.flags & 2u) != 0u;
- float4 acc = ((p.flags & 1u) != 0u) ? b[ch] : float4(0.0f);
- for (uint ky = 0u; ky < p.kh; ky++)
- {
- int sy = int(oy * p.sh + ky) - int(p.pt);
- for (uint kx = 0u; kx < p.kw; kx++)
- {
- int sx = int(ox * p.sw + kx) - int(p.pl);
- float4 wv = float4(w[(ky * p.kw + kx) * c4 + ch]);
- int yy = sy, xx = sx;
- if (uint(sy) >= p.ih || uint(sx) >= p.iw)
- {
- if (!refl) { acc += wv * p.padVal; continue; }
- if (uint(sy) >= p.ih) yy = mirror1(sy, int(p.ih));
- if (uint(sx) >= p.iw) xx = mirror1(sx, int(p.iw));
- }
- acc += wv * float4(x[(uint(yy) * p.iw + uint(xx)) * c4 + ch]);
- }
- }
- if ((p.flags & 4u) != 0u) acc = max(acc, float4(0.0f));
- if ((p.flags & 16u) != 0u)
- for (int l = 0; l < 4; l++) acc[l] = 0.5f * acc[l] * (1.0f + ferf1(acc[l] * 0.7071067811865475f));
- o[gid] = half4(acc);
-}
-
-// ---- convpool / convpool_k3: small-ic stem conv + fused 2x2/2 maxpool ----
-struct PcConvPool {
- uint ic, ih, iw, oc, ph, pw, kh, kw, pt, pl, flags;
- float padv;
-};
-kernel void convpool(device const half* x [[buffer(0)]],
- device const half* w [[buffer(1)]],
- device const float* b [[buffer(2)]],
- device half* o [[buffer(3)]],
- constant PcConvPool& p [[buffer(4)]],
- uint gid [[thread_position_in_grid]],
- uint tid [[thread_position_in_threadgroup]])
-{
- threadgroup float4 ws[1024];
- uint K = p.kh * p.kw * p.ic, oc4 = p.oc >> 2;
- for (uint e = tid; e < K * p.oc; e += 256u) ((threadgroup float*)ws)[e] = float(w[e]);
- threadgroup_barrier(mem_flags::mem_threadgroup);
- uint groups = p.oc >> 3;
- if (gid >= p.ph * p.pw * groups) return;
- uint g = gid % groups, pp = gid / groups;
- uint pxx = pp % p.pw, pyy = pp / p.pw;
- bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u;
- uint n0 = g * 8u;
- float4 b0 = float4(0.0f), b1 = float4(0.0f);
- if ((p.flags & 1u) != 0u)
- {
- b0 = float4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]);
- b1 = float4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]);
- }
- float4 m0 = float4(-3.4e38f), m1 = float4(-3.4e38f);
- for (uint d = 0u; d < 4u; d++)
- {
- uint oy = pyy * 2u + (d >> 1), ox = pxx * 2u + (d & 1u);
- float4 a0 = b0, a1 = b1;
- for (uint ky = 0u; ky < p.kh; ky++)
- {
- int iy = int(oy + ky) - int(p.pt);
- bool vy = iy >= 0 && iy < int(p.ih);
- if (!vy && reflect) { iy = mirror1(iy, int(p.ih)); vy = true; }
- for (uint kx = 0u; kx < p.kw; kx++)
- {
- int ix = int(ox + kx) - int(p.pl);
- bool vx = ix >= 0 && ix < int(p.iw);
- if (!vx && reflect) { ix = mirror1(ix, int(p.iw)); vx = true; }
- bool valid = vy && vx;
- uint base = valid ? (uint(iy) * p.iw + uint(ix)) * p.ic : 0u;
- uint kb = (ky * p.kw + kx) * p.ic;
- for (uint ci = 0u; ci < p.ic; ci++)
- {
- float xv = valid ? float(x[base + ci]) : p.padv;
- uint wr = (kb + ci) * oc4 + g * 2u;
- a0 += xv * ws[wr];
- a1 += xv * ws[wr + 1u];
- }
- }
- }
- if (crelu) { a0 = max(a0, float4(0.0f)); a1 = max(a1, float4(0.0f)); }
- m0 = max(m0, a0); m1 = max(m1, a1);
- }
- if ((p.flags & 8u) != 0u) { m0 = max(m0, float4(0.0f)); m1 = max(m1, float4(0.0f)); }
- uint ob = pp * p.oc + n0;
- for (uint l = 0u; l < 4u; l++) { o[ob + l] = half(m0[l]); o[ob + 4u + l] = half(m1[l]); }
-}
-
-kernel void convpool_k3(device const half* x [[buffer(0)]],
- device const half* w [[buffer(1)]],
- device const float* b [[buffer(2)]],
- device half* o [[buffer(3)]],
- constant PcConvPool& p [[buffer(4)]],
- uint gid [[thread_position_in_grid]],
- uint tid [[thread_position_in_threadgroup]])
-{
- threadgroup float4 ws[1024];
- uint K = 9u * p.ic, oc4 = p.oc >> 2;
- for (uint e = tid; e < K * p.oc; e += 256u) ((threadgroup float*)ws)[e] = float(w[e]);
- threadgroup_barrier(mem_flags::mem_threadgroup);
- uint groups = p.oc >> 3;
- if (gid >= p.ph * p.pw * groups) return;
- uint g = gid % groups, pp = gid / groups;
- uint pxx = pp % p.pw, pyy = pp / p.pw;
- bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u;
- int ro[4], co[4];
- bool rv[4], cv[4];
- for (uint r = 0u; r < 4u; r++)
- {
- int iy = int(pyy * 2u + r) - int(p.pt);
- bool vy = iy >= 0 && iy < int(p.ih);
- if (!vy && reflect) { iy = mirror1(iy, int(p.ih)); vy = true; }
- rv[r] = vy; ro[r] = vy ? iy * int(p.iw) : 0;
- int ix = int(pxx * 2u + r) - int(p.pl);
- bool vx = ix >= 0 && ix < int(p.iw);
- if (!vx && reflect) { ix = mirror1(ix, int(p.iw)); vx = true; }
- cv[r] = vx; co[r] = vx ? ix : 0;
- }
- uint n0 = g * 8u;
- float4 b0 = float4(0.0f), b1 = float4(0.0f);
- if ((p.flags & 1u) != 0u)
- {
- b0 = float4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]);
- b1 = float4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]);
- }
- float4 a0[4], a1[4];
- for (uint d = 0u; d < 4u; d++) { a0[d] = b0; a1[d] = b1; }
- for (uint r = 0u; r < 4u; r++)
- {
- float xr[4][8];
- for (uint c = 0u; c < 4u; c++)
- {
- bool v = rv[r] && cv[c];
- uint base = uint(ro[r] + co[c]) * p.ic;
- for (uint ci = 0u; ci < 8u; ci++)
- xr[c][ci] = ci < p.ic ? (v ? float(x[base + ci]) : p.padv) : 0.0f;
- }
- for (uint dy = 0u; dy < 2u; dy++)
- {
- if (r < dy || r - dy > 2u) continue;
- uint ky = r - dy;
- for (uint kx = 0u; kx < 3u; kx++)
- {
- uint kb = (ky * 3u + kx) * p.ic;
- for (uint ci = 0u; ci < 8u; ci++)
- {
- if (ci >= p.ic) break;
- uint wr = (kb + ci) * oc4 + g * 2u;
- float4 w0 = ws[wr], w1 = ws[wr + 1u];
- for (uint dx = 0u; dx < 2u; dx++)
- {
- float xv = xr[kx + dx][ci];
- a0[dy * 2u + dx] += xv * w0;
- a1[dy * 2u + dx] += xv * w1;
- }
- }
- }
- }
- }
- float4 m0 = float4(-3.4e38f), m1 = float4(-3.4e38f);
- for (uint d = 0u; d < 4u; d++)
- {
- if (crelu) { a0[d] = max(a0[d], float4(0.0f)); a1[d] = max(a1[d], float4(0.0f)); }
- m0 = max(m0, a0[d]); m1 = max(m1, a1[d]);
- }
- if ((p.flags & 8u) != 0u) { m0 = max(m0, float4(0.0f)); m1 = max(m1, float4(0.0f)); }
- uint ob = pp * p.oc + n0;
- for (uint l = 0u; l < 4u; l++) { o[ob + l] = half(m0[l]); o[ob + 4u + l] = half(m1[l]); }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv_mm.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv_mm.metal
deleted file mode 100644
index f784851..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv_mm.metal
+++ /dev/null
@@ -1,229 +0,0 @@
-// conv_mm: simdgroup-MMA implicit-GEMM NHWC conv (Metal port of conv_cm.comp).
-// A gathers input pixels through tap-major im2col addressing — no im2col
-// materialization; W is backend-packed [oc][Kd] tap-major, Kd = K rounded up
-// to 8 (zero-filled rows make pad k-slices contribute 0 for any gathered
-// pixel). BM = 64 output pixels x BN output channels, 8 simdgroups
-// (256 threads), fp16 storage + simdgroup fp16xfp32 MMA.
-// cflags: bit0 bias, bit1 residual, bit2 reflect pad, bits4-6 act (1 relu, 2 gelu).
-struct PcConvMm {
- uint mDim, nDim, kDim, flags;
- uint outW, inW, inH, cin, kW;
- uint sH, sW, pT, pL, kReal;
-};
-
-template
-static inline void conv_mm_body(device const half* x,
- device const half* w,
- device const float* bias,
- device const half* res,
- device half* o,
- constant PcConvMm& p,
- uint2 wg, uint tid,
- threadgroup half* tA, threadgroup half* tB,
- threadgroup float* outst)
-{
- const uint JN = BN / 16u; // acc columns per simdgroup row-half
- uint sgid = tid >> 5;
- uint sm = (sgid & 3u) * 16u; // 4 rows of 16 -> 64 M
- uint sn = (sgid >> 2) * (BN / 2u); // 2 cols of BN/2
- uint m0 = wg.x * 64u + sm; // m tiles on gx (grid allows 2^31 in x)
- uint n0 = wg.y * BN + sn;
-
- simdgroup_float8x8 acc[2][8];
- for (uint i = 0u; i < 2u; i++) for (uint j = 0u; j < 8u; j++)
- acc[i][j] = simdgroup_float8x8(0.0f);
-
- bool reflect = (p.flags & 4u) != 0u;
-
- for (uint kb = 0u; kb < p.kDim; kb += 16u)
- {
- if (SCK == 0)
- {
- // vec4 gather: 64 rows x 16 k as one half4 per thread
- uint am = tid >> 2, ak = (tid & 3u) * 4u;
- uint gm = wg.x * 64u + am;
- uint gk = kb + ak;
- half4 av = half4(0.0h);
- if (gm < p.mDim && gk < p.kReal)
- {
- uint tap = gk / p.cin, ci = gk - tap * p.cin;
- uint dy = tap / p.kW, dx = tap - dy * p.kW;
- uint oy = gm / p.outW, ox = gm - oy * p.outW;
- int iy = int(oy * p.sH + dy) - int(p.pT);
- int ix = int(ox * p.sW + dx) - int(p.pL);
- if (reflect)
- {
- if (uint(iy) >= p.inH) iy = mirror1(iy, int(p.inH));
- if (uint(ix) >= p.inW) ix = mirror1(ix, int(p.inW));
- av = *(device const half4*)(x + (((uint(iy) * p.inW + uint(ix)) * (p.cin >> 2)) + (ci >> 2)) * 4u);
- }
- else if (iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW))
- av = *(device const half4*)(x + (((uint(iy) * p.inW + uint(ix)) * (p.cin >> 2)) + (ci >> 2)) * 4u);
- }
- *(threadgroup half4*)(tA + am * 20 + ak) = av;
- }
- else
- {
- // scalar gather for cin % 4 != 0 (RGB stems): 64x16 one half per thread x4
- for (uint e = tid; e < 64u * 16u; e += 256u)
- {
- uint am = e >> 4, ak = e & 15u;
- uint gm = wg.x * 64u + am;
- uint gk = kb + ak;
- half av = 0.0h;
- if (gm < p.mDim && gk < p.kReal)
- {
- uint tap = gk / p.cin, ci = gk - tap * p.cin;
- uint dy = tap / p.kW, dx = tap - dy * p.kW;
- uint oy = gm / p.outW, ox = gm - oy * p.outW;
- int iy = int(oy * p.sH + dy) - int(p.pT);
- int ix = int(ox * p.sW + dx) - int(p.pL);
- if (reflect)
- {
- if (uint(iy) >= p.inH) iy = mirror1(iy, int(p.inH));
- if (uint(ix) >= p.inW) ix = mirror1(ix, int(p.inW));
- av = x[(uint(iy) * p.inW + uint(ix)) * p.cin + ci];
- }
- else if (iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW))
- av = x[(uint(iy) * p.inW + uint(ix)) * p.cin + ci];
- }
- tA[am * 20 + ak] = av;
- }
- }
- // B stage: [N][Kd] rows, one half4 per thread covers 64x16; BN>64 loops
- for (uint rr = tid >> 2; rr < uint(BN); rr += 64u)
- {
- uint bk = (tid & 3u) * 4u;
- uint gn = wg.y * BN + rr;
- uint gkB = kb + bk;
- half4 wv = half4(0.0h);
- if (gn < p.nDim && gkB < p.kDim)
- wv = *(device const half4*)(w + (ulong)gn * p.kDim + gkB);
- *(threadgroup half4*)(tB + rr * 20 + bk) = wv;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint kk = 0u; kk < 16u; kk += 8u)
- {
- simdgroup_half8x8 a0, a1;
- simdgroup_load(a0, tA + sm * 20 + kk, 20);
- simdgroup_load(a1, tA + (sm + 8u) * 20 + kk, 20);
- simdgroup_half8x8 bfr[8];
- for (uint j = 0u; j < JN; j++)
- simdgroup_load(bfr[j], tB + (sn + j * 8u) * 20 + kk, 20, ulong2(0, 0), true);
- for (uint j = 0u; j < JN; j++)
- {
- simdgroup_multiply_accumulate(acc[0][j], a0, bfr[j], acc[0][j]);
- simdgroup_multiply_accumulate(acc[1][j], a1, bfr[j], acc[1][j]);
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- uint lane = tid & 31u;
- for (uint i = 0u; i < 2u; i++)
- for (uint j = 0u; j < JN; j++)
- {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- simdgroup_store(acc[i][j], outst + sgid * 64, 8);
- threadgroup_barrier(mem_flags::mem_threadgroup);
- uint r = lane >> 2, c2 = (lane & 3u) * 2u;
- for (uint e = 0u; e < 2u; e++)
- {
- uint mm = m0 + i * 8u + r;
- uint nn = n0 + j * 8u + c2 + e;
- if (mm >= p.mDim || nn >= p.nDim) continue;
- float v = outst[sgid * 64 + r * 8 + c2 + e];
- if ((p.flags & 1u) != 0u) v += bias[nn];
- if ((p.flags & 2u) != 0u) v += float(res[mm * p.nDim + nn]);
- o[mm * p.nDim + nn] = half(cact(v, p.flags));
- }
- }
-}
-
-#define CONV_MM_INST(BN, SCK) \
-kernel void conv_mm_##BN##_##SCK(device const half* x [[buffer(0)]], \
- device const half* w [[buffer(1)]], device const float* bias [[buffer(2)]], \
- device const half* res [[buffer(3)]], device half* o [[buffer(4)]], \
- constant PcConvMm& p [[buffer(5)]], uint2 wg [[threadgroup_position_in_grid]], \
- uint2 tp [[thread_position_in_threadgroup]]) \
-{ \
- threadgroup half tA[64 * 20]; \
- threadgroup half tB[BN * 20]; \
- threadgroup float outst[8 * 64]; \
- conv_mm_body(x, w, bias, res, o, p, wg, tp.x, tA, tB, outst); \
-}
-
-CONV_MM_INST(128, 0)
-CONV_MM_INST(64, 0)
-CONV_MM_INST(32, 0)
-CONV_MM_INST(128, 1)
-CONV_MM_INST(64, 1)
-CONV_MM_INST(32, 1)
-
-// ---- gemm: C[M,N] = A[M,K]*B + bias[N]; B [K,N] or [N,K] (transB) ----
-// tiled 64x64x16, 256 threads -> 4x4 each (fallback; MMA path uses conv_mm).
-struct PcGemm { uint m, n, k, flags; }; // bit0 transB bit1 bias bit2 relu bit3 gelu
-kernel void gemm(device const half* a [[buffer(0)]],
- device const half* b [[buffer(1)]],
- device const float* bs [[buffer(2)]],
- device half* o [[buffer(3)]],
- constant PcGemm& p [[buffer(4)]],
- uint2 wg [[threadgroup_position_in_grid]],
- uint2 tp [[thread_position_in_threadgroup]])
-{
- const uint tid = tp.x;
- const uint BM = 64u, BN = 64u, BK = 16u;
- threadgroup float As[BK * BM];
- threadgroup float Bs[BK * BN];
- uint m0 = wg.y * BM;
- uint n0 = wg.x * BN;
- uint ty = tid >> 4, tx = tid & 15u;
- bool transB = (p.flags & 1u) != 0u;
- float acc[4][4];
- for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) acc[i][j] = 0.0f;
- for (uint k0 = 0u; k0 < p.k; k0 += BK)
- {
- for (uint e = tid; e < BK * BM; e += 256u)
- {
- uint kk = e >> 6, mm = e & 63u;
- uint gm = m0 + mm, gk = k0 + kk;
- As[e] = (gm < p.m && gk < p.k) ? float(a[gm * p.k + gk]) : 0.0f;
- }
- for (uint e = tid; e < BK * BN; e += 256u)
- {
- uint kk = e >> 6, nn = e & 63u;
- uint gn = n0 + nn, gk = k0 + kk;
- float v = 0.0f;
- if (gn < p.n && gk < p.k)
- v = float(transB ? b[gn * p.k + gk] : b[gk * p.n + gn]);
- Bs[e] = v;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint kk = 0u; kk < BK; kk++)
- {
- float av[4], bv[4];
- for (int i = 0; i < 4; i++) av[i] = As[kk * BM + ty * 4u + uint(i)];
- for (int j = 0; j < 4; j++) bv[j] = Bs[kk * BN + tx * 4u + uint(j)];
- for (int i = 0; i < 4; i++)
- for (int j = 0; j < 4; j++)
- acc[i][j] += av[i] * bv[j];
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
- bool hasBias = (p.flags & 2u) != 0u;
- for (int i = 0; i < 4; i++)
- {
- uint gm = m0 + ty * 4u + uint(i);
- if (gm >= p.m) continue;
- for (int j = 0; j < 4; j++)
- {
- uint gn = n0 + tx * 4u + uint(j);
- if (gn >= p.n) continue;
- float v = acc[i][j];
- if (hasBias) v += bs[gn];
- if ((p.flags & 4u) != 0u) v = max(v, 0.0f);
- if ((p.flags & 8u) != 0u) v = 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f));
- o[gm * p.n + gn] = half(v);
- }
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal
index 02142af..e0bd563 100644
--- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal
+++ b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal
@@ -1,4 +1,4 @@
-// Pipeline glue + elementwise/shape kernels. All activation storage is fp16
+// GCDR pipeline glue kernels. All activation storage is fp16
// (half/half4); math fp32 — see common.metal.
// ---- bgr2nhwc: BGR u8 HWC -> NHWC [h][w][3] /255, top/left replicate pad ----
@@ -67,333 +67,3 @@ kernel void nhwc2bgr(device const half* x [[buffer(0)]],
o[gid] = packed;
}
-// ---- unary: mode 0 relu, 1 sigmoid, 2 erf, 3 gelu, 4 copy ----
-struct PcUnary { uint n, mode; };
-kernel void unary(device const half* x [[buffer(0)]],
- device half* o [[buffer(1)]],
- constant PcUnary& p [[buffer(2)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= p.n) return;
- float v = float(x[gid]);
- float r;
- if (p.mode == 0u) r = max(v, 0.0f);
- else if (p.mode == 1u) r = 1.0f / (1.0f + fexp1(-v));
- else if (p.mode == 2u) r = ferf1(v);
- else if (p.mode == 3u) r = 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f));
- else r = v;
- o[gid] = half(r);
-}
-
-// ---- ew2: elementwise binary + act, rank<=4 broadcast ----
-struct PcEw2 {
- uint n, s0, s1, s2;
- uint as0, as1, as2, as3;
- uint bs0, bs1, bs2, bs3;
- uint op, act;
-};
-kernel void ew2(device const half* a [[buffer(0)]],
- device const half* b [[buffer(1)]],
- device half* o [[buffer(2)]],
- constant PcEw2& p [[buffer(3)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= p.n) return;
- uint d0 = gid / p.s0, r0 = gid % p.s0;
- uint d1 = r0 / p.s1, r1 = r0 % p.s1;
- uint d2 = r1 / p.s2, d3 = r1 % p.s2;
- float va = float(a[d0 * p.as0 + d1 * p.as1 + d2 * p.as2 + d3 * p.as3]);
- float vb = float(b[d0 * p.bs0 + d1 * p.bs1 + d2 * p.bs2 + d3 * p.bs3]);
- float v = p.op == 0u ? va + vb : p.op == 1u ? va - vb : p.op == 2u ? va * vb : va / vb;
- if (p.act == 1u) v = max(v, 0.0f);
- else if (p.act == 2u) v = 1.0f / (1.0f + exp(-v));
- o[gid] = half(v);
-}
-
-// ---- ew2v: same-shape vec4 fast path ----
-struct PcEw2v { uint nv4, op, act; };
-kernel void ew2v(device const half4* a [[buffer(0)]],
- device const half4* b [[buffer(1)]],
- device half4* o [[buffer(2)]],
- constant PcEw2v& p [[buffer(3)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= p.nv4) return;
- float4 va = float4(a[gid]), vb = float4(b[gid]), v;
- if (p.op == 0u) v = va + vb;
- else if (p.op == 1u) v = va - vb;
- else if (p.op == 2u) v = va * vb;
- else v = va / vb;
- if (p.act == 1u) v = max(v, float4(0.0f));
- else if (p.act == 2u) v = 1.0f / (1.0f + exp(-v));
- o[gid] = half4(v);
-}
-
-// ---- transpose: rank<=4 permute ----
-struct PcTranspose { uint n, os0, os1, os2, ps0, ps1, ps2, ps3; };
-kernel void transpose(device const half* x [[buffer(0)]],
- device half* o [[buffer(1)]],
- constant PcTranspose& p [[buffer(2)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= p.n) return;
- uint d0 = gid / p.os0, r0 = gid % p.os0;
- uint d1 = r0 / p.os1, r1 = r0 % p.os1;
- uint d2 = r1 / p.os2, d3 = r1 % p.os2;
- o[gid] = x[d0 * p.ps0 + d1 * p.ps1 + d2 * p.ps2 + d3 * p.ps3];
-}
-
-// ---- slice: rank<=4 strided slice ----
-struct PcSlice {
- uint n, os0, os1, os2;
- uint st0, st1, st2, st3;
- uint sp0, sp1, sp2, sp3;
- uint is0, is1, is2, is3;
-};
-kernel void slice(device const half* x [[buffer(0)]],
- device half* o [[buffer(1)]],
- constant PcSlice& p [[buffer(2)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= p.n) return;
- uint d0 = gid / p.os0, r0 = gid % p.os0;
- uint d1 = r0 / p.os1, r1 = r0 % p.os1;
- uint d2 = r1 / p.os2, d3 = r1 % p.os2;
- o[gid] = x[(p.st0 + d0 * p.sp0) * p.is0 + (p.st1 + d1 * p.sp1) * p.is1
- + (p.st2 + d2 * p.sp2) * p.is2 + (p.st3 + d3 * p.sp3) * p.is3];
-}
-
-// ---- concat_ax: copy one input into dst at axis offset ----
-struct PcConcat { uint n, inner, srcAxis, dstAxis, axisOff; };
-kernel void concat_ax(device const half* x [[buffer(0)]],
- device half* o [[buffer(1)]],
- constant PcConcat& p [[buffer(2)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= p.n) return;
- uint ii = gid % p.inner;
- uint a = (gid / p.inner) % p.srcAxis;
- uint oi = gid / (p.inner * p.srcAxis);
- o[(oi * p.dstAxis + a + p.axisOff) * p.inner + ii] = x[gid];
-}
-
-// ---- pad: rank-4 all-dims pad, constant or reflect ----
-struct PcPad {
- uint id0, id1, id2, id3;
- uint od0, od1, od2, od3;
- uint pb0, pb1, pb2, pb3;
- uint mode; // 0 const 1 reflect
- float value;
-};
-static inline int padcoord(uint oidx, uint pb, uint idim, uint mode, thread bool& miss)
-{
- int s = int(oidx) - int(pb);
- if (uint(s) >= idim)
- {
- if (mode == 0u) { miss = true; return 0; }
- s = mirror1(s, int(idim));
- }
- miss = false; return s;
-}
-kernel void pad(device const half* x [[buffer(0)]],
- device half* o [[buffer(1)]],
- constant PcPad& p [[buffer(2)]],
- uint gid [[thread_position_in_grid]])
-{
- uint n = p.od0 * p.od1 * p.od2 * p.od3;
- if (gid >= n) return;
- uint s2 = p.od3, s1 = p.od2 * s2, s0 = p.od1 * s1;
- uint d0 = gid / s0, r0 = gid % s0;
- uint d1 = r0 / s1, r1 = r0 % s1;
- uint d2 = r1 / s2, d3 = r1 % s2;
- bool m0, m1, m2, m3;
- int c0 = padcoord(d0, p.pb0, p.id0, p.mode, m0);
- int c1 = padcoord(d1, p.pb1, p.id1, p.mode, m1);
- int c2 = padcoord(d2, p.pb2, p.id2, p.mode, m2);
- int c3 = padcoord(d3, p.pb3, p.id3, p.mode, m3);
- if (m0 || m1 || m2 || m3) { o[gid] = half(p.value); return; }
- uint si2 = p.id3, si1 = p.id2 * si2, si0 = p.id1 * si1;
- o[gid] = x[uint(c0) * si0 + uint(c1) * si1 + uint(c2) * si2 + uint(c3)];
-}
-
-// ---- shift: NHWC channel-shift (fused shift-MLP) ----
-struct PcShift {
- uint c, h, w, axis;
- uint ge0, ge1, ge2, ge3, ge4;
- int of0, of1, of2, of3, of4;
-};
-static inline int shiftoff(uint ch, constant PcShift& p)
-{
- return ch < p.ge0 ? p.of0 : ch < p.ge1 ? p.of1 : ch < p.ge2 ? p.of2 : ch < p.ge3 ? p.of3 : ch < p.ge4 ? p.of4 : 0;
-}
-kernel void shift(device const half* x [[buffer(0)]],
- device half* o [[buffer(1)]],
- constant PcShift& p [[buffer(2)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= p.c * p.h * p.w) return;
- uint ch = gid % p.c, px = gid / p.c;
- uint ox = px % p.w, oy = px / p.w;
- int off = shiftoff(ch, p);
- int sy = int(oy), sx = int(ox);
- if (p.axis == 2u) sy += off; else sx += off;
- o[gid] = half((uint(sy) < p.h && uint(sx) < p.w) ? float(x[(uint(sy) * p.w + uint(sx)) * p.c + ch]) : 0.0f);
-}
-kernel void shift4(device const half4* x [[buffer(0)]],
- device half4* o [[buffer(1)]],
- constant PcShift& p [[buffer(2)]],
- uint gid [[thread_position_in_grid]])
-{
- uint c4 = p.c >> 2;
- if (gid >= c4 * p.h * p.w) return;
- uint q = gid % c4, px = gid / c4, ch = q * 4u;
- uint ox = px % p.w, oy = px / p.w;
- int off = shiftoff(ch, p);
- int sy = int(oy), sx = int(ox);
- if (p.axis == 2u) sy += off; else sx += off;
- o[gid] = (uint(sy) < p.h && uint(sx) < p.w) ? x[(uint(sy) * p.w + uint(sx)) * c4 + q] : half4(0.0h);
-}
-
-// ---- bilinear / bilinear4: NHWC half-pixel bilinear + fused add ----
-struct PcBilinear { uint c, ih, iw, oh, ow, flags; float shy, shx; };
-kernel void bilinear(device const half* x [[buffer(0)]],
- device const half* a [[buffer(1)]],
- device half* o [[buffer(2)]],
- constant PcBilinear& p [[buffer(3)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= p.c * p.oh * p.ow) return;
- uint ch = gid % p.c, px = gid / p.c;
- uint ox = px % p.ow, oy = px / p.ow;
- float fy = (float(oy) + 0.5f) / p.shy - 0.5f;
- float fx = (float(ox) + 0.5f) / p.shx - 0.5f;
- int fy0 = int(floor(fy)), fx0 = int(floor(fx));
- float dy = fy - float(fy0), dx = fx - float(fx0);
- uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1));
- uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1));
- float v00 = float(x[(y0 * p.iw + x0) * p.c + ch]);
- float v01 = float(x[(y0 * p.iw + x1) * p.c + ch]);
- float v10 = float(x[(y1 * p.iw + x0) * p.c + ch]);
- float v11 = float(x[(y1 * p.iw + x1) * p.c + ch]);
- float v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy);
- if ((p.flags & 1u) != 0u) v = max(v, 0.0f);
- if ((p.flags & 2u) != 0u)
- {
- float s = float(a[gid]);
- v += (p.flags & 4u) != 0u ? max(s, 0.0f) : s;
- }
- o[gid] = half(v);
-}
-kernel void bilinear4(device const half4* x [[buffer(0)]],
- device const half4* a [[buffer(1)]],
- device half4* o [[buffer(2)]],
- constant PcBilinear& p [[buffer(3)]],
- uint gid [[thread_position_in_grid]])
-{
- uint c4 = p.c >> 2;
- if (gid >= c4 * p.oh * p.ow) return;
- uint ch = gid % c4, px = gid / c4;
- uint ox = px % p.ow, oy = px / p.ow;
- float fy = (float(oy) + 0.5f) / p.shy - 0.5f;
- float fx = (float(ox) + 0.5f) / p.shx - 0.5f;
- int fy0 = int(floor(fy)), fx0 = int(floor(fx));
- float dy = fy - float(fy0), dx = fx - float(fx0);
- uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1));
- uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1));
- float4 v00 = float4(x[(y0 * p.iw + x0) * c4 + ch]);
- float4 v01 = float4(x[(y0 * p.iw + x1) * c4 + ch]);
- float4 v10 = float4(x[(y1 * p.iw + x0) * c4 + ch]);
- float4 v11 = float4(x[(y1 * p.iw + x1) * c4 + ch]);
- float4 v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy);
- if ((p.flags & 1u) != 0u) v = max(v, float4(0.0f));
- if ((p.flags & 2u) != 0u)
- {
- float4 s = float4(a[gid]);
- v += (p.flags & 4u) != 0u ? max(s, float4(0.0f)) : s;
- }
- o[gid] = half4(v);
-}
-
-// ---- maxpool / maxpool4: NHWC max pool + optional relu ----
-struct PcPool {
- uint c, ih, iw, oh, ow;
- uint kh, kw, sh, sw, pt, pl, relu;
-};
-kernel void maxpool(device const half* x [[buffer(0)]],
- device half* o [[buffer(1)]],
- constant PcPool& p [[buffer(2)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= p.c * p.oh * p.ow) return;
- uint ch = gid % p.c, px = gid / p.c;
- uint ox = px % p.ow, oy = px / p.ow;
- float m = -3.4e38f;
- for (uint ky = 0u; ky < p.kh; ky++)
- {
- int sy = int(oy * p.sh + ky) - int(p.pt);
- if (uint(sy) >= p.ih) continue;
- for (uint kx = 0u; kx < p.kw; kx++)
- {
- int sx = int(ox * p.sw + kx) - int(p.pl);
- if (uint(sx) >= p.iw) continue;
- m = max(m, float(x[(uint(sy) * p.iw + uint(sx)) * p.c + ch]));
- }
- }
- if (m == -3.4e38f) m = 0.0f;
- if (p.relu != 0u && m < 0.0f) m = 0.0f;
- o[gid] = half(m);
-}
-kernel void maxpool4(device const half4* x [[buffer(0)]],
- device half4* o [[buffer(1)]],
- constant PcPool& p [[buffer(2)]],
- uint gid [[thread_position_in_grid]])
-{
- uint c4 = p.c >> 2;
- if (gid >= c4 * p.oh * p.ow) return;
- uint ch = gid % c4, px = gid / c4;
- uint ox = px % p.ow, oy = px / p.ow;
- float4 m = float4(-3.4e38f);
- for (uint ky = 0u; ky < p.kh; ky++)
- {
- int sy = int(oy * p.sh + ky) - int(p.pt);
- if (uint(sy) >= p.ih) continue;
- for (uint kx = 0u; kx < p.kw; kx++)
- {
- int sx = int(ox * p.sw + kx) - int(p.pl);
- if (uint(sx) >= p.iw) continue;
- m = max(m, float4(x[(uint(sy) * p.iw + uint(sx)) * c4 + ch]));
- }
- }
- if (m.x == -3.4e38f) m = float4(0.0f);
- if (p.relu != 0u) m = select(m, float4(0.0f), m < float4(0.0f));
- o[gid] = half4(m);
-}
-
-// ---- layernorm: one workgroup (256 threads) per row ----
-struct PcLn { uint len; uint rows; float eps; };
-kernel void layernorm(device const half* x [[buffer(0)]],
- device const float* w [[buffer(1)]],
- device const float* b [[buffer(2)]],
- device half* o [[buffer(3)]],
- constant PcLn& p [[buffer(4)]],
- uint wg [[threadgroup_position_in_grid]],
- uint tid [[thread_position_in_threadgroup]])
-{
- // one simdgroup (32 lanes) per row, 8 rows per threadgroup: lane-strided
- // accumulate + simd_sum only, no threadgroup barrier at all.
- uint lane = tid & 31u;
- uint row = wg * 8u + (tid >> 5);
- if (row >= p.rows) return;
- uint base = row * p.len;
- float s = 0.0f, s2 = 0.0f;
- for (uint i = lane; i < p.len; i += 32u)
- {
- float v = float(x[base + i]);
- s += v; s2 += v * v;
- }
- s = simd_sum(s); s2 = simd_sum(s2);
- float mean = s / float(p.len);
- float inv = rsqrt(s2 / float(p.len) - mean * mean + p.eps);
- for (uint i = lane; i < p.len; i += 32u)
- o[base + i] = half((float(x[base + i]) - mean) * inv * w[i] + b[i]);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/shift.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/shift.metal
new file mode 100644
index 0000000..a6509c3
--- /dev/null
+++ b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/shift.metal
@@ -0,0 +1,38 @@
+// ---- shift: NHWC channel-shift (fused shift-MLP) ----
+struct PcShift {
+ uint c, h, w, axis;
+ uint ge0, ge1, ge2, ge3, ge4;
+ int of0, of1, of2, of3, of4;
+};
+static inline int shiftoff(uint ch, constant PcShift& p)
+{
+ return ch < p.ge0 ? p.of0 : ch < p.ge1 ? p.of1 : ch < p.ge2 ? p.of2 : ch < p.ge3 ? p.of3 : ch < p.ge4 ? p.of4 : 0;
+}
+kernel void shift(device const half* x [[buffer(0)]],
+ device half* o [[buffer(1)]],
+ constant PcShift& p [[buffer(2)]],
+ uint gid [[thread_position_in_grid]])
+{
+ if (gid >= p.c * p.h * p.w) return;
+ uint ch = gid % p.c, px = gid / p.c;
+ uint ox = px % p.w, oy = px / p.w;
+ int off = shiftoff(ch, p);
+ int sy = int(oy), sx = int(ox);
+ if (p.axis == 2u) sy += off; else sx += off;
+ o[gid] = half((uint(sy) < p.h && uint(sx) < p.w) ? float(x[(uint(sy) * p.w + uint(sx)) * p.c + ch]) : 0.0f);
+}
+kernel void shift4(device const half4* x [[buffer(0)]],
+ device half4* o [[buffer(1)]],
+ constant PcShift& p [[buffer(2)]],
+ uint gid [[thread_position_in_grid]])
+{
+ uint c4 = p.c >> 2;
+ if (gid >= c4 * p.h * p.w) return;
+ uint q = gid % c4, px = gid / c4, ch = q * 4u;
+ uint ox = px % p.w, oy = px / p.w;
+ int off = shiftoff(ch, p);
+ int sy = int(oy), sx = int(ox);
+ if (p.axis == 2u) sy += off; else sx += off;
+ o[gid] = (uint(sy) < p.h && uint(sx) < p.w) ? x[(uint(sy) * p.w + uint(sx)) * c4 + q] : half4(0.0h);
+}
+
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.comp
deleted file mode 100644
index 28ef98b..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.comp
+++ /dev/null
@@ -1,44 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// NHWC bilinear resize, half-pixel centers: src_coord = (dst + 0.5) / scale - 0.5.
-// flags: bit0 relu on the resized value, bit1 add same-shape A, bit2 relu on A.
-// out = [relu](resize(x)) + [relu](a)
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; };
-layout(set = 0, binding = 1) readonly buffer A { ET a[]; }; // addend (dummy if none)
-layout(set = 0, binding = 2) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC {
- uint c, ih, iw, oh, ow, flags;
- float shy, shx; // scales
-} p;
-
-void main()
-{
- uint id = gl_GlobalInvocationID.x;
- if (id >= p.c * p.oh * p.ow) return;
- uint ch = id % p.c, px = id / p.c;
- uint ox = px % p.ow, oy = px / p.ow;
-
- float fy = (float(oy) + 0.5) / p.shy - 0.5;
- float fx = (float(ox) + 0.5) / p.shx - 0.5;
- int fy0 = int(floor(fy)), fx0 = int(floor(fx));
- float dy = fy - float(fy0), dx = fx - float(fx0);
- uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1));
- uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1));
-
- float v00 = float(x[(y0 * p.iw + x0) * p.c + ch]);
- float v01 = float(x[(y0 * p.iw + x1) * p.c + ch]);
- float v10 = float(x[(y1 * p.iw + x0) * p.c + ch]);
- float v11 = float(x[(y1 * p.iw + x1) * p.c + ch]);
- float v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy);
- if ((p.flags & 1u) != 0u) v = max(v, 0.0);
- if ((p.flags & 2u) != 0u)
- {
- float s = float(a[id]);
- v += (p.flags & 4u) != 0u ? max(s, 0.0) : s;
- }
- o[id] = ET(v);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.spv
deleted file mode 100644
index a252fef..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.comp
deleted file mode 100644
index 0a84eb4..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.comp
+++ /dev/null
@@ -1,44 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// bilinear.comp with 4 channels per invocation (c % 4 == 0), same per-element math.
-// flags: bit0 relu on the resized value, bit1 add same-shape A, bit2 relu on A.
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET4 x[]; };
-layout(set = 0, binding = 1) readonly buffer A { ET4 a[]; }; // addend (dummy if none)
-layout(set = 0, binding = 2) writeonly buffer O { ET4 o[]; };
-
-layout(push_constant) uniform PC {
- uint c, ih, iw, oh, ow, flags;
- float shy, shx; // scales
-} p;
-
-void main()
-{
- uint id = gl_GlobalInvocationID.x;
- uint c4 = p.c >> 2;
- if (id >= c4 * p.oh * p.ow) return;
- uint ch = id % c4, px = id / c4;
- uint ox = px % p.ow, oy = px / p.ow;
-
- float fy = (float(oy) + 0.5) / p.shy - 0.5;
- float fx = (float(ox) + 0.5) / p.shx - 0.5;
- int fy0 = int(floor(fy)), fx0 = int(floor(fx));
- float dy = fy - float(fy0), dx = fx - float(fx0);
- uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1));
- uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1));
-
- vec4 v00 = vec4(x[(y0 * p.iw + x0) * c4 + ch]);
- vec4 v01 = vec4(x[(y0 * p.iw + x1) * c4 + ch]);
- vec4 v10 = vec4(x[(y1 * p.iw + x0) * c4 + ch]);
- vec4 v11 = vec4(x[(y1 * p.iw + x1) * c4 + ch]);
- vec4 v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy);
- if ((p.flags & 1u) != 0u) v = max(v, vec4(0.0));
- if ((p.flags & 2u) != 0u)
- {
- vec4 s = vec4(a[id]);
- v += (p.flags & 4u) != 0u ? max(s, vec4(0.0)) : s;
- }
- o[id] = ET4(v);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.spv
deleted file mode 100644
index 274db15..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4_h.spv
deleted file mode 100644
index bbbaff9..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear_h.spv
deleted file mode 100644
index c7258e9..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.comp
deleted file mode 100644
index 8b1f173..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.comp
+++ /dev/null
@@ -1,27 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// Concat along an axis: one dispatch per input. Copies this input's slice into
-// dst at axisOffset. dst[o, a, i] = src[o, a - axisOff, i].
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; };
-layout(set = 0, binding = 1) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC {
- uint n; // src element count
- uint inner; // product of dims after axis
- uint srcAxis; // src extent on axis
- uint dstAxis; // dst extent on axis
- uint axisOff; // dst offset of this input on axis
-} p;
-
-void main()
-{
- uint i = gl_GlobalInvocationID.x;
- if (i >= p.n) return;
- uint ii = i % p.inner;
- uint a = (i / p.inner) % p.srcAxis;
- uint oi = i / (p.inner * p.srcAxis);
- o[(oi * p.dstAxis + a + p.axisOff) * p.inner + ii] = x[i];
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.spv
deleted file mode 100644
index efde93b..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax_h.spv
deleted file mode 100644
index 3fdf16a..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm.comp
deleted file mode 100644
index c843790..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm.comp
+++ /dev/null
@@ -1,358 +0,0 @@
-#version 450
-
-#extension GL_KHR_cooperative_matrix : require
-#extension GL_KHR_memory_scope_semantics : require
-#extension GL_KHR_shader_subgroup_basic : require
-#extension GL_EXT_shader_16bit_storage : require
-#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
-#extension GL_EXT_control_flow_attributes : require
-
-// Tensor-core implicit-GEMM dense KxK conv on NHWC fp16 activations
-// (after SimdPaddleOCR's convk_cm_sg32):
-// OUT[m = oy*outW + ox, n] = act(sum_k A[m, k] * W[n, k] + bias[n] + res[m, n])
-// k = (dy*kW + dx)*cin + ci (tap-major), A gathered from the input (no im2col),
-// zero padding or reflect padding (flag), fp16 in memory, fp32 accumulation.
-// SGX*SGY 32-lane subgroups, each owning a TSM*16 x TSN*16 strip of 16x16
-// accumulators; A/B k-slices of 32 halves are staged through double-buffered
-// shared memory with coalesced 16 B loads. Requires cin % 8 == 0 (a 16 B quad
-// never straddles a tap), nDim % 8 == 0 and 16 B aligned bindings. W is the
-// [N][kDim] tap-major repack (kDim = kH*kW*cin rounded up to 8, zero-filled).
-// SCALARK: any cin (the RGB/6-channel stems) — each k element of a quad is
-// gathered individually, k >= kReal reads as zero.
-// LITE (wave64-capable parts / 32 KB shared memory, after SimdPaddleOCR's sg32l): single-
-// buffered staging, and the epilogue stage aliases the staging memory (one uvec4 array; the
-// coopmat load/store element type may differ from the matrix type), so a 512-thread / 16-
-// subgroup tile fits in 20 KB; scalar epilogue; workgroups rastered in groups of RASTER
-// N-columns so the tiles in flight share A strips in L2. Same k order and epilogue math as
-// the default build.
-// SG16 (with LITE; Intel Xe2 XMX, after SimdPaddleOCR's conv1x1_cm): 16-lane subgroups
-// (requiredSubgroupSize = 16) and the 8x16x16 shape — A and accumulator fragments are 8
-// rows, so a TSM*8 x TSN*16 strip per subgroup; staging, k order and epilogue as LITE.
-// flags: bit0 bias, bit1 residual, bit2 reflect pad, bits[4:6] act (0 none, 1 relu, 2 gelu)
-
-#ifndef TSM
-#define TSM 4u
-#endif
-#ifndef TSN
-#define TSN 2u
-#endif
-#ifndef SGX
-#define SGX 4u
-#endif
-#ifndef SGY
-#define SGY 2u
-#endif
-#ifndef NTHR
-#define NTHR 256
-#endif
-#ifdef SG16
-#ifndef LITE
-#error SG16 requires LITE
-#endif
-#define SGSZ 16u
-#define AM 8 // A / accumulator fragment rows
-#define AMU 8u
-#else
-#define SGSZ 32u
-#define AM 16
-#define AMU 16u
-#endif
-#define RASTER 4u
-
-layout(local_size_x = NTHR) in;
-
-const uint STRM = TSM * AMU;
-const uint STRN = TSN * 16u;
-const uint BM = SGY * STRM;
-const uint BN = SGX * STRN;
-const uint BKV = 8u; // k-slice in vec4 units (32 halves)
-const uint LDS = BKV + 2u; // padded smem row stride (vec4 units)
-const uint BKQ = BKV / 2u; // 16 B quads per slice row
-const uint RSTEP = NTHR / BKQ;
-const uint LA = BM / RSTEP;
-const uint LB = BN / RSTEP;
-
-layout(set = 0, binding = 0) readonly buffer X8 { uvec4 x8[]; };
-layout(set = 0, binding = 0) readonly buffer X1 { float16_t x1[]; };
-layout(set = 0, binding = 1) readonly buffer W8 { uvec4 w8[]; };
-layout(set = 0, binding = 2) readonly buffer Bias { float bias[]; };
-layout(set = 0, binding = 3) readonly buffer Res8 { uvec4 res8[]; };
-layout(set = 0, binding = 4) writeonly buffer O8 { uvec4 o8[]; };
-#ifdef LITE
-layout(set = 0, binding = 3) readonly buffer Res1 { float16_t res1[]; };
-layout(set = 0, binding = 4) writeonly buffer O1 { float16_t o1[]; };
-#endif
-
-layout(push_constant) uniform PC {
- uint mDim; uint nDim; uint kDim; uint flags;
- uint outW; uint inW; uint inH; uint cin; uint kW;
- uint sH; uint sW; uint pT; uint pL; uint kReal;
-} p;
-
-#ifdef LITE
-const uint LDS4 = LDS / 2u; // row stride in 16 B units
-const uint SM_AB = (BM + BN) * LDS4; // A rows, then B rows (single buffer)
-const uint SM_ST = NTHR / SGSZ * (AMU * 4u); // one AMx16 fp32 tile per subgroup (epilogue)
-shared uvec4 smem[SM_AB > SM_ST ? SM_AB : SM_ST];
-#else
-shared f16vec4 sA[2u * BM * LDS];
-shared f16vec4 sB[2u * BN * LDS];
-shared float stage[NTHR / SGSZ * 16 * 16];
-#endif
-
-float ferf(float v)
-{
- float s = v < 0.0 ? -1.0 : 1.0;
- float ax = abs(v);
- float t = 1.0 / (1.0 + 0.3275911 * ax);
- float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
- return (1.0 - poly * exp(-ax * ax)) * s;
-}
-
-float actf(float v, uint act)
-{
- if (act == 1u) return max(v, 0.0);
- if (act == 2u) return 0.5 * v * (1.0 + ferf(v * 0.7071067811865475));
- return v;
-}
-
-int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; }
-
-int rowY[LA], rowX[LA];
-bool rowOk[LA];
-
-void loadSlice(out uvec4 ra[LA], out uvec4 rb[LB], out uint va, out uint vb,
- uint kt, uint n0, uint r0, uint kq, bool reflect)
-{
- uint q = kt * BKQ + kq; // global 16 B quad index along k
- uint kv2 = p.kDim >> 3;
- bool kok = q < kv2;
- uint qc = min(q, kv2 - 1u);
- va = 0u; vb = 0u;
- uint k8 = qc * 8u;
-#ifdef SCALARK
-#ifdef LITE
- // the k -> (tap, ci) split does not depend on the row: once per element, not per row
- uint edy[8], edx[8], eci[8];
- [[unroll]] for (uint e = 0u; e < 8u; e++) {
- uint k = k8 + e;
- uint tap = k / p.cin;
- eci[e] = k - tap * p.cin;
- edy[e] = tap / p.kW;
- edx[e] = tap - edy[e] * p.kW;
- }
-#endif
- [[unroll]] for (uint i = 0u; i < LA; i++) {
- float16_t h[8];
- [[unroll]] for (uint e = 0u; e < 8u; e++) {
- uint k = k8 + e;
-#ifdef LITE
- uint ci = eci[e], dy = edy[e], dx = edx[e];
-#else
- uint tap = k / p.cin, ci = k - tap * p.cin;
- uint dy = tap / p.kW, dx = tap - dy * p.kW;
-#endif
- int iy = rowY[i] + int(dy), ix = rowX[i] + int(dx);
- bool inb = iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW);
- if (reflect && rowOk[i] && !inb) { iy = mirror(iy, int(p.inH)); ix = mirror(ix, int(p.inW)); inb = true; }
- bool ok = kok && rowOk[i] && inb && k < p.kReal;
- h[e] = x1[ok ? (uint(iy) * p.inW + uint(ix)) * p.cin + ci : 0u];
- h[e] = ok ? h[e] : float16_t(0.0);
- }
- ra[i] = uvec4(packFloat2x16(f16vec2(h[0], h[1])), packFloat2x16(f16vec2(h[2], h[3])),
- packFloat2x16(f16vec2(h[4], h[5])), packFloat2x16(f16vec2(h[6], h[7])));
- va |= kok && rowOk[i] ? 1u << i : 0u;
- }
-#else
- uint tap = k8 / p.cin, ci = k8 - tap * p.cin;
- uint dy = tap / p.kW, dx = tap - dy * p.kW;
- uint cq = p.cin >> 3;
- [[unroll]] for (uint i = 0u; i < LA; i++) {
- int iy = rowY[i] + int(dy), ix = rowX[i] + int(dx);
- bool inb = iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW);
- if (reflect && rowOk[i] && !inb) { iy = mirror(iy, int(p.inH)); ix = mirror(ix, int(p.inW)); inb = true; }
- bool ok = kok && rowOk[i] && inb;
- uint px = ok ? uint(iy) * p.inW + uint(ix) : 0u;
- ra[i] = x8[px * cq + (ci >> 3)];
- va |= ok ? 1u << i : 0u;
- }
-#endif
- [[unroll]] for (uint i = 0u; i < LB; i++) {
- uint gn = n0 + r0 + i * RSTEP;
- bool rok = gn < p.nDim;
- gn = min(gn, p.nDim - 1u);
- rb[i] = w8[gn * kv2 + qc];
- vb |= rok && kok ? 1u << i : 0u;
- }
-}
-
-void storeSlice(uvec4 ra[LA], uvec4 rb[LB], uint va, uint vb, uint buf, uint r0, uint kq)
-{
-#ifdef LITE
- [[unroll]] for (uint i = 0u; i < LA; i++)
- smem[(r0 + i * RSTEP) * LDS4 + kq] = (va & (1u << i)) != 0u ? ra[i] : uvec4(0u);
- [[unroll]] for (uint i = 0u; i < LB; i++)
- smem[(BM + r0 + i * RSTEP) * LDS4 + kq] = (vb & (1u << i)) != 0u ? rb[i] : uvec4(0u);
-#else
- [[unroll]] for (uint i = 0u; i < LA; i++) {
- uint o = buf * BM * LDS + (r0 + i * RSTEP) * LDS + kq * 2u;
- bool ok = (va & (1u << i)) != 0u;
- sA[o] = ok ? f16vec4(unpackFloat2x16(ra[i].x), unpackFloat2x16(ra[i].y)) : f16vec4(0.0hf);
- sA[o + 1u] = ok ? f16vec4(unpackFloat2x16(ra[i].z), unpackFloat2x16(ra[i].w)) : f16vec4(0.0hf);
- }
- [[unroll]] for (uint i = 0u; i < LB; i++) {
- uint o = buf * BN * LDS + (r0 + i * RSTEP) * LDS + kq * 2u;
- bool ok = (vb & (1u << i)) != 0u;
- sB[o] = ok ? f16vec4(unpackFloat2x16(rb[i].x), unpackFloat2x16(rb[i].y)) : f16vec4(0.0hf);
- sB[o + 1u] = ok ? f16vec4(unpackFloat2x16(rb[i].z), unpackFloat2x16(rb[i].w)) : f16vec4(0.0hf);
- }
-#endif
-}
-
-void main()
-{
- uint tid = gl_LocalInvocationID.x;
- uint sg = gl_SubgroupID;
- uint lane = gl_SubgroupInvocationID;
- uint sgx = sg & (SGX - 1u), sgy = sg / SGX;
-#ifdef LITE
- // walk RASTER N-columns at a time, M fastest within the group
- uint gmt = gl_NumWorkGroups.x;
- uint lin = gl_WorkGroupID.y * gmt + gl_WorkGroupID.x;
- uint grp = lin / (RASTER * gmt);
- uint gw = min(gl_NumWorkGroups.y - grp * RASTER, RASTER);
- uint gr = lin - grp * RASTER * gmt;
- uint m0 = gr / gw * BM;
- uint n0 = (grp * RASTER + gr % gw) * BN;
-#else
- uint m0 = gl_WorkGroupID.x * BM;
- uint n0 = gl_WorkGroupID.y * BN;
-#endif
-
- uint act = (p.flags >> 4) & 7u;
- bool hasBias = (p.flags & 1u) != 0u;
- bool hasRes = (p.flags & 2u) != 0u;
- bool reflect = (p.flags & 4u) != 0u;
-
- coopmat acc[TSM][TSN];
- [[unroll]] for (uint i = 0u; i < TSM; i++)
- [[unroll]] for (uint j = 0u; j < TSN; j++)
- acc[i][j] = coopmat(0.0f);
-
- uint kq = tid % BKQ;
- uint r0 = tid / BKQ;
- [[unroll]] for (uint i = 0u; i < LA; i++) {
- uint gm = m0 + r0 + i * RSTEP;
- uint oy = gm / p.outW, ox = gm - oy * p.outW;
- rowOk[i] = gm < p.mDim;
- rowY[i] = int(oy * p.sH) - int(p.pT);
- rowX[i] = int(ox * p.sW) - int(p.pL);
- }
- uint nk = (p.kDim + 31u) / 32u;
-
- uvec4 ra[LA], rb[LB];
- uint va, vb;
- loadSlice(ra, rb, va, vb, 0u, n0, r0, kq, reflect);
- storeSlice(ra, rb, va, vb, 0u, r0, kq);
- barrier();
-
- coopmat ma[TSM];
- coopmat mb[TSN];
-#ifdef LITE
- uint ab = sgy * STRM * LDS4, bb = (BM + sgx * STRN) * LDS4;
- for (uint kt = 0u; kt < nk; kt++) {
- bool more = kt + 1u < nk;
- if (more) loadSlice(ra, rb, va, vb, kt + 1u, n0, r0, kq, reflect);
- [[unroll]] for (uint kk = 0u; kk < BKV / 2u; kk += 2u) {
- [[unroll]] for (uint i = 0u; i < TSM; i++)
- coopMatLoad(ma[i], smem, ab + i * AMU * LDS4 + kk, LDS4, gl_CooperativeMatrixLayoutRowMajor);
- [[unroll]] for (uint j = 0u; j < TSN; j++)
- coopMatLoad(mb[j], smem, bb + j * 16u * LDS4 + kk, LDS4, gl_CooperativeMatrixLayoutColumnMajor);
- [[unroll]] for (uint i = 0u; i < TSM; i++)
- [[unroll]] for (uint j = 0u; j < TSN; j++)
- acc[i][j] = coopMatMulAdd(ma[i], mb[j], acc[i][j]);
- }
- barrier(); // one buffer: every read done before the refill
- if (more) storeSlice(ra, rb, va, vb, 0u, r0, kq);
- barrier();
- }
-#else
- uint aw = sgy * STRM * LDS, bw = sgx * STRN * LDS;
- for (uint kt = 0u; kt < nk; kt++) {
- uint cur = kt & 1u;
- bool more = kt + 1u < nk;
- if (more) loadSlice(ra, rb, va, vb, kt + 1u, n0, r0, kq, reflect);
- uint ab = cur * BM * LDS + aw, bb = cur * BN * LDS + bw;
- [[unroll]] for (uint kk = 0u; kk < BKV; kk += 4u) {
- [[unroll]] for (uint i = 0u; i < TSM; i++)
- coopMatLoad(ma[i], sA, ab + i * 16u * LDS + kk, LDS, gl_CooperativeMatrixLayoutRowMajor);
- [[unroll]] for (uint j = 0u; j < TSN; j++)
- coopMatLoad(mb[j], sB, bb + j * 16u * LDS + kk, LDS, gl_CooperativeMatrixLayoutColumnMajor);
- [[unroll]] for (uint i = 0u; i < TSM; i++)
- [[unroll]] for (uint j = 0u; j < TSN; j++)
- acc[i][j] = coopMatMulAdd(ma[i], mb[j], acc[i][j]);
- }
- if (more) storeSlice(ra, rb, va, vb, cur ^ 1u, r0, kq);
- barrier();
- }
-#endif
-
-#ifdef LITE
- // lane owns elements lane + SGSZ*l of each AMx16 tile: scalar fp16 stores, coalesced rows
- [[unroll]] for (uint i = 0u; i < TSM; i++) {
- [[unroll]] for (uint j = 0u; j < TSN; j++) {
- coopMatStore(acc[i][j], smem, sg * (AMU * 4u), 4u, gl_CooperativeMatrixLayoutRowMajor);
- subgroupMemoryBarrierShared();
- subgroupBarrier();
- uint fr = m0 + sgy * STRM + i * AMU, cn = n0 + sgx * STRN + j * 16u;
- for (uint l = 0u; l < AMU * 16u / SGSZ; l++) {
- uint e = l * SGSZ + lane;
- uint gmr = fr + (e >> 4), gn = cn + (e & 15u);
- if (gmr < p.mDim && gn < p.nDim) {
- float v = uintBitsToFloat(smem[sg * (AMU * 4u) + (e >> 2)][e & 3u]);
- if (hasBias) v += bias[gn];
- uint oe = gmr * p.nDim + gn;
- if (hasRes) v += float(res1[oe]);
- o1[oe] = float16_t(actf(v, act));
- }
- }
- subgroupMemoryBarrierShared();
- subgroupBarrier();
- }
- }
-#else
- // epilogue through a per-subgroup row-major stage: each lane owns 8
- // consecutive columns of one row, so bias / residual / outputs move as 16 B
- uint mm = lane >> 1, nb = (lane & 1u) * 8u;
- [[unroll]] for (uint i = 0u; i < TSM; i++) {
- [[unroll]] for (uint j = 0u; j < TSN; j++) {
- coopMatStore(acc[i][j], stage, sg * 256u, 16u, gl_CooperativeMatrixLayoutRowMajor);
- subgroupMemoryBarrierShared();
- subgroupBarrier();
- uint gmr = m0 + sgy * STRM + i * 16u + mm;
- uint gnc = n0 + sgx * STRN + j * 16u + nb;
- uint sb = sg * 256u + mm * 16u + nb;
- if (gmr < p.mDim && gnc < p.nDim) {
- float v[8];
- [[unroll]] for (uint l = 0u; l < 8u; l++) v[l] = stage[sb + l];
- if (hasBias)
- [[unroll]] for (uint l = 0u; l < 8u; l++) v[l] += bias[gnc + l];
- uint oi = (gmr * p.nDim + gnc) >> 3;
- if (hasRes) {
- uvec4 rq = res8[oi];
- [[unroll]] for (uint l = 0u; l < 8u; l += 2u) {
- vec2 r2 = vec2(unpackFloat2x16(rq[l >> 1]));
- v[l] += r2.x; v[l + 1u] += r2.y;
- }
- }
- uvec4 oq;
- [[unroll]] for (uint l = 0u; l < 8u; l += 2u)
- oq[l >> 1] = packFloat2x16(f16vec2(actf(v[l], act), actf(v[l + 1u], act)));
- o8[oi] = oq;
- }
- subgroupMemoryBarrierShared();
- subgroupBarrier();
- }
- }
-#endif
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_h.spv
deleted file mode 100644
index 8b3f3b8..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_h.spv
deleted file mode 100644
index 9cc7c44..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32_h.spv
deleted file mode 100644
index 53a037b..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32s_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32s_h.spv
deleted file mode 100644
index 756df1d..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32s_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n64_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n64_h.spv
deleted file mode 100644
index 424f6fb..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n64_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32_h.spv
deleted file mode 100644
index 12a63e4..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32s_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32s_h.spv
deleted file mode 100644
index a025194..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32s_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n64_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n64_h.spv
deleted file mode 100644
index ca2e6f7..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n64_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_h.spv
deleted file mode 100644
index 41ebb12..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32_h.spv
deleted file mode 100644
index c99e99f..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32s_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32s_h.spv
deleted file mode 100644
index 33d4f36..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32s_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n64_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n64_h.spv
deleted file mode 100644
index dae8855..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n64_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot.comp
deleted file mode 100644
index f6ae97b..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot.comp
+++ /dev/null
@@ -1,146 +0,0 @@
-#version 450
-#extension GL_EXT_shader_16bit_storage : require
-#extension GL_EXT_control_flow_attributes : require
-// Direct NHWC 3x3 conv for GPUs without cooperative matrices whose wide waves
-// (Adreno: 64/128 lanes, 32 KB shared memory) starve on conv_tile's shared-memory
-// staging: no shared memory, every operand is an f16vec4 load that the L1/texture
-// cache serves. Each invocation computes TM consecutive output pixels of one row x
-// TN*4 output channels; the TM*SW+2 input pixels of a kernel row are loaded once
-// per 4 input channels and reused by the three kx taps. Invocations of one wave
-// share pixels and step through oc (n4 fastest), so input loads broadcast and
-// weight loads coalesce. Requires fp16 storage, ic % 4 == 0 and oc % (4*TN) == 0.
-// Accumulation is fp32. Same push constants / weight layout ([K][OC], k = tap*IC + ic)
-// as conv_tile.comp. Compile-time: -DSW= (1 or 2), -DTM=, -DTN=.
-layout(local_size_x = 64) in;
-
-layout(set = 0, binding = 0) readonly buffer X { f16vec4 x[]; };
-layout(set = 0, binding = 1) readonly buffer W { f16vec4 w[]; };
-layout(set = 0, binding = 2) readonly buffer B { float b[]; };
-layout(set = 0, binding = 3) readonly buffer R { f16vec4 r[]; };
-layout(set = 0, binding = 4) writeonly buffer O { f16vec4 o[]; };
-
-layout(push_constant) uniform PC {
- uint ic, ih, iw;
- uint oc, oh, ow;
- uint kh, kw, sh, sw;
- uint pt, pl;
- uint flags; // bit0 hasBias, bit1 reflectPad, bit2 relu, bit3 hasAdd, bit4 gelu
- uint pad0;
-} p;
-
-#ifndef SW
-#define SW 1u
-#endif
-#ifndef TM
-#define TM 4
-#endif
-#ifndef TN
-#define TN 2
-#endif
-const int NX = (TM - 1) * int(SW) + 3; // input pixels of one kernel row
-
-int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; }
-
-float ferf(float v)
-{
- float s = v < 0.0 ? -1.0 : 1.0;
- float ax = abs(v);
- float t = 1.0 / (1.0 + 0.3275911 * ax);
- float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
- return (1.0 - poly * exp(-ax * ax)) * s;
-}
-
-vec4 gelu4(vec4 v)
-{
- return vec4(0.5 * v.x * (1.0 + ferf(v.x * 0.7071067811865475)),
- 0.5 * v.y * (1.0 + ferf(v.y * 0.7071067811865475)),
- 0.5 * v.z * (1.0 + ferf(v.z * 0.7071067811865475)),
- 0.5 * v.w * (1.0 + ferf(v.w * 0.7071067811865475)));
-}
-
-void main()
-{
- uint ng = p.oc / uint(4 * TN); // oc groups
- uint xg = (p.ow + uint(TM) - 1u) / uint(TM); // pixel groups per row
- uint gid = gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * gl_WorkGroupSize.x;
- if (gid >= ng * xg * p.oh) return;
- uint g = gid % ng;
- uint t = gid / ng;
- uint xb = t % xg, oy = t / xg;
- uint ox0 = xb * uint(TM);
- uint c4 = p.ic >> 2, o4 = p.oc >> 2;
- uint n40 = g * uint(TN);
- bool reflect = (p.flags & 2u) != 0u;
-
- vec4 acc[TM * TN];
- [[unroll]] for (int i = 0; i < TM * TN; i++) acc[i] = vec4(0.0);
-
- // input column per window slot (clamped / mirrored once; zero-pad slots masked)
- int sx0 = int(ox0 * SW) - int(p.pl);
- uint col[NX];
- bool cok[NX];
- [[unroll]] for (int i = 0; i < NX; i++)
- {
- int sx = sx0 + i;
- cok[i] = reflect || uint(sx) < p.iw;
- if (reflect && uint(sx) >= p.iw) sx = mirror(sx, int(p.iw));
- col[i] = uint(clamp(sx, 0, int(p.iw) - 1));
- }
-
- // no `continue`: Adreno 750 mis-executed a `continue` in this ky loop (one accumulator
- // of the 4x2 tile was lost) and in depthwise's tap loop; zero-pad rows are masked loads
- for (uint ky = 0u; ky < 3u; ky++)
- {
- int sy = int(oy * p.sh + ky) - int(p.pt);
- bool rok = reflect || uint(sy) < p.ih;
- sy = reflect ? mirror(sy, int(p.ih)) : clamp(sy, 0, int(p.ih) - 1);
- uint rowBase = uint(sy) * p.iw;
- uint wk = ky * 3u * p.ic; // k of (ky, kx = 0, ci = 0)
- for (uint ci = 0u; ci < c4; ci++)
- {
- vec4 xs[NX];
- [[unroll]] for (int i = 0; i < NX; i++)
- xs[i] = cok[i] && rok ? vec4(x[(rowBase + col[i]) * c4 + ci]) : vec4(0.0);
- [[unroll]] for (uint kx = 0u; kx < 3u; kx++)
- {
- uint wrow = (wk + kx * p.ic + ci * 4u) * o4 + n40;
- [[unroll]] for (int j = 0; j < TN; j++)
- {
- vec4 w0 = vec4(w[wrow + uint(j)]);
- vec4 w1 = vec4(w[wrow + o4 + uint(j)]);
- vec4 w2 = vec4(w[wrow + 2u * o4 + uint(j)]);
- vec4 w3 = vec4(w[wrow + 3u * o4 + uint(j)]);
- [[unroll]] for (int m = 0; m < TM; m++)
- {
- vec4 a = xs[m * int(SW) + int(kx)];
- acc[m * TN + j] += a.x * w0 + a.y * w1 + a.z * w2 + a.w * w3;
- }
- }
- }
- }
- }
-
- bool hasBias = (p.flags & 1u) != 0u;
- bool relu = (p.flags & 4u) != 0u;
- bool hasAdd = (p.flags & 8u) != 0u;
- bool gelu = (p.flags & 16u) != 0u;
- [[unroll]] for (int j = 0; j < TN; j++)
- {
- uint n4 = n40 + uint(j);
- uint oc0 = n4 * 4u;
- vec4 bias = hasBias ? vec4(b[oc0], b[oc0 + 1u], b[oc0 + 2u], b[oc0 + 3u]) : vec4(0.0);
- [[unroll]] for (int m = 0; m < TM; m++)
- {
- uint ox = ox0 + uint(m);
- if (ox < p.ow)
- {
- uint oi = (oy * p.ow + ox) * o4 + n4;
- vec4 v = acc[m * TN + j] + bias;
- if (hasAdd) v += vec4(r[oi]);
- if (relu) v = max(v, vec4(0.0));
- if (gelu) v = gelu4(v);
- o[oi] = f16vec4(v);
- }
- }
- }
-}
\ No newline at end of file
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x1_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x1_h.spv
deleted file mode 100644
index 38307d1..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x1_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x2_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x2_h.spv
deleted file mode 100644
index 3a5a359..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x2_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x1_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x1_h.spv
deleted file mode 100644
index 8923448..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x1_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x2_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x2_h.spv
deleted file mode 100644
index bfcc4bc..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x2_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen.spv
deleted file mode 100644
index 3e630a8..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen_h.spv
deleted file mode 100644
index 1255943..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1.spv
deleted file mode 100644
index 95df567..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1_h.spv
deleted file mode 100644
index d94d45a..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1.spv
deleted file mode 100644
index 8d3b620..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1_h.spv
deleted file mode 100644
index ec1fc64..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2.spv
deleted file mode 100644
index e47861f..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2_h.spv
deleted file mode 100644
index 5740460..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_tile.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_tile.comp
deleted file mode 100644
index 52ca6ab..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_tile.comp
+++ /dev/null
@@ -1,166 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-#extension GL_EXT_control_flow_attributes : require
-// NHWC implicit-GEMM conv: O[px, oc] = sum_k X[px, k] * W[k, oc] with
-// k = (ky*KW + kx)*IC + ic, so consecutive k are consecutive channels of one
-// input pixel (coalesced loads). W is backend-packed [K][OC] (oc fastest).
-// 3D dispatch: gx = x-tile(16), gy = y-tile(8), gz = oc-tile(64); 256 threads,
-// each 4 oc x 8 pixels (vec4 accumulators over oc). K-chunk 32 staged in
-// shared: As[k][m] (vec4 over m), Bs[n][k] (row stride 36 floats, vec4 over k).
-// Accumulation is always fp32. Compile-time kernel/stride: -DKH= -DKW= -DSH= -DSW=.
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; };
-layout(set = 0, binding = 1) readonly buffer W { ET w[]; };
-layout(set = 0, binding = 2) readonly buffer B { float b[]; }; // per-OC bias (dummy if none)
-layout(set = 0, binding = 3) readonly buffer R { ET r[]; }; // same-shape residual (dummy if none)
-layout(set = 0, binding = 4) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC {
- uint ic, ih, iw;
- uint oc, oh, ow;
- uint kh, kw, sh, sw;
- uint pt, pl;
- uint flags; // bit0 hasBias, bit1 reflectPad, bit2 relu, bit3 hasAdd, bit4 gelu
- uint pad0;
-} p;
-
-#ifndef KH
-#define KH (p.kh)
-#endif
-#ifndef KW
-#define KW (p.kw)
-#endif
-#ifndef SH
-#define SH (p.sh)
-#endif
-#ifndef SW
-#define SW (p.sw)
-#endif
-
-const uint BM = 64u; // oc tile
-const uint BH = 8u, BW = 16u; // spatial tile
-const uint BN = BH * BW; // 128 pixels per wg
-const uint BK = 32u;
-const uint BKP = 36u; // Bs row stride in floats (vec4 aligned)
-shared vec4 Asv[BK * BM / 4];
-shared vec4 Bsv[BN * BKP / 4];
-
-int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; }
-
-float ferf(float v)
-{
- float s = v < 0.0 ? -1.0 : 1.0;
- float ax = abs(v);
- float t = 1.0 / (1.0 + 0.3275911 * ax);
- float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
- return (1.0 - poly * exp(-ax * ax)) * s;
-}
-
-void main()
-{
- uint oc0 = gl_WorkGroupID.z * BM;
- uint oy0 = gl_WorkGroupID.y * BH;
- uint ox0 = gl_WorkGroupID.x * BW;
- uint K = KH * KW * p.ic;
- uint tid = gl_LocalInvocationID.x;
- uint tm = tid & 15u, tn = tid >> 4; // tm: 4 oc each, tn: 8 pixels each
-
- vec4 acc[8] = vec4[8](vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0));
-
- bool reflect = (p.flags & 2u) != 0u;
- uint bk = tid & 31u; // fixed k lane for Bs loads (256 % 32 == 0)
-
- for (uint k0 = 0u; k0 < K; k0 += BK)
- {
- // As[k][m] = W[k0+k][oc0+m]
- for (uint e = tid; e < BK * BM; e += 256u)
- {
- uint k = e >> 6, m = e & 63u;
- uint gm = oc0 + m, gk = k0 + k;
- Asv[e >> 2][e & 3u] = (gm < p.oc && gk < K) ? float(w[gk * p.oc + gm]) : 0.0;
- }
- // Bs[n][k]: a warp reads 32 consecutive k = channels of one pixel tap
- uint gk = k0 + bk;
- bool kok = gk < K;
- uint tap = kok ? gk / p.ic : 0u;
- uint ch = gk - tap * p.ic;
- uint ky = tap / KW, kx = tap - ky * KW;
- for (uint n = tid >> 5; n < BN; n += 8u)
- {
- float v = 0.0;
- uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u);
- if (kok && oy < p.oh && ox < p.ow)
- {
- int sy = int(oy * SH + ky) - int(p.pt);
- int sx = int(ox * SW + kx) - int(p.pl);
- if (reflect)
- {
- if (uint(sy) >= p.ih) sy = mirror(sy, int(p.ih));
- if (uint(sx) >= p.iw) sx = mirror(sx, int(p.iw));
- v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]);
- }
- else if (uint(sy) < p.ih && uint(sx) < p.iw)
- v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]);
- }
- uint si = n * BKP + bk;
- Bsv[si >> 2][si & 3u] = v;
- }
- barrier();
-
- [[unroll]] for (uint k = 0u; k < BK; k += 4u)
- {
- vec4 a0 = Asv[k * 16u + tm];
- vec4 a1 = Asv[(k + 1u) * 16u + tm];
- vec4 a2 = Asv[(k + 2u) * 16u + tm];
- vec4 a3 = Asv[(k + 3u) * 16u + tm];
- [[unroll]] for (int j = 0; j < 8; j++)
- {
- vec4 bv = Bsv[(tn * 8u + uint(j)) * (BKP / 4u) + (k >> 2)];
- acc[j] += a0 * bv.x + a1 * bv.y + a2 * bv.z + a3 * bv.w;
- }
- }
- barrier();
- }
-
- bool hasBias = (p.flags & 1u) != 0u;
- bool relu = (p.flags & 4u) != 0u;
- bool hasAdd = (p.flags & 8u) != 0u;
- bool gelu = (p.flags & 16u) != 0u;
- uint gm0 = oc0 + tm * 4u;
- // constant component indices only: keeps acc[] in registers (no local-memory spill)
- vec4 bias = vec4(
- hasBias && gm0 < p.oc ? b[gm0] : 0.0,
- hasBias && gm0 + 1u < p.oc ? b[gm0 + 1u] : 0.0,
- hasBias && gm0 + 2u < p.oc ? b[gm0 + 2u] : 0.0,
- hasBias && gm0 + 3u < p.oc ? b[gm0 + 3u] : 0.0);
- bool full = gm0 + 3u < p.oc;
- [[unroll]] for (int j = 0; j < 8; j++)
- {
- uint n = tn * 8u + uint(j);
- uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u);
- if (oy >= p.oh || ox >= p.ow || gm0 >= p.oc) continue;
- uint base = (oy * p.ow + ox) * p.oc + gm0;
- vec4 v = acc[j] + bias;
- if (hasAdd)
- {
- v.x += float(r[base]);
- if (full) { v.y += float(r[base + 1u]); v.z += float(r[base + 2u]); v.w += float(r[base + 3u]); }
- else
- {
- if (gm0 + 1u < p.oc) v.y += float(r[base + 1u]);
- if (gm0 + 2u < p.oc) v.z += float(r[base + 2u]);
- }
- }
- if (relu) v = max(v, vec4(0.0));
- if (gelu) v = vec4(0.5 * v.x * (1.0 + ferf(v.x * 0.7071067811865475)),
- 0.5 * v.y * (1.0 + ferf(v.y * 0.7071067811865475)),
- 0.5 * v.z * (1.0 + ferf(v.z * 0.7071067811865475)),
- 0.5 * v.w * (1.0 + ferf(v.w * 0.7071067811865475)));
- o[base] = ET(v.x);
- if (gm0 + 1u < p.oc) o[base + 1u] = ET(v.y);
- if (gm0 + 2u < p.oc) o[base + 2u] = ET(v.z);
- if (full) o[base + 3u] = ET(v.w);
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.comp
deleted file mode 100644
index 85421b3..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.comp
+++ /dev/null
@@ -1,135 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-#extension GL_EXT_control_flow_attributes : require
-// Small-ic conv (stem: ic < 8, stride 1) with a fused 2x2/2 max pool, NHWC:
-// o[py][px][n] = [relu] max_{dy,dx<2} [relu](b[n] + sum_k x[2py+dy, 2px+dx; k] w[k][n])
-// One thread per (pooled pixel, 8 output channels); weights staged in shared
-// memory as vec4 over n ([K][oc/4], K * oc <= 4096). The full-resolution conv
-// output is never stored. fp32 math.
-// flags: bit0 bias, bit1 reflect pad, bit2 conv relu, bit3 pool relu
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; // [ih][iw][ic]
-layout(set = 0, binding = 1) readonly buffer W { ET w[]; }; // [K][oc], k = (ky*kw + kx)*ic + ci
-layout(set = 0, binding = 2) readonly buffer B { float b[]; }; // [oc] (dummy if none)
-layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; // [ph][pw][oc]
-
-layout(push_constant) uniform PC {
- uint ic, ih, iw, oc, ph, pw, kh, kw, pt, pl, flags;
- float padv;
-} p;
-
-shared vec4 ws[1024];
-
-int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; }
-
-void main()
-{
- uint tid = gl_LocalInvocationID.x;
- uint K = p.kh * p.kw * p.ic, oc4 = p.oc >> 2;
- for (uint e = tid; e < K * p.oc; e += 256u) ws[e >> 2][e & 3u] = float(w[e]);
- barrier();
-
- uint groups = p.oc >> 3;
- uint idx = gl_GlobalInvocationID.x;
- if (idx >= p.ph * p.pw * groups) return;
- uint g = idx % groups, pp = idx / groups;
- uint pxx = pp % p.pw, pyy = pp / p.pw;
- bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u;
- uint n0 = g * 8u;
- vec4 b0 = vec4(0.0), b1 = vec4(0.0);
- if ((p.flags & 1u) != 0u)
- {
- b0 = vec4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]);
- b1 = vec4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]);
- }
- vec4 m0 = vec4(-3.4e38), m1 = vec4(-3.4e38);
- uint oh = p.ph * 2u, ow = p.pw * 2u; // conv output = input size (stride 1, 'same')
-#ifdef PIXELS_INNER
- // the four pool pixels share each weight load (wide-wave GPUs: shared/L1 loads, not
- // FMAs, bound the per-pixel loop); same per-pixel summation order
- vec4 a[8];
- [[unroll]] for (uint d = 0u; d < 4u; d++) { a[2u * d] = b0; a[2u * d + 1u] = b1; }
- for (uint ky = 0u; ky < p.kh; ky++)
- {
- uint rb[2];
- bool rv[2];
- [[unroll]] for (uint dy = 0u; dy < 2u; dy++)
- {
- int iy = int(pyy * 2u + dy + ky) - int(p.pt);
- rv[dy] = iy >= 0 && iy < int(p.ih);
- if (!rv[dy] && reflect) { iy = mirror(iy, int(p.ih)); rv[dy] = true; }
- rb[dy] = uint(clamp(iy, 0, int(p.ih) - 1)) * p.iw;
- }
- for (uint kx = 0u; kx < p.kw; kx++)
- {
- uint base[4];
- bool valid[4];
- [[unroll]] for (uint dx = 0u; dx < 2u; dx++)
- {
- int ix = int(pxx * 2u + dx + kx) - int(p.pl);
- bool vx = ix >= 0 && ix < int(p.iw);
- if (!vx && reflect) { ix = mirror(ix, int(p.iw)); vx = true; }
- uint cx = uint(clamp(ix, 0, int(p.iw) - 1));
- [[unroll]] for (uint dy = 0u; dy < 2u; dy++)
- {
- valid[dy * 2u + dx] = rv[dy] && vx;
- base[dy * 2u + dx] = (rb[dy] + cx) * p.ic;
- }
- }
- uint kb = (ky * p.kw + kx) * p.ic;
- for (uint ci = 0u; ci < p.ic; ci++)
- {
- uint wr = (kb + ci) * oc4 + g * 2u;
- vec4 w0 = ws[wr], w1 = ws[wr + 1u];
- [[unroll]] for (uint d = 0u; d < 4u; d++)
- {
- float xv = valid[d] ? float(x[base[d] + ci]) : p.padv;
- a[2u * d] += xv * w0;
- a[2u * d + 1u] += xv * w1;
- }
- }
- }
- }
- [[unroll]] for (uint d = 0u; d < 4u; d++)
- {
- vec4 a0 = a[2u * d], a1 = a[2u * d + 1u];
- if (crelu) { a0 = max(a0, vec4(0.0)); a1 = max(a1, vec4(0.0)); }
- m0 = max(m0, a0); m1 = max(m1, a1);
- }
-#else
- [[unroll]] for (uint d = 0u; d < 4u; d++)
- {
- uint oy = pyy * 2u + (d >> 1), ox = pxx * 2u + (d & 1u);
- vec4 a0 = b0, a1 = b1;
- for (uint ky = 0u; ky < p.kh; ky++)
- {
- int iy = int(oy + ky) - int(p.pt);
- bool vy = iy >= 0 && iy < int(p.ih);
- if (!vy && reflect) { iy = mirror(iy, int(p.ih)); vy = true; }
- for (uint kx = 0u; kx < p.kw; kx++)
- {
- int ix = int(ox + kx) - int(p.pl);
- bool vx = ix >= 0 && ix < int(p.iw);
- if (!vx && reflect) { ix = mirror(ix, int(p.iw)); vx = true; }
- bool valid = vy && vx;
- uint base = valid ? (uint(iy) * p.iw + uint(ix)) * p.ic : 0u;
- uint kb = (ky * p.kw + kx) * p.ic;
- for (uint ci = 0u; ci < p.ic; ci++)
- {
- float xv = valid ? float(x[base + ci]) : p.padv;
- uint wr = (kb + ci) * oc4 + g * 2u;
- a0 += xv * ws[wr];
- a1 += xv * ws[wr + 1u];
- }
- }
- }
- if (crelu) { a0 = max(a0, vec4(0.0)); a1 = max(a1, vec4(0.0)); }
- m0 = max(m0, a0); m1 = max(m1, a1);
- }
-#endif
- if ((p.flags & 8u) != 0u) { m0 = max(m0, vec4(0.0)); m1 = max(m1, vec4(0.0)); }
- uint ob = pp * p.oc + n0;
- [[unroll]] for (uint l = 0u; l < 4u; l++) { o[ob + l] = ET(m0[l]); o[ob + 4u + l] = ET(m1[l]); }
-}
\ No newline at end of file
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.spv
deleted file mode 100644
index c90b3a6..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_h.spv
deleted file mode 100644
index 7435cd6..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.comp
deleted file mode 100644
index d904a8b..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.comp
+++ /dev/null
@@ -1,104 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-#extension GL_EXT_control_flow_attributes : require
-// convpool.comp specialised for 3x3 kernels (ic <= 7): the 4x4 input window is read one row
-// at a time into registers and each staged weight pair is applied to both pool columns, so
-// every input element is loaded once (16 vs 36 taps) and shared-memory weight reads halve.
-// Same thread mapping (pooled pixel, 8 output channels) and the same per-output accumulation
-// order (bias, then k = (ky*3 + kx)*ic + ci ascending) as convpool.comp:
-// o[py][px][n] = [relu] max_{dy,dx<2} [relu](b[n] + sum_k x[2py+dy, 2px+dx; k] w[k][n])
-// flags: bit0 bias, bit1 reflect pad, bit2 conv relu, bit3 pool relu
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; // [ih][iw][ic]
-layout(set = 0, binding = 1) readonly buffer W { ET w[]; }; // [K][oc], k = (ky*3 + kx)*ic + ci
-layout(set = 0, binding = 2) readonly buffer B { float b[]; }; // [oc] (dummy if none)
-layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; // [ph][pw][oc]
-
-layout(push_constant) uniform PC {
- uint ic, ih, iw, oc, ph, pw, kh, kw, pt, pl, flags;
- float padv;
-} p;
-
-shared vec4 ws[1024];
-
-int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; }
-
-void main()
-{
- uint tid = gl_LocalInvocationID.x;
- uint K = 9u * p.ic, oc4 = p.oc >> 2;
- for (uint e = tid; e < K * p.oc; e += 256u) ws[e >> 2][e & 3u] = float(w[e]);
- barrier();
-
- uint groups = p.oc >> 3;
- uint idx = gl_GlobalInvocationID.x;
- if (idx >= p.ph * p.pw * groups) return;
- uint g = idx % groups, pp = idx / groups;
- uint pxx = pp % p.pw, pyy = pp / p.pw;
- bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u;
- int ro[4], co[4]; // window rows (pre-multiplied by iw) / columns
- bool rv[4], cv[4];
- [[unroll]] for (uint r = 0u; r < 4u; r++)
- {
- int iy = int(pyy * 2u + r) - int(p.pt);
- bool vy = iy >= 0 && iy < int(p.ih);
- if (!vy && reflect) { iy = mirror(iy, int(p.ih)); vy = true; }
- rv[r] = vy; ro[r] = vy ? iy * int(p.iw) : 0;
- int ix = int(pxx * 2u + r) - int(p.pl);
- bool vx = ix >= 0 && ix < int(p.iw);
- if (!vx && reflect) { ix = mirror(ix, int(p.iw)); vx = true; }
- cv[r] = vx; co[r] = vx ? ix : 0;
- }
- uint n0 = g * 8u;
- vec4 b0 = vec4(0.0), b1 = vec4(0.0);
- if ((p.flags & 1u) != 0u)
- {
- b0 = vec4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]);
- b1 = vec4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]);
- }
- vec4 a0[4], a1[4];
- [[unroll]] for (uint d = 0u; d < 4u; d++) { a0[d] = b0; a1[d] = b1; }
- [[unroll]] for (uint r = 0u; r < 4u; r++)
- {
- float xr[4][8];
- [[unroll]] for (uint c = 0u; c < 4u; c++)
- {
- bool v = rv[r] && cv[c];
- uint base = uint(ro[r] + co[c]) * p.ic;
- [[unroll]] for (uint ci = 0u; ci < 8u; ci++)
- xr[c][ci] = ci < p.ic ? (v ? float(x[base + ci]) : p.padv) : 0.0;
- }
- [[unroll]] for (uint dy = 0u; dy < 2u; dy++)
- {
- if (r < dy || r - dy > 2u) continue;
- uint ky = r - dy;
- [[unroll]] for (uint kx = 0u; kx < 3u; kx++)
- {
- uint kb = (ky * 3u + kx) * p.ic;
- [[unroll]] for (uint ci = 0u; ci < 8u; ci++)
- {
- if (ci >= p.ic) break;
- uint wr = (kb + ci) * oc4 + g * 2u;
- vec4 w0 = ws[wr], w1 = ws[wr + 1u];
- [[unroll]] for (uint dx = 0u; dx < 2u; dx++)
- {
- float xv = xr[kx + dx][ci];
- a0[dy * 2u + dx] += xv * w0;
- a1[dy * 2u + dx] += xv * w1;
- }
- }
- }
- }
- }
- vec4 m0 = vec4(-3.4e38), m1 = vec4(-3.4e38);
- [[unroll]] for (uint d = 0u; d < 4u; d++)
- {
- if (crelu) { a0[d] = max(a0[d], vec4(0.0)); a1[d] = max(a1[d], vec4(0.0)); }
- m0 = max(m0, a0[d]); m1 = max(m1, a1[d]);
- }
- if ((p.flags & 8u) != 0u) { m0 = max(m0, vec4(0.0)); m1 = max(m1, vec4(0.0)); }
- uint ob = pp * p.oc + n0;
- [[unroll]] for (uint l = 0u; l < 4u; l++) { o[ob + l] = ET(m0[l]); o[ob + 4u + l] = ET(m1[l]); }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.spv
deleted file mode 100644
index 6073da8..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3_h.spv
deleted file mode 100644
index fe50d31..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_pi_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_pi_h.spv
deleted file mode 100644
index 8153e1e..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_pi_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.comp
deleted file mode 100644
index 383d084..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.comp
+++ /dev/null
@@ -1,57 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// NHWC depthwise conv: out[oy,ox,c] = bias[c] + sum w[tap,c] * x[sy,sx,c]
-// (w backend-packed [tap][C]); channel-fastest flat index -> coalesced.
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; };
-layout(set = 0, binding = 1) readonly buffer W { ET w[]; };
-layout(set = 0, binding = 2) readonly buffer B { float b[]; };
-layout(set = 0, binding = 3) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC {
- uint c, ih, iw, oh, ow;
- uint kh, kw, sh, sw, pt, pl;
- uint flags; // bit0 hasBias, bit1 reflect, bit2 relu, bit4 gelu
- float padVal;
-} p;
-
-int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; }
-
-float ferf(float v)
-{
- float s = v < 0.0 ? -1.0 : 1.0;
- float ax = abs(v);
- float t = 1.0 / (1.0 + 0.3275911 * ax);
- float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
- return (1.0 - poly * exp(-ax * ax)) * s;
-}
-
-void main()
-{
- uint id = gl_GlobalInvocationID.x;
- if (id >= p.c * p.oh * p.ow) return;
- uint ch = id % p.c, px = id / p.c;
- uint ox = px % p.ow, oy = px / p.ow;
- bool refl = (p.flags & 2u) != 0u;
- float acc = ((p.flags & 1u) != 0u) ? b[ch] : 0.0;
- for (uint ky = 0u; ky < p.kh; ky++)
- {
- int sy = int(oy * p.sh + ky) - int(p.pt);
- for (uint kx = 0u; kx < p.kw; kx++)
- {
- int sx = int(ox * p.sw + kx) - int(p.pl);
- float wv = float(w[(ky * p.kw + kx) * p.c + ch]);
- // branch-free border: Adreno 750 mis-executed the former `acc += pad; continue;` form
- bool oob = uint(sy) >= p.ih || uint(sx) >= p.iw;
- int yy = refl ? mirror(sy, int(p.ih)) : clamp(sy, 0, int(p.ih) - 1);
- int xx = refl ? mirror(sx, int(p.iw)) : clamp(sx, 0, int(p.iw) - 1);
- float xv = float(x[(uint(yy) * p.iw + uint(xx)) * p.c + ch]);
- acc += wv * (oob && !refl ? p.padVal : xv);
- }
- }
- if ((p.flags & 4u) != 0u) acc = max(acc, 0.0);
- if ((p.flags & 16u) != 0u) acc = 0.5 * acc * (1.0 + ferf(acc * 0.7071067811865475));
- o[id] = ET(acc);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.spv
deleted file mode 100644
index ec4f43b..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.comp
deleted file mode 100644
index b158de4..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.comp
+++ /dev/null
@@ -1,58 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// depthwise.comp with 4 channels per invocation (c % 4 == 0), same per-element math.
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET4 x[]; };
-layout(set = 0, binding = 1) readonly buffer W { ET4 w[]; };
-layout(set = 0, binding = 2) readonly buffer B { vec4 b[]; };
-layout(set = 0, binding = 3) writeonly buffer O { ET4 o[]; };
-
-layout(push_constant) uniform PC {
- uint c, ih, iw, oh, ow;
- uint kh, kw, sh, sw, pt, pl;
- uint flags; // bit0 hasBias, bit1 reflect, bit2 relu, bit4 gelu
- float padVal;
-} p;
-
-int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; }
-
-float ferf(float v)
-{
- float s = v < 0.0 ? -1.0 : 1.0;
- float ax = abs(v);
- float t = 1.0 / (1.0 + 0.3275911 * ax);
- float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
- return (1.0 - poly * exp(-ax * ax)) * s;
-}
-
-void main()
-{
- uint id = gl_GlobalInvocationID.x;
- uint c4 = p.c >> 2;
- if (id >= c4 * p.oh * p.ow) return;
- uint ch = id % c4, px = id / c4;
- uint ox = px % p.ow, oy = px / p.ow;
- bool refl = (p.flags & 2u) != 0u;
- vec4 acc = ((p.flags & 1u) != 0u) ? b[ch] : vec4(0.0);
- for (uint ky = 0u; ky < p.kh; ky++)
- {
- int sy = int(oy * p.sh + ky) - int(p.pt);
- for (uint kx = 0u; kx < p.kw; kx++)
- {
- int sx = int(ox * p.sw + kx) - int(p.pl);
- vec4 wv = vec4(w[(ky * p.kw + kx) * c4 + ch]);
- // branch-free border: Adreno 750 mis-executed the former `acc += pad; continue;` form
- bool oob = uint(sy) >= p.ih || uint(sx) >= p.iw;
- int yy = refl ? mirror(sy, int(p.ih)) : clamp(sy, 0, int(p.ih) - 1);
- int xx = refl ? mirror(sx, int(p.iw)) : clamp(sx, 0, int(p.iw) - 1);
- vec4 xv = vec4(x[(uint(yy) * p.iw + uint(xx)) * c4 + ch]);
- acc += wv * (oob && !refl ? vec4(p.padVal) : xv);
- }
- }
- if ((p.flags & 4u) != 0u) acc = max(acc, vec4(0.0));
- if ((p.flags & 16u) != 0u)
- for (int l = 0; l < 4; l++) acc[l] = 0.5 * acc[l] * (1.0 + ferf(acc[l] * 0.7071067811865475));
- o[id] = ET4(acc);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.spv
deleted file mode 100644
index ff663e4..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4_h.spv
deleted file mode 100644
index ca0dfb6..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise_h.spv
deleted file mode 100644
index f476c95..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.comp
deleted file mode 100644
index 3eb4856..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.comp
+++ /dev/null
@@ -1,33 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// Elementwise binary + fused activation, general rank<=4 broadcast.
-// dst flat index decomposed by out dims (d0..d3); operand index = sum(dk * stride_k).
-// op: 0 add 1 sub 2 mul 3 div; act: 0 none 1 relu 2 sigmoid
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer A { ET a[]; };
-layout(set = 0, binding = 1) readonly buffer B { ET b[]; };
-layout(set = 0, binding = 2) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC {
- uint n, s0, s1, s2; // out strides (s3 = 1 implied)
- uint as0, as1, as2, as3;
- uint bs0, bs1, bs2, bs3;
- uint op, act;
-} p;
-
-void main()
-{
- uint i = gl_GlobalInvocationID.x;
- if (i >= p.n) return;
- uint d0 = i / p.s0, r0 = i % p.s0;
- uint d1 = r0 / p.s1, r1 = r0 % p.s1;
- uint d2 = r1 / p.s2, d3 = r1 % p.s2;
- float va = float(a[d0 * p.as0 + d1 * p.as1 + d2 * p.as2 + d3 * p.as3]);
- float vb = float(b[d0 * p.bs0 + d1 * p.bs1 + d2 * p.bs2 + d3 * p.bs3]);
- float v = p.op == 0u ? va + vb : p.op == 1u ? va - vb : p.op == 2u ? va * vb : va / vb;
- if (p.act == 1u) v = max(v, 0.0);
- else if (p.act == 2u) v = 1.0 / (1.0 + exp(-v));
- o[i] = ET(v);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.spv
deleted file mode 100644
index fc007c5..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2_h.spv
deleted file mode 100644
index 87a2dfe..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.comp
deleted file mode 100644
index de8d905..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.comp
+++ /dev/null
@@ -1,26 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// Elementwise binary + fused activation, contiguous same-shape fast path (4 floats/thread).
-// op: 0 add 1 sub 2 mul 3 div; act: 0 none 1 relu 2 sigmoid
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer A { ET4 a[]; };
-layout(set = 0, binding = 1) readonly buffer B { ET4 b[]; };
-layout(set = 0, binding = 2) writeonly buffer O { ET4 o[]; };
-
-layout(push_constant) uniform PC { uint nv4; uint op, act; } p;
-
-void main()
-{
- uint i = gl_GlobalInvocationID.x;
- if (i >= p.nv4) return;
- vec4 va = vec4(a[i]), vb = vec4(b[i]), v;
- if (p.op == 0u) v = va + vb;
- else if (p.op == 1u) v = va - vb;
- else if (p.op == 2u) v = va * vb;
- else v = va / vb;
- if (p.act == 1u) v = max(v, vec4(0.0));
- else if (p.act == 2u) v = 1.0 / (1.0 + exp(-v));
- o[i] = ET4(v);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.spv
deleted file mode 100644
index da05819..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v_h.spv
deleted file mode 100644
index c108578..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.comp
deleted file mode 100644
index e37fd69..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.comp
+++ /dev/null
@@ -1,88 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// Tiled GEMM: C[M,N] = A[M,K] * B + bias[N]; B is [K,N] or [N,K] when transB.
-// Tile 64x64, K-chunk 16, 256 threads -> 4x4 each.
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer A { ET a[]; };
-layout(set = 0, binding = 1) readonly buffer B { ET b[]; };
-layout(set = 0, binding = 2) readonly buffer BS { float bs[]; }; // bias[N] (dummy if none)
-layout(set = 0, binding = 3) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC {
- uint m, n, k;
- uint flags; // bit0 transB, bit1 hasBias, bit2 relu, bit3 gelu
-} p;
-
-const uint BM = 64u, BN = 64u, BK = 16u;
-shared float As[BK * BM];
-shared float Bs[BK * BN];
-
-float ferf(float v)
-{
- float s = v < 0.0 ? -1.0 : 1.0;
- float ax = abs(v);
- float t = 1.0 / (1.0 + 0.3275911 * ax);
- float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
- return (1.0 - poly * exp(-ax * ax)) * s;
-}
-
-void main()
-{
- uint m0 = gl_WorkGroupID.y * BM;
- uint n0 = gl_WorkGroupID.x * BN;
- uint tid = gl_LocalInvocationID.x;
- uint ty = tid >> 4, tx = tid & 15u;
- bool transB = (p.flags & 1u) != 0u;
-
- float acc[4][4];
- for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) acc[i][j] = 0.0;
-
- for (uint k0 = 0u; k0 < p.k; k0 += BK)
- {
- for (uint e = tid; e < BK * BM; e += 256u)
- {
- uint kk = e >> 6, mm = e & 63u;
- uint gm = m0 + mm, gk = k0 + kk;
- As[e] = (gm < p.m && gk < p.k) ? float(a[gm * p.k + gk]) : 0.0;
- }
- for (uint e = tid; e < BK * BN; e += 256u)
- {
- uint kk = e >> 6, nn = e & 63u;
- uint gn = n0 + nn, gk = k0 + kk;
- float v = 0.0;
- if (gn < p.n && gk < p.k)
- v = float(transB ? b[gn * p.k + gk] : b[gk * p.n + gn]);
- Bs[e] = v;
- }
- barrier();
- for (uint kk = 0u; kk < BK; kk++)
- {
- float av[4], bv[4];
- for (int i = 0; i < 4; i++) av[i] = As[kk * BM + ty * 4u + uint(i)];
- for (int j = 0; j < 4; j++) bv[j] = Bs[kk * BN + tx * 4u + uint(j)];
- for (int i = 0; i < 4; i++)
- for (int j = 0; j < 4; j++)
- acc[i][j] += av[i] * bv[j];
- }
- barrier();
- }
-
- bool hasBias = (p.flags & 2u) != 0u;
- for (int i = 0; i < 4; i++)
- {
- uint gm = m0 + ty * 4u + uint(i);
- if (gm >= p.m) continue;
- for (int j = 0; j < 4; j++)
- {
- uint gn = n0 + tx * 4u + uint(j);
- if (gn >= p.n) continue;
- float v = acc[i][j];
- if (hasBias) v += bs[gn];
- if ((p.flags & 4u) != 0u) v = max(v, 0.0);
- if ((p.flags & 8u) != 0u) v = 0.5 * v * (1.0 + ferf(v * 0.7071067811865475));
- o[gm * p.n + gn] = ET(v);
- }
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.spv
deleted file mode 100644
index ff9d30d..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot.comp
deleted file mode 100644
index 3ef44c9..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot.comp
+++ /dev/null
@@ -1,89 +0,0 @@
-#version 450
-#extension GL_EXT_shader_16bit_storage : require
-#extension GL_EXT_control_flow_attributes : require
-// GEMM C[M,N] = A[M,K] * B[K,N] + bias[N] for the conv_dot devices (wide waves, no
-// cooperative matrices): conv_dot.comp's register tiling for a 1x1 "conv" without
-// shared memory. Each invocation computes TM rows x TN*4 columns from f16vec4 loads
-// (n4 fastest across the wave: A loads broadcast, B loads coalesce). Requires fp16
-// storage, !transB, K % 4 == 0 and N % (4*TN) == 0. fp32 accumulation, K in order.
-// Push constants as gemm.comp (flags: bit0 transB (unsupported), bit1 bias, bit2 relu,
-// bit3 gelu).
-layout(local_size_x = 64) in;
-
-layout(set = 0, binding = 0) readonly buffer A { f16vec4 a[]; };
-layout(set = 0, binding = 1) readonly buffer B { f16vec4 b[]; };
-layout(set = 0, binding = 2) readonly buffer BS { float bs[]; };
-layout(set = 0, binding = 3) writeonly buffer O { f16vec4 o[]; };
-
-layout(push_constant) uniform PC {
- uint m, n, k;
- uint flags;
-} p;
-
-#ifndef TM
-#define TM 4
-#endif
-#ifndef TN
-#define TN 2
-#endif
-
-float ferf(float v)
-{
- float s = v < 0.0 ? -1.0 : 1.0;
- float ax = abs(v);
- float t = 1.0 / (1.0 + 0.3275911 * ax);
- float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
- return (1.0 - poly * exp(-ax * ax)) * s;
-}
-
-void main()
-{
- uint ng = p.n / uint(4 * TN);
- uint gid = gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * gl_WorkGroupSize.x;
- uint mg = (p.m + uint(TM) - 1u) / uint(TM);
- if (gid >= ng * mg) return;
- uint g = gid % ng, m0 = (gid / ng) * uint(TM);
- uint k4 = p.k >> 2, n4s = p.n >> 2, n40 = g * uint(TN);
-
- vec4 acc[TM * TN];
- [[unroll]] for (int i = 0; i < TM * TN; i++) acc[i] = vec4(0.0);
- uint arow[TM];
- [[unroll]] for (int i = 0; i < TM; i++) arow[i] = min(m0 + uint(i), p.m - 1u) * k4;
-
- for (uint kk = 0u; kk < k4; kk++)
- {
- vec4 av[TM];
- [[unroll]] for (int i = 0; i < TM; i++) av[i] = vec4(a[arow[i] + kk]);
- uint brow = kk * 4u * n4s + n40;
- [[unroll]] for (int j = 0; j < TN; j++)
- {
- vec4 w0 = vec4(b[brow + uint(j)]);
- vec4 w1 = vec4(b[brow + n4s + uint(j)]);
- vec4 w2 = vec4(b[brow + 2u * n4s + uint(j)]);
- vec4 w3 = vec4(b[brow + 3u * n4s + uint(j)]);
- [[unroll]] for (int i = 0; i < TM; i++)
- acc[i * TN + j] += av[i].x * w0 + av[i].y * w1 + av[i].z * w2 + av[i].w * w3;
- }
- }
-
- bool hasBias = (p.flags & 2u) != 0u, relu = (p.flags & 4u) != 0u, gelu = (p.flags & 8u) != 0u;
- [[unroll]] for (int j = 0; j < TN; j++)
- {
- uint n4 = n40 + uint(j), c0 = n4 * 4u;
- vec4 bias = hasBias ? vec4(bs[c0], bs[c0 + 1u], bs[c0 + 2u], bs[c0 + 3u]) : vec4(0.0);
- [[unroll]] for (int i = 0; i < TM; i++)
- {
- uint row = m0 + uint(i);
- if (row < p.m)
- {
- vec4 v = acc[i * TN + j] + bias;
- if (relu) v = max(v, vec4(0.0));
- if (gelu) v = vec4(0.5 * v.x * (1.0 + ferf(v.x * 0.7071067811865475)),
- 0.5 * v.y * (1.0 + ferf(v.y * 0.7071067811865475)),
- 0.5 * v.z * (1.0 + ferf(v.z * 0.7071067811865475)),
- 0.5 * v.w * (1.0 + ferf(v.w * 0.7071067811865475)));
- o[row * n4s + n4] = f16vec4(v);
- }
- }
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot_h.spv
deleted file mode 100644
index c20ca7c..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_h.spv
deleted file mode 100644
index 2aecca4..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.comp
deleted file mode 100644
index 09e4b94..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.comp
+++ /dev/null
@@ -1,41 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// LayerNorm over last dim: one workgroup (256 threads) per row.
-// o[r,i] = (x[r,i] - mean) * rsqrt(var + eps) * w[i] + b[i]
-// Single reduction pass: sum and sum-of-squares accumulate together in vec2 lanes.
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; };
-layout(set = 0, binding = 1) readonly buffer W { float w[]; };
-layout(set = 0, binding = 2) readonly buffer B { float b[]; };
-layout(set = 0, binding = 3) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC { uint len; float eps; } p;
-
-shared vec2 red[256];
-
-void main()
-{
- uint row = gl_WorkGroupID.x;
- uint tid = gl_LocalInvocationID.x;
- uint base = row * p.len;
-
- float s = 0.0, s2 = 0.0;
- for (uint i = tid; i < p.len; i += 256u)
- {
- float v = float(x[base + i]);
- s += v; s2 += v * v;
- }
- red[tid] = vec2(s, s2);
- barrier();
- for (uint st = 128u; st > 0u; st >>= 1)
- {
- if (tid < st) red[tid] += red[tid + st];
- barrier();
- }
- float mean = red[0].x / float(p.len);
- float inv = inversesqrt(red[0].y / float(p.len) - mean * mean + p.eps);
- for (uint i = tid; i < p.len; i += 256u)
- o[base + i] = ET((float(x[base + i]) - mean) * inv * w[i] + b[i]);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.spv
deleted file mode 100644
index 6f5d4a9..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm_h.spv
deleted file mode 100644
index 38e49d0..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.comp
deleted file mode 100644
index 96dce91..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.comp
+++ /dev/null
@@ -1,37 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// NHWC maxpool + optional relu (channel-fastest flat index).
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; };
-layout(set = 0, binding = 1) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC {
- uint c, ih, iw, oh, ow;
- uint kh, kw, sh, sw, pt, pl;
- uint relu;
-} p;
-
-void main()
-{
- uint id = gl_GlobalInvocationID.x;
- if (id >= p.c * p.oh * p.ow) return;
- uint ch = id % p.c, px = id / p.c;
- uint ox = px % p.ow, oy = px / p.ow;
- float m = -3.4e38;
- for (uint ky = 0u; ky < p.kh; ky++)
- {
- int sy = int(oy * p.sh + ky) - int(p.pt);
- if (uint(sy) >= p.ih) continue;
- for (uint kx = 0u; kx < p.kw; kx++)
- {
- int sx = int(ox * p.sw + kx) - int(p.pl);
- if (uint(sx) >= p.iw) continue;
- m = max(m, float(x[(uint(sy) * p.iw + uint(sx)) * p.c + ch]));
- }
- }
- if (m == -3.4e38) m = 0.0;
- if (p.relu != 0u && m < 0.0) m = 0.0;
- o[id] = ET(m);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.spv
deleted file mode 100644
index 38f34c4..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.comp
deleted file mode 100644
index 649b334..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.comp
+++ /dev/null
@@ -1,39 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// maxpool.comp with 4 channels per invocation (c % 4 == 0): same per-element math,
-// a quarter of the index arithmetic and load instructions (bandwidth-bound iGPUs).
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET4 x[]; };
-layout(set = 0, binding = 1) writeonly buffer O { ET4 o[]; };
-
-layout(push_constant) uniform PC {
- uint c, ih, iw, oh, ow;
- uint kh, kw, sh, sw, pt, pl;
- uint relu;
-} p;
-
-void main()
-{
- uint id = gl_GlobalInvocationID.x;
- uint c4 = p.c >> 2;
- if (id >= c4 * p.oh * p.ow) return;
- uint ch = id % c4, px = id / c4;
- uint ox = px % p.ow, oy = px / p.ow;
- vec4 m = vec4(-3.4e38);
- for (uint ky = 0u; ky < p.kh; ky++)
- {
- int sy = int(oy * p.sh + ky) - int(p.pt);
- if (uint(sy) >= p.ih) continue;
- for (uint kx = 0u; kx < p.kw; kx++)
- {
- int sx = int(ox * p.sw + kx) - int(p.pl);
- if (uint(sx) >= p.iw) continue;
- m = max(m, vec4(x[(uint(sy) * p.iw + uint(sx)) * c4 + ch]));
- }
- }
- if (m.x == -3.4e38) m = vec4(0.0);
- if (p.relu != 0u) m = mix(m, vec4(0.0), lessThan(m, vec4(0.0))); // as the scalar m < 0 test
- o[id] = ET4(m);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.spv
deleted file mode 100644
index 59aff0e..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4_h.spv
deleted file mode 100644
index e577d02..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool_h.spv
deleted file mode 100644
index f8fa66d..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.comp
deleted file mode 100644
index c015c7a..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.comp
+++ /dev/null
@@ -1,47 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// Rank-4 pad on all dims (spatial in practice), constant or reflect.
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; };
-layout(set = 0, binding = 1) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC {
- uint id0, id1, id2, id3; // in dims
- uint od0, od1, od2, od3; // out dims
- uint pb0, pb1, pb2, pb3; // pad before per dim
- uint mode; // 0 const 1 reflect
- float value;
-} p;
-
-int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; }
-int coord(uint oidx, uint pb, uint idim, out bool miss)
-{
- int s = int(oidx) - int(pb);
- if (uint(s) >= idim)
- {
- if (p.mode == 0u) { miss = true; return 0; }
- s = mirror(s, int(idim));
- }
- miss = false; return s;
-}
-
-void main()
-{
- uint i = gl_GlobalInvocationID.x;
- uint n = p.od0 * p.od1 * p.od2 * p.od3;
- if (i >= n) return;
- uint s2 = p.od3, s1 = p.od2 * s2, s0 = p.od1 * s1;
- uint d0 = i / s0, r0 = i % s0;
- uint d1 = r0 / s1, r1 = r0 % s1;
- uint d2 = r1 / s2, d3 = r1 % s2;
- bool m0, m1, m2, m3;
- int c0 = coord(d0, p.pb0, p.id0, m0);
- int c1 = coord(d1, p.pb1, p.id1, m1);
- int c2 = coord(d2, p.pb2, p.id2, m2);
- int c3 = coord(d3, p.pb3, p.id3, m3);
- if (m0 || m1 || m2 || m3) { o[i] = ET(p.value); return; }
- uint si2 = p.id3, si1 = p.id2 * si2, si0 = p.id1 * si1;
- o[i] = x[uint(c0) * si0 + uint(c1) * si1 + uint(c2) * si2 + uint(c3)];
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.spv
deleted file mode 100644
index b5e56ec..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad_h.spv
deleted file mode 100644
index 683332e..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.comp
deleted file mode 100644
index 56a9127..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.comp
+++ /dev/null
@@ -1,13 +0,0 @@
-#version 450
-#extension GL_KHR_shader_subgroup_basic : require
-
-// Device probe for pipelines created with a requiredSubgroupSize (32, or 16 for SG16): every
-// invocation reports subgroup size | count << 8 | id << 16 | lane << 24 (VkDevice.ProbeSubgroup).
-layout(local_size_x = 256) in;
-layout(set = 0, binding = 0) writeonly buffer O { uint o[]; };
-
-void main()
-{
- o[gl_LocalInvocationID.x] = gl_SubgroupSize | (gl_NumSubgroups << 8)
- | (gl_SubgroupID << 16) | (gl_SubgroupInvocationID << 24);
-}
\ No newline at end of file
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.spv
deleted file mode 100644
index 913cb72..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.comp
deleted file mode 100644
index 484e0ac..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.comp
+++ /dev/null
@@ -1,27 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// Generic rank<=4 strided slice: dst[k0,k1,k2,k3] = src[start_k + k*step_k].
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; };
-layout(set = 0, binding = 1) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC {
- uint n;
- uint os0, os1, os2; // out strides (os3 = 1)
- uint st0, st1, st2, st3; // start per out dim
- uint sp0, sp1, sp2, sp3; // step per out dim
- uint is0, is1, is2, is3; // in strides
-} p;
-
-void main()
-{
- uint i = gl_GlobalInvocationID.x;
- if (i >= p.n) return;
- uint d0 = i / p.os0, r0 = i % p.os0;
- uint d1 = r0 / p.os1, r1 = r0 % p.os1;
- uint d2 = r1 / p.os2, d3 = r1 % p.os2;
- o[i] = x[(p.st0 + d0 * p.sp0) * p.is0 + (p.st1 + d1 * p.sp1) * p.is1
- + (p.st2 + d2 * p.sp2) * p.is2 + (p.st3 + d3 * p.sp3) * p.is3];
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.spv
deleted file mode 100644
index 459b1b8..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice_h.spv
deleted file mode 100644
index 6a0e3a4..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.comp
deleted file mode 100644
index 2448009..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.comp
+++ /dev/null
@@ -1,24 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// Generic rank<=4 permute: dst flat idx -> src idx via permuted in-strides.
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; };
-layout(set = 0, binding = 1) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC {
- uint n;
- uint os0, os1, os2; // out strides (os3 = 1)
- uint ps0, ps1, ps2, ps3; // in strides in out-dim order: inStr[perm[k]]
-} p;
-
-void main()
-{
- uint i = gl_GlobalInvocationID.x;
- if (i >= p.n) return;
- uint d0 = i / p.os0, r0 = i % p.os0;
- uint d1 = r0 / p.os1, r1 = r0 % p.os1;
- uint d2 = r1 / p.os2, d3 = r1 % p.os2;
- o[i] = x[d0 * p.ps0 + d1 * p.ps1 + d2 * p.ps2 + d3 * p.ps3];
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.spv
deleted file mode 100644
index 7d17649..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose_h.spv
deleted file mode 100644
index be61c6c..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.comp
deleted file mode 100644
index ade3618..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.comp
+++ /dev/null
@@ -1,44 +0,0 @@
-#version 450
-#extension GL_GOOGLE_include_directive : require
-#include "et.glsl"
-// Elementwise unary: mode 0 relu, 1 sigmoid, 2 erf, 3 gelu, 4 copy
-layout(local_size_x = 256) in;
-
-layout(set = 0, binding = 0) readonly buffer X { ET x[]; };
-layout(set = 0, binding = 1) writeonly buffer O { ET o[]; };
-
-layout(push_constant) uniform PC { uint n, mode; } p;
-
-float fexp(float v)
-{
- v = clamp(v, -87.3, 88.7);
- float fx = v * 1.4426950408889634 + 0.5;
- float nf = floor(fx);
- float r = v - nf * 0.693359375 - nf * -0.00021219444005469058;
- float poly = (((((1.9875691500e-4 * r + 1.3981999507e-3) * r + 8.3334519073e-3) * r
- + 4.1665795894e-2) * r + 1.6666665459e-1) * r + 5.0000001201e-1) * r * r + r + 1.0;
- return poly * uintBitsToFloat((uint(int(nf)) + 127u) << 23);
-}
-
-float ferf(float v)
-{
- float s = v < 0.0 ? -1.0 : 1.0;
- float ax = abs(v);
- float t = 1.0 / (1.0 + 0.3275911 * ax);
- float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
- return (1.0 - poly * fexp(-ax * ax)) * s;
-}
-
-void main()
-{
- uint i = gl_GlobalInvocationID.x;
- if (i >= p.n) return;
- float v = float(x[i]);
- float r;
- if (p.mode == 0u) r = max(v, 0.0);
- else if (p.mode == 1u) r = 1.0 / (1.0 + fexp(-v));
- else if (p.mode == 2u) r = ferf(v);
- else if (p.mode == 3u) r = 0.5 * v * (1.0 + ferf(v * 0.7071067811865475));
- else r = v;
- o[i] = ET(r);
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.spv
deleted file mode 100644
index 72742dd..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary_h.spv
deleted file mode 100644
index b6a9285..0000000
Binary files a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary_h.spv and /dev/null differ
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Vk.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Vk.cs
deleted file mode 100644
index 98f3a54..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Vk.cs
+++ /dev/null
@@ -1,589 +0,0 @@
-// Minimal Vulkan 1.1 P/Invoke for compute-only use (SimdPaddleOCR Vulkan backend).
-// Loads vulkan-1.dll (Windows) / libvulkan.so.1 (Linux) / libvulkan.so (Android)
-// via SetDllImportResolver.
-// Only the entry points and structs we need; all structs are blittable.
-using System.Runtime.InteropServices;
-
-namespace Sdcb.SimdGcdrNet.Backends.Vulkan;
-
-internal static unsafe partial class Vk
-{
- public const string LibName = "vulkan-1";
-
- private static int _resolverRegistered;
-
- // Android ships only the unversioned NDK name (/system/lib64/libvulkan.so)
- private static readonly string[] s_candidates = OperatingSystem.IsWindows()
- ? ["vulkan-1.dll"] : ["libvulkan.so.1", "libvulkan.so"];
-
- public static void RegisterResolver()
- {
- if (Interlocked.Exchange(ref _resolverRegistered, 1) != 0)
- return;
- NativeLibrary.SetDllImportResolver(typeof(Vk).Assembly, (name, asm, path) =>
- {
- if (name != LibName)
- return IntPtr.Zero;
- foreach (string candidate in s_candidates)
- if (NativeLibrary.TryLoad(candidate, out IntPtr h))
- {
- LoadedLibrary = candidate;
- return h;
- }
- throw new DllNotFoundException(
- $"Vulkan loader not found (tried {string.Join(", ", s_candidates)})");
- });
- }
-
- /// File name the resolver loaded, or null before the first call.
- public static string? LoadedLibrary { get; private set; }
-
- public const ulong WholeSize = ~0UL;
- public const uint QueueFamilyIgnored = ~0u;
-
- public static void Check(VkResult r, string what)
- {
- if (r != VkResult.Success)
- throw new InvalidOperationException($"Vulkan {what} failed: {r}");
- }
-
- // ---- structs (field order must match the Vulkan spec) ----
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkApplicationInfo
- {
- public uint SType; public void* PNext; public void* PAppName;
- public uint AppVersion; public void* PEngineName; public uint EngineVersion; public uint ApiVersion;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkInstanceCreateInfo
- {
- public uint SType; public void* PNext; public uint Flags;
- public VkApplicationInfo* PAppInfo;
- public uint EnabledLayerCount; public void* PpEnabledLayerNames;
- public uint EnabledExtensionCount; public void* PpEnabledExtensionNames;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDeviceProperties
- {
- public uint ApiVersion, DriverVersion, VendorID, DeviceID, DeviceType;
- public fixed byte DeviceName[256];
- public fixed byte PipelineCacheUUID[16];
- public fixed byte LimitsAndSparse[1024]; // VkPhysicalDeviceLimits then Sparse; timestampPeriod float @428
- public float TimestampPeriodNs
- {
- get { fixed (byte* p = LimitsAndSparse) return *(float*)(p + 428); }
- }
- // limits start 4 bytes in (8-byte aligned natively); maxComputeSharedMemorySize @216
- public uint MaxComputeSharedMemorySize
- {
- get { fixed (byte* p = LimitsAndSparse) return *(uint*)(p + 4 + 216); }
- }
- public uint MaxStorageBufferRange
- {
- get { fixed (byte* p = LimitsAndSparse) return *(uint*)(p + 4 + 28); }
- }
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDeviceProperties2
- {
- public uint SType; public void* PNext;
- public VkPhysicalDeviceProperties Properties;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDeviceSubgroupSizeControlProperties
- {
- public uint SType; public void* PNext;
- public uint MinSubgroupSize, MaxSubgroupSize;
- public uint MaxComputeWorkgroupSubgroups, RequiredSubgroupSizeStages;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDeviceSubgroupProperties
- {
- public uint SType; public void* PNext;
- public uint SubgroupSize;
- public uint SupportedStages; // VkShaderStageFlags
- public uint SupportedOperations; // VkSubgroupFeatureFlags
- public uint QuadOperationsInAllStages;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkMemoryType { public uint PropertyFlags; public uint HeapIndex; }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkMemoryHeap { public ulong Size; public uint Flags; }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDeviceMemoryProperties
- {
- public uint MemoryTypeCount;
- public fixed byte MemoryTypes[32 * 8]; // VkMemoryType[32]
- public uint MemoryHeapCount;
- public fixed byte MemoryHeaps[16 * 16]; // VkMemoryHeap is 16 bytes (8-aligned), not 12
-
- public VkMemoryType TypeAt(int i)
- {
- fixed (byte* p = MemoryTypes) return ((VkMemoryType*)p)[i];
- }
- public VkMemoryHeap HeapAt(int i)
- {
- // native VkMemoryHeap: { VkDeviceSize size; VkMemoryHeapFlags flags; } = 16 bytes
- fixed (byte* p = MemoryHeaps)
- {
- byte* h = p + i * 16;
- return new() { Size = *(ulong*)h, Flags = *(uint*)(h + 8) };
- }
- }
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkQueueFamilyProperties
- {
- public uint QueueFlags, QueueCount, TimestampValidBits;
- public uint MinImageTransferGranularityX, MinImageTransferGranularityY, MinImageTransferGranularityZ;
- }
-
- /// VkPhysicalDeviceFeatures as raw uint[55]; shaderInt16 is index 41.
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDeviceFeatures
- {
- public fixed uint F[55];
- public uint ShaderInt16 { get => F[41]; set => F[41] = value; }
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDeviceFeatures2
- {
- public uint SType; public void* PNext;
- public VkPhysicalDeviceFeatures Features;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDevice16BitStorageFeatures
- {
- public uint SType; public void* PNext;
- public uint StorageBuffer16BitAccess, UniformAndStorageBuffer16BitAccess, StoragePushConstant16, StorageInputOutput16;
- }
-
- // VK_KHR_shader_float16_int8 (Vulkan 1.2: shaderFloat16/shaderInt8)
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDeviceVulkanMemoryModelFeatures
- {
- public uint SType; public void* PNext;
- public uint VulkanMemoryModel, VulkanMemoryModelDeviceScope, VulkanMemoryModelAvailabilityVisibilityChains;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDeviceShaderFloat16Int8Features
- {
- public uint SType; public void* PNext;
- public uint ShaderFloat16, ShaderInt8;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkDeviceQueueCreateInfo
- {
- public uint SType; public void* PNext; public uint Flags;
- public uint QueueFamilyIndex, QueueCount; public float* PQueuePriorities;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkDeviceCreateInfo
- {
- public uint SType; public void* PNext; public uint Flags;
- public uint QueueCreateInfoCount; public VkDeviceQueueCreateInfo* PQueueCreateInfos;
- public uint EnabledLayerCount; public void* PpEnabledLayerNames;
- public uint EnabledExtensionCount; public void* PpEnabledExtensionNames;
- public void* PEnabledFeatures;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkBufferCreateInfo
- {
- public uint SType; public void* PNext; public uint Flags;
- public ulong Size; public uint Usage; public uint SharingMode;
- public uint QueueFamilyIndexCount; public uint* PQueueFamilyIndices;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkMemoryRequirements { public ulong Size; public ulong Alignment; public uint MemoryTypeBits; }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkMemoryAllocateInfo
- {
- public uint SType; public void* PNext;
- public ulong AllocationSize; public uint MemoryTypeIndex;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkMappedMemoryRange
- {
- public uint SType; public void* PNext;
- public IntPtr Memory; public ulong Offset; public ulong Size;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkShaderModuleCreateInfo
- {
- public uint SType; public void* PNext; public uint Flags;
- public nuint CodeSize; public uint* PCode;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPushConstantRange { public uint StageFlags; public uint Offset; public uint Size; }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPipelineLayoutCreateInfo
- {
- public uint SType; public void* PNext; public uint Flags;
- public uint SetLayoutCount; public IntPtr* PSetLayouts;
- public uint PushConstantRangeCount; public VkPushConstantRange* PPushConstantRanges;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPipelineShaderStageCreateInfo
- {
- public uint SType; public void* PNext; public uint Flags;
- public uint Stage; public IntPtr Module; public byte* PName; public void* PSpecializationInfo;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDeviceCooperativeMatrixFeaturesKHR
- {
- public uint SType; public void* PNext;
- public uint CooperativeMatrix; public uint CooperativeMatrixRobustBufferAccess;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkCooperativeMatrixPropertiesKHR
- {
- public uint SType; public void* PNext;
- public uint MSize, NSize, KSize;
- public uint AType, BType, CType, ResultType;
- public uint SaturatingAccumulation, Scope;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPhysicalDeviceSubgroupSizeControlFeaturesEXT
- {
- public uint SType; public void* PNext;
- public uint SubgroupSizeControl; public uint ComputeFullSubgroups;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkPipelineShaderStageRequiredSubgroupSizeCreateInfo
- {
- public uint SType; public void* PNext; public uint RequiredSubgroupSize;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkComputePipelineCreateInfo
- {
- public uint SType; public void* PNext; public uint Flags;
- public VkPipelineShaderStageCreateInfo Stage;
- public IntPtr Layout; public IntPtr BasePipelineHandle; public int BasePipelineIndex;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkDescriptorSetLayoutBinding
- {
- public uint Binding; public uint DescriptorType; public uint DescriptorCount;
- public uint StageFlags; public IntPtr* PImmutableSamplers;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkDescriptorSetLayoutCreateInfo
- {
- public uint SType; public void* PNext; public uint Flags;
- public uint BindingCount; public VkDescriptorSetLayoutBinding* PBindings;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkDescriptorPoolSize { public uint Type; public uint DescriptorCount; }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkDescriptorPoolCreateInfo
- {
- public uint SType; public void* PNext; public uint Flags;
- public uint MaxSets; public uint PoolSizeCount; public VkDescriptorPoolSize* PPoolSizes;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkDescriptorSetAllocateInfo
- {
- public uint SType; public void* PNext; public IntPtr DescriptorPool;
- public uint DescriptorSetCount; public IntPtr* PSetLayouts;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkDescriptorBufferInfo { public IntPtr Buffer; public ulong Offset; public ulong Range; }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkExtensionProperties
- {
- public unsafe fixed byte ExtensionName[256];
- public uint SpecVersion;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkWriteDescriptorSet
- {
- public uint SType; public void* PNext; public IntPtr DstSet;
- public uint DstBinding; public uint DstArrayElement; public uint DescriptorCount;
- public uint DescriptorType; public void* PImageInfo; public VkDescriptorBufferInfo* PBufferInfo;
- public void* PTexelBufferView;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkDeviceQueueGlobalPriorityCreateInfo
- {
- public uint SType; public void* PNext; public uint GlobalPriority;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkCommandPoolCreateInfo
- {
- public uint SType; public void* PNext; public uint Flags; public uint QueueFamilyIndex;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkCommandBufferAllocateInfo
- {
- public uint SType; public void* PNext; public IntPtr CommandPool;
- public uint Level; public uint CommandBufferCount;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkCommandBufferBeginInfo
- {
- public uint SType; public void* PNext; public uint Flags; public void* PInheritanceInfo;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkSubmitInfo
- {
- public uint SType; public void* PNext;
- public uint WaitSemaphoreCount; public IntPtr* PWaitSemaphores; public uint* PWaitDstStageMask;
- public uint CommandBufferCount; public IntPtr* PCommandBuffers;
- public uint SignalSemaphoreCount; public IntPtr* PSignalSemaphores;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkFenceCreateInfo { public uint SType; public void* PNext; public uint Flags; }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkMemoryBarrier
- {
- public uint SType; public void* PNext; public uint SrcAccessMask; public uint DstAccessMask;
- }
-
- // ---- entry points ----
-
- [LibraryImport(LibName)] public static partial VkResult vkCreateInstance(VkInstanceCreateInfo* ci, void* alloc, out IntPtr instance);
- [LibraryImport(LibName)] public static partial void vkDestroyInstance(IntPtr instance, void* alloc);
- [LibraryImport(LibName)] public static partial VkResult vkEnumeratePhysicalDevices(IntPtr instance, uint* count, IntPtr* devices);
- [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceProperties(IntPtr device, VkPhysicalDeviceProperties* props);
- [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceProperties2(IntPtr device, VkPhysicalDeviceProperties2* props);
- [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceMemoryProperties(IntPtr device, VkPhysicalDeviceMemoryProperties* props);
- [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceQueueFamilyProperties(IntPtr device, uint* count, VkQueueFamilyProperties* props);
- [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceFeatures(IntPtr device, VkPhysicalDeviceFeatures* features);
- [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceFeatures2(IntPtr device, VkPhysicalDeviceFeatures2* features);
- [LibraryImport(LibName)] public static partial VkResult vkCreateDevice(IntPtr physicalDevice, VkDeviceCreateInfo* ci, void* alloc, out IntPtr device);
- [LibraryImport(LibName)] public static partial void vkDestroyDevice(IntPtr device, void* alloc);
- [LibraryImport(LibName)] public static partial void vkGetDeviceQueue(IntPtr device, uint family, uint index, out IntPtr queue);
-
- [LibraryImport(LibName)] public static partial VkResult vkCreateBuffer(IntPtr device, VkBufferCreateInfo* ci, void* alloc, out IntPtr buffer);
- [LibraryImport(LibName)] public static partial void vkDestroyBuffer(IntPtr device, IntPtr buffer, void* alloc);
- [LibraryImport(LibName)] public static partial void vkFreeCommandBuffers(IntPtr device, IntPtr pool, uint count, IntPtr* buffers);
- [LibraryImport(LibName)] public static partial VkResult vkFreeDescriptorSets(IntPtr device, IntPtr pool, uint count, IntPtr* sets);
- [LibraryImport(LibName)] public static partial void vkDestroyCommandPool(IntPtr device, IntPtr pool, void* alloc);
- [LibraryImport(LibName)] public static partial void vkGetBufferMemoryRequirements(IntPtr device, IntPtr buffer, VkMemoryRequirements* req);
- [LibraryImport(LibName)] public static partial VkResult vkAllocateMemory(IntPtr device, VkMemoryAllocateInfo* ai, void* alloc, out IntPtr memory);
- [LibraryImport(LibName)] public static partial void vkFreeMemory(IntPtr device, IntPtr memory, void* alloc);
- [LibraryImport(LibName)] public static partial VkResult vkBindBufferMemory(IntPtr device, IntPtr buffer, IntPtr memory, ulong offset);
- [LibraryImport(LibName)] public static partial VkResult vkMapMemory(IntPtr device, IntPtr memory, ulong offset, ulong size, uint flags, void** data);
- [LibraryImport(LibName)] public static partial void vkUnmapMemory(IntPtr device, IntPtr memory);
- [LibraryImport(LibName)] public static partial VkResult vkFlushMappedMemoryRanges(IntPtr device, uint count, VkMappedMemoryRange* ranges);
- [LibraryImport(LibName)] public static partial VkResult vkInvalidateMappedMemoryRanges(IntPtr device, uint count, VkMappedMemoryRange* ranges);
-
- [LibraryImport(LibName)] public static partial VkResult vkCreateShaderModule(IntPtr device, VkShaderModuleCreateInfo* ci, void* alloc, out IntPtr module);
- [LibraryImport(LibName)] public static partial void vkDestroyShaderModule(IntPtr device, IntPtr module, void* alloc);
- [LibraryImport(LibName)] public static partial VkResult vkCreatePipelineLayout(IntPtr device, VkPipelineLayoutCreateInfo* ci, void* alloc, out IntPtr layout);
- [LibraryImport(LibName)] public static partial VkResult vkCreateComputePipelines(IntPtr device, IntPtr cache, uint count, VkComputePipelineCreateInfo* ci, void* alloc, IntPtr* pipelines);
- [LibraryImport(LibName)] public static partial void vkDestroyPipeline(IntPtr device, IntPtr pipeline, void* alloc);
-
- [LibraryImport(LibName)] public static partial VkResult vkCreateDescriptorSetLayout(IntPtr device, VkDescriptorSetLayoutCreateInfo* ci, void* alloc, out IntPtr setLayout);
- [LibraryImport(LibName)] public static partial void vkDestroyPipelineLayout(IntPtr device, IntPtr layout, void* alloc);
- [LibraryImport(LibName)] public static partial void vkDestroyDescriptorSetLayout(IntPtr device, IntPtr setLayout, void* alloc);
- [LibraryImport(LibName)] public static partial VkResult vkCreateDescriptorPool(IntPtr device, VkDescriptorPoolCreateInfo* ci, void* alloc, out IntPtr pool);
- [LibraryImport(LibName)] public static partial VkResult vkAllocateDescriptorSets(IntPtr device, VkDescriptorSetAllocateInfo* ai, IntPtr* sets);
- [LibraryImport(LibName)] public static partial VkResult vkResetDescriptorPool(IntPtr device, IntPtr pool, uint flags);
- [LibraryImport(LibName)] public static partial void vkDestroyDescriptorPool(IntPtr device, IntPtr pool, void* alloc);
- [LibraryImport(LibName)] public static partial void vkUpdateDescriptorSets(IntPtr device, uint writeCount, VkWriteDescriptorSet* writes, uint copyCount, void* copies);
- [LibraryImport(LibName)] public static partial VkResult vkEnumerateDeviceExtensionProperties(IntPtr physicalDevice, byte* layerName, uint* count, VkExtensionProperties* props);
- [LibraryImport(LibName)] public static partial IntPtr vkGetDeviceProcAddr(IntPtr device, byte* name);
- [LibraryImport(LibName)] public static partial IntPtr vkGetInstanceProcAddr(IntPtr instance, byte* name);
-
- [LibraryImport(LibName)] public static partial VkResult vkCreateCommandPool(IntPtr device, VkCommandPoolCreateInfo* ci, void* alloc, out IntPtr pool);
- [LibraryImport(LibName)] public static partial VkResult vkAllocateCommandBuffers(IntPtr device, VkCommandBufferAllocateInfo* ai, IntPtr* buffers);
- [LibraryImport(LibName)] public static partial VkResult vkResetCommandBuffer(IntPtr cmd, uint flags);
- [LibraryImport(LibName)] public static partial VkResult vkBeginCommandBuffer(IntPtr cmd, VkCommandBufferBeginInfo* bi);
- [LibraryImport(LibName)] public static partial VkResult vkEndCommandBuffer(IntPtr cmd);
- [LibraryImport(LibName)] public static partial void vkCmdBindPipeline(IntPtr cmd, uint bindPoint, IntPtr pipeline);
- [LibraryImport(LibName)] public static partial void vkCmdBindDescriptorSets(IntPtr cmd, uint bindPoint, IntPtr layout, uint firstSet, uint count, IntPtr* sets, uint dynCount, uint* dynOffsets);
- [LibraryImport(LibName)] public static partial void vkCmdPushConstants(IntPtr cmd, IntPtr layout, uint stageFlags, uint offset, uint size, void* values);
- [LibraryImport(LibName)] public static partial void vkCmdDispatch(IntPtr cmd, uint x, uint y, uint z);
- [LibraryImport(LibName)] public static partial void vkCmdPipelineBarrier(IntPtr cmd, uint srcStage, uint dstStage, uint depFlags,
- uint memBarrierCount, VkMemoryBarrier* memBarriers, uint bufBarrierCount, void* bufBarriers, uint imgBarrierCount, void* imgBarriers);
- [LibraryImport(LibName)] public static partial void vkCmdFillBuffer(IntPtr cmd, IntPtr buffer, ulong offset, ulong size, uint data);
- [LibraryImport(LibName)] public static partial void vkCmdCopyBuffer(IntPtr cmd, IntPtr src, IntPtr dst, uint count, VkBufferCopy* regions);
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkQueryPoolCreateInfo { public uint SType; public void* PNext; public uint Flags; public uint QueryType; public uint QueryCount; public uint PipelineStatistics; }
- [LibraryImport(LibName)] public static partial VkResult vkCreateQueryPool(IntPtr device, VkQueryPoolCreateInfo* ci, void* alloc, out IntPtr pool);
- [LibraryImport(LibName)] public static partial void vkDestroyQueryPool(IntPtr device, IntPtr pool, void* alloc);
- [LibraryImport(LibName)] public static partial void vkCmdResetQueryPool(IntPtr cmd, IntPtr pool, uint first, uint count);
- [LibraryImport(LibName)] public static partial void vkCmdWriteTimestamp(IntPtr cmd, uint stage, IntPtr pool, uint query);
- [LibraryImport(LibName)] public static partial VkResult vkGetQueryPoolResults(IntPtr device, IntPtr pool, uint first, uint count, nuint dataSize, void* data, ulong stride, uint flags);
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkBufferCopy { public ulong SrcOffset, DstOffset, Size; }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct VkBufferMemoryBarrier
- {
- public uint SType; public void* PNext;
- public uint SrcAccessMask, DstAccessMask;
- public uint SrcQueueFamilyIndex, DstQueueFamilyIndex;
- public IntPtr Buffer; public ulong Offset, Size;
- }
-
- [LibraryImport(LibName)] public static partial VkResult vkQueueSubmit(IntPtr queue, uint submitCount, VkSubmitInfo* submits, IntPtr fence);
- [LibraryImport(LibName)] public static partial VkResult vkCreateFence(IntPtr device, VkFenceCreateInfo* ci, void* alloc, out IntPtr fence);
- [LibraryImport(LibName)] public static partial VkResult vkWaitForFences(IntPtr device, uint count, IntPtr* fences, uint waitAll, ulong timeout);
- [LibraryImport(LibName)] public static partial VkResult vkResetFences(IntPtr device, uint count, IntPtr* fences);
- [LibraryImport(LibName)] public static partial void vkDestroyFence(IntPtr device, IntPtr fence, void* alloc);
- [LibraryImport(LibName)] public static partial VkResult vkGetFenceStatus(IntPtr device, IntPtr fence);
- [LibraryImport(LibName)] public static partial VkResult vkQueueWaitIdle(IntPtr queue);
- [LibraryImport(LibName)] public static partial VkResult vkDeviceWaitIdle(IntPtr device);
-}
-
-internal enum VkResult : int
-{
- Success = 0,
- NotReady = 1,
- Timeout = 2,
- ErrorOutOfHostMemory = -1,
- ErrorOutOfDeviceMemory = -2,
- ErrorInitializationFailed = -3,
- ErrorDeviceLost = -4,
- ErrorExtensionNotPresent = -7,
- ErrorFeatureNotPresent = -8,
- ErrorIncompatibleDriver = -9,
-}
-
-internal static class VkConst
-{
- // VkStructureType
- public const uint StApplicationInfo = 0;
- public const uint StInstanceCreateInfo = 1;
- public const uint StDeviceQueueCreateInfo = 2;
- public const uint StDeviceCreateInfo = 3;
- public const uint StSubmitInfo = 4;
- public const uint StMemoryAllocateInfo = 5;
- public const uint StMappedMemoryRange = 6;
- public const uint StFenceCreateInfo = 8;
- public const uint StQueryPoolCreateInfo = 11;
- public const uint StBufferCreateInfo = 12;
- public const uint StShaderModuleCreateInfo = 16;
- public const uint StPipelineShaderStageCreateInfo = 18;
- public const uint StComputePipelineCreateInfo = 29;
- public const uint StPipelineLayoutCreateInfo = 30;
- public const uint StDescriptorSetLayoutCreateInfo = 32;
- public const uint StDescriptorPoolCreateInfo = 33;
- public const uint StDescriptorSetAllocateInfo = 34;
- public const uint StWriteDescriptorSet = 35;
- public const uint StCommandPoolCreateInfo = 39;
- public const uint StCommandBufferAllocateInfo = 40;
- public const uint StCommandBufferBeginInfo = 42;
- public const uint StMemoryBarrier = 46;
- public const uint StBufferMemoryBarrier = 44;
- public const uint QueueFamilyIgnored = 0xFFFFFFFFu;
- public const uint StPhysicalDeviceFeatures2 = 49;
- public const uint StPhysicalDevice16BitStorageFeatures = 1000083000u;
- public const uint StPhysicalDeviceShaderFloat16Int8Features = 1000082000u;
- public const uint StPhysicalDeviceVulkanMemoryModelFeatures = 1000211000u;
- public const uint StPhysicalDeviceCooperativeMatrixFeaturesKHR = 1000506000u;
- public const uint StCooperativeMatrixPropertiesKHR = 1000506001u;
- public const uint StPhysicalDeviceSubgroupProperties = 1000094000u;
- public const uint StPhysicalDeviceSubgroupSizeControlPropertiesEXT = 1000225000u;
- public const uint StPhysicalDeviceSubgroupSizeControlFeaturesEXT = 1000225002u;
- public const uint StPhysicalDeviceProperties2 = 1000059000u;
- public const uint StPipelineShaderStageRequiredSubgroupSizeCreateInfo = 1000225001u;
-
- // VkPhysicalDeviceType
- public const uint PhysDeviceIntegrated = 1;
- public const uint PhysDeviceDiscrete = 2;
- public const uint PhysDeviceVirtualGpu = 3;
- public const uint PhysDeviceCpu = 4;
-
- // VkQueueFlagBits
- public const uint QueueGraphics = 0x1;
- public const uint QueueCompute = 0x2;
- public const uint QueueTransfer = 0x4;
-
- // VkBufferUsageFlagBits
- public const uint BufferUsageTransferSrc = 0x1;
- public const uint BufferUsageTransferDst = 0x2;
- public const uint BufferUsageStorageBuffer = 0x20;
-
- // VkMemoryPropertyFlagBits
- public const uint MemDeviceLocal = 0x1;
- public const uint MemHostVisible = 0x2;
- public const uint MemHostCoherent = 0x4;
- public const uint MemHostCached = 0x8;
-
- // VkDescriptorType
- public const uint DescStorageBuffer = 7;
- public const uint DescUniformBuffer = 6;
-
- // VkShaderStageFlagBits / VkPipelineStageFlagBits
- public const uint StageComputeShader = 0x20;
- public const uint PipelineStageTransfer = 0x1000;
- public const uint PipelineStageComputeShader = 0x800;
- public const uint PipelineStageHost = 0x4000;
- public const uint PipelineStageTopOfPipe = 0x1;
- public const uint PipelineStageBottomOfPipe = 0x2000;
-
- // VkAccessFlagBits
- public const uint AccessShaderRead = 0x20;
- public const uint AccessShaderWrite = 0x40;
- public const uint AccessTransferRead = 0x800;
- public const uint AccessTransferWrite = 0x1000;
- public const uint AccessHostRead = 0x2000;
- public const uint AccessHostWrite = 0x4000;
- public const uint StageTopOfPipe = 0x1;
- public const uint StageAllCommands = 0x10000;
-
- // VkPipelineBindPoint
- public const uint BindPointCompute = 1;
-
- // VkCommandBufferLevel
- public const uint CmdBufferLevelPrimary = 0;
-
- // VkCommandPoolCreateFlagBits
- public const uint CmdPoolResetCommandBuffer = 0x2;
-
- // VkSharingMode
- public const uint SharingExclusive = 0;
-
- // VkQueryType / VkQueryResultFlagBits
- public const uint QueryTypeTimestamp = 1;
- public const uint QueryResult64 = 0x1;
- public const uint QueryResultWait = 0x2;
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkBackend.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkBackend.cs
deleted file mode 100644
index 661321e..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkBackend.cs
+++ /dev/null
@@ -1,91 +0,0 @@
-using System.Collections.Concurrent;
-using Sdcb.SimdGcdrNet.Engine;
-using Sdcb.SimdGcdrNet.Onnx;
-
-namespace Sdcb.SimdGcdrNet.Backends.Vulkan;
-
-///
-/// Process-wide Vulkan device + per-(model,size) compiled graph cache.
-/// Constructed lazily on first Vulkan use.
-///
-internal sealed unsafe class VkBackend : IDisposable
-{
- internal readonly VkDevice Dev;
- // (model, c, h, w) -> compiled graph (weights upload + dispatch list)
- private readonly ConcurrentDictionary<(OnnxModel m, int c, int h, int w, bool physIo), VkGraphModel> _graphs = new();
-
- private static readonly Lazy s_default = new(() => new VkBackend());
- internal static VkBackend Default => s_default.Value;
-
- ///
- /// Plan options for GPU compiles: always channels-last (shaders address NHWC).
- /// Standalone sessions keep logical NCHW I/O (boundary relayouts in the plan);
- /// the fused pipeline keeps I/O physical and its glue shaders speak NHWC.
- ///
- internal static CompileOptions PlanOptions => new(Nhwc: true, PhysicalIo: false, Gpu: true);
- internal static CompileOptions PipeOptions => new(Nhwc: true, PhysicalIo: true, Gpu: true);
-
- /// fp16 activation/weight storage (needs storageBuffer16BitAccess; GCDRNET_VK_F16=0 forces fp32).
- internal bool F16 => Dev.StorageBuffer16 && Environment.GetEnvironmentVariable("GCDRNET_VK_F16") != "0";
-
- internal VkBackend()
- {
- Dev = VkDevice.Create();
- }
-
- /// Standalone session for one model at [1,c,h,w] (test path).
- public VkInferenceSession NewSession(OnnxModel m, int[] shape)
- {
- VkGraphModel g = AcquireGraph(m, shape, physIo: false); // caller ref: later evictions can't free it under us
- try { return new VkInferenceSession(Dev, g.Plan, g); }
- finally { g.Release(); }
- }
-
- /// Compiled GPU graph for a model+shape (cached, bounded).
- internal VkGraphModel GraphFor(OnnxModel m, int[] shape, bool physIo = false)
- {
- var key = (m, shape[1], shape[2], shape[3], physIo);
- if (_graphs.TryGetValue(key, out var g)) return g;
- // weights live in a per-model const store shared across sizes, so a graph is
- // just a plan + dispatch list (no per-size device memory)
- g = new VkGraphModel(Dev, GraphCompiler.Compile(m, [1, shape[1], shape[2], shape[3]],
- physIo ? PipeOptions : PlanOptions), F16, constKey: m);
- if (!_graphs.TryAdd(key, g)) { g.Dispose(); return _graphs[key]; }
- // bound the cache: evict an arbitrary other graph beyond the cap
- while (_graphs.Count > MaxGraphs)
- {
- var victim = _graphs.Keys.FirstOrDefault(k => !k.Equals(key));
- if (victim.Equals(default)) break;
- if (_graphs.TryRemove(victim, out var old)) old.Release();
- }
- return g;
- }
-
- // graphs hold no device memory beyond the shared const store; the bound is for
- // the managed plans (each holds its own copy of the model's const floats)
- private const int MaxGraphs = 32;
-
- /// plus a caller reference. Retries when a concurrent
- /// eviction freed the cached graph between lookup and acquire (never resurrects it).
- internal VkGraphModel AcquireGraph(OnnxModel m, int[] shape, bool physIo)
- {
- while (true)
- {
- VkGraphModel g = GraphFor(m, shape, physIo);
- if (g.TryAcquire()) return g;
- }
- }
-
- /// New size-agnostic GPU worker for a (gc, dr) model pair (stateful; callers pool them).
- internal VkGcdrPipeline NewWorker(OnnxModel gc, OnnxModel dr) => new(this, gc, dr);
-
- /// New GPU worker pre-bound to one source size (debug/bench helper).
- internal VkGcdrPipeline NewPipe(OnnxModel gc, OnnxModel dr, int ih, int iw)
- {
- var p = new VkGcdrPipeline(this, gc, dr);
- try { p.Bind(ih, iw); return p; }
- catch { p.Dispose(); throw; }
- }
-
- public void Dispose() => Dev.Dispose();
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkConstStore.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkConstStore.cs
deleted file mode 100644
index 5c46aeb..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkConstStore.cs
+++ /dev/null
@@ -1,91 +0,0 @@
-namespace Sdcb.SimdGcdrNet.Backends.Vulkan;
-
-///
-/// Device-side const tensors (backend-packed weights) plus a 64B zero buffer,
-/// shared by every of one model: per-size plans copy
-/// the same initializers, so instead of re-packing and re-uploading ~20 MB per
-/// input size, a new graph whose const list is element-for-element identical
-/// (same consumer packing, same contents) reuses the existing buffer. A graph
-/// whose consts differ gets a private, unshared store. Ref-counted; the
-/// buffers are freed when the last graph releases.
-///
-internal sealed class VkConstStore
-{
- private static readonly Dictionary<(VkDevice dev, object key, bool f16), VkConstStore> s_shared = new();
-
- private readonly (VkDevice, object, bool)? _key;
- private readonly float[][] _src;
- private readonly int[] _kinds;
- private int _refs = 1;
- internal readonly VkBuffer Consts, Dummy;
- internal readonly ulong[] Offsets;
- internal readonly ulong Bytes;
-
- private VkConstStore(VkDevice dev, (VkDevice, object, bool)? key, float[][] src, int[] kinds,
- ulong[] offsets, byte[] host)
- {
- _key = key;
- _src = src;
- _kinds = kinds;
- Offsets = offsets;
- Bytes = (ulong)host.LongLength;
- Consts = dev.NewStorageBuffer(Bytes, hostVisible: dev.CoherentDeviceLocal);
- Dummy = dev.NewStorageBuffer(64, hostVisible: dev.CoherentDeviceLocal);
- unsafe
- {
- fixed (byte* hp = host) dev.Upload(Consts, hp, Bytes);
- byte* z = stackalloc byte[64];
- new Span(z, 64).Clear();
- dev.Upload(Dummy, z, 64);
- }
- }
-
- ///
- /// Returns a store holding (kinds = consumer packing ids),
- /// shared under when an identical list was uploaded before.
- /// (offsets, host bytes) only runs when a new store is built.
- ///
- internal static VkConstStore Acquire(VkDevice dev, object? key, bool f16, float[][] src, int[] kinds,
- Func<(ulong[] offsets, byte[] host)> pack)
- {
- if (key is null)
- {
- var (o, h) = pack();
- return new VkConstStore(dev, null, src, kinds, o, h);
- }
- var k = (dev, key, f16);
- lock (s_shared)
- {
- if (s_shared.TryGetValue(k, out VkConstStore? s) && s.Matches(src, kinds))
- {
- s._refs++;
- return s;
- }
- var (offs, host) = pack();
- bool share = s is null; // keep the first registered store on a mismatch
- var ns = new VkConstStore(dev, share ? k : null, src, kinds, offs, host);
- if (share) s_shared[k] = ns;
- return ns;
- }
- }
-
- private bool Matches(float[][] src, int[] kinds)
- {
- if (src.Length != _src.Length || !kinds.AsSpan().SequenceEqual(_kinds)) return false;
- for (int i = 0; i < src.Length; i++)
- if (!ReferenceEquals(src[i], _src[i]) && !src[i].AsSpan().SequenceEqual(_src[i])) return false;
- return true;
- }
-
- internal void Release()
- {
- lock (s_shared)
- {
- if (--_refs > 0) return;
- if (_key is { } k && s_shared.TryGetValue(k, out VkConstStore? cur) && cur == this)
- s_shared.Remove(k);
- }
- Consts.Free();
- Dummy.Free();
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkDevice.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkDevice.cs
deleted file mode 100644
index 34f6d51..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkDevice.cs
+++ /dev/null
@@ -1,938 +0,0 @@
-// Thin object wrappers over Vk.cs: instance/device/queue, buffers with
-// memory-type selection (ReBAR fast path vs host-visible staging), shader
-// modules from embedded SPIR-V, compute pipelines, descriptor sets, and
-// one-shot command submission. Adapted from Sdcb.SimdPaddleOCR.
-using System.Runtime.InteropServices;
-
-namespace Sdcb.SimdGcdrNet.Backends.Vulkan;
-
-internal unsafe sealed class VkDevice : IDisposable
-{
- public IntPtr Instance, PhysDevice, Device, Queue;
- public string DeviceName = "";
- public uint VendorId;
- public uint SubgroupSize;
- public uint QueueFamily;
- public Vk.VkPhysicalDeviceMemoryProperties MemProps;
- public bool CoherentDeviceLocal; // DEVICE_LOCAL|HOST_VISIBLE|HOST_COHERENT exists (ReBAR)
- public bool StorageBuffer16; // VK_KHR_16bit_storage: activations may be stored as float16
- public uint MaxSharedMemory;
- public ulong MaxStorageRange = uint.MaxValue;
- public bool PushDescriptors;
- /// Tensor-core convs (conv_cm.comp; GCDRNET_VK_COOPMAT=0 disables): VK_KHR_cooperative_matrix
- /// plus shaderFloat16, in one of two shapes. Default: a 16x16x16 fp16 x fp16 -> fp32 subgroup config
- /// on 32-lane subgroups; devices whose default compute subgroup is not 32 lanes (RDNA wave64) qualify
- /// only when VK_EXT_subgroup_size_control can pin it to 32 ().
- /// Otherwise : the 8x16x16 config on subgroups pinned to 16 lanes (Intel Xe2).
- /// A probe dispatch must confirm the pinned width; failing that the convs stay on conv_tile.
- public bool CoopMat16;
- /// The coopmat convs use the SG16 build (8x16x16 fragments, 16-lane subgroups; Intel Arc
- /// XMX): taken only when the 16x16x16 / 32-lane path is unavailable (GCDRNET_VK_SG16=0 disables).
- public bool CoopM8;
- /// requiredSubgroupSize for the coopmat pipelines: 0 = device default (already 32),
- /// 32 / 16 = pinned through VK_EXT_subgroup_size_control.
- public uint CoopSubgroupSize;
- /// Coopmat convs use the LITE variants (single-buffered 28 KB staging, scalar epilogue,
- /// grouped raster): wave64-capable parts or less than the 48 KB shared memory the
- /// double-buffered kernels take.
- public bool CoopLite;
- /// VK_EXT_subgroup_size_control range (1..128 when the extension is absent).
- public uint SubgroupMin = 1, SubgroupMax = 128;
- /// Memory-bound NHWC shaders (maxpool, bilinear, depthwise, shift) take 4 channels per
- /// invocation when c % 4 == 0 — integrated GPUs, where they are bandwidth/issue bound, and Intel
- /// GPUs (Arc B580: 10.4 -> 6.7 ms for the four; per-element index math is costly on SIMD16)
- /// (GCDRNET_VK_VEC4=0 disables; other discrete GPUs keep the per-element shaders).
- public bool Vec4Eltwise;
- /// 3x3 stem conv+pool uses convpool_k3 (input window rows in registers): Intel GPUs
- /// (Arc B580: 7.5 -> 4.9 ms for the two stems) (GCDRNET_VK_CONVPOOL3=0 disables).
- public bool ConvPoolK3;
- /// Register-tiled kernels for GPUs without cooperative matrices whose compute subgroups
- /// are at least 64 lanes (Adreno 750: 64/128, 32 KB shared memory), where conv_tile / gemm's
- /// shared-memory staging is load-bound: 3x3 convs on conv_dot.comp, Gemm on gemm_dot.comp,
- /// the stem conv+pool with the four pool pixels per weight load (convpool_pi) and the UpHead
- /// tail two pixels per invocation (uphead_sc2). fp16 storage only (GCDRNET_VK_DOT=0 disables).
- public bool ConvDot;
- private const uint VendorIntel = 0x8086;
- /// Largest DEVICE_LOCAL heap (bytes).
- public ulong DeviceLocalHeap;
- /// Integrated GPU: "device-local" memory is system RAM shared with the CPU.
- public bool UnifiedMemory;
- private unsafe delegate* unmanaged[Cdecl] _pushDesc;
- public double TimestampPeriodNs = 1;
- private IntPtr _commandPool;
- private IntPtr _descPool;
-
- // live vkAllocateMemory bytes across the process (diagnostic — buffers are pooled)
- private static long _liveBytes, _peakBytes;
- public static long LiveDeviceBytes => Interlocked.Read(ref _liveBytes);
- public static long PeakDeviceBytes => Interlocked.Read(ref _peakBytes);
- public static void ResetPeakDeviceBytes() => Interlocked.Exchange(ref _peakBytes, _liveBytes);
- private static void TrackAlloc(long bytes)
- {
- long now = Interlocked.Add(ref _liveBytes, bytes), p;
- while ((p = Interlocked.Read(ref _peakBytes)) < now)
- Interlocked.CompareExchange(ref _peakBytes, now, p);
- }
- internal static void TrackFree(long bytes) => Interlocked.Add(ref _liveBytes, -bytes);
- /// Guards the device-level command pool / descriptor pool (uploads).
- public readonly object Sync = new();
- /// vkQueueSubmit on the single compute queue is externally synchronized.
- public readonly object QueueLock = new();
- private readonly Dictionary<(string name, int bindings, int pcBytes, uint sg), VkPipeline> _pipes = new();
-
- /// Process-wide pipeline cache: every graph/session shares one
- /// VkPipeline per (shader, binding-count, push-const-size). Set layouts carry
- /// PUSH_DESCRIPTOR_BIT when the extension is available.
- public VkPipeline GetPipeline(string name, int bindings, int pcBytes, uint requiredSubgroupSize = 0)
- {
- var key = (name, bindings, pcBytes, requiredSubgroupSize);
- lock (_pipes)
- {
- if (_pipes.TryGetValue(key, out VkPipeline? p)) return p;
- p = NewPipeline(NewShaderModule(VkShaders.Load(name)), bindings, pcBytes, PushDescriptors, requiredSubgroupSize);
- p.Name = name;
- p.Bindings = bindings;
- _pipes[key] = p;
- return p;
- }
- }
-
- /// Pipeline of a conv_cm variant ("conv_cm", "conv_cm_n64", ...; fp16 storage): the
- /// LITE build on devices (or when forces it),
- /// the SG16 build (always LITE-style) on devices, with the subgroup pinned
- /// to . Every variant keeps BM = 128 rows per workgroup.
- public VkPipeline GetCoopPipeline(string variant, int bindings, int pcBytes, bool? lite = null)
- {
- if (!variant.StartsWith("conv_cm", StringComparison.Ordinal)) throw new ArgumentException(variant);
- string suffix = variant["conv_cm".Length..];
- string name = CoopM8 ? "conv_cm_sg16" + suffix : (lite ?? CoopLite) ? "conv_cm_l" + suffix : variant;
- return GetPipeline(name + "_h", bindings, pcBytes, CoopSubgroupSize);
- }
-
- public static VkDevice Create(uint deviceIndex = 0)
- {
- Vk.RegisterResolver();
- var d = new VkDevice();
-
- Vk.VkApplicationInfo app = new()
- {
- SType = VkConst.StApplicationInfo,
- AppVersion = 1, EngineVersion = 1, ApiVersion = (1u << 22) | (1u << 12), // Vulkan 1.1
- };
- Vk.VkInstanceCreateInfo ici = new() { SType = VkConst.StInstanceCreateInfo, PAppInfo = &app };
- Vk.Check(Vk.vkCreateInstance(&ici, null, out d.Instance), "vkCreateInstance");
-
- uint ndev = 0;
- Vk.Check(Vk.vkEnumeratePhysicalDevices(d.Instance, &ndev, null), "enum physical devices");
- if (ndev == 0) throw new PlatformNotSupportedException("Vulkan: no physical devices (no Vulkan-capable GPU/driver)");
- IntPtr* devs = stackalloc IntPtr[(int)ndev];
- Vk.Check(Vk.vkEnumeratePhysicalDevices(d.Instance, &ndev, devs), "enum physical devices");
-
- // Prefer discrete GPU; honor deviceIndex as "the nth discrete first, else nth overall".
- int chosen = -1;
- for (int pass = 0; pass < 2 && chosen < 0; pass++)
- {
- uint seen = 0;
- for (int i = 0; i < (int)ndev; i++)
- {
- Vk.VkPhysicalDeviceProperties p;
- Vk.vkGetPhysicalDeviceProperties(devs[i], &p);
- bool want = pass == 0 ? p.DeviceType == VkConst.PhysDeviceDiscrete : p.DeviceType != VkConst.PhysDeviceCpu;
- if (!want) continue;
- if (seen++ == deviceIndex) { chosen = i; break; }
- }
- }
- if (chosen < 0) throw new PlatformNotSupportedException($"Vulkan: device index {deviceIndex} out of range ({ndev} devices)");
- d.PhysDevice = devs[chosen];
-
- Vk.VkPhysicalDeviceProperties props;
- Vk.vkGetPhysicalDeviceProperties(d.PhysDevice, &props);
- d.DeviceName = Marshal.PtrToStringAnsi((IntPtr)props.DeviceName) ?? "?";
- d.VendorId = props.VendorID;
- d.TimestampPeriodNs = props.TimestampPeriodNs;
- d.MaxSharedMemory = props.MaxComputeSharedMemorySize;
- d.MaxStorageRange = props.MaxStorageBufferRange;
-
- uint nqf = 0;
- Vk.vkGetPhysicalDeviceQueueFamilyProperties(d.PhysDevice, &nqf, null);
- Vk.VkQueueFamilyProperties* qf = stackalloc Vk.VkQueueFamilyProperties[(int)nqf];
- Vk.vkGetPhysicalDeviceQueueFamilyProperties(d.PhysDevice, &nqf, qf);
- if (Environment.GetEnvironmentVariable("GCDRNET_VK_DBG") == "1")
- for (uint i = 0; i < nqf; i++)
- Console.Error.WriteLine($"[vk] qf{i}: flags={qf[i].QueueFlags:x} count={qf[i].QueueCount} tsbits={qf[i].TimestampValidBits}");
- // Prefer a compute-only family (no GRAPHICS bit) for async-compute —
- // but only if it supports timestamps; otherwise fall back to graphics+compute.
- d.QueueFamily = uint.MaxValue;
- for (uint i = 0; i < nqf; i++)
- if ((qf[i].QueueFlags & VkConst.QueueCompute) != 0 && (qf[i].QueueFlags & 1u) == 0
- && qf[i].TimestampValidBits > 0)
- { d.QueueFamily = i; break; }
- for (uint i = 0; i < nqf && d.QueueFamily == uint.MaxValue; i++)
- if ((qf[i].QueueFlags & VkConst.QueueCompute) != 0 && qf[i].TimestampValidBits > 0)
- { d.QueueFamily = i; break; }
- for (uint i = 0; i < nqf && d.QueueFamily == uint.MaxValue; i++)
- if ((qf[i].QueueFlags & VkConst.QueueCompute) != 0) { d.QueueFamily = i; break; }
- if (d.QueueFamily == uint.MaxValue)
- throw new PlatformNotSupportedException("Vulkan: no compute queue family");
-
- // Extension probe: VK_KHR_push_descriptor (inline descriptor writes,
- // skips per-dispatch descriptor-set binds).
- uint next = 0;
- Vk.vkEnumerateDeviceExtensionProperties(d.PhysDevice, null, &next, null);
- Vk.VkExtensionProperties* exts = stackalloc Vk.VkExtensionProperties[(int)next];
- Vk.vkEnumerateDeviceExtensionProperties(d.PhysDevice, null, &next, exts);
- byte* wantPush = stackalloc byte[] { (byte)'V', (byte)'K', (byte)'_', (byte)'K', (byte)'H', (byte)'R',
- (byte)'_', (byte)'p', (byte)'u', (byte)'s', (byte)'h', (byte)'_', (byte)'d', (byte)'e', (byte)'s', (byte)'c',
- (byte)'r', (byte)'i', (byte)'p', (byte)'t', (byte)'o', (byte)'r', 0 };
- bool hasPush = false, hasCoop = false, hasF16Int8 = false, hasVmm = false, hasSgc = false;
- for (int i = 0; i < (int)next; i++)
- {
- string en = new((sbyte*)exts[i].ExtensionName);
- if (en == "VK_KHR_push_descriptor") hasPush = true;
- else if (en == "VK_KHR_cooperative_matrix") hasCoop = true;
- else if (en == "VK_KHR_shader_float16_int8") hasF16Int8 = true;
- else if (en == "VK_KHR_vulkan_memory_model") hasVmm = true;
- else if (en == "VK_EXT_subgroup_size_control") hasSgc = true;
- }
- if (Environment.GetEnvironmentVariable("GCDRNET_VK_NOPUSH") == "1") hasPush = false;
-
- float prio = 1f;
- Vk.VkDeviceQueueCreateInfo qci = new()
- {
- SType = VkConst.StDeviceQueueCreateInfo,
- QueueFamilyIndex = d.QueueFamily, QueueCount = 1, PQueuePriorities = &prio,
- };
- Vk.VkPhysicalDeviceSubgroupProperties sgP = new() { SType = VkConst.StPhysicalDeviceSubgroupProperties };
- Vk.VkPhysicalDeviceSubgroupSizeControlProperties sgcP = new() { SType = VkConst.StPhysicalDeviceSubgroupSizeControlPropertiesEXT };
- if (hasSgc) sgP.PNext = &sgcP;
- Vk.VkPhysicalDeviceProperties2 p2 = new() { SType = VkConst.StPhysicalDeviceProperties2, PNext = &sgP };
- Vk.vkGetPhysicalDeviceProperties2(d.PhysDevice, &p2);
- d.SubgroupSize = sgP.SubgroupSize;
- bool sgcCompute = false;
- if (hasSgc)
- {
- d.SubgroupMin = sgcP.MinSubgroupSize;
- d.SubgroupMax = sgcP.MaxSubgroupSize;
- sgcCompute = (sgcP.RequiredSubgroupSizeStages & VkConst.StageComputeShader) != 0;
- }
-
- // Query storageBuffer16BitAccess (VK 1.1 core) for fp16 activation storage.
- Vk.VkPhysicalDevice16BitStorageFeatures feat16 = new()
- {
- SType = VkConst.StPhysicalDevice16BitStorageFeatures,
- };
- Vk.VkPhysicalDeviceFeatures2 feats2 = new()
- {
- SType = VkConst.StPhysicalDeviceFeatures2,
- PNext = &feat16,
- };
- Vk.vkGetPhysicalDeviceFeatures2(d.PhysDevice, &feats2);
- d.StorageBuffer16 = feat16.StorageBuffer16BitAccess != 0;
-
- // tensor-core path: coopmat + shaderFloat16 on top of 16-bit storage, 32-lane subgroups;
- // coopmat SPIR-V uses the Vulkan memory model, so that feature is required as well
- Vk.VkPhysicalDeviceShaderFloat16Int8Features f16q = new() { SType = VkConst.StPhysicalDeviceShaderFloat16Int8Features };
- Vk.VkPhysicalDeviceCooperativeMatrixFeaturesKHR coopQ = new() { SType = VkConst.StPhysicalDeviceCooperativeMatrixFeaturesKHR };
- Vk.VkPhysicalDeviceVulkanMemoryModelFeatures vmmQ = new() { SType = VkConst.StPhysicalDeviceVulkanMemoryModelFeatures };
- Vk.VkPhysicalDeviceSubgroupSizeControlFeaturesEXT sgcQ = new() { SType = VkConst.StPhysicalDeviceSubgroupSizeControlFeaturesEXT };
- // a default other than 32 lanes (RDNA: wave64) needs the compute subgroup pinned to 32
- bool pinSg32 = d.SubgroupSize != 32 && hasSgc && sgcCompute && d.SubgroupMin <= 32 && d.SubgroupMax >= 32
- && Environment.GetEnvironmentVariable("GCDRNET_VK_SG32") != "0";
- // the SG16 build (8x16x16, Intel Xe2 XMX) needs 16 lanes in the pinnable compute range
- bool canSg16 = hasSgc && sgcCompute && d.SubgroupMin <= 16 && d.SubgroupMax >= 16
- && Environment.GetEnvironmentVariable("GCDRNET_VK_SG16") != "0";
- bool wantCm = hasCoop && hasF16Int8 && hasVmm && d.StorageBuffer16 && (d.SubgroupSize == 32 || pinSg32 || canSg16)
- && Environment.GetEnvironmentVariable("GCDRNET_VK_COOPMAT") != "0";
- bool pinSg16 = false;
- if (wantCm)
- {
- f16q.PNext = &coopQ;
- coopQ.PNext = &vmmQ;
- if (pinSg32 || canSg16) vmmQ.PNext = &sgcQ;
- Vk.VkPhysicalDeviceFeatures2 fq = new() { SType = VkConst.StPhysicalDeviceFeatures2, PNext = &f16q };
- Vk.vkGetPhysicalDeviceFeatures2(d.PhysDevice, &fq);
- bool feats = f16q.ShaderFloat16 != 0 && coopQ.CooperativeMatrix != 0 && vmmQ.VulkanMemoryModel != 0;
- CoopShapes(d, out bool m16, out bool m8);
- // 16x16x16 on 32 lanes first (NVIDIA, RDNA pinned to 32); 8x16x16 on 16 lanes only without it
- bool cm32 = feats && (d.SubgroupSize == 32 || pinSg32) && (!pinSg32 || sgcQ.SubgroupSizeControl != 0) && m16;
- pinSg16 = !cm32 && feats && canSg16 && sgcQ.SubgroupSizeControl != 0 && m8;
- wantCm = cm32 || pinSg16;
- pinSg32 &= cm32;
- }
- else pinSg32 = false;
- bool pinSg = pinSg32 || pinSg16;
-
- byte** extsToEnable = stackalloc byte*[5];
- uint nExt = 0;
- if (hasPush) extsToEnable[nExt++] = wantPush;
- byte* pCm = stackalloc byte[32], pF16 = stackalloc byte[32], pVmm = stackalloc byte[32], pSgc = stackalloc byte[32];
- System.Text.Encoding.ASCII.GetBytes("VK_KHR_cooperative_matrix\0", new Span(pCm, 32));
- System.Text.Encoding.ASCII.GetBytes("VK_KHR_shader_float16_int8\0", new Span(pF16, 32));
- System.Text.Encoding.ASCII.GetBytes("VK_KHR_vulkan_memory_model\0", new Span(pVmm, 32));
- System.Text.Encoding.ASCII.GetBytes("VK_EXT_subgroup_size_control\0", new Span(pSgc, 32));
- if (wantCm) { extsToEnable[nExt++] = pCm; extsToEnable[nExt++] = pF16; extsToEnable[nExt++] = pVmm; }
- if (pinSg) extsToEnable[nExt++] = pSgc;
- Vk.VkPhysicalDeviceShaderFloat16Int8Features f16en = new()
- {
- SType = VkConst.StPhysicalDeviceShaderFloat16Int8Features, ShaderFloat16 = 1,
- };
- Vk.VkPhysicalDeviceCooperativeMatrixFeaturesKHR coopEn = new()
- {
- SType = VkConst.StPhysicalDeviceCooperativeMatrixFeaturesKHR, CooperativeMatrix = 1,
- };
- Vk.VkPhysicalDeviceVulkanMemoryModelFeatures vmmEn = new()
- {
- SType = VkConst.StPhysicalDeviceVulkanMemoryModelFeatures, VulkanMemoryModel = 1,
- };
- Vk.VkPhysicalDeviceSubgroupSizeControlFeaturesEXT sgcEn = new()
- {
- SType = VkConst.StPhysicalDeviceSubgroupSizeControlFeaturesEXT, SubgroupSizeControl = 1,
- ComputeFullSubgroups = sgcQ.ComputeFullSubgroups,
- };
- if (pinSg) vmmEn.PNext = &sgcEn;
- coopEn.PNext = &vmmEn;
- f16en.PNext = &coopEn;
- feat16.PNext = wantCm ? &f16en : null;
-
- // To enable 16-bit storage we chain feat16 into VkDeviceCreateInfo.pNext
- // (VK 1.1 core: no extension needed). PEnabledFeatures must be null when
- // using Features2 pNext; we inline a zero VkPhysicalDeviceFeatures in feat2.
- Vk.VkPhysicalDeviceFeatures2 feat2enable = new()
- {
- SType = VkConst.StPhysicalDeviceFeatures2,
- PNext = d.StorageBuffer16 ? &feat16 : null,
- };
- Vk.VkDeviceCreateInfo dci = new()
- {
- SType = VkConst.StDeviceCreateInfo,
- QueueCreateInfoCount = 1, PQueueCreateInfos = &qci,
- PEnabledFeatures = null,
- PNext = &feat2enable,
- EnabledExtensionCount = nExt,
- PpEnabledExtensionNames = nExt > 0 ? extsToEnable : null,
- };
- Vk.Check(Vk.vkCreateDevice(d.PhysDevice, &dci, null, out d.Device), "vkCreateDevice");
- d.CoopMat16 = wantCm;
- d.CoopM8 = pinSg16;
- d.CoopSubgroupSize = pinSg32 ? 32u : pinSg16 ? 16u : 0u;
- d._fullSubgroups = pinSg && sgcQ.ComputeFullSubgroups != 0;
- d.CoopLite = wantCm && !pinSg16 && ((hasSgc ? d.SubgroupMax : d.SubgroupSize) >= 64 || d.MaxSharedMemory < 48 * 1024)
- && Environment.GetEnvironmentVariable("GCDRNET_VK_LITE") != "0";
- Vk.vkGetDeviceQueue(d.Device, d.QueueFamily, 0, out d.Queue);
-
- if (hasPush)
- {
- byte* fnName = stackalloc byte[] { (byte)'v', (byte)'k', (byte)'C', (byte)'m', (byte)'d',
- (byte)'P', (byte)'u', (byte)'s', (byte)'h', (byte)'D', (byte)'e', (byte)'s', (byte)'c',
- (byte)'r', (byte)'i', (byte)'p', (byte)'t', (byte)'o', (byte)'r', (byte)'S', (byte)'e',
- (byte)'t', (byte)'K', (byte)'H', (byte)'R', 0 };
- IntPtr fn = Vk.vkGetDeviceProcAddr(d.Device, fnName);
- if (fn != IntPtr.Zero)
- {
- d.PushDescriptors = true;
- d._pushDesc = (delegate* unmanaged[Cdecl])fn;
- }
- }
-
- fixed (Vk.VkPhysicalDeviceMemoryProperties* mp = &d.MemProps)
- Vk.vkGetPhysicalDeviceMemoryProperties(d.PhysDevice, mp);
- for (int i = 0; i < (int)d.MemProps.MemoryTypeCount; i++)
- {
- uint f = d.MemProps.TypeAt(i).PropertyFlags;
- if ((f & (VkConst.MemDeviceLocal | VkConst.MemHostVisible | VkConst.MemHostCoherent)) == (VkConst.MemDeviceLocal | VkConst.MemHostVisible | VkConst.MemHostCoherent))
- {
- d.CoherentDeviceLocal = true;
- break;
- }
- }
- d.UnifiedMemory = props.DeviceType == VkConst.PhysDeviceIntegrated;
- d.Vec4Eltwise = (d.UnifiedMemory || d.VendorId == VendorIntel) && Environment.GetEnvironmentVariable("GCDRNET_VK_VEC4") != "0";
- d.ConvPoolK3 = d.VendorId == VendorIntel && Environment.GetEnvironmentVariable("GCDRNET_VK_CONVPOOL3") != "0";
- for (int i = 0; i < (int)d.MemProps.MemoryHeapCount; i++)
- {
- var hp = d.MemProps.HeapAt(i);
- if ((hp.Flags & 1u) != 0) d.DeviceLocalHeap = Math.Max(d.DeviceLocalHeap, hp.Size); // HEAP_DEVICE_LOCAL
- }
-
- Vk.VkCommandPoolCreateInfo cpci = new()
- {
- SType = VkConst.StCommandPoolCreateInfo,
- Flags = VkConst.CmdPoolResetCommandBuffer,
- QueueFamilyIndex = d.QueueFamily,
- };
- Vk.Check(Vk.vkCreateCommandPool(d.Device, &cpci, null, out d._commandPool), "vkCreateCommandPool");
-
- Vk.VkDescriptorPoolSize psz = new() { Type = VkConst.DescStorageBuffer, DescriptorCount = 1 << 20 };
- Vk.VkDescriptorPoolCreateInfo dpci = new()
- {
- SType = VkConst.StDescriptorPoolCreateInfo,
- Flags = 1, MaxSets = 1 << 16, PoolSizeCount = 1, PPoolSizes = &psz,
- };
- Vk.Check(Vk.vkCreateDescriptorPool(d.Device, &dpci, null, out d._descPool), "vkCreateDescriptorPool");
-
- // never trust a pinned subgroup size blindly: the coopmat lane mapping is hard-wired to
- // the pinned width (32, or 16 for SG16), so anything else would silently compute garbage
- if (pinSg && !d.ProbeSubgroup(d.CoopSubgroupSize))
- {
- d.CoopMat16 = false;
- d.CoopM8 = false;
- d.CoopLite = false;
- d.CoopSubgroupSize = 0;
- }
- d.ConvDot = !d.CoopMat16 && hasSgc && d.SubgroupMin >= 64 && d.StorageBuffer16
- && Environment.GetEnvironmentVariable("GCDRNET_VK_DOT") != "0";
- if (Environment.GetEnvironmentVariable("GCDRNET_VK_VERBOSE") == "1")
- {
- Console.Error.WriteLine($"[vk] device={d.DeviceName} type={props.DeviceType} vendor=0x{d.VendorId:x4} " +
- $"api={props.ApiVersion >> 22}.{(props.ApiVersion >> 12) & 0x3ff}.{props.ApiVersion & 0xfff} driver=0x{props.DriverVersion:x8} qfamily={d.QueueFamily} " +
- $"sg={d.SubgroupSize} sgRange={d.SubgroupMin}-{d.SubgroupMax} sgcCompute={sgcCompute} push={d.PushDescriptors} " +
- $"coopmat={d.CoopMat16} coopSg={(d.CoopSubgroupSize == 0 ? "default" : d.CoopSubgroupSize.ToString())} " +
- $"m8={d.CoopM8} fullSg={d._fullSubgroups} lite={d.CoopLite} vec4={d.Vec4Eltwise} cp3={d.ConvPoolK3} " +
- $"dot={d.ConvDot} lds={d.MaxSharedMemory} sb16={d.StorageBuffer16}");
- for (uint i = 0; i < nqf; i++)
- Console.Error.WriteLine($"[vk] qf{i}: flags=0x{qf[i].QueueFlags:x} count={qf[i].QueueCount} tsbits={qf[i].TimestampValidBits}"
- + (i == d.QueueFamily ? " <- selected" : ""));
- if (hasCoop)
- foreach (var cp in CoopProps(d))
- Console.Error.WriteLine($"[vk] coopmat {cp.MSize}x{cp.NSize}x{cp.KSize} a={cp.AType} b={cp.BType} c={cp.CType} r={cp.ResultType} scope={cp.Scope}");
- for (int i = 0; i < (int)d.MemProps.MemoryHeapCount; i++)
- {
- var hp = d.MemProps.HeapAt(i);
- Console.Error.WriteLine($"[vk] heap{i}: {hp.Size >> 20} MB flags={hp.Flags:x}");
- }
- for (int i = 0; i < (int)d.MemProps.MemoryTypeCount; i++)
- Console.Error.WriteLine($"[vk] type{i}: heap{d.MemProps.TypeAt(i).HeapIndex} flags={d.MemProps.TypeAt(i).PropertyFlags:x}");
- }
- return d;
- }
-
- private bool _fullSubgroups;
-
- /// Runs sgprobe.comp (256 threads) with requiredSubgroupSize = and
- /// checks every invocation reports -lane subgroups, 256 / lanes per workgroup.
- private bool ProbeSubgroup(uint lanes)
- {
- VkPipeline? pipe = null;
- VkBuffer? buf = null;
- uint count = 256 / lanes;
- try
- {
- pipe = NewPipeline(NewShaderModule(VkShaders.Load("sgprobe")), 1, 0, PushDescriptors, lanes);
- buf = NewStorageBuffer(256 * 4, hostVisible: true, preferHost: true);
- IntPtr set = IntPtr.Zero, pool = IntPtr.Zero;
- lock (Sync)
- {
- IntPtr cmd = NewCommandBuffer();
- IntPtr fence = NewFence();
- var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo };
- Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer");
- Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, pipe.Pipeline);
- Vk.VkDescriptorBufferInfo bi = new() { Buffer = buf.Buffer, Offset = 0, Range = 256 * 4 };
- Vk.VkWriteDescriptorSet w = new()
- {
- SType = VkConst.StWriteDescriptorSet, DstBinding = 0,
- DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, PBufferInfo = &bi,
- };
- if (PushDescriptors) CmdPushDescriptors(cmd, pipe.Layout, &w, 1);
- else
- {
- pool = NewDescriptorPool(1);
- set = AllocateDescriptorSet(pool, pipe.SetLayout);
- BindBuffer(set, 0, buf, 0, 256 * 4);
- Vk.vkCmdBindDescriptorSets(cmd, VkConst.BindPointCompute, pipe.Layout, 0, 1, &set, 0, null);
- }
- Vk.vkCmdDispatch(cmd, 1, 1, 1);
- Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer");
- Submit(cmd, fence);
- WaitFence(fence);
- Vk.vkDestroyFence(Device, fence, null);
- FreeCommandBuffer(cmd);
- if (pool != IntPtr.Zero) Vk.vkDestroyDescriptorPool(Device, pool, null);
- }
- uint* r = (uint*)buf.Map();
- buf.Invalidate(0, 256 * 4);
- bool ok = true;
- for (int i = 0; i < 256; i++)
- ok &= (r[i] & 0xff) == lanes && ((r[i] >> 8) & 0xff) == count && ((r[i] >> 16) & 0xff) < count && (r[i] >> 24) < lanes;
- buf.Unmap();
- return ok;
- }
- catch (InvalidOperationException) { return false; }
- finally
- {
- buf?.Free();
- if (pipe is not null)
- {
- Vk.vkDestroyPipeline(Device, pipe.Pipeline, null);
- Vk.vkDestroyPipelineLayout(Device, pipe.Layout, null);
- Vk.vkDestroyDescriptorSetLayout(Device, pipe.SetLayout, null);
- }
- }
- }
-
- /// Which fp16 x fp16 -> fp32 (C and result) subgroup coopmat shapes the device offers:
- /// 16x16x16 (the default conv_cm builds) and 8x16x16 (SG16).
- private static void CoopShapes(VkDevice d, out bool m16, out bool m8)
- {
- m16 = m8 = false;
- // component types: 0 = float16, 1 = float32; scope 3 = subgroup
- foreach (var p in CoopProps(d))
- if (p.NSize == 16 && p.KSize == 16 && p.AType == 0 && p.BType == 0
- && p.CType == 1 && p.ResultType == 1 && p.Scope == 3)
- {
- m16 |= p.MSize == 16;
- m8 |= p.MSize == 8;
- }
- }
-
- /// vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR (empty when unavailable).
- private static Vk.VkCooperativeMatrixPropertiesKHR[] CoopProps(VkDevice d)
- {
- byte[] name = System.Text.Encoding.ASCII.GetBytes("vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR\0");
- IntPtr fn;
- fixed (byte* pn = name) fn = Vk.vkGetInstanceProcAddr(d.Instance, pn);
- if (fn == IntPtr.Zero) return [];
- var query = (delegate* unmanaged[Cdecl])fn;
- uint np = 0;
- if (query(d.PhysDevice, &np, null) != VkResult.Success || np == 0) return [];
- var props = new Vk.VkCooperativeMatrixPropertiesKHR[np];
- for (int i = 0; i < props.Length; i++) props[i].SType = VkConst.StCooperativeMatrixPropertiesKHR;
- fixed (Vk.VkCooperativeMatrixPropertiesKHR* pp = props)
- if (query(d.PhysDevice, &np, pp) != VkResult.Success) return [];
- return props;
- }
-
- /// Allocate a storage buffer. hostVisible=false prefers DEVICE_LOCAL;
- /// hostVisible=true prefers DEVICE_LOCAL|HOST_VISIBLE|HOST_COHERENT (ReBAR zero-copy), falling
- /// back to HOST_VISIBLE|HOST_COHERENT. preferHost skips DEVICE_LOCAL types — for buffers the
- /// CPU reads (uncached WC reads from VRAM are ~15MB/s, catastrophic).
- public VkBuffer NewStorageBuffer(ulong bytes, bool hostVisible, bool preferHost = false)
- {
- var b = new VkBuffer { Dev = Device, Size = bytes };
- Vk.VkBufferCreateInfo bci = new()
- {
- SType = VkConst.StBufferCreateInfo,
- Size = bytes,
- Usage = VkConst.BufferUsageStorageBuffer | VkConst.BufferUsageTransferSrc | VkConst.BufferUsageTransferDst,
- SharingMode = VkConst.SharingExclusive,
- };
- Vk.Check(Vk.vkCreateBuffer(Device, &bci, null, out b.Buffer), "vkCreateBuffer");
- Vk.VkMemoryRequirements req;
- Vk.vkGetBufferMemoryRequirements(Device, b.Buffer, &req);
-
- uint memType = uint.MaxValue;
- for (int i = 0; i < (int)MemProps.MemoryTypeCount; i++)
- {
- if ((req.MemoryTypeBits & (1u << i)) == 0) continue;
- uint f = MemProps.TypeAt(i).PropertyFlags;
- bool ok;
- if (!hostVisible)
- ok = (f & VkConst.MemDeviceLocal) != 0;
- else if (preferHost)
- ok = (f & (VkConst.MemHostVisible | VkConst.MemHostCoherent)) == (VkConst.MemHostVisible | VkConst.MemHostCoherent)
- && (f & VkConst.MemDeviceLocal) == 0
- && (f & VkConst.MemHostCached) != 0;
- else
- ok = (f & VkConst.MemHostVisible) != 0 && (f & VkConst.MemHostCoherent) != 0;
- if (ok && (memType == uint.MaxValue || (!preferHost && (f & VkConst.MemDeviceLocal) != 0)))
- {
- memType = (uint)i;
- if (preferHost || (f & VkConst.MemDeviceLocal) != 0) break;
- }
- }
- if (memType == uint.MaxValue && preferHost)
- for (int i = 0; i < (int)MemProps.MemoryTypeCount && memType == uint.MaxValue; i++)
- {
- if ((req.MemoryTypeBits & (1u << i)) == 0) continue;
- uint f = MemProps.TypeAt(i).PropertyFlags;
- if ((f & (VkConst.MemHostVisible | VkConst.MemHostCoherent))
- == (VkConst.MemHostVisible | VkConst.MemHostCoherent)
- && (f & VkConst.MemDeviceLocal) == 0)
- memType = (uint)i;
- }
- if (memType == uint.MaxValue && preferHost)
- {
- for (int i = 0; i < (int)MemProps.MemoryTypeCount && memType == uint.MaxValue; i++)
- {
- if ((req.MemoryTypeBits & (1u << i)) == 0) continue;
- uint f = MemProps.TypeAt(i).PropertyFlags;
- if ((f & (VkConst.MemHostVisible | VkConst.MemHostCoherent | VkConst.MemHostCached))
- == (VkConst.MemHostVisible | VkConst.MemHostCoherent | VkConst.MemHostCached))
- memType = (uint)i;
- }
- for (int i = 0; i < (int)MemProps.MemoryTypeCount && memType == uint.MaxValue; i++)
- {
- if ((req.MemoryTypeBits & (1u << i)) == 0) continue;
- uint f = MemProps.TypeAt(i).PropertyFlags;
- if ((f & (VkConst.MemHostVisible | VkConst.MemHostCoherent))
- == (VkConst.MemHostVisible | VkConst.MemHostCoherent))
- memType = (uint)i;
- }
- }
- if (memType == uint.MaxValue)
- throw new PlatformNotSupportedException($"Vulkan: no suitable memory type for {bytes}-byte buffer (hostVisible={hostVisible})");
-
- Vk.VkMemoryAllocateInfo mai = new()
- {
- SType = VkConst.StMemoryAllocateInfo,
- AllocationSize = req.Size, MemoryTypeIndex = memType,
- };
- Vk.Check(Vk.vkAllocateMemory(Device, &mai, null, out b.Memory), "vkAllocateMemory");
- b.AllocSize = req.Size;
- TrackAlloc((long)req.Size);
- Vk.Check(Vk.vkBindBufferMemory(Device, b.Buffer, b.Memory, 0), "vkBindBufferMemory");
- b.Flags = MemProps.TypeAt((int)memType).PropertyFlags;
- return b;
- }
-
- public IntPtr NewShaderModule(byte[] spirv)
- {
- if ((spirv.Length & 3) != 0) throw new ArgumentException("SPIR-V size must be a multiple of 4");
- fixed (byte* p = spirv)
- {
- Vk.VkShaderModuleCreateInfo ci = new()
- {
- SType = VkConst.StShaderModuleCreateInfo,
- CodeSize = (nuint)spirv.Length, PCode = (uint*)p,
- };
- Vk.Check(Vk.vkCreateShaderModule(Device, &ci, null, out IntPtr m), "vkCreateShaderModule");
- return m;
- }
- }
-
- /// One set layout with `bindings` storage buffers (0..bindings-1) + push constants.
- /// requiredSubgroupSize != 0 pins the compute subgroup width (VK_EXT_subgroup_size_control).
- public VkPipeline NewPipeline(IntPtr shaderModule, int bindings, int pushConstBytes, bool pushLayout,
- uint requiredSubgroupSize = 0)
- {
- var dslb = stackalloc Vk.VkDescriptorSetLayoutBinding[bindings];
- for (int i = 0; i < bindings; i++)
- dslb[i] = new Vk.VkDescriptorSetLayoutBinding
- {
- Binding = (uint)i, DescriptorType = VkConst.DescStorageBuffer,
- DescriptorCount = 1, StageFlags = VkConst.StageComputeShader,
- };
- IntPtr setLayout;
- Vk.VkDescriptorSetLayoutCreateInfo dslci = new()
- {
- SType = VkConst.StDescriptorSetLayoutCreateInfo,
- BindingCount = (uint)bindings, PBindings = dslb,
- Flags = pushLayout ? 1u : 0u, // VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR
- };
- Vk.Check(Vk.vkCreateDescriptorSetLayout(Device, &dslci, null, out setLayout), "vkCreateDescriptorSetLayout");
-
- Vk.VkPushConstantRange pcr = new()
- {
- StageFlags = VkConst.StageComputeShader, Offset = 0, Size = (uint)pushConstBytes,
- };
- IntPtr layout;
- Vk.VkPipelineLayoutCreateInfo plci = new()
- {
- SType = VkConst.StPipelineLayoutCreateInfo,
- SetLayoutCount = 1, PSetLayouts = &setLayout,
- PushConstantRangeCount = pushConstBytes > 0 ? 1u : 0u,
- PPushConstantRanges = pushConstBytes > 0 ? &pcr : null,
- };
- Vk.Check(Vk.vkCreatePipelineLayout(Device, &plci, null, out layout), "vkCreatePipelineLayout");
-
- byte* name = stackalloc byte[5] { (byte)'m', (byte)'a', (byte)'i', (byte)'n', 0 };
- Vk.VkPipelineShaderStageRequiredSubgroupSizeCreateInfo sgc = new()
- {
- SType = VkConst.StPipelineShaderStageRequiredSubgroupSizeCreateInfo,
- RequiredSubgroupSize = requiredSubgroupSize,
- };
- Vk.VkComputePipelineCreateInfo cpci = new()
- {
- SType = VkConst.StComputePipelineCreateInfo,
- Stage = new Vk.VkPipelineShaderStageCreateInfo
- {
- SType = VkConst.StPipelineShaderStageCreateInfo,
- Stage = VkConst.StageComputeShader, Module = shaderModule, PName = name,
- PNext = requiredSubgroupSize != 0 ? &sgc : null,
- // REQUIRE_FULL_SUBGROUPS: every subgroup of a pinned pipeline has all its lanes
- Flags = requiredSubgroupSize != 0 && _fullSubgroups ? 2u : 0u,
- },
- Layout = layout,
- };
- IntPtr pipeline;
- Vk.Check(Vk.vkCreateComputePipelines(Device, IntPtr.Zero, 1, &cpci, null, &pipeline), "vkCreateComputePipelines");
- return new VkPipeline { Pipeline = pipeline, Layout = layout, SetLayout = setLayout, PushLayout = pushLayout };
- }
-
- public IntPtr NewDescriptorPool(uint maxSets)
- {
- Vk.VkDescriptorPoolSize psz = new() { Type = VkConst.DescStorageBuffer, DescriptorCount = maxSets * 16 };
- Vk.VkDescriptorPoolCreateInfo dpci = new()
- {
- SType = VkConst.StDescriptorPoolCreateInfo,
- Flags = 0, MaxSets = maxSets, PoolSizeCount = 1, PPoolSizes = &psz,
- };
- Vk.Check(Vk.vkCreateDescriptorPool(Device, &dpci, null, out IntPtr pool), "vkCreateDescriptorPool");
- return pool;
- }
-
- public IntPtr AllocateDescriptorSet(IntPtr pool, IntPtr setLayout)
- {
- IntPtr set;
- Vk.VkDescriptorSetAllocateInfo ai = new()
- {
- SType = VkConst.StDescriptorSetAllocateInfo,
- DescriptorPool = pool, DescriptorSetCount = 1, PSetLayouts = &setLayout,
- };
- Vk.Check(Vk.vkAllocateDescriptorSets(Device, &ai, &set), "vkAllocateDescriptorSets");
- return set;
- }
-
- public IntPtr NewDescriptorSet(IntPtr setLayout)
- {
- IntPtr set;
- Vk.VkDescriptorSetAllocateInfo ai = new()
- {
- SType = VkConst.StDescriptorSetAllocateInfo,
- DescriptorPool = _descPool, DescriptorSetCount = 1, PSetLayouts = &setLayout,
- };
- Vk.Check(Vk.vkAllocateDescriptorSets(Device, &ai, &set), "vkAllocateDescriptorSets");
- return set;
- }
-
- /// VK_KHR_push_descriptor: write bindings inline into the command buffer
- /// (DstSet in each write is ignored).
- public void CmdPushDescriptors(IntPtr cmd, IntPtr layout, Vk.VkWriteDescriptorSet* writes, uint count)
- => _pushDesc(cmd, VkConst.BindPointCompute, layout, 0, count, writes);
-
- /// Record the standard write->read barrier between two compute dispatches.
- public void CmdComputeBarrier(IntPtr cmd)
- {
- Vk.VkMemoryBarrier mb = new()
- {
- SType = VkConst.StMemoryBarrier,
- SrcAccessMask = VkConst.AccessShaderWrite, DstAccessMask = VkConst.AccessShaderRead,
- };
- Vk.vkCmdPipelineBarrier(cmd, VkConst.PipelineStageComputeShader, VkConst.PipelineStageComputeShader, 0,
- 1, &mb, 0, null, 0, null);
- }
-
- /// Upload raw bytes into a buffer (host memcpy when mappable, else staging + copy cmd).
- public void Upload(VkBuffer buf, void* src, ulong bytes)
- {
- if ((buf.Flags & VkConst.MemHostVisible) != 0)
- {
- void* p = buf.Map();
- Buffer.MemoryCopy(src, p, bytes, bytes);
- buf.Flush(0, bytes);
- buf.Unmap();
- return;
- }
- VkBuffer staging = NewStorageBuffer(bytes, hostVisible: true);
- void* sp = staging.Map();
- Buffer.MemoryCopy(src, sp, bytes, bytes);
- staging.Flush(0, bytes);
- staging.Unmap();
- lock (Sync)
- {
- IntPtr cmd = NewCommandBuffer();
- IntPtr fence = NewFence();
- var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo };
- Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer");
- Vk.VkBufferCopy r = new() { Size = bytes };
- Vk.vkCmdCopyBuffer(cmd, staging.Buffer, buf.Buffer, 1, &r);
- Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer");
- Submit(cmd, fence);
- WaitFence(fence);
- Vk.vkDestroyFence(Device, fence, null);
- FreeCommandBuffer(cmd);
- }
- staging.Free();
- }
-
- /// Download raw bytes from a buffer into dst (host memcpy or staging).
- public void Download(VkBuffer buf, void* dst, ulong bytes, ulong srcOffset = 0)
- {
- if ((buf.Flags & VkConst.MemHostVisible) != 0)
- {
- void* p = buf.Map();
- buf.Invalidate(srcOffset, bytes);
- Buffer.MemoryCopy((byte*)p + srcOffset, dst, bytes, bytes);
- buf.Unmap();
- return;
- }
- VkBuffer staging = NewStorageBuffer(bytes, hostVisible: true, preferHost: true);
- lock (Sync)
- {
- IntPtr cmd = NewCommandBuffer();
- IntPtr fence = NewFence();
- var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo };
- Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer");
- Vk.VkBufferCopy r = new() { Size = bytes, SrcOffset = srcOffset };
- Vk.vkCmdCopyBuffer(cmd, buf.Buffer, staging.Buffer, 1, &r);
- Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer");
- Submit(cmd, fence);
- WaitFence(fence);
- Vk.vkDestroyFence(Device, fence, null);
- FreeCommandBuffer(cmd);
- }
- void* sp = staging.Map();
- staging.Invalidate(0, bytes);
- Buffer.MemoryCopy(sp, dst, bytes, bytes);
- staging.Unmap();
- staging.Free();
- }
-
- public void BindBuffer(IntPtr set, uint binding, VkBuffer buf, ulong offset, ulong size)
- {
- Vk.VkDescriptorBufferInfo bi = new() { Buffer = buf.Buffer, Offset = offset, Range = size };
- Vk.VkWriteDescriptorSet w = new()
- {
- SType = VkConst.StWriteDescriptorSet, DstSet = set, DstBinding = binding,
- DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, PBufferInfo = &bi,
- };
- Vk.vkUpdateDescriptorSets(Device, 1, &w, 0, null);
- }
-
- public IntPtr NewCommandBuffer()
- {
- Vk.VkCommandBufferAllocateInfo ai = new()
- {
- SType = VkConst.StCommandBufferAllocateInfo,
- CommandPool = _commandPool, Level = VkConst.CmdBufferLevelPrimary, CommandBufferCount = 1,
- };
- IntPtr cmd;
- Vk.Check(Vk.vkAllocateCommandBuffers(Device, &ai, &cmd), "vkAllocateCommandBuffers");
- return cmd;
- }
-
- public void FreeCommandBuffer(IntPtr cmd)
- {
- if (cmd != IntPtr.Zero) Vk.vkFreeCommandBuffers(Device, _commandPool, 1, &cmd);
- }
-
- /// Private resettable command pool (command pools are externally synchronized:
- /// a worker that re-records on its own thread must not share the device pool).
- public IntPtr NewCommandPool()
- {
- Vk.VkCommandPoolCreateInfo ci = new()
- {
- SType = VkConst.StCommandPoolCreateInfo,
- Flags = VkConst.CmdPoolResetCommandBuffer,
- QueueFamilyIndex = QueueFamily,
- };
- Vk.Check(Vk.vkCreateCommandPool(Device, &ci, null, out IntPtr pool), "vkCreateCommandPool");
- return pool;
- }
-
- public IntPtr AllocateCommandBuffer(IntPtr pool)
- {
- Vk.VkCommandBufferAllocateInfo ai = new()
- {
- SType = VkConst.StCommandBufferAllocateInfo,
- CommandPool = pool, Level = VkConst.CmdBufferLevelPrimary, CommandBufferCount = 1,
- };
- IntPtr cmd;
- Vk.Check(Vk.vkAllocateCommandBuffers(Device, &ai, &cmd), "vkAllocateCommandBuffers");
- return cmd;
- }
-
- /// Destroys a private command pool (frees its command buffers).
- public void DestroyCommandPool(IntPtr pool)
- {
- if (pool != IntPtr.Zero) Vk.vkDestroyCommandPool(Device, pool, null);
- }
-
- public IntPtr NewFence(bool signaled = false)
- {
- Vk.VkFenceCreateInfo ci = new() { SType = VkConst.StFenceCreateInfo, Flags = signaled ? 1u : 0u };
- Vk.Check(Vk.vkCreateFence(Device, &ci, null, out IntPtr f), "vkCreateFence");
- return f;
- }
-
- public void Submit(IntPtr cmd, IntPtr fence)
- {
- Vk.VkSubmitInfo si = new()
- {
- SType = VkConst.StSubmitInfo,
- CommandBufferCount = 1, PCommandBuffers = &cmd,
- };
- VkResult r;
- lock (QueueLock) r = Vk.vkQueueSubmit(Queue, 1, &si, fence);
- Vk.Check(r, "vkQueueSubmit");
- }
-
- public void DestroyFence(IntPtr fence)
- {
- if (fence != IntPtr.Zero) Vk.vkDestroyFence(Device, fence, null);
- }
-
- public void WaitFence(IntPtr fence)
- {
- Vk.Check(Vk.vkWaitForFences(Device, 1, &fence, 1, ulong.MaxValue), "vkWaitForFences");
- Vk.Check(Vk.vkResetFences(Device, 1, &fence), "vkResetFences");
- }
-
- public void Dispose()
- {
- if (Device != IntPtr.Zero)
- {
- Vk.vkDeviceWaitIdle(Device);
- if (_descPool != IntPtr.Zero) { Vk.vkDestroyDescriptorPool(Device, _descPool, null); _descPool = IntPtr.Zero; }
- if (_commandPool != IntPtr.Zero) { Vk.vkDestroyCommandPool(Device, _commandPool, null); _commandPool = IntPtr.Zero; }
- lock (_pipes)
- foreach (VkPipeline p in _pipes.Values)
- {
- Vk.vkDestroyPipeline(Device, p.Pipeline, null);
- Vk.vkDestroyPipelineLayout(Device, p.Layout, null);
- Vk.vkDestroyDescriptorSetLayout(Device, p.SetLayout, null);
- }
- Vk.vkDestroyDevice(Device, null); Device = IntPtr.Zero;
- }
- if (Instance != IntPtr.Zero) { Vk.vkDestroyInstance(Instance, null); Instance = IntPtr.Zero; }
- }
-}
-
-internal sealed class VkPipeline
-{
- public IntPtr Pipeline, Layout, SetLayout;
- public string Name = "";
- public int Bindings;
- public bool PushLayout;
-}
-
-internal unsafe sealed class VkBuffer
-{
- public IntPtr Dev, Buffer, Memory;
- public ulong Size, AllocSize;
- public uint Flags;
-
- public bool HostCoherent => (Flags & VkConst.MemHostCoherent) != 0;
-
- public void* Map()
- {
- void* p;
- Vk.Check(Vk.vkMapMemory(Dev, Memory, 0, Size, 0, &p), "vkMapMemory");
- return p;
- }
- public void Unmap() => Vk.vkUnmapMemory(Dev, Memory);
-
- public void Flush(ulong offset, ulong size)
- {
- if (HostCoherent) return;
- Vk.VkMappedMemoryRange r = new() { SType = VkConst.StMappedMemoryRange, Memory = Memory, Offset = offset, Size = size };
- Vk.vkFlushMappedMemoryRanges(Dev, 1, &r);
- }
- public void Invalidate(ulong offset, ulong size)
- {
- if (HostCoherent) return;
- Vk.VkMappedMemoryRange r = new() { SType = VkConst.StMappedMemoryRange, Memory = Memory, Offset = offset, Size = size };
- Vk.vkInvalidateMappedMemoryRanges(Dev, 1, &r);
- }
-
- public void Free()
- {
- if (Buffer != IntPtr.Zero) { Vk.vkDestroyBuffer(Dev, Buffer, null); Buffer = IntPtr.Zero; }
- if (Memory != IntPtr.Zero)
- {
- Vk.vkFreeMemory(Dev, Memory, null); Memory = IntPtr.Zero;
- VkDevice.TrackFree((long)AllocSize);
- AllocSize = 0;
- }
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkGraph.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkGraph.cs
deleted file mode 100644
index d686644..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkGraph.cs
+++ /dev/null
@@ -1,738 +0,0 @@
-using System.Runtime.InteropServices;
-using Sdcb.SimdGcdrNet.Engine;
-
-namespace Sdcb.SimdGcdrNet.Backends.Vulkan;
-
-/// Binding slots resolved per-session: which underlying buffer.
-internal enum VkSlot : int { Arena = 0, Consts = 1, Dummy = 2, StageIn = 3, StageOut = 4, Direct = 5 }
-
-internal readonly struct VkBind
-{
- public readonly VkSlot Slot;
- public readonly VkBuffer? Buf; // when Slot == Direct
- public readonly ulong Offset, Size;
- public VkBind(VkSlot s, ulong off, ulong size) { Slot = s; Offset = off; Size = size; }
- public VkBind(VkBuffer b, ulong off, ulong size) { Slot = VkSlot.Direct; Buf = b; Offset = off; Size = size; }
-}
-
-/// One recorded compute dispatch.
-internal sealed class VkDispatch
-{
- public required VkPipeline Pipe;
- public required VkBind[] Binds;
- public required byte[] Pc;
- public uint Gx, Gy = 1, Gz = 1;
- public string Name = "";
-}
-
-///
-/// Per-(device, ExecPlan) half of the Vulkan backend: backend-private constant
-/// packing/upload and the node list compiled to records.
-/// Sessions own the activation arena and resolve slot ids to live buffers.
-///
-/// Plans are layout-planned (NHWC, see ); every
-/// addressing decision reads . With
-/// activations and weights are stored as float16 (16-bit storage only — all
-/// shader math, reductions included, is fp32); conv bias and LayerNorm params
-/// stay fp32. Const bytes therefore differ per backend and per precision.
-///
-internal sealed class VkGraphModel : IDisposable
-{
- private readonly VkDevice _dev;
- private readonly ExecPlan _plan;
- private int _refs = 1; // the graph cache holds the first reference
- private readonly VkConstStore _store;
- internal readonly VkBuffer Consts; // all const tensors, backend-packed (shared store)
- private readonly Dictionary<(int v, CK k), ulong> _constOff = new();
- internal readonly VkBuffer Dummy; // 64B zero buffer for unused bindings
- internal readonly VkDispatch[] Dispatches;
- internal readonly ulong ConstBytes;
- /// Activations + weights stored as float16 (fp32 math in every shader).
- internal readonly bool F16;
- /// Bytes per stored activation element.
- internal int Es => F16 ? 2 : 4;
-
- /// How a const is laid out on the device (decided by its consumer).
- private enum CK { Raw, F32, ConvW, DwW, ConvWT, GemmNK }
-
- /// Vulkan device the graph is lowered for.
- /// Compiled execution plan to lower.
- /// Store activations and (non-fp32) weights as fp16; math stays fp32.
- /// Sharing key for the const store (the source model): graphs of the
- /// same model at other sizes reuse its uploaded weights. Null = private store.
- internal VkGraphModel(VkDevice dev, ExecPlan plan, bool f16, object? constKey = null)
- {
- _dev = dev;
- _plan = plan;
- F16 = f16;
- if (!plan.Nhwc) throw new NotSupportedException("Vulkan backend requires a layout-planned (NHWC) plan.");
-
- // ---- consts: per-consumer packing ----
- var need = new List<(int v, CK k)>();
- void Need(int v, CK k)
- {
- if (v < 0 || plan.Values[v].Floats is null) return;
- if (!need.Contains((v, k))) need.Add((v, k));
- }
- foreach (ExecNode n in plan.Nodes)
- {
- switch (n.P)
- {
- case ConvP cp:
- Need(n.In[0], CK.Raw);
- Need(n.In[1], cp.Group > 1 ? CK.DwW : UseCoopConv(dev, f16, plan, n, cp) ? CK.ConvWT : CK.ConvW);
- Need(cp.BiasValue, CK.F32);
- Need(cp.AddValue, CK.Raw);
- break;
- case GemmP gp:
- Need(n.In[0], CK.Raw); Need(n.In[1], UseCoopGemm(dev, f16, plan, n, gp) ? CK.GemmNK : CK.Raw);
- Need(gp.BiasValue, CK.F32);
- break;
- case LnP:
- Need(n.In[0], CK.Raw); Need(n.In[1], CK.F32); Need(n.In[2], CK.F32);
- break;
- case UpHeadP up:
- Need(n.In[2], CK.Raw);
- if (n.In.Length > 3) Need(n.In[3], CK.F32);
- if (up.SkipConv is not null) { Need(up.SkipW, CK.ConvW); Need(up.SkipB, CK.F32); }
- break;
- default:
- foreach (int iv in n.In) Need(iv, CK.Raw);
- break;
- }
- }
- foreach (var (v, k) in need)
- {
- CValue cv = plan.Values[v];
- if (k == CK.Raw && cv.Layout is { } l && !Engine.Layout.IsIdentity(l))
- throw new NotSupportedException($"Vulkan: const '{cv.Name}' with non-identity layout");
- }
- float[][] srcs = need.Select(x => plan.Values[x.v].Floats!).ToArray();
- int[] kinds = need.Select(x => (int)x.k).ToArray();
- _store = VkConstStore.Acquire(dev, constKey, f16, srcs, kinds, () =>
- {
- ulong total = 0;
- var offs = new ulong[need.Count];
- var items = new List<(ulong off, float[] data, bool half)>();
- for (int i = 0; i < need.Count; i++)
- {
- var (v, k) = need[i];
- CValue cv = plan.Values[v];
- float[] data = k switch
- {
- CK.ConvW => PackConv(cv),
- CK.ConvWT => PackConvT(cv),
- CK.GemmNK => PackGemmNK(cv, plan.Nodes.First(m => m.P is GemmP && m.In[1] == v)),
- CK.DwW => PackDw(cv),
- _ => cv.Floats!,
- };
- bool half = f16 && k != CK.F32;
- total = (total + 63) & ~63UL;
- offs[i] = total;
- items.Add((total, data, half));
- total += (ulong)data.Length * (half ? 2UL : 4UL);
- }
- byte[] host = new byte[Math.Max((total + 63) & ~63UL, 64)];
- foreach (var (off, data, half) in items)
- {
- if (half)
- {
- Span dst = MemoryMarshal.Cast(host.AsSpan(checked((int)off), data.Length * 2));
- for (int i = 0; i < data.Length; i++) dst[i] = (Half)data[i];
- }
- else
- MemoryMarshal.AsBytes(data.AsSpan()).CopyTo(host.AsSpan(checked((int)off)));
- }
- return (offs, host);
- });
- for (int i = 0; i < need.Count; i++) _constOff[need[i]] = _store.Offsets[i];
- Consts = _store.Consts;
- Dummy = _store.Dummy;
- ConstBytes = _store.Bytes;
-
- var disps = new List();
- NodeRanges = new (int start, int count)[plan.Nodes.Length];
- for (int i = 0; i < plan.Nodes.Length; i++)
- {
- int start = disps.Count;
- disps.AddRange(Emit(plan.Nodes[i]));
- NodeRanges[i] = (start, disps.Count - start);
- }
- Dispatches = disps.ToArray();
- }
-
- // ONNX [OC][IC][KH][KW] -> [K][OC], k = (ky*KW + kx)*IC + ic (matches conv_tile.comp)
- private static float[] PackConv(CValue w)
- {
- int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3];
- float[] src = w.Floats!, dst = new float[src.Length];
- for (int o = 0; o < oc; o++)
- for (int c = 0; c < ic; c++)
- for (int t = 0; t < taps; t++)
- dst[(t * ic + c) * oc + o] = src[(o * ic + c) * taps + t];
- return dst;
- }
-
- // ONNX [OC][IC][KH][KW] -> [OC][Kpad], k = (ky*KW + kx)*IC + ic, Kpad = K rounded up to 8
- // (zero-filled; matches conv_cm.comp)
- private static float[] PackConvT(CValue w)
- {
- int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3], k = (taps * ic + 7) & ~7;
- float[] src = w.Floats!, dst = new float[oc * k];
- for (int o = 0; o < oc; o++)
- for (int c = 0; c < ic; c++)
- for (int t = 0; t < taps; t++)
- dst[o * k + t * ic + c] = src[(o * ic + c) * taps + t];
- return dst;
- }
-
- /// Tensor-core conv eligibility (conv_cm.comp): fp16 storage on a coopmat device,
- /// dense, channel counts in 16 B quads, an epilogue it implements, M within the grid limit.
- private static bool UseCoopConv(VkDevice dev, bool f16, ExecPlan plan, ExecNode n, ConvP cp)
- {
- if (!dev.CoopMat16 || dev.ConvDot || !f16 || cp.Group != 1 || cp.Pool is not null || cp.Dilations.Any(d => d != 1)) return false;
- if (cp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false;
- if (cp.PadMode == PadMode.Constant && cp.PadValue != 0f) return false;
- int[] si = plan.Values[n.In[0]].PhysShape, so = plan.Values[n.Out[0]].PhysShape; // [1,h,w,c]
- long m = (long)so[1] * so[2];
- return so[3] % 8 == 0 && (m + 127) / 128 <= 65535; // cin % 8 != 0 -> scalar-gather stem variant
- }
-
- // Gemm B -> [N][K] (k fastest), the conv_cm weight layout
- private static float[] PackGemmNK(CValue b, ExecNode n)
- {
- var gp = (GemmP)n.P!;
- if (gp.TransB) return b.Floats!; // already [N][K]
- int k = b.Shape[0], nn = b.Shape[1];
- float[] src = b.Floats!, dst = new float[src.Length];
- for (int i = 0; i < k; i++)
- for (int j = 0; j < nn; j++) dst[j * k + i] = src[i * nn + j];
- return dst;
- }
-
- /// Tensor-core Gemm (a 1x1 conv over M "pixels" through conv_cm.comp): fp16 storage
- /// on a coopmat device, const B, K/N in 16 B quads, relu/gelu/none epilogue.
- private static bool UseCoopGemm(VkDevice dev, bool f16, ExecPlan plan, ExecNode n, GemmP gp)
- {
- if (!dev.CoopMat16 || dev.ConvDot || !f16 || plan.Values[n.In[1]].Floats is null || plan.Values[n.In[1]].Shape.Length != 2) return false;
- // B packed per value: only when no other node reads it (its transB could differ)
- if (plan.Nodes.Count(m => m.In.Contains(n.In[1])) != 1) return false;
- if (gp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false;
- CValue a = plan.Values[n.In[0]];
- int k = a.Shape[^1], nn = plan.Values[n.Out[0]].Shape[^1];
- if (a.PhysShape[^1] != k) return false;
- long m = a.ElementCount / k;
- return k % 8 == 0 && nn % 8 == 0 && (m + 127) / 128 <= 65535;
- }
-
- // ONNX [C][1][KH][KW] -> [tap][C]
- private static float[] PackDw(CValue w)
- {
- int c = w.Shape[0], taps = w.Shape[2] * w.Shape[3];
- float[] src = w.Floats!, dst = new float[src.Length];
- for (int ch = 0; ch < c; ch++)
- for (int t = 0; t < taps; t++) dst[t * c + ch] = src[ch * taps + t];
- return dst;
- }
-
- /// Dispatch index range per source ExecNode (for per-node debug runs).
- internal readonly (int start, int count)[] NodeRanges;
-
- // ---- value → bind resolution (byte offsets/sizes: all long math) ----
- private VkBind Bind(int valueId, CK kind = CK.Raw)
- {
- CValue v = _plan.Values[valueId];
- if (v.Buf >= 0)
- return new VkBind(VkSlot.Arena, (ulong)_plan.BufOffset[v.Buf] * (ulong)Es,
- (ulong)Math.Max(v.ElementCount, 1) * (ulong)Es);
- if (_constOff.TryGetValue((valueId, kind), out ulong off))
- {
- ulong es = F16 && kind != CK.F32 ? 2UL : 4UL;
- // ConvWT rows are K-padded to 8: bind the packed length, not the logical one
- long elems = kind == CK.ConvWT
- ? (long)v.Shape[0] * ((v.Shape[1] * v.Shape[2] * v.Shape[3] + 7) & ~7)
- : Math.Max(v.ElementCount, 1);
- return new VkBind(VkSlot.Consts, off, (ulong)elems * es);
- }
- if (v.Floats != null)
- throw new InvalidOperationException($"const value '{v.Name}' not packed as {kind}");
- throw new InvalidOperationException($"value '{v.Name}' has no GPU storage (ints must be folded)");
- }
-
- private static byte[] PcOf(params uint[] ints)
- {
- byte[] b = new byte[ints.Length * 4];
- Buffer.BlockCopy(ints, 0, b, 0, b.Length);
- return b;
- }
-
- private static byte[] PcOfF(uint[] ints, params float[] floats)
- {
- byte[] b = new byte[(ints.Length + floats.Length) * 4];
- Buffer.BlockCopy(ints, 0, b, 0, ints.Length * 4);
- Buffer.BlockCopy(floats, 0, b, ints.Length * 4, floats.Length * 4);
- return b;
- }
-
- private static uint[] U(params int[] v) { uint[] u = new uint[v.Length]; for (int i = 0; i < v.Length; i++) u[i] = (uint)v[i]; return u; }
-
- private static uint G(uint n) => (n + 255) / 256;
-
- ///
- /// Broadcast strides (padded rank-4, storage order of the output) of operand
- /// onto output . Output physical dim i
- /// holds logical dim o.Layout[i]; the operand's logical dim is that minus the
- /// numpy right-alignment offset.
- ///
- private static int[] PhysBroadcastStrides(CValue a, CValue o)
- {
- int rank = o.Shape.Length, off = rank - a.Shape.Length;
- if (rank > 4) throw new NotSupportedException("Vulkan elementwise: rank > 4");
- int[] lo = o.Layout ?? Engine.Layout.Identity(rank);
- int[] la = a.Layout ?? Engine.Layout.Identity(a.Shape.Length);
- int[] ap = a.PhysShape;
- int[] aStr = new int[ap.Length];
- for (int i = ap.Length - 1, acc = 1; i >= 0; i--) { aStr[i] = acc; acc *= ap[i]; }
- int[] r4 = new int[4];
- for (int i = 0; i < rank; i++)
- {
- int d = lo[i] - off;
- r4[4 - rank + i] = d < 0 || a.Shape[d] == 1 ? 0 : aStr[Array.IndexOf(la, d)];
- }
- return r4;
- }
-
- private string Sh(string name) => F16 ? name + "_h" : name;
-
- private IEnumerable Emit(ExecNode n)
- {
- CValue[] V = _plan.Values;
- // all addressing on storage-order shapes
- int[] shapeIn(int k) => V[n.In[k]].PhysShape;
- int[] shapeOut() => V[n.Out[0]].PhysShape;
- int nOut() => checked((int)V[n.Out[0]].ElementCount);
-
- switch (n.Op)
- {
- case ExecOp.Copy:
- yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf((uint)nOut(), 4), G((uint)nOut()));
- break;
-
- case ExecOp.Relu or ExecOp.Sigmoid or ExecOp.Erf or ExecOp.Gelu:
- {
- uint mode = n.Op switch
- {
- ExecOp.Relu => 0, ExecOp.Sigmoid => 1, ExecOp.Erf => 2, _ => 3,
- };
- yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf((uint)nOut(), mode), G((uint)nOut()));
- break;
- }
-
- case ExecOp.Elementwise:
- {
- var p = (EwP)n.P!;
- CValue a = V[n.In[0]], bv = V[n.In[1]], o = V[n.Out[0]];
- int[] os = Pad4(shapeOut());
- int[] ostr = Strides(os);
- int[] astr = PhysBroadcastStrides(a, o);
- int[] bstr = PhysBroadcastStrides(bv, o);
- uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, _ => 0u };
- // same-storage fast path: vec4 (n % 4 guard — no OOB tail writes)
- bool dense = astr.SequenceEqual(ostr) && bstr.SequenceEqual(ostr);
- if (nOut() % 4 == 0 && dense)
- {
- yield return ND("ew2v", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])],
- PcOf((uint)(nOut() / 4), (uint)p.Op, act), G((uint)(nOut() / 4)));
- break;
- }
- yield return ND("ew2", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])],
- PcOf(U(nOut(), ostr[0], ostr[1], ostr[2],
- astr[0], astr[1], astr[2], astr[3],
- bstr[0], bstr[1], bstr[2], bstr[3], (int)p.Op, (int)act)),
- G((uint)nOut()));
- break;
- }
-
- case ExecOp.Conv:
- {
- var p = (ConvP)n.P!;
- RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]);
- if (p.Pool is { } pool)
- {
- // small-ic stem conv + 2x2/2 max pool (GraphCompiler.GpuSmallConv)
- int[] ci = shapeIn(0), co = shapeOut(), cw = V[n.In[1]].Shape;
- if (!GraphCompiler.GpuSmallConv(p, cw, ci[3]) || p.AddValue >= 0 || ci[1] != co[1] * 2 || ci[2] != co[2] * 2)
- throw new NotSupportedException("Vulkan convpool: unsupported conv");
- uint pf = (p.BiasValue >= 0 ? 1u : 0u) | (p.PadMode == PadMode.Reflect ? 2u : 0u)
- | (p.Act == ActivationKind.Relu ? 4u : 0u) | (pool.FuseRelu ? 8u : 0u);
- bool k3 = _dev.ConvPoolK3 && p.Kernel[0] == 3 && p.Kernel[1] == 3;
- string cpName = k3 ? "convpool_k3" : _dev.ConvDot && F16 && cw[0] % 8 == 0 ? "convpool_pi" : "convpool";
- yield return ND(cpName,
- [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), Bind(n.Out[0])],
- PcOfF(U(ci[3], ci[1], ci[2], cw[0], co[1], co[2], p.Kernel[0], p.Kernel[1], p.Pads[0], p.Pads[1], (int)pf),
- p.PadMode == PadMode.Constant ? p.PadValue : 0f),
- G((uint)co[1] * (uint)co[2] * (uint)(cw[0] / 8)), name: "convpool");
- break;
- }
- if (p.AddValue >= 0) RequireNhwc(p.AddValue);
- int[] si = shapeIn(0), sw_ = V[n.In[1]].Shape, so = shapeOut(); // [1,h,w,c]
- uint ih = (uint)si[1], iw = (uint)si[2], ic = (uint)si[3];
- uint oc = (uint)sw_[0], oh = (uint)so[1], ow = (uint)so[2];
- if (p.PadMode == PadMode.Constant && p.PadValue != 0f)
- throw new NotSupportedException("Vulkan conv: nonzero constant pad");
- if (p.Dilations.Any(d => d != 1))
- throw new NotSupportedException("Vulkan conv: dilation");
- if (p.Act == ActivationKind.Sigmoid)
- throw new NotSupportedException("Vulkan conv: sigmoid epilogue");
- uint flags = (p.BiasValue >= 0 ? 1u : 0u)
- | (p.PadMode == PadMode.Reflect ? 2u : 0u)
- | (p.Act == ActivationKind.Relu ? 4u : 0u)
- | (p.AddValue >= 0 ? 8u : 0u)
- | (p.Act == ActivationKind.Gelu ? 16u : 0u);
- if (p.Group > 1)
- {
- if (p.Group != ic || oc != ic) throw new NotSupportedException("grouped non-depthwise conv");
- if (p.AddValue >= 0) throw new NotSupportedException("depthwise conv + residual");
- bool v4 = _dev.Vec4Eltwise && ic % 4 == 0;
- yield return ND(v4 ? "depthwise4" : "depthwise",
- [Bind(n.In[0]), Bind(n.In[1], CK.DwW), B(p.BiasValue, CK.F32), Bind(n.Out[0])],
- PcOfF(U((int)ic, (int)ih, (int)iw, (int)oh, (int)ow, p.Kernel[0], p.Kernel[1],
- p.Strides[0], p.Strides[1], p.Pads[0], p.Pads[1], (int)(flags & ~8u)), p.PadValue),
- G(ic * oh * ow / (v4 ? 4u : 1u)), name: "depthwise");
- break;
- }
- if (UseCoopConv(_dev, F16, _plan, n, p))
- {
- // BM = 128 rows (pixels) in every variant; BN by oc divisibility
- (string cm, uint bn) = ic % 8 != 0 ? ("conv_cm_n32s", 32u)
- : oc % 128 == 0 ? ("conv_cm", 128u) : oc % 64 == 0 ? ("conv_cm_n64", 64u) : ("conv_cm_n32", 32u);
- uint m = oh * ow, kr = (uint)(p.Kernel[0] * p.Kernel[1]) * ic, kd = (kr + 7u) & ~7u;
- uint cflags = (p.BiasValue >= 0 ? 1u : 0u) | (p.AddValue >= 0 ? 2u : 0u)
- | (p.PadMode == PadMode.Reflect ? 4u : 0u)
- | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4);
- yield return ND(cm,
- [Bind(n.In[0]), Bind(n.In[1], CK.ConvWT), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])],
- PcOf(m, oc, kd, cflags, ow, iw, ih, ic, (uint)p.Kernel[1],
- (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], kr),
- (m + 127) / 128, (oc + bn - 1) / bn, name: "conv_cm");
- break;
- }
- string convShader =
- p.Kernel[0] == 3 && p.Kernel[1] == 3 && p.Strides[0] == 1 && p.Strides[1] == 1 ? "conv_k33s1" :
- p.Kernel[0] == 3 && p.Kernel[1] == 3 && p.Strides[0] == 2 && p.Strides[1] == 2 ? "conv_k33s2" :
- p.Kernel[0] == 1 && p.Kernel[1] == 1 && p.Strides[0] == 1 && p.Strides[1] == 1 ? "conv_k11s1" :
- "conv_gen";
- if (_dev.ConvDot && F16 && convShader is "conv_k33s1" or "conv_k33s2" && ic % 4 == 0 && oc % 4 == 0)
- {
- // Adreno 750 sweep (ms, 1600x1152 gc+dr): stride 1 at 4 px x 8 oc, stride 2 at 2 px x 8 oc
- uint tm = p.Strides[1] == 1 ? 4u : 2u, tn = oc % 8 == 0 ? 2u : 1u;
- uint threads = oc / (4 * tn) * ((ow + tm - 1) / tm) * oh;
- uint wgs = (threads + 63) / 64, gx = Math.Min(wgs, 32768u);
- yield return ND($"conv_dot_s{p.Strides[1]}_{tm}x{tn}",
- [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])],
- PcOf(ic, ih, iw, oc, oh, ow, (uint)p.Kernel[0], (uint)p.Kernel[1],
- (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1],
- flags, 0),
- gx, (wgs + gx - 1) / gx, name: "conv");
- break;
- }
- yield return ND(convShader,
- [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])],
- PcOf(ic, ih, iw, oc, oh, ow, (uint)p.Kernel[0], (uint)p.Kernel[1],
- (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1],
- flags, 0),
- (ow + 15) / 16, (oh + 7) / 8, (oc + 63) / 64, name: "conv");
- break;
- }
-
- case ExecOp.MaxPool:
- {
- var p = (PoolP)n.P!;
- RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]);
- int[] si = shapeIn(0), so = shapeOut();
- bool v4 = _dev.Vec4Eltwise && si[3] % 4 == 0;
- yield return ND(v4 ? "maxpool4" : "maxpool", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf((uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2],
- (uint)p.Kernel[0], (uint)p.Kernel[1], (uint)p.Strides[0], (uint)p.Strides[1],
- (uint)p.Pads[0], (uint)p.Pads[1], p.FuseRelu ? 1u : 0u),
- G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "maxpool");
- break;
- }
-
- case ExecOp.LayerNorm:
- {
- var p = (LnP)n.P!;
- uint len = (uint)V[n.In[0]].Shape[^1];
- if ((uint)shapeIn(0)[^1] != len) throw new NotSupportedException("Vulkan LayerNorm: last dim not innermost");
- uint rows = (uint)(V[n.In[0]].ElementCount / len);
- yield return ND("layernorm",
- [Bind(n.In[0]), Bind(n.In[1], CK.F32), Bind(n.In[2], CK.F32), Bind(n.Out[0])],
- PcOfF([len], p.Eps), rows, name: "layernorm");
- break;
- }
-
- case ExecOp.Gemm:
- {
- var p = (GemmP)n.P!;
- uint k = (uint)V[n.In[0]].Shape[^1];
- if ((uint)shapeIn(0)[^1] != k) throw new NotSupportedException("Vulkan Gemm: K not innermost");
- uint m = (uint)(V[n.In[0]].ElementCount / k);
- uint nn = (uint)V[n.Out[0]].Shape[^1];
- if (UseCoopGemm(_dev, F16, _plan, n, p))
- {
- (string cm, uint bn) = nn % 128 == 0 ? ("conv_cm", 128u) : nn % 64 == 0 ? ("conv_cm_n64", 64u) : ("conv_cm_n32", 32u);
- uint cflags = (p.BiasValue >= 0 ? 1u : 0u)
- | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4);
- // [M,K] x [N,K]^T as a 1x1 conv over a 1 x M image
- yield return ND(cm,
- [Bind(n.In[0]), Bind(n.In[1], CK.GemmNK), B(p.BiasValue, CK.F32), B(-1), Bind(n.Out[0])],
- PcOf(m, nn, k, cflags, m, m, 1u, k, 1u, 1u, 1u, 0u, 0u, k),
- (m + 127) / 128, (nn + bn - 1) / bn, name: "gemm_cm");
- break;
- }
- uint gflags = (p.TransB ? 1u : 0u) | (p.BiasValue >= 0 ? 2u : 0u)
- | (p.Act == ActivationKind.Relu ? 4u : 0u)
- | (p.Act == ActivationKind.Gelu ? 8u : 0u);
- if (_dev.ConvDot && F16 && !p.TransB && k % 4 == 0 && nn % 8 == 0)
- {
- // 4 rows x 8 columns per invocation
- uint gth = nn / 8 * ((m + 3) / 4);
- uint gwg = (gth + 63) / 64, ggx = Math.Min(gwg, 32768u);
- yield return ND("gemm_dot",
- [Bind(n.In[0]), Bind(n.In[1]), B(p.BiasValue, CK.F32), Bind(n.Out[0])],
- PcOf(m, nn, k, gflags), ggx, (gwg + ggx - 1) / ggx, name: "gemm");
- break;
- }
- yield return ND("gemm",
- [Bind(n.In[0]), Bind(n.In[1]), B(p.BiasValue, CK.F32), Bind(n.Out[0])],
- PcOf(m, nn, k, gflags),
- (nn + 63) / 64, (m + 63) / 64, name: "gemm");
- break;
- }
-
- case ExecOp.ResizeBilinear:
- {
- var p = (ResizeP)n.P!;
- RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]);
- bool hasAdd = n.In.Length > 1;
- if (hasAdd) RequireNhwc(n.In[1]);
- int[] si = shapeIn(0), so = shapeOut();
- uint flags = (p.FuseRelu ? 1u : 0u) | (hasAdd ? 2u : 0u) | (p.AddRelu ? 4u : 0u);
- bool v4 = _dev.Vec4Eltwise && si[3] % 4 == 0;
- yield return ND(v4 ? "bilinear4" : "bilinear", [Bind(n.In[0]), hasAdd ? Bind(n.In[1]) : B(-1), Bind(n.Out[0])],
- PcOfF([(uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], flags],
- p.Scales[2], p.Scales[3]),
- G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "bilinear");
- break;
- }
-
- case ExecOp.UpHead:
- {
- var p = (UpHeadP)n.P!;
- RequireNhwc(n.In[0]); RequireNhwc(n.In[1]); RequireNhwc(n.Out[0]);
- int[] si = shapeIn(0), so = shapeOut();
- uint c = (uint)si[3], oc = (uint)so[3];
- if (oc > 8 || c > 128 || c % 4 != 0) throw new NotSupportedException($"Vulkan UpHead: c={c} oc={oc}");
- bool hasBias = n.In.Length > 3;
- uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, ActivationKind.None => 0u,
- _ => throw new NotSupportedException($"Vulkan UpHead: {p.Act}") };
- uint flags = (p.ResizeRelu ? 1u : 0u) | (p.SkipRelu ? 2u : 0u) | (hasBias ? 4u : 0u) | (act << 3);
- if (p.SkipConv is { } sc)
- {
- // skip recomputed from the stem conv's input (GraphCompiler.GpuSmallConv, c <= 32)
- int[] xi = shapeIn(1), cw = V[p.SkipW].Shape;
- if (!GraphCompiler.GpuSmallConv(sc, cw, xi[3]) || cw[0] != c || c > 32 || xi[1] != so[1] || xi[2] != so[2])
- throw new NotSupportedException("Vulkan UpHead: unsupported skip conv");
- uint cf = (sc.BiasValue >= 0 ? 1u : 0u) | (sc.PadMode == PadMode.Reflect ? 2u : 0u)
- | (sc.Act == ActivationKind.Relu ? 4u : 0u);
- uint np = _dev.ConvDot && F16 ? 2u : 1u;
- yield return ND(np > 1 ? $"uphead_sc{np}" : "uphead_sc",
- [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0]),
- Bind(p.SkipW, CK.ConvW), B(p.SkipB, CK.F32)],
- PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags],
- p.Scales[2], p.Scales[3]).Concat(PcOfF(U(xi[3], sc.Kernel[0], sc.Kernel[1], sc.Pads[0], sc.Pads[1], (int)cf),
- sc.PadMode == PadMode.Constant ? sc.PadValue : 0f)).ToArray(),
- G((uint)so[1] * (((uint)so[2] + np - 1) / np)), name: "uphead_sc");
- break;
- }
- yield return ND("uphead",
- [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0])],
- PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags],
- p.Scales[2], p.Scales[3]),
- G((uint)so[1] * (uint)so[2]));
- break;
- }
-
- case ExecOp.Pad:
- {
- // generic rank-4 pad in storage order (pads permuted to physical dims)
- var p = (PadP)n.P!;
- CValue x = V[n.In[0]];
- if (x.Shape.Length != 4) throw new NotSupportedException("Vulkan pad: rank != 4");
- int[] l = x.Layout ?? Engine.Layout.Identity(4);
- int[] id4 = shapeIn(0), od4 = shapeOut();
- int[] pb4 = new int[4];
- for (int d = 0; d < 4; d++) pb4[d] = p.Pads[l[d]];
- yield return ND("pad", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOfF(U(id4[0], id4[1], id4[2], id4[3], od4[0], od4[1], od4[2], od4[3],
- pb4[0], pb4[1], pb4[2], pb4[3], p.Mode == PadMode.Reflect ? 1 : 0),
- p.Value),
- G((uint)V[n.Out[0]].ElementCount));
- break;
- }
-
- case ExecOp.Transpose:
- {
- // only planner relayouts reach here; the perm is physical
- var p = (TransposeP)n.P!;
- int[] is4 = Pad4(shapeIn(0));
- int[] os4 = Pad4(shapeOut());
- int[] ostr = Strides(os4), istr = Strides(is4);
- int[] perm4 = PermuteTo4(p.Perm, shapeIn(0).Length);
- yield return ND("transpose", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf(U(nOut(), ostr[0], ostr[1], ostr[2],
- istr[perm4[0]], istr[perm4[1]], istr[perm4[2]], istr[perm4[3]])),
- G((uint)nOut()));
- break;
- }
-
- case ExecOp.Slice:
- {
- // SliceP.Axes are physical after the layout pass
- var p = (SliceP)n.P!;
- int[] is4 = Pad4(shapeIn(0)), os4 = Pad4(shapeOut());
- int[] ostr = Strides(os4), istr = Strides(is4);
- uint[] st = [0, 0, 0, 0], sp = [1, 1, 1, 1];
- int shift = 4 - shapeIn(0).Length;
- for (int i = 0; i < p.Axes.Length; i++)
- {
- st[p.Axes[i] + shift] = (uint)p.Starts[i];
- sp[p.Axes[i] + shift] = (uint)p.Steps[i];
- }
- yield return ND("slice", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf(U(nOut(), ostr[0], ostr[1], ostr[2],
- (int)st[0], (int)st[1], (int)st[2], (int)st[3],
- (int)sp[0], (int)sp[1], (int)sp[2], (int)sp[3],
- istr[0], istr[1], istr[2], istr[3])),
- G((uint)nOut()));
- break;
- }
-
- case ExecOp.Concat:
- {
- // ConcatP.Axis is physical after the layout pass
- var p = (ConcatP)n.P!;
- int[] os = shapeOut();
- int axis = p.Axis;
- int inner = 1, dstAxis = os[axis];
- for (int d = axis + 1; d < os.Length; d++) inner *= os[d];
- int axisOff = 0;
- foreach (int iv in n.In)
- {
- int[] ishape = V[iv].PhysShape;
- int cnt = checked((int)V[iv].ElementCount);
- if (cnt > 0)
- {
- yield return ND("concat_ax", [Bind(iv), Bind(n.Out[0])],
- PcOf((uint)cnt, (uint)inner, (uint)ishape[axis], (uint)dstAxis, (uint)axisOff),
- G((uint)cnt));
- }
- axisOff += ishape[axis];
- }
- break;
- }
-
- case ExecOp.ChannelShift:
- {
- var p = (ShiftP)n.P!;
- RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]);
- int[] si = shapeIn(0); // [1,h,w,c]
- var ge = p.GroupEnds; var of = p.Offsets;
- if (ge.Length > 5) throw new NotSupportedException("Vulkan shift: > 5 groups");
- var pc = new List { (uint)si[3], (uint)si[1], (uint)si[2], (uint)p.Axis };
- for (int i = 0; i < 5; i++) pc.Add((uint)(i < ge.Length ? ge[i] : si[3]));
- for (int i = 0; i < 5; i++) pc.Add((uint)(i < of.Length ? of[i] : 0));
- bool v4 = _dev.Vec4Eltwise && si[3] % 4 == 0 && ge.All(e => e % 4 == 0);
- yield return ND(v4 ? "shift4" : "shift", [Bind(n.In[0]), Bind(n.Out[0])],
- PcOf(pc.ToArray()), G((uint)nOut() / (v4 ? 4u : 1u)), name: "shift");
- break;
- }
-
- default:
- throw new NotSupportedException($"Vulkan: ExecOp {n.Op} not implemented");
- }
-
- VkDispatch ND(string shader, VkBind[] binds, byte[] pc, uint gx, uint gy = 1, uint gz = 1, string? name = null)
- {
- return new VkDispatch
- {
- Pipe = shader.StartsWith("conv_cm", StringComparison.Ordinal)
- ? _dev.GetCoopPipeline(shader, binds.Length, pc.Length)
- : _dev.GetPipeline(Sh(shader), binds.Length, pc.Length),
- Pc = pc, Binds = binds, Gx = gx, Gy = gy, Gz = gz,
- Name = $"{shader}({n.Name})",
- };
- }
-
- VkBind B(int valueId, CK kind = CK.Raw) => valueId >= 0 ? Bind(valueId, kind) : new VkBind(VkSlot.Dummy, 0, 64);
- }
-
- private void RequireNhwc(int v)
- {
- CValue cv = _plan.Values[v];
- if (cv.Shape.Length != 4 || !Engine.Layout.Equivalent(cv.Shape, cv.Layout ?? Engine.Layout.Identity(4), Engine.Layout.Nhwc))
- throw new NotSupportedException($"Vulkan: '{cv.Name}' expected NHWC storage, got {Engine.Layout.Str(cv.Layout)}");
- }
-
- // pad a shape to rank 4 with leading 1s
- private static int[] Pad4(int[] s)
- {
- if (s.Length == 4) return s;
- int[] r = [1, 1, 1, 1];
- s.CopyTo(r, 4 - s.Length);
- return r;
- }
-
- private static int[] Strides(int[] shape4)
- {
- int[] s = new int[4]; s[3] = 1;
- for (int i = 2; i >= 0; i--) s[i] = s[i + 1] * shape4[i + 1];
- return s;
- }
-
- // lift an original-rank perm into padded rank-4 (leading dims map to themselves)
- private static int[] PermuteTo4(int[] perm, int rank)
- {
- int shift = 4 - rank;
- int[] p4 = [0, 1, 2, 3];
- for (int i = 0; i < rank; i++) p4[i + shift] = perm[i] + shift;
- return p4;
- }
-
- /// Sessions hold a reference so cache eviction can't free live const/dummy buffers.
- public void Acquire()
- {
- if (Interlocked.Increment(ref _refs) <= 1)
- throw new ObjectDisposedException(nameof(VkGraphModel), "const buffers already freed");
- }
- /// Acquire unless already freed (no resurrection; for racing cache readers).
- public bool TryAcquire()
- {
- while (true)
- {
- int r = Volatile.Read(ref _refs);
- if (r <= 0) return false;
- if (Interlocked.CompareExchange(ref _refs, r + 1, r) == r) return true;
- }
- }
- public void Release()
- {
- if (Interlocked.Decrement(ref _refs) == 0) _store.Release();
- }
-
- /// The plan this graph was compiled from (arena offsets, I/O values).
- internal ExecPlan Plan => _plan;
- public void Dispose() => Release();
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs
index dd6a5d7..154a0a3 100644
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs
+++ b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs
@@ -1,5 +1,7 @@
-using Sdcb.SimdGcdrNet.Engine;
-using Sdcb.SimdGcdrNet.Onnx;
+using Sdcb.SimdGcdrNet.Extensions;
+using Sdcb.SimdOnnx.Backends.Vulkan;
+using Sdcb.SimdOnnx.Engine;
+using Sdcb.SimdOnnx.Onnx;
namespace Sdcb.SimdGcdrNet.Backends.Vulkan;
@@ -45,9 +47,9 @@ internal VkGcdrPipeline(VkBackend be, OnnxModel gc, OnnxModel dr)
_drModel = dr;
_f16 = be.F16;
string sfx = _f16 ? "_h" : "";
- _pPre = _dev.GetPipeline("bgr2nhwc" + sfx, 2, 9 * 4);
- _pDiv6 = _dev.GetPipeline("div6" + sfx, 3, 4);
- _pPost = _dev.GetPipeline("nhwc2bgr" + sfx, 3, 9 * 4);
+ _pPre = _dev.GetPipeline("bgr2nhwc" + sfx, 2, 9 * 4, owner: GcdrExtensions.Owner);
+ _pDiv6 = _dev.GetPipeline("div6" + sfx, 3, 4, owner: GcdrExtensions.Owner);
+ _pPost = _dev.GetPipeline("nhwc2bgr" + sfx, 3, 9 * 4, owner: GcdrExtensions.Owner);
_cmdPool = _dev.NewCommandPool();
try
{
@@ -80,9 +82,9 @@ internal void Bind(int ih, int iw, GcdrPixelFormat fmt = GcdrPixelFormat.Bgr24)
if (_gcG is null || h != _h || w != _w)
{
// a new size compiles two plans + graphs: build them side by side
- var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], physIo: true));
+ var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], GraphOptions));
VkGraphModel gd;
- try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], physIo: true); }
+ try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], GraphOptions); }
catch { try { tg.GetAwaiter().GetResult().Release(); } catch { } throw; }
VkGraphModel gg;
try { gg = tg.GetAwaiter().GetResult(); }
@@ -100,7 +102,9 @@ internal void Bind(int ih, int iw, GcdrPixelFormat fmt = GcdrPixelFormat.Bgr24)
ulong es = (ulong)Es;
// gc base aligned to 256 B (any minStorageBufferOffsetAlignment)
- var (gcOff, total) = GcdrEnhancer.StackArenas(GcPlan, DrPlan, (long)(256 / es));
+ // the graphs' full arenas: the plan's values plus scratch / tile-read slack past them
+ var (gcOff, total) = GcdrEnhancer.StackArenas((long)((_gcG!.ArenaBytes + es - 1) / es), DrPlan,
+ (long)((_drG!.ArenaBytes + es - 1) / es), (long)(256 / es));
_gcBase = (ulong)gcOff * es;
ulong need = (ulong)total * es + 64;
// the arena only grows while within ArenaBudget; an over-budget one (huge image)
@@ -118,6 +122,12 @@ internal void Bind(int ih, int iw, GcdrPixelFormat fmt = GcdrPixelFormat.Bgr24)
_recorded = true;
}
+ /// Fused-pipeline graphs: physical I/O, the GcdrNet extension ops, and the staged
+ /// tensor-core convs on SG16 (Intel Xe2) devices — twice as fast here as the direct family and
+ /// stress-clean on this model (the OCR models are not; see CompileOptions.StagedCoopConv).
+ private static CompileOptions GraphOptions { get; } =
+ VkBackend.PipeOptions with { Extensions = GcdrExtensions.Instance, StagedCoopConv = true };
+
/// Per-worker device-arena budget (gc + dr). Within it the arena is grow-only
/// (12.5% headroom on the first allocation, 25% on regrowth) so mixed sizes don't reallocate.
/// 3 GB; on integrated GPUs the "device-local" heap is system RAM shared with the CPU, so
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkSession.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkSession.cs
deleted file mode 100644
index edfc0ae..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkSession.cs
+++ /dev/null
@@ -1,394 +0,0 @@
-using Sdcb.SimdGcdrNet.Engine;
-
-namespace Sdcb.SimdGcdrNet.Backends.Vulkan;
-
-///
-/// GPU counterpart of : one activation arena in
-/// device-local memory, persistent host-visible staging buffers, and a single
-/// pre-recorded command buffer (staging copy-in → all dispatches → copy-out).
-/// Not thread-safe; one per thread or serialize externally.
-///
-public sealed unsafe class VkInferenceSession : IDisposable
-{
- private readonly VkDevice _dev;
- private readonly VkGraphModel _graph;
- private readonly ExecPlan _plan;
-
- internal readonly VkBuffer Arena; // device-local activation arena
- private readonly VkBuffer _stageIn; // host-visible float input staging
- private readonly VkBuffer _stageOut; // host-visible float output staging
- private float* _stageInMap, _stageOutMap;
- private readonly float[] _inHost, _outHost;
- private readonly IntPtr _cmd, _fence;
- private readonly IntPtr[] _descSets; // per-dispatch sets when !PushDescriptors
- private readonly ulong _inOff, _outOff;
- private readonly int _inFloats, _outFloats;
- private readonly int _es; // bytes per stored element (2 = fp16 storage)
- internal bool F16 => _es == 2;
-
- internal VkInferenceSession(VkDevice dev, ExecPlan plan, VkGraphModel graph)
- {
- _dev = dev;
- _plan = plan;
- _graph = graph;
- _graph.Acquire();
- _es = graph.Es;
-
- Arena = dev.NewStorageBuffer((ulong)plan.ArenaFloats * (ulong)_es + 64, hostVisible: false);
- _inFloats = checked((int)plan.Values[plan.InputValue].ElementCount);
- _outFloats = checked((int)plan.Values[plan.OutputValue].ElementCount);
- _inOff = (ulong)plan.BufOffset[plan.Values[plan.InputValue].Buf] * (ulong)_es;
- _outOff = (ulong)plan.BufOffset[plan.Values[plan.OutputValue].Buf] * (ulong)_es;
- _stageIn = dev.NewStorageBuffer(InputBytes, hostVisible: true, preferHost: true);
- _stageOut = dev.NewStorageBuffer(OutputBytes, hostVisible: true, preferHost: true);
- _stageInMap = (float*)_stageIn.Map();
- _stageOutMap = (float*)_stageOut.Map();
- _inHost = new float[_inFloats];
- _outHost = new float[_outFloats];
- _fence = dev.NewFence();
-
- if (!dev.PushDescriptors)
- {
- _descSets = new IntPtr[_graph.Dispatches.Length];
- for (int i = 0; i < _graph.Dispatches.Length; i++)
- {
- var d = _graph.Dispatches[i];
- IntPtr set = dev.NewDescriptorSet(d.Pipe.SetLayout);
- for (uint b = 0; b < d.Binds.Length; b++)
- {
- (VkBuffer buf, _, _) = Resolve(d.Binds[b]);
- dev.BindBuffer(set, b, buf, d.Binds[b].Offset, d.Binds[b].Size);
- }
- _descSets[i] = set;
- }
- }
- else _descSets = [];
-
- _cmd = dev.NewCommandBuffer();
- Record();
- }
-
- internal VkBuffer ResolveBuf(VkBind b) => Resolve(b).buf;
- internal ulong ResolveOff(VkBind b) => b.Offset;
-
- internal (VkBuffer buf, ulong off, ulong size) Resolve(VkBind b) => b.Slot switch
- {
- VkSlot.Arena => (Arena, b.Offset, b.Size),
- VkSlot.Consts => (_graph.Consts, b.Offset, b.Size),
- VkSlot.StageIn => (_stageIn, b.Offset, b.Size),
- VkSlot.StageOut => (_stageOut, b.Offset, b.Size),
- VkSlot.Direct => (b.Buf!, b.Offset, b.Size),
- _ => (_graph.Dummy, b.Offset, b.Size),
- };
-
- /// Input tensor shape (e.g. [1,3,H,W]).
- public int[] InputShape => _plan.Values[_plan.InputValue].Shape;
- /// Output tensor shape (e.g. [1,3,H,W]).
- public int[] OutputShape => _plan.Values[_plan.OutputValue].Shape;
-
- /// The writable input window (host copy staged on ).
- public Span Input => _inHost;
- /// The output tensor after .
- public ReadOnlySpan Output => _outHost;
-
- internal ulong InputOffset => _inOff;
- internal ulong OutputOffset => _outOff;
- internal ulong InputBytes => (ulong)_inFloats * (ulong)_es;
- internal ulong OutputBytes => (ulong)_outFloats * (ulong)_es;
- internal int ElemBytes => _es;
-
- /// Host float -> staged storage elements (fp16 narrowing when F16).
- internal static void ToStorage(ReadOnlySpan src, void* dst, bool f16)
- {
- if (!f16) { src.CopyTo(new Span(dst, src.Length)); return; }
- var d = new Span(dst, src.Length);
- for (int i = 0; i < src.Length; i++) d[i] = (Half)src[i];
- }
-
- /// Stored elements -> host float (fp16 widening when F16).
- internal static void FromStorage(void* src, Span dst, bool f16)
- {
- if (!f16) { new ReadOnlySpan(src, dst.Length).CopyTo(dst); return; }
- var sp = new ReadOnlySpan(src, dst.Length);
- for (int i = 0; i < dst.Length; i++) dst[i] = (float)sp[i];
- }
-
- /// Downloads dst.Length stored elements at byte offset .
- internal void DownloadElems(VkBuffer buf, ulong off, Span dst)
- {
- ulong bytes = (ulong)dst.Length * (ulong)_es;
- byte[] tmp = new byte[bytes];
- fixed (byte* t = tmp)
- {
- _dev.Download(buf, t, bytes, off);
- FromStorage(t, dst, F16);
- }
- }
-
- /// Record one dispatch (push-descriptor path only; caller emits barriers).
- internal static void RecordOne(VkDevice dev, IntPtr cmd, VkPipeline pipe,
- (VkBuffer buf, ulong off, ulong size)[] binds, byte[] pc, uint gx, uint gy, uint gz = 1)
- {
- Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, pipe.Pipeline);
- var writes = stackalloc Vk.VkWriteDescriptorSet[8];
- var infos = stackalloc Vk.VkDescriptorBufferInfo[8];
- for (uint b = 0; b < binds.Length; b++)
- {
- infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = binds[b].buf.Buffer, Offset = binds[b].off, Range = binds[b].size };
- writes[b] = new Vk.VkWriteDescriptorSet
- {
- SType = VkConst.StWriteDescriptorSet, DstBinding = b,
- DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer,
- PBufferInfo = infos + b,
- };
- }
- dev.CmdPushDescriptors(cmd, pipe.Layout, writes, (uint)binds.Length);
- if (pc.Length > 0)
- fixed (byte* p = pc)
- Vk.vkCmdPushConstants(cmd, pipe.Layout, VkConst.StageComputeShader, 0, (uint)pc.Length, p);
- Vk.vkCmdDispatch(cmd, gx, gy, gz);
- }
-
- /// Record all dispatches with barriers into an open command buffer.
- internal void RecordCompute(IntPtr cmd)
- {
- var writes = stackalloc Vk.VkWriteDescriptorSet[8];
- var infos = stackalloc Vk.VkDescriptorBufferInfo[8];
- bool first = true;
- for (int i = 0; i < _graph.Dispatches.Length; i++)
- {
- var d = _graph.Dispatches[i];
- if (!first) _dev.CmdComputeBarrier(cmd);
- first = false;
- Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, d.Pipe.Pipeline);
- if (_dev.PushDescriptors)
- {
- for (uint b = 0; b < d.Binds.Length; b++)
- {
- (VkBuffer buf, ulong off, ulong size) = Resolve(d.Binds[b]);
- infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = buf.Buffer, Offset = off, Range = size };
- writes[b] = new Vk.VkWriteDescriptorSet
- {
- SType = VkConst.StWriteDescriptorSet, DstBinding = b,
- DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer,
- PBufferInfo = infos + b,
- };
- }
- _dev.CmdPushDescriptors(cmd, d.Pipe.Layout, writes, (uint)d.Binds.Length);
- }
- else
- {
- IntPtr set = _descSets[i];
- Vk.vkCmdBindDescriptorSets(cmd, VkConst.BindPointCompute, d.Pipe.Layout, 0, 1, &set, 0, null);
- }
- if (d.Pc.Length > 0)
- fixed (byte* pc = d.Pc)
- Vk.vkCmdPushConstants(cmd, d.Pipe.Layout, VkConst.StageComputeShader, 0, (uint)d.Pc.Length, pc);
- Vk.vkCmdDispatch(cmd, d.Gx, d.Gy, d.Gz);
- }
- }
-
- private void Record()
- {
- var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo };
- Vk.Check(Vk.vkBeginCommandBuffer(_cmd, &begin), "vkBeginCommandBuffer");
- // staging -> arena(input)
- Vk.VkBufferCopy cin = new() { Size = InputBytes, DstOffset = _inOff };
- Vk.vkCmdCopyBuffer(_cmd, _stageIn.Buffer, Arena.Buffer, 1, &cin);
- // barrier: transfer-write -> shader-read
- Vk.VkMemoryBarrier mb = new()
- {
- SType = VkConst.StMemoryBarrier,
- SrcAccessMask = VkConst.AccessTransferWrite, DstAccessMask = VkConst.AccessShaderRead,
- };
- Vk.vkCmdPipelineBarrier(_cmd, VkConst.PipelineStageTransfer, VkConst.PipelineStageComputeShader, 0,
- 1, &mb, 0, null, 0, null);
- RecordCompute(_cmd);
- // barrier: shader-write -> transfer-read
- Vk.VkMemoryBarrier mb2 = new()
- {
- SType = VkConst.StMemoryBarrier,
- SrcAccessMask = VkConst.AccessShaderWrite, DstAccessMask = VkConst.AccessTransferRead,
- };
- Vk.vkCmdPipelineBarrier(_cmd, VkConst.PipelineStageComputeShader, VkConst.PipelineStageTransfer, 0,
- 1, &mb2, 0, null, 0, null);
- Vk.VkBufferCopy cout = new() { Size = OutputBytes, SrcOffset = _outOff };
- Vk.vkCmdCopyBuffer(_cmd, Arena.Buffer, _stageOut.Buffer, 1, &cout);
- Vk.Check(Vk.vkEndCommandBuffer(_cmd), "vkEndCommandBuffer");
- }
-
- /// Debug/test: record a single dispatch list slice and run it.
- internal void DebugRunRange(int dStart, int dCount)
- {
- IntPtr cmd = _dev.NewCommandBuffer();
- IntPtr fence = _dev.NewFence();
- var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo };
- Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer");
- var writes = stackalloc Vk.VkWriteDescriptorSet[8];
- var infos = stackalloc Vk.VkDescriptorBufferInfo[8];
- for (int i = dStart; i < dStart + dCount; i++)
- {
- var d = _graph.Dispatches[i];
- if (i > dStart) _dev.CmdComputeBarrier(cmd);
- Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, d.Pipe.Pipeline);
- for (uint b = 0; b < d.Binds.Length; b++)
- {
- (VkBuffer buf, ulong off, ulong size) = Resolve(d.Binds[b]);
- infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = buf.Buffer, Offset = off, Range = size };
- writes[b] = new Vk.VkWriteDescriptorSet
- {
- SType = VkConst.StWriteDescriptorSet, DstBinding = b,
- DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer,
- PBufferInfo = infos + b,
- };
- }
- _dev.CmdPushDescriptors(cmd, d.Pipe.Layout, writes, (uint)d.Binds.Length);
- if (d.Pc.Length > 0)
- fixed (byte* pc = d.Pc)
- Vk.vkCmdPushConstants(cmd, d.Pipe.Layout, VkConst.StageComputeShader, 0, (uint)d.Pc.Length, pc);
- Vk.vkCmdDispatch(cmd, d.Gx, d.Gy, d.Gz);
- }
- Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer");
- _dev.Submit(cmd, fence);
- _dev.WaitFence(fence);
- _dev.DestroyFence(fence);
- _dev.FreeCommandBuffer(cmd);
- }
-
- ///
- /// Debug/test: time each dispatch by submitting it alone (barrier + dispatch) and
- /// wall-timing the fenced submission. Includes ~20-40us launch overhead per dispatch;
- /// accurate enough to rank ms-scale kernels. Returns name→best-of-2 ms.
- ///
- internal Dictionary DebugProfile()
- {
- int n = _graph.Dispatches.Length;
- IntPtr cmd = _dev.NewCommandBuffer();
- IntPtr fence = _dev.NewFence();
- var writes = stackalloc Vk.VkWriteDescriptorSet[8];
- var infos = stackalloc Vk.VkDescriptorBufferInfo[8];
- var best = new double[n];
- Array.Fill(best, double.MaxValue);
- void RecordDispatch(VkDispatch d)
- {
- _dev.CmdComputeBarrier(cmd);
- Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, d.Pipe.Pipeline);
- for (uint b = 0; b < d.Binds.Length; b++)
- {
- (VkBuffer buf, ulong off, ulong size) = Resolve(d.Binds[b]);
- infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = buf.Buffer, Offset = off, Range = size };
- writes[b] = new Vk.VkWriteDescriptorSet
- {
- SType = VkConst.StWriteDescriptorSet, DstBinding = b,
- DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer,
- PBufferInfo = infos + b,
- };
- }
- _dev.CmdPushDescriptors(cmd, d.Pipe.Layout, writes, (uint)d.Binds.Length);
- if (d.Pc.Length > 0)
- fixed (byte* pc = d.Pc)
- Vk.vkCmdPushConstants(cmd, d.Pipe.Layout, VkConst.StageComputeShader, 0, (uint)d.Pc.Length, pc);
- Vk.vkCmdDispatch(cmd, d.Gx, d.Gy, d.Gz);
- }
- var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo };
- // warm pass to keep state identical to production (input already in arena)
- Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer");
- foreach (var d in _graph.Dispatches) RecordDispatch(d);
- Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer");
- _dev.Submit(cmd, fence);
- _dev.WaitFence(fence);
- // timed passes: one dispatch per submit, min of 2
- var sw = new System.Diagnostics.Stopwatch();
- for (int rep = 0; rep < 2; rep++)
- for (int i = 0; i < n; i++)
- {
- var d = _graph.Dispatches[i];
- Vk.vkResetCommandBuffer(cmd, 0);
- Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer");
- RecordDispatch(d);
- Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer");
- sw.Restart();
- _dev.Submit(cmd, fence);
- _dev.WaitFence(fence);
- double ms = sw.Elapsed.TotalMilliseconds;
- if (ms < best[i]) best[i] = ms;
- }
- _dev.DestroyFence(fence);
- _dev.FreeCommandBuffer(cmd);
- var agg = new Dictionary();
- PerDispatch = new (string, double)[n];
- for (int i = 0; i < n; i++)
- {
- string name = _graph.Dispatches[i].Name ?? $"d{i}";
- PerDispatch[i] = (name, best[i]);
- name = name[..(name.IndexOf('(') is int cut && cut > 0 ? cut : name.Length)];
- agg[name] = agg.TryGetValue(name, out double v) ? v + best[i] : best[i];
- }
- return agg;
- }
-
- /// Per-dispatch timings from the last call.
- internal (string Name, double Ms)[]? PerDispatch;
-
- /// Debug: the compiled plan backing this session.
- internal ExecPlan DebugPlan => _plan;
-
- /// Debug: download the whole arena (ArenaFloats floats) into dst.
- internal void DebugArena(Span dst)
- {
- DownloadElems(Arena, 0, dst.Slice(0, _plan.ArenaFloats));
- }
-
- /// Debug: graph consts buffer + the session's arena buffer.
- internal (VkBuffer consts, VkBuffer arena) DebugBuffers => (_graph.Consts, Arena);
-
- /// Debug/test: download an activation value's region into dst.
- internal void DebugDownload(int valueId, float[] dst)
- {
- CValue v = _plan.Values[valueId];
- DownloadElems(Arena, (ulong)_plan.BufOffset[v.Buf] * (ulong)_es, dst.AsSpan(0, checked((int)v.ElementCount)));
- }
-
- /// Debug/test: copy host input into the arena's input region.
- internal void DebugUploadInput()
- {
- ToStorage(_inHost, _stageInMap, F16);
- _stageIn.Flush(0, InputBytes);
- IntPtr cmd = _dev.NewCommandBuffer();
- IntPtr fence = _dev.NewFence();
- var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo };
- Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer");
- Vk.VkBufferCopy cin = new() { Size = InputBytes, DstOffset = _inOff };
- Vk.vkCmdCopyBuffer(cmd, _stageIn.Buffer, Arena.Buffer, 1, &cin);
- Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer");
- _dev.Submit(cmd, fence);
- _dev.WaitFence(fence);
- _dev.DestroyFence(fence);
- _dev.FreeCommandBuffer(cmd);
- }
-
- /// Copy host input into staging and run the recorded pipeline.
- public void Run()
- {
- ToStorage(_inHost, _stageInMap, F16);
- _stageIn.Flush(0, InputBytes);
- _dev.Submit(_cmd, _fence);
- _dev.WaitFence(_fence);
- _stageOut.Invalidate(0, OutputBytes);
- FromStorage(_stageOutMap, _outHost, F16);
- }
-
- /// Releases the session's device buffers, fence and command buffer, and its
- /// reference to the shared graph.
- public void Dispose()
- {
- if (_stageInMap != null) { _stageIn.Unmap(); _stageInMap = null; }
- if (_stageOutMap != null) { _stageOut.Unmap(); _stageOutMap = null; }
- _dev.DestroyFence(_fence);
- _dev.FreeCommandBuffer(_cmd);
- // descriptor sets live in the device pool; pool teardown frees them
- Arena.Free(); _stageIn.Free(); _stageOut.Free();
- _graph.Release();
- }
-}
-
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkShaders.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkShaders.cs
deleted file mode 100644
index 28c5b78..0000000
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkShaders.cs
+++ /dev/null
@@ -1,21 +0,0 @@
-using System.Reflection;
-
-namespace Sdcb.SimdGcdrNet.Backends.Vulkan;
-
-/// Loads SPIR-V blobs embedded next to their .comp sources.
-internal static class VkShaders
-{
- private static readonly Assembly s_asm = typeof(VkShaders).Assembly;
- private const string Prefix = "Sdcb.SimdGcdrNet.Backends.Vulkan.Shaders.";
-
- public static byte[] Load(string name)
- {
- using Stream? s = s_asm.GetManifestResourceStream(Prefix + name + ".spv");
- if (s == null)
- throw new InvalidOperationException(
- $"Embedded shader '{name}' not found. Available: {string.Join(", ", s_asm.GetManifestResourceNames().Where(n => n.StartsWith(Prefix)))}");
- byte[] b = new byte[s.Length];
- s.ReadExactly(b);
- return b;
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat
index 017cbf1..914ae88 100644
--- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat
+++ b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat
@@ -1,8 +1,6 @@
@echo off
-rem Dev-time only: compile .comp GLSL -> .spv next to each source (committed).
-rem Not part of `dotnet build` - SPIR-V is embedded as a resource.
-rem Every shader is built twice: name.spv (fp32 storage) and name_h.spv
-rem (-DF16: 16-bit storage, fp32 math; see Shaders/et.glsl).
+rem Dev-time only: compile the GCDR-specific .comp GLSL -> .spv next to each source (committed).
+rem Every shader is built twice: name.spv (fp32 storage) and name_h.spv (-DF16, see Shaders/et.glsl).
setlocal
set "GLSLC=C:\VulkanSDK\1.4.357.0\Bin\glslc.exe"
if not exist "%GLSLC%" for %%i in (glslc.exe) do set "GLSLC=%%~$PATH:i"
@@ -13,53 +11,9 @@ if not exist "%GLSLC%" (
set "DIR=%~dp0..\Shaders"
set "O=-O --target-env=vulkan1.1"
for %%f in ("%DIR%\*.comp") do (
- if not "%%~nxf"=="conv_tile.comp" if not "%%~nxf"=="conv_cm.comp" if not "%%~nxf"=="sgprobe.comp" if not "%%~nxf"=="conv_dot.comp" if not "%%~nxf"=="gemm_dot.comp" (
- echo glslc %%~nxf
- "%GLSLC%" %O% "%%f" -o "%%~dpnf.spv" || exit /b 1
- "%GLSLC%" %O% -DF16 "%%f" -o "%%~dpnf_h.spv" || exit /b 1
- )
+ echo glslc %%~nxf
+ "%GLSLC%" %O% "%%f" -o "%%~dpnf.spv" || exit /b 1
+ "%GLSLC%" %O% -DF16 "%%f" -o "%%~dpnf_h.spv" || exit /b 1
)
-rem conv_tile specialization variants (constant kernel/stride -> fast index math)
-call :conv "" "" || exit /b 1
-call :conv -DF16 _h || exit /b 1
-rem tensor-core conv (fp16 storage only): BN=128 / 64 / 32 output-channel tiles
-set "CM=%DIR%\conv_cm.comp"
-"%GLSLC%" %O% "%CM%" -o "%DIR%\conv_cm_h.spv" || exit /b 1
-"%GLSLC%" %O% -DSGX=2u -DSGY=2u -DTSM=4u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_n64_h.spv" || exit /b 1
-"%GLSLC%" %O% -DSGX=1u -DSGY=4u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_n32_h.spv" || exit /b 1
-rem any-cin stems (k elements gathered one by one)
-"%GLSLC%" %O% -DSCALARK -DSGX=1u -DSGY=4u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_n32s_h.spv" || exit /b 1
-rem LITE: the same four for wave64-capable / 32 KB shared-memory parts (subgroup pinned to 32);
-rem more, narrower (64x16) subgroup strips per tile measured fastest on the Radeon 880M
-"%GLSLC%" %O% -DLITE -DSGX=8u -DSGY=2u -DTSM=4u -DTSN=1u -DNTHR=512 "%CM%" -o "%DIR%\conv_cm_l_h.spv" || exit /b 1
-"%GLSLC%" %O% -DLITE -DSGX=4u -DSGY=2u -DTSM=4u -DTSN=1u -DNTHR=256 "%CM%" -o "%DIR%\conv_cm_l_n64_h.spv" || exit /b 1
-"%GLSLC%" %O% -DLITE -DSGX=2u -DSGY=2u -DTSM=4u -DTSN=1u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_l_n32_h.spv" || exit /b 1
-"%GLSLC%" %O% -DLITE -DSCALARK -DSGX=1u -DSGY=4u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_l_n32s_h.spv" || exit /b 1
-rem SG16: 8x16x16 fragments on 16-lane subgroups (Intel Xe2 XMX), LITE staging, BM = 128;
-rem a 16x32 strip per subgroup, eight stacked along M, measured fastest on the Arc B580
-"%GLSLC%" %O% -DLITE -DSG16 -DSGX=4u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=512 "%CM%" -o "%DIR%\conv_cm_sg16_h.spv" || exit /b 1
-"%GLSLC%" %O% -DLITE -DSG16 -DSGX=2u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=256 "%CM%" -o "%DIR%\conv_cm_sg16_n64_h.spv" || exit /b 1
-"%GLSLC%" %O% -DLITE -DSG16 -DSGX=1u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_sg16_n32_h.spv" || exit /b 1
-"%GLSLC%" %O% -DLITE -DSG16 -DSCALARK -DSGX=1u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_sg16_n32s_h.spv" || exit /b 1
-rem wide-wave no-coopmat GPUs (Adreno): register-tiled direct kernels, fp16 storage only
-set "CD=%DIR%\conv_dot.comp"
-"%GLSLC%" %O% -DSW=1u -DTM=4 -DTN=2 "%CD%" -o "%DIR%\conv_dot_s1_4x2_h.spv" || exit /b 1
-"%GLSLC%" %O% -DSW=1u -DTM=4 -DTN=1 "%CD%" -o "%DIR%\conv_dot_s1_4x1_h.spv" || exit /b 1
-"%GLSLC%" %O% -DSW=2u -DTM=2 -DTN=2 "%CD%" -o "%DIR%\conv_dot_s2_2x2_h.spv" || exit /b 1
-"%GLSLC%" %O% -DSW=2u -DTM=2 -DTN=1 "%CD%" -o "%DIR%\conv_dot_s2_2x1_h.spv" || exit /b 1
-"%GLSLC%" %O% "%DIR%\gemm_dot.comp" -o "%DIR%\gemm_dot_h.spv" || exit /b 1
-"%GLSLC%" %O% -DF16 -DPIXELS_INNER "%DIR%\convpool.comp" -o "%DIR%\convpool_pi_h.spv" || exit /b 1
"%GLSLC%" %O% -DF16 -DNP=2u "%DIR%\uphead_sc.comp" -o "%DIR%\uphead_sc2_h.spv" || exit /b 1
-rem subgroup-size probe for pinned (requiredSubgroupSize = 32 / 16) pipelines
-"%GLSLC%" %O% "%DIR%\sgprobe.comp" -o "%DIR%\sgprobe.spv" || exit /b 1
-exit /b 0
-
-:conv
-set "CT=%DIR%\conv_tile.comp"
-set "F=%~1"
-set "S=%~2"
-"%GLSLC%" %O% %F% -DKH=3u -DKW=3u -DSH=1u -DSW=1u "%CT%" -o "%DIR%\conv_k33s1%S%.spv" || exit /b 1
-"%GLSLC%" %O% %F% -DKH=3u -DKW=3u -DSH=2u -DSW=2u "%CT%" -o "%DIR%\conv_k33s2%S%.spv" || exit /b 1
-"%GLSLC%" %O% %F% -DKH=1u -DKW=1u -DSH=1u -DSW=1u "%CT%" -o "%DIR%\conv_k11s1%S%.spv" || exit /b 1
-"%GLSLC%" %O% %F% "%CT%" -o "%DIR%\conv_gen%S%.spv" || exit /b 1
exit /b 0
diff --git a/src/Sdcb.SimdGcdrNet/Engine/CompiledModel.cs b/src/Sdcb.SimdGcdrNet/Engine/CompiledModel.cs
deleted file mode 100644
index 59f24c9..0000000
--- a/src/Sdcb.SimdGcdrNet/Engine/CompiledModel.cs
+++ /dev/null
@@ -1,30 +0,0 @@
-using Sdcb.SimdGcdrNet.Onnx;
-
-namespace Sdcb.SimdGcdrNet.Engine;
-
-///
-/// An specialized to a concrete input shape: all
-/// dynamic-shape plumbing folded, ops lowered to the exec set, buffers
-/// planned. Immutable; share across sessions freely.
-///
-public sealed class CompiledModel
-{
- internal ExecPlan Plan { get; }
-
- private CompiledModel(ExecPlan plan) => Plan = plan;
-
- /// Compiles for a fixed input shape.
- /// The ONNX model to compile.
- /// Concrete input shape, e.g. [1, 3, H, W].
- public static CompiledModel Compile(OnnxModel model, ReadOnlySpan inputShape) =>
- new(GraphCompiler.Compile(model, inputShape));
-
- /// Creates a new CPU with its own arena.
- public InferenceSession CreateSession() => new(Plan);
-
- /// Number of executable nodes after folding and fusion.
- public int NodeCount => Plan.Nodes.Length;
-
- /// Size of the activation arena, in floats, a session needs.
- public int ArenaFloats => Plan.ArenaFloats;
-}
diff --git a/src/Sdcb.SimdGcdrNet/Engine/CpuKernels.cs b/src/Sdcb.SimdGcdrNet/Engine/CpuKernels.cs
deleted file mode 100644
index 7a6e336..0000000
--- a/src/Sdcb.SimdGcdrNet/Engine/CpuKernels.cs
+++ /dev/null
@@ -1,76 +0,0 @@
-using Sdcb.SimdGcdrNet.Kernels;
-
-namespace Sdcb.SimdGcdrNet.Engine;
-
-///
-/// CPU-private packed weights (conv panels + Winograd transforms, Gemm panels,
-/// UpHead lanes) stored in the plan's op params. Per-size plans of one model
-/// carry identical weights, so a shared across sizes packs
-/// each layer once: entries are keyed by node identity and only reused when the
-/// weights (and bias) are element-for-element equal.
-///
-internal static class CpuKernels
-{
- internal sealed class Cache
- {
- private readonly Dictionary<(ExecOp op, string name, int src), (float[] w, float[]? b, object kp)> _d = new();
-
- internal object GetOrPack(ExecNode n, float[] w, float[]? b, Func
- internal static (long gcOff, long total) StackArenas(ExecPlan gc, ExecPlan dr, long align = 16)
+ internal static (long gcOff, long total) StackArenas(ExecPlan gc, ExecPlan dr, long align = 16) =>
+ StackArenas(gc.ArenaFloats, dr, dr.ArenaFloats, align);
+
+ /// over each stage's full arena
+ /// (GPU graphs append scratch / tile-read slack past the plan's values).
+ internal static (long gcOff, long total) StackArenas(long gcArena, ExecPlan dr, long drArena, long align = 16)
{
CValue din = dr.Values[dr.InputValue];
long o = dr.BufOffset[din.Buf], end = (o + din.ElementCount + align - 1) / align * align;
- long gcOff = o >= gc.ArenaFloats ? 0 : end;
- return (gcOff, Math.Max(dr.ArenaFloats, gcOff + gc.ArenaFloats));
+ long gcOff = o >= gcArena ? 0 : end;
+ return (gcOff, Math.Max(drArena, gcOff + gcArena));
}
// browser/WASI: unlinked P/Invokes (vulkan-1, libobjc) trap the wasm module with
@@ -275,7 +292,7 @@ internal static (long gcOff, long total) StackArenas(ExecPlan gc, ExecPlan dr, l
private static bool VulkanOk()
{
- try { _ = Sdcb.SimdGcdrNet.Backends.Vulkan.VkBackend.Default.Dev.DeviceName; return true; }
+ try { _ = VkBackend.Default.Dev.DeviceName; return true; }
catch { return false; }
}
@@ -283,11 +300,11 @@ private static bool MetalOk()
{
try
{
- var dev = Backends.Metal.MetalBackend.Default.Dev;
+ var dev = MetalBackend.Default.Dev;
// Apple7+ (M1+) for simdgroup matrix ops; Pipe() forces the embedded
// MSL library compile now rather than mid-Enhance on macOS 13. Intel Macs can
// claim Apple7 yet build the simdgroup-MMA conv with 0 threads/group (Pipe throws).
- return dev.SupportsApple7 && dev.Pipe("bgr2nhwc") is not null && dev.Pipe("conv_mm_32_0") is not null;
+ return dev.SupportsApple7 && dev.Pipe("bgr2nhwc", GcdrExtensions.Owner) is not null && dev.Pipe("conv_mm_32_0") is not null;
}
catch { return false; }
}
@@ -312,7 +329,7 @@ public void Dispose()
///
/// Size-agnostic CPU worker: one native arena holding both stages (see
- /// ), borrowed by per-call sessions. Within
+ /// ), borrowed by per-call sessions. Within
/// it only grows (25% headroom on regrowth), so mixed sizes
/// neither reallocate nor re-fault pages; an over-budget arena (one huge image) drops
/// to exact size once a size that fits comes along.
diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Avx2Kernels.cs b/src/Sdcb.SimdGcdrNet/Kernels/Avx2Kernels.cs
deleted file mode 100644
index c8615d5..0000000
--- a/src/Sdcb.SimdGcdrNet/Kernels/Avx2Kernels.cs
+++ /dev/null
@@ -1,970 +0,0 @@
-using System.Runtime.CompilerServices;
-using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.X86;
-using Sdcb.SimdGcdrNet.Engine;
-
-using F32x8 = System.Runtime.Intrinsics.Vector256;
-using V256 = System.Runtime.Intrinsics.Vector256;
-
-namespace Sdcb.SimdGcdrNet.Kernels;
-
-///
-/// AVX2+FMA kernels: multi-threaded, register-tiled NCHW conv (OC4 x YB2),
-/// deinterleaved stride-2 conv, depthwise, 1x1, maxpool, GEMM, LayerNorm.
-/// Inputs are pinned for the duration of each call so Parallel.For lambdas
-/// can capture raw pointers (spans are not capturable). Edge pixels go
-/// through a scalar spill path so every pad mode stays exact.
-///
-internal static unsafe partial class Avx2Kernels
-{
- internal static readonly int Threads = InitThreads();
- private static int InitThreads()
- {
- string? s = Environment.GetEnvironmentVariable("GCDRNET_THREADS");
- return int.TryParse(s, out int t) && t > 0 ? t : Environment.ProcessorCount;
- }
- private static readonly ParallelOptions PO = new() { MaxDegreeOfParallelism = Threads };
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static F32x8 Fma(F32x8 a, F32x8 b, F32x8 c) => V256.FusedMultiplyAdd(a, b, c);
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static F32x8 Ld(float* p) => Avx.LoadVector256(p);
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void St(float* p, F32x8 v) => Avx.Store(p, v);
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static F32x8 Bc(float* p) => Avx2.BroadcastScalarToVector256(p);
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static int Reflect(int i, int dim)
- {
- if (dim <= 1) return 0;
- while (i < 0 || i >= dim) i = i < 0 ? -i : 2 * dim - 2 - i;
- return i;
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void ApplyAct(ref F32x8 v, ActivationKind act)
- {
- if (act == ActivationKind.Relu) v = V256.Max(v, F32x8.Zero);
- else if (act == ActivationKind.Sigmoid) v = SimdMath.SigmoidCore(v);
- else if (act == ActivationKind.Gelu) v = SimdMath.GeluCore(v);
- }
-
- // lane-wise fetch with pad resolution (edge chunks only)
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static F32x8 Spill8(float* sp, int ih, int iw, int sy, int sx,
- PadMode mode, float padVal, float* spill)
- {
- for (int l = 0; l < 8; l++)
- {
- int px = sx + l;
- spill[l] = (uint)px < (uint)iw ? sp[sy * iw + px]
- : mode == PadMode.Reflect ? sp[sy * iw + Reflect(px, iw)]
- : padVal;
- }
- return Ld(spill);
- }
-
- private static void StoreRow(float* dp, int oStride,
- int xw, ActivationKind act, F32x8 a0, F32x8 a1, F32x8 a2, F32x8 a3)
- {
- ApplyAct(ref a0, act); ApplyAct(ref a1, act);
- ApplyAct(ref a2, act); ApplyAct(ref a3, act);
- if (xw == 8)
- {
- St(dp, a0);
- St(dp + oStride, a1);
- St(dp + 2 * oStride, a2);
- St(dp + 3 * oStride, a3);
- return;
- }
- float* tmp = stackalloc float[32];
- St(tmp, a0); St(tmp + 8, a1); St(tmp + 16, a2); St(tmp + 24, a3);
- for (int l = 0; l < xw; l++)
- for (int j = 0; j < 4; j++)
- dp[j * oStride + l] = tmp[j * 8 + l];
- }
-
- // ======================== dense conv 3x3 s1 ========================
- public static void Conv3x3S1(ReadOnlySpan src, int ic, int ih, int iw,
- ReadOnlySpan w, ReadOnlySpan bias, Span dst,
- int oc, int oh, int ow, int pt, int pl,
- PadMode mode, float padVal, ActivationKind act)
- {
- int yb = (oh + 1) / 2, ob = oc / 4;
- bool hasBias = !bias.IsEmpty;
- fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst)
- {
- float* sp = sp0, wp = wp0, bp = bp0, dp = dp0;
- Parallel.For(0, ob * yb, PO, tile =>
- {
- Conv3x3Tile(sp, ic, ih, iw, wp, hasBias ? bp : null, dp, oc, oh, ow,
- pt, pl, mode, padVal, act, tile / yb * 4, tile % yb * 2);
- });
- }
- }
-
- private static void Conv3x3Tile(float* sp, int ic, int ih, int iw,
- float* wp, float* bp, float* dp,
- int oc, int oh, int ow, int pt, int pl,
- PadMode mode, float padVal, ActivationKind act, int o0, int y0)
- {
- bool two = y0 + 1 < oh;
- int y1 = two ? y0 + 1 : y0;
- int wStride = ic * 9;
- int ostride = oh * ow;
- float* spill = stackalloc float[8];
-
- for (int x0 = 0; x0 < ow; x0 += 8)
- {
- int xw = Math.Min(8, ow - x0);
- F32x8 a0, a1, a2, a3, b0, b1, b2, b3;
- if (bp is null)
- {
- a0 = a1 = a2 = a3 = b0 = b1 = b2 = b3 = F32x8.Zero;
- }
- else
- {
- a0 = b0 = Bc(bp + o0);
- a1 = b1 = Bc(bp + o0 + 1);
- a2 = b2 = Bc(bp + o0 + 2);
- a3 = b3 = Bc(bp + o0 + 3);
- }
- for (int c = 0; c < ic; c++)
- {
- float* sbase = sp + c * ih * iw;
- float* wb = wp + (o0 * ic + c) * 9;
- for (int ky = 0; ky < 3; ky++)
- {
- int syA = y0 + ky - pt, syB = y1 + ky - pt;
- int ryA = (uint)syA < (uint)ih ? syA
- : mode == PadMode.Reflect ? Reflect(syA, ih) : -1;
- int ryB = !two ? -1
- : (uint)syB < (uint)ih ? syB
- : mode == PadMode.Reflect ? Reflect(syB, ih) : -1;
- for (int kx = 0; kx < 3; kx++)
- {
- int sx = x0 + kx - pl;
- int wi = ky * 3 + kx;
- F32x8 wv0 = Bc(wb + wi);
- F32x8 wv1 = Bc(wb + wi + wStride);
- F32x8 wv2 = Bc(wb + wi + 2 * wStride);
- F32x8 wv3 = Bc(wb + wi + 3 * wStride);
- bool interior = (uint)sx < (uint)iw && sx + 8 <= iw;
- if (ryA >= 0)
- {
- F32x8 v = interior
- ? Ld(sbase + ryA * iw + sx)
- : Spill8(sbase, ih, iw, ryA, sx, mode, padVal, spill);
- a0 = Fma(v, wv0, a0); a1 = Fma(v, wv1, a1); a2 = Fma(v, wv2, a2); a3 = Fma(v, wv3, a3);
- }
- else if (padVal != 0f)
- {
- F32x8 pv = V256.Create(padVal);
- a0 = Fma(pv, wv0, a0); a1 = Fma(pv, wv1, a1);
- a2 = Fma(pv, wv2, a2); a3 = Fma(pv, wv3, a3);
- }
- if (ryB >= 0)
- {
- F32x8 v = interior
- ? Ld(sbase + ryB * iw + sx)
- : Spill8(sbase, ih, iw, ryB, sx, mode, padVal, spill);
- b0 = Fma(v, wv0, b0); b1 = Fma(v, wv1, b1); b2 = Fma(v, wv2, b2); b3 = Fma(v, wv3, b3);
- }
- else if (ryB < 0 && padVal != 0f)
- {
- F32x8 pv = V256.Create(padVal);
- b0 = Fma(pv, wv0, b0); b1 = Fma(pv, wv1, b1);
- b2 = Fma(pv, wv2, b2); b3 = Fma(pv, wv3, b3);
- }
- }
- }
- }
- float* db = dp + (o0 * oh + y0) * ow + x0;
- StoreRow(db, ostride, xw, act, a0, a1, a2, a3);
- if (two) StoreRow(db + ow, ostride, xw, act, b0, b1, b2, b3);
- }
- }
-
- // ======================== conv 1x1 ========================
- public static void Conv1x1(ReadOnlySpan src, int ic, int hw,
- ReadOnlySpan w, ReadOnlySpan bias, Span dst,
- int oc, ActivationKind act)
- {
- int stripes = Math.Clamp(hw / 32768, 1, 8);
- int stride = hw / stripes;
- bool hasBias = !bias.IsEmpty;
- fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst)
- {
- float* sp = sp0, wp = wp0, bp = bp0, dp = dp0;
- Parallel.For(0, ((oc + 3) / 4) * stripes, PO, tile =>
- {
- int o0 = tile / stripes * 4;
- int hw0 = tile % stripes * stride;
- int hw1 = tile % stripes == stripes - 1 ? hw : hw0 + stride;
- Conv1x1Strip(sp, ic, wp, hasBias ? bp : null, dp,
- o0, Math.Min(4, oc - o0), hw0, hw1, hw, act);
- });
- }
- }
-
- private static void Conv1x1Strip(float* sp, int ic,
- float* wp, float* bp, float* dp,
- int o0, int ocLim, int hw0, int hw1, int hw, ActivationKind act)
- {
- int i = hw0;
- for (; i + 8 <= hw1; i += 8)
- {
- F32x8 a0 = V256.Create(bp is null ? 0f : bp[o0]);
- F32x8 a1 = V256.Create(ocLim > 1 && bp is not null ? bp[o0 + 1] : 0f);
- F32x8 a2 = V256.Create(ocLim > 2 && bp is not null ? bp[o0 + 2] : 0f);
- F32x8 a3 = V256.Create(ocLim > 3 && bp is not null ? bp[o0 + 3] : 0f);
- for (int c = 0; c < ic; c++)
- {
- F32x8 v = Ld(sp + c * hw + i);
- a0 = Fma(v, Bc(wp + o0 * ic + c), a0);
- if (ocLim > 1) a1 = Fma(v, Bc(wp + (o0 + 1) * ic + c), a1);
- if (ocLim > 2) a2 = Fma(v, Bc(wp + (o0 + 2) * ic + c), a2);
- if (ocLim > 3) a3 = Fma(v, Bc(wp + (o0 + 3) * ic + c), a3);
- }
- float* db = dp + o0 * hw + i;
- ApplyAct(ref a0, act);
- St(db, a0);
- if (ocLim > 1) { ApplyAct(ref a1, act); St(db + hw, a1); }
- if (ocLim > 2) { ApplyAct(ref a2, act); St(db + 2 * hw, a2); }
- if (ocLim > 3) { ApplyAct(ref a3, act); St(db + 3 * hw, a3); }
- }
- for (; i < hw1; i++)
- for (int j = 0; j < ocLim; j++)
- {
- float acc = bp is null ? 0f : bp[o0 + j];
- for (int c = 0; c < ic; c++)
- acc += sp[c * hw + i] * wp[(o0 + j) * ic + c];
- float r = act == ActivationKind.Relu ? MathF.Max(acc, 0f)
- : act == ActivationKind.Sigmoid ? 1f / (1f + MathF.Exp(-acc)) : acc;
- dp[(o0 + j) * hw + i] = r;
- }
- }
-
- // ======================== conv 3x3 s2 (patch embed) ========================
- private static readonly Vector256 Evens = Vector256.Create(0, 2, 4, 6, 0, 0, 0, 0);
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static F32x8 LoadStride2(float* p)
- {
- F32x8 a = Ld(p);
- F32x8 b = Ld(p + 8);
- var ea = Avx2.PermuteVar8x32(a.AsInt32(), Evens);
- var eb = Avx2.PermuteVar8x32(b.AsInt32(), Evens);
- return V256.Create(ea.GetLower().AsSingle(), eb.GetLower().AsSingle());
- }
-
- public static void Conv3x3S2(ReadOnlySpan src, int ic, int ih, int iw,
- ReadOnlySpan w, ReadOnlySpan bias, Span dst,
- int oc, int oh, int ow, int pt, int pl,
- PadMode mode, float padVal, ActivationKind act)
- {
- int yb = (oh + 1) / 2, ob = oc / 4;
- bool hasBias = !bias.IsEmpty;
- fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst)
- {
- float* sp = sp0, wp = wp0, bp = bp0, dp = dp0;
- Parallel.For(0, ob * yb, PO, tile =>
- {
- ConvS2Tile(sp, ic, ih, iw, wp, hasBias ? bp : null, dp, oc, oh, ow,
- pt, pl, mode, padVal, act, tile / yb * 4, tile % yb * 2);
- });
- }
- }
-
- private static void ConvS2Tile(float* sp, int ic, int ih, int iw,
- float* wp, float* bp, float* dp,
- int oc, int oh, int ow, int pt, int pl,
- PadMode mode, float padVal, ActivationKind act, int o0, int y0)
- {
- bool two = y0 + 1 < oh;
- int y1 = two ? y0 + 1 : y0;
- int wStride = ic * 9;
- int ostride = oh * ow;
- float* spill = stackalloc float[16];
-
- for (int x0 = 0; x0 < ow; x0 += 8)
- {
- int xw = Math.Min(8, ow - x0);
- F32x8 a0, a1, a2, a3, b0, b1, b2, b3;
- if (bp is null)
- {
- a0 = a1 = a2 = a3 = b0 = b1 = b2 = b3 = F32x8.Zero;
- }
- else
- {
- a0 = b0 = Bc(bp + o0);
- a1 = b1 = Bc(bp + o0 + 1);
- a2 = b2 = Bc(bp + o0 + 2);
- a3 = b3 = Bc(bp + o0 + 3);
- }
- for (int c = 0; c < ic; c++)
- {
- float* sbase = sp + c * ih * iw;
- float* wb = wp + (o0 * ic + c) * 9;
- for (int ky = 0; ky < 3; ky++)
- {
- int syA = y0 * 2 + ky - pt, syB = y1 * 2 + ky - pt;
- int ryA = (uint)syA < (uint)ih ? syA
- : mode == PadMode.Reflect ? Reflect(syA, ih) : -1;
- int ryB = !two ? -1
- : (uint)syB < (uint)ih ? syB
- : mode == PadMode.Reflect ? Reflect(syB, ih) : -1;
- for (int kx = 0; kx < 3; kx++)
- {
- int sx = x0 * 2 + kx - pl;
- int wi = ky * 3 + kx;
- F32x8 wv0 = Bc(wb + wi);
- F32x8 wv1 = Bc(wb + wi + wStride);
- F32x8 wv2 = Bc(wb + wi + 2 * wStride);
- F32x8 wv3 = Bc(wb + wi + 3 * wStride);
- bool interior = sx >= 0 && sx + 15 < iw;
- if (ryA >= 0)
- {
- F32x8 v;
- if (interior)
- {
- v = LoadStride2(sbase + ryA * iw + sx);
- }
- else
- {
- for (int l = 0; l < 8; l++)
- {
- int px = sx + 2 * l;
- spill[l] = x0 + l < ow && (uint)px < (uint)iw
- ? sbase[ryA * iw + px]
- : x0 + l < ow && mode == PadMode.Reflect
- ? sbase[ryA * iw + Reflect(px, iw)]
- : padVal;
- }
- v = Ld(spill);
- }
- a0 = Fma(v, wv0, a0); a1 = Fma(v, wv1, a1); a2 = Fma(v, wv2, a2); a3 = Fma(v, wv3, a3);
- }
- else if (padVal != 0f)
- {
- F32x8 pv = V256.Create(padVal);
- a0 = Fma(pv, wv0, a0); a1 = Fma(pv, wv1, a1);
- a2 = Fma(pv, wv2, a2); a3 = Fma(pv, wv3, a3);
- }
- if (ryB >= 0)
- {
- F32x8 v;
- if (interior)
- {
- v = LoadStride2(sbase + ryB * iw + sx);
- }
- else
- {
- for (int l = 0; l < 8; l++)
- {
- int px = sx + 2 * l;
- spill[l] = x0 + l < ow && (uint)px < (uint)iw
- ? sbase[ryB * iw + px]
- : x0 + l < ow && mode == PadMode.Reflect
- ? sbase[ryB * iw + Reflect(px, iw)]
- : padVal;
- }
- v = Ld(spill);
- }
- b0 = Fma(v, wv0, b0); b1 = Fma(v, wv1, b1); b2 = Fma(v, wv2, b2); b3 = Fma(v, wv3, b3);
- }
- else if (ryB < 0 && padVal != 0f)
- {
- F32x8 pv = V256.Create(padVal);
- b0 = Fma(pv, wv0, b0); b1 = Fma(pv, wv1, b1);
- b2 = Fma(pv, wv2, b2); b3 = Fma(pv, wv3, b3);
- }
- }
- }
- }
- float* db = dp + (o0 * oh + y0) * ow + x0;
- StoreRow(db, ostride, xw, act, a0, a1, a2, a3);
- if (two) StoreRow(db + ow, ostride, xw, act, b0, b1, b2, b3);
- }
- }
-
- // ======================== depthwise conv ========================
- public static void Depthwise(ReadOnlySpan src, int ic, int ih, int iw,
- ReadOnlySpan w, ReadOnlySpan bias, Span dst,
- int oh, int ow, int kh, int kw, int pt, int pl,
- PadMode mode, float padVal, ActivationKind act)
- {
- bool hasBias = !bias.IsEmpty;
- fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst)
- {
- float* sp = sp0, wp = wp0, bp = bp0, dp = dp0;
- Parallel.For(0, ic * oh, PO, t =>
- {
- int c = t / oh, y = t % oh;
- float* sbase = sp + c * ih * iw;
- float* drow = dp + (c * oh + y) * ow;
- float* wk = wp + c * kh * kw;
- float b = bp is null ? 0f : bp[c];
- float* spill = stackalloc float[16];
- for (int x0 = 0; x0 < ow; x0 += 8)
- {
- int xw = Math.Min(8, ow - x0);
- F32x8 acc = V256.Create(b);
- for (int ky = 0; ky < kh; ky++)
- {
- int sy = y + ky - pt;
- int ry = (uint)sy < (uint)ih ? sy
- : mode == PadMode.Reflect ? Reflect(sy, ih) : -1;
- for (int kx = 0; kx < kw; kx++)
- {
- int sx = x0 + kx - pl;
- float wv = wk[ky * kw + kx];
- F32x8 v;
- if (ry >= 0 && sx >= 0 && sx + 8 <= iw)
- {
- v = Ld(sbase + ry * iw + sx);
- }
- else
- {
- for (int l = 0; l < 8; l++)
- {
- int px = sx + l;
- spill[l] = x0 + l < ow && ry >= 0 && (uint)px < (uint)iw
- ? sbase[ry * iw + px]
- : x0 + l < ow && ry >= 0 && mode == PadMode.Reflect
- ? sbase[ry * iw + Reflect(px, iw)]
- : padVal;
- }
- v = Ld(spill);
- }
- acc = Fma(v, V256.Create(wv), acc);
- }
- }
- ApplyAct(ref acc, act);
- if (xw == 8)
- {
- St(drow + x0, acc);
- }
- else
- {
- St(spill, acc);
- for (int l = 0; l < xw; l++) drow[x0 + l] = spill[l];
- }
- }
- });
- }
- }
-
- // ======================== maxpool 2x2 s2 ========================
- public static void MaxPool2x2(ReadOnlySpan src, int c, int ih, int iw,
- Span dst, int oh, int ow, bool fuseRelu)
- {
- fixed (float* sp0 = src, dp0 = dst)
- {
- float* sp = sp0, dp = dp0;
- Parallel.For(0, c * oh, PO, t =>
- {
- int ch = t / oh, y = t % oh;
- float* sbase = sp + ch * ih * iw;
- float* drow = dp + (ch * oh + y) * ow;
- int r0 = Math.Min(y * 2, ih - 1) * iw, r1 = Math.Min(y * 2 + 1, ih - 1) * iw;
- float* spill = stackalloc float[16];
- for (int x0 = 0; x0 < ow; x0 += 8)
- {
- int xw = Math.Min(8, ow - x0);
- int sx = x0 * 2;
- F32x8 m;
- if (sx + 15 < iw)
- {
- m = V256.Max(LoadStride2(sbase + r0 + sx), LoadStride2(sbase + r0 + sx + 1));
- m = V256.Max(m, V256.Max(LoadStride2(sbase + r1 + sx), LoadStride2(sbase + r1 + sx + 1)));
- }
- else
- {
- for (int l = 0; l < 8; l++)
- {
- if (x0 + l >= ow) { spill[l] = float.NegativeInfinity; continue; }
- int xx = Math.Min(sx + 2 * l, iw - 2);
- float mx = MathF.Max(sbase[r0 + xx], sbase[r0 + xx + 1]);
- spill[l] = MathF.Max(mx, MathF.Max(sbase[r1 + xx], sbase[r1 + xx + 1]));
- }
- m = Ld(spill);
- }
- if (fuseRelu) m = V256.Max(m, F32x8.Zero);
- if (xw == 8)
- {
- St(drow + x0, m);
- }
- else
- {
- St(spill, m);
- for (int l = 0; l < xw; l++) drow[x0 + l] = spill[l];
- }
- }
- });
- }
- }
-
- // ======================== GEMM (B row-major [K,N]) ========================
- public static void GemmKN(ReadOnlySpan a, int M, int K,
- ReadOnlySpan b, int N, ReadOnlySpan bias, Span dst)
- {
- bool hasBias = !bias.IsEmpty;
- fixed (float* ap0 = a, bp0 = b, cp0 = bias, dp0 = dst)
- {
- float* ap = ap0, bp = bp0, cp = cp0, dp = dp0;
- Parallel.For(0, (M + 3) / 4, PO, t =>
- {
- int m0 = t * 4;
- GemmTile(ap, m0, Math.Min(4, M - m0), K, bp, N, hasBias ? cp : null, dp);
- });
- }
- }
-
- private static void GemmTile(float* ap, int m0, int mc, int K,
- float* bp, int N, float* cp, float* dp)
- {
- int nv = N - N % 16;
- for (int n0 = 0; n0 < nv; n0 += 16)
- {
- F32x8 c00, c01;
- if (cp is null)
- {
- c00 = c01 = F32x8.Zero;
- }
- else
- {
- c00 = Ld(cp + n0);
- c01 = Ld(cp + n0 + 8);
- }
- F32x8 c10 = c00, c11 = c01, c20 = c00, c21 = c01, c30 = c00, c31 = c01;
- for (int k = 0; k < K; k++)
- {
- F32x8 bv0 = Ld(bp + k * N + n0);
- F32x8 bv1 = Ld(bp + k * N + n0 + 8);
- F32x8 av = Bc(ap + m0 * K + k);
- c00 = Fma(av, bv0, c00); c01 = Fma(av, bv1, c01);
- if (mc > 1) { av = Bc(ap + (m0 + 1) * K + k); c10 = Fma(av, bv0, c10); c11 = Fma(av, bv1, c11); }
- if (mc > 2) { av = Bc(ap + (m0 + 2) * K + k); c20 = Fma(av, bv0, c20); c21 = Fma(av, bv1, c21); }
- if (mc > 3) { av = Bc(ap + (m0 + 3) * K + k); c30 = Fma(av, bv0, c30); c31 = Fma(av, bv1, c31); }
- }
- float* db = dp + m0 * N + n0;
- St(db, c00); St(db + 8, c01);
- if (mc > 1) { St(db + N, c10); St(db + N + 8, c11); }
- if (mc > 2) { St(db + 2 * N, c20); St(db + 2 * N + 8, c21); }
- if (mc > 3) { St(db + 3 * N, c30); St(db + 3 * N + 8, c31); }
- }
- for (int n0 = nv; n0 < N; n0++)
- for (int m = 0; m < mc; m++)
- {
- float s = cp is null ? 0f : cp[n0];
- for (int k = 0; k < K; k++)
- s += ap[(m0 + m) * K + k] * bp[k * N + n0];
- dp[(m0 + m) * N + n0] = s;
- }
- }
-
- // ======================== LayerNorm ========================
- public static void LayerNorm(ReadOnlySpan x, int rows, int c,
- ReadOnlySpan w, ReadOnlySpan b, Span dst, float eps)
- {
- fixed (float* xp0 = x, wp0 = w, bp0 = b, dp0 = dst)
- {
- float* xp = xp0, wp = wp0, bp = bp0, dp = dp0;
- Parallel.For(0, rows, PO, r =>
- {
- float* xr = xp + r * c;
- float* dr = dp + r * c;
- F32x8 sum = F32x8.Zero, sq = F32x8.Zero;
- int i = 0;
- for (; i + 8 <= c; i += 8)
- {
- F32x8 v = Ld(xr + i);
- sum = V256.Add(sum, v);
- sq = Fma(v, v, sq);
- }
- float s = 0, ss = 0;
- for (int l = 0; l < 8; l++) { s += sum[l]; ss += sq[l]; }
- for (; i < c; i++) { float v = xr[i]; s += v; ss += v * v; }
- float mean = s / c;
- float inv = 1f / MathF.Sqrt(MathF.Max(ss / c - mean * mean, 0f) + eps);
- F32x8 vm = V256.Create(mean), vi = V256.Create(inv);
- for (i = 0; i + 8 <= c; i += 8)
- {
- F32x8 v = Ld(xr + i);
- F32x8 wv = Ld(wp + i);
- F32x8 bv = Ld(bp + i);
- F32x8 t = V256.Subtract(v, vm);
- St(dr + i, Fma(t, V256.Multiply(vi, wv), bv));
- }
- for (; i < c; i++) dr[i] = (xr[i] - mean) * inv * wp[i] + bp[i];
- });
- }
- }
-
- // ======================== parallel unary ========================
- // op: 0 relu, 1 sigmoid, 2 gelu, 3 erf
- public static void Unary(ReadOnlySpan src, Span dst, int n, int op)
- {
- int chunks = Math.Clamp(n / 8192, 1, Threads * 4);
- int stride = (n + chunks - 1) / chunks;
- fixed (float* sp0 = src, dp0 = dst)
- {
- float* sp = sp0, dp = dp0;
- Parallel.For(0, chunks, PO, t =>
- {
- int i0 = t * stride, i1 = Math.Min(n, i0 + stride);
- int i = i0;
- for (; i + 8 <= i1; i += 8)
- {
- F32x8 v = Ld(sp + i);
- ApplyUnary(ref v, op);
- St(dp + i, v);
- }
- for (; i < i1; i++) dp[i] = UnaryScalar(sp[i], op);
- });
- }
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void ApplyUnary(ref F32x8 v, int op)
- {
- if (op == 0) v = V256.Max(v, F32x8.Zero);
- else if (op == 1) v = SimdMath.SigmoidCore(v);
- else if (op == 2) v = SimdMath.GeluCore(v);
- else v = SimdMath.ErfCore(v);
- }
-
- private static float UnaryScalar(float x, int op) => op switch
- {
- 0 => MathF.Max(x, 0f),
- 1 => 1f / (1f + MathF.Exp(-x)),
- 2 => 0.5f * x * (1f + SimdMath.ErfScalar(x * 0.70710678118654752f)),
- _ => SimdMath.ErfScalar(x),
- };
-
- // ======================== parallel binary elementwise ========================
- // op: 0 add, 1 sub, 2 mul, 3 div
- public static void Ew2(ReadOnlySpan a, ReadOnlySpan b,
- Span dst, int n, int op, ActivationKind act = ActivationKind.None)
- {
- int chunks = Math.Clamp(n / 8192, 1, Threads * 4);
- int stride = (n + chunks - 1) / chunks;
- fixed (float* ap0 = a, bp0 = b, dp0 = dst)
- {
- float* ap = ap0, bp = bp0, dp = dp0;
- Parallel.For(0, chunks, PO, t =>
- {
- int i0 = t * stride, i1 = Math.Min(n, i0 + stride);
- int i = i0;
- for (; i + 8 <= i1; i += 8)
- {
- F32x8 v = EwCore(Ld(ap + i), Ld(bp + i), op);
- ApplyAct(ref v, act);
- St(dp + i, v);
- }
- for (; i < i1; i++)
- {
- float v = EwScalar(ap[i], bp[i], op);
- dp[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v;
- }
- });
- }
- }
-
- // scalar broadcast: dst[i] = src[i] op s (rev: dst[i] = s op src[i])
- public static void Ew1(ReadOnlySpan src, Span dst,
- int n, float s, int op, bool rev = false, ActivationKind act = ActivationKind.None)
- {
- int chunks = Math.Clamp(n / 8192, 1, Threads * 4);
- int stride = (n + chunks - 1) / chunks;
- fixed (float* ap0 = src, dp0 = dst)
- {
- float* ap = ap0, dp = dp0;
- F32x8 vs = V256.Create(s);
- Parallel.For(0, chunks, PO, t =>
- {
- int i0 = t * stride, i1 = Math.Min(n, i0 + stride);
- int i = i0;
- for (; i + 8 <= i1; i += 8)
- {
- F32x8 v = Ld(ap + i);
- v = rev ? EwCore(vs, v, op) : EwCore(v, vs, op);
- ApplyAct(ref v, act);
- St(dp + i, v);
- }
- for (; i < i1; i++)
- {
- float v = rev ? EwScalar(s, ap[i], op) : EwScalar(ap[i], s, op);
- dp[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v;
- }
- });
- }
- }
-
- // broadcast over a contiguous inner block: b[bo] constant per outer row
- public static void EwInner(ReadOnlySpan a, ReadOnlySpan b,
- int[] bStr, int[] outShape, int[] outStr, int rank, int inner,
- Span dst, int n, int op, ActivationKind act = ActivationKind.None)
- {
- int outer = n / inner;
- fixed (float* ap0 = a, bp0 = b, dp0 = dst)
- {
- float* ap = ap0, bp = bp0, dp = dp0;
- Parallel.For(0, outer, PO, e =>
- {
- int rem = e, bo = 0;
- for (int d = 0; d < rank - 1; d++)
- {
- int cc = rem / (outStr[d] / inner);
- rem %= outStr[d] / inner;
- bo += cc * bStr[d];
- }
- float s = bp[bo];
- F32x8 vs = V256.Create(s);
- float* ar = ap + e * inner, dr = dp + e * inner;
- int i = 0;
- for (; i + 8 <= inner; i += 8)
- {
- F32x8 v = EwCore(Ld(ar + i), vs, op);
- ApplyAct(ref v, act);
- St(dr + i, v);
- }
- for (; i < inner; i++)
- {
- float v = EwScalar(ar[i], s, op);
- dr[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v;
- }
- });
- }
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static F32x8 EwCore(F32x8 a, F32x8 b, int op) => op switch
- {
- 0 => V256.Add(a, b),
- 1 => V256.Subtract(a, b),
- 2 => V256.Multiply(a, b),
- _ => V256.Divide(a, b),
- };
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static float EwScalar(float a, float b, int op) => op switch
- {
- 0 => a + b, 1 => a - b, 2 => a * b, _ => a / b,
- };
-
- // ======================== bilinear resize (half-pixel) ========================
- public static void Bilinear(ReadOnlySpan src, int c, int ih, int iw,
- Span dst, int oh, int ow,
- int[] y0, int[] y1, float[] wy, int[] x0, int[] x1, float[] wx,
- ActivationKind act = ActivationKind.None)
- {
- fixed (float* sp0 = src, dp0 = dst)
- fixed (int* y0p = y0, y1p = y1, x0p = x0, x1p = x1)
- fixed (float* wyp = wy, wxp = wx)
- {
- float* sp = sp0, dp = dp0;
- int* y0a = y0p, y1a = y1p, x0a = x0p, x1a = x1p;
- float* wya = wyp, wxa = wxp;
- Parallel.For(0, c * oh, PO, t =>
- {
- int ch = t / oh, y = t % oh;
- float* r0 = sp + (ch * ih + y0a[y]) * iw;
- float* r1 = sp + (ch * ih + y1a[y]) * iw;
- float* dr = dp + (ch * oh + y) * ow;
- F32x8 wyv = V256.Create(wya[y]);
- int x = 0;
- for (; x + 8 <= ow; x += 8)
- {
- var i0 = Avx.LoadVector256(x0a + x);
- var i1 = Avx.LoadVector256(x1a + x);
- F32x8 wxv = Ld(wxa + x);
- F32x8 t0 = Avx2.GatherVector256(r0, i0, 4);
- F32x8 t1 = Avx2.GatherVector256(r0, i1, 4);
- F32x8 b0 = Avx2.GatherVector256(r1, i0, 4);
- F32x8 b1 = Avx2.GatherVector256(r1, i1, 4);
- F32x8 top = Fma(V256.Subtract(t1, t0), wxv, t0);
- F32x8 bot = Fma(V256.Subtract(b1, b0), wxv, b0);
- F32x8 v = Fma(V256.Subtract(bot, top), wyv, top);
- ApplyAct(ref v, act);
- St(dr + x, v);
- }
- for (; x < ow; x++)
- {
- float t0 = r0[x0a[x]], t1v = r0[x1a[x]];
- float b0 = r1[x0a[x]], b1v = r1[x1a[x]];
- float top = t0 + (t1v - t0) * wxa[x];
- float bot = b0 + (b1v - b0) * wxa[x];
- float v = top + (bot - top) * wya[y];
- dr[x] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v;
- }
- });
- }
- }
-
- // ======================== tiled transpose [a,b] -> [b,a] ========================
- public static void Transpose2D(ReadOnlySpan src, int a, int b,
- Span dst, int copies = 1, int srcStride = 0, int dstStride = 0)
- {
- if (srcStride == 0) srcStride = a * b;
- if (dstStride == 0) dstStride = a * b;
- int ya = (a + 7) / 8, xb = (b + 7) / 8;
- fixed (float* sp0 = src, dp0 = dst)
- {
- float* sp = sp0, dp = dp0;
- Parallel.For(0, copies * ya, PO, t =>
- {
- int o = t / ya, ty = t % ya;
- float* sb = sp + o * srcStride, db = dp + o * dstStride;
- for (int tx = 0; tx < xb; tx++)
- {
- int y0 = ty * 8, x0 = tx * 8;
- if (y0 + 8 <= a && x0 + 8 <= b)
- {
- T8(sb + y0 * b + x0, b, db + x0 * a + y0, a);
- }
- else
- {
- int ye = Math.Min(y0 + 8, a), xe = Math.Min(x0 + 8, b);
- for (int y = y0; y < ye; y++)
- for (int x = x0; x < xe; x++)
- db[x * a + y] = sb[y * b + x];
- }
- }
- });
- }
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void T8(float* s, int ls, float* d, int ld)
- {
- F32x8 r0 = Ld(s), r1 = Ld(s + ls), r2 = Ld(s + 2 * ls), r3 = Ld(s + 3 * ls);
- F32x8 r4 = Ld(s + 4 * ls), r5 = Ld(s + 5 * ls), r6 = Ld(s + 6 * ls), r7 = Ld(s + 7 * ls);
- F32x8 t0 = Avx.UnpackLow(r0, r1), t1 = Avx.UnpackHigh(r0, r1);
- F32x8 t2 = Avx.UnpackLow(r2, r3), t3 = Avx.UnpackHigh(r2, r3);
- F32x8 t4 = Avx.UnpackLow(r4, r5), t5 = Avx.UnpackHigh(r4, r5);
- F32x8 t6 = Avx.UnpackLow(r6, r7), t7 = Avx.UnpackHigh(r6, r7);
- const byte LO = 0x44, HI = 0xEE;
- F32x8 u0 = Avx.Shuffle(t0, t2, LO), u1 = Avx.Shuffle(t0, t2, HI);
- F32x8 u2 = Avx.Shuffle(t1, t3, LO), u3 = Avx.Shuffle(t1, t3, HI);
- F32x8 u4 = Avx.Shuffle(t4, t6, LO), u5 = Avx.Shuffle(t4, t6, HI);
- F32x8 u6 = Avx.Shuffle(t5, t7, LO), u7 = Avx.Shuffle(t5, t7, HI);
- const byte L = 0x20, H = 0x31;
- St(d, Avx.Permute2x128(u0, u4, L));
- St(d + ld, Avx.Permute2x128(u1, u5, L));
- St(d + 2 * ld, Avx.Permute2x128(u2, u6, L));
- St(d + 3 * ld, Avx.Permute2x128(u3, u7, L));
- St(d + 4 * ld, Avx.Permute2x128(u0, u4, H));
- St(d + 5 * ld, Avx.Permute2x128(u1, u5, H));
- St(d + 6 * ld, Avx.Permute2x128(u2, u6, H));
- St(d + 7 * ld, Avx.Permute2x128(u3, u7, H));
- }
-
- // ======================== pad ========================
- public static void Pad4(ReadOnlySpan src, int nc, int ih, int iw,
- Span dst, int oh, int ow, int[] mapY, int[] mapX, float padVal)
- {
- // contiguous interior run of mapX (identity-mapped region)
- int xl = 0; while (xl < ow && mapX[xl] < 0) xl++;
- int xr = ow; while (xr > xl && mapX[xr - 1] < 0) xr--;
- bool interior = true;
- for (int x = xl; x < xr; x++) if (mapX[x] != x - xl + mapX[xl]) { interior = false; break; }
- int srcX0 = xl < xr ? mapX[xl] : 0;
- fixed (float* sp0 = src, dp0 = dst)
- fixed (int* myp = mapY, mxp = mapX)
- {
- float* sp = sp0, dp = dp0;
- int* mya = myp, mxa = mxp;
- Parallel.For(0, nc * oh, PO, t =>
- {
- int ch = t / oh, y = t % oh;
- float* dr = dp + (ch * oh + y) * ow;
- int my = mya[y];
- if (my < 0)
- {
- F32x8 pv = V256.Create(padVal);
- int x = 0;
- for (; x + 8 <= ow; x += 8) St(dr + x, pv);
- for (; x < ow; x++) dr[x] = padVal;
- return;
- }
- float* sr = sp + (ch * ih + my) * iw;
- for (int x = 0; x < xl; x++) dr[x] = mxa[x] < 0 ? padVal : sr[mxa[x]];
- for (int x = xr; x < ow; x++) dr[x] = mxa[x] < 0 ? padVal : sr[mxa[x]];
- if (interior)
- {
- float* s = sr + srcX0;
- int i = 0, len = xr - xl;
- for (; i + 8 <= len; i += 8) St(dr + xl + i, Ld(s + i));
- for (; i < len; i++) dr[xl + i] = s[i];
- }
- else
- {
- for (int x = xl; x < xr; x++) dr[x] = sr[mxa[x]];
- }
- });
- }
- }
-
- // ======================== channel shift ========================
- public static void ChannelShift(ReadOnlySpan src, int c, int h, int w,
- Span dst, int axis, int[] offsets, int[] groupEnds)
- {
- fixed (float* sp0 = src, dp0 = dst)
- fixed (int* offp = offsets, endp = groupEnds)
- {
- float* sp = sp0, dp = dp0;
- int* offa = offp, enda = endp;
- int groups = offsets.Length;
- Parallel.For(0, c * h, PO, t =>
- {
- int ch = t / h, y = t % h;
- int off = 0;
- for (int g = 0, g0 = 0; g < groups; g++)
- {
- int g1 = enda[g];
- if (ch >= g0 && ch < g1) { off = offa[g]; break; }
- g0 = g1;
- }
- float* sr = sp + (ch * h + y) * w;
- float* dr = dp + (ch * h + y) * w;
- if (axis == 2)
- {
- int sy = y + off;
- if ((uint)sy < (uint)h)
- Buffer.MemoryCopy(sp + (ch * h + sy) * w, dr, w * 4L, w * 4L);
- else
- {
- F32x8 z = F32x8.Zero;
- int x = 0;
- for (; x + 8 <= w; x += 8) St(dr + x, z);
- for (; x < w; x++) dr[x] = 0f;
- }
- }
- else if (Math.Abs(off) >= w)
- {
- int x = 0;
- for (; x + 8 <= w; x += 8) St(dr + x, F32x8.Zero);
- for (; x < w; x++) dr[x] = 0f;
- }
- else if (off >= 0)
- {
- // dr[x] = sr[x+off]; tail zero
- Buffer.MemoryCopy(sr + off, dr, (w - off) * 4L, (w - off) * 4L);
- int x = w - off;
- for (; x < w; x++) dr[x] = 0f;
- }
- else
- {
- // dr[x] = sr[x+off] with off<0: head zero, then copy
- int x = 0;
- for (; x + 8 <= -off; x += 8) St(dr + x, F32x8.Zero);
- for (; x < -off; x++) dr[x] = 0f;
- Buffer.MemoryCopy(sr, dr - off, (w + off) * 4L, (w + off) * 4L);
- }
- });
- }
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Kernels/NeonKernels.cs b/src/Sdcb.SimdGcdrNet/Kernels/NeonKernels.cs
deleted file mode 100644
index c578885..0000000
--- a/src/Sdcb.SimdGcdrNet/Kernels/NeonKernels.cs
+++ /dev/null
@@ -1,54 +0,0 @@
-using System.Runtime.CompilerServices;
-using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.Arm;
-
-using V4 = System.Runtime.Intrinsics.Vector128;
-
-namespace Sdcb.SimdGcdrNet.Kernels;
-
-/// AArch64 AdvSimd (NEON) versions of the non-conv ops that have AVX2 kernels.
-internal static unsafe class NeonKernels
-{
- private static readonly ParallelOptions PO = new() { MaxDegreeOfParallelism = Avx2Kernels.Threads };
-
- /// Row-wise LayerNorm over the last axis (c); rows split across threads in groups.
- public static void LayerNorm(ReadOnlySpan x, int rows, int c,
- ReadOnlySpan w, ReadOnlySpan b, Span dst, float eps)
- {
- int per = Math.Max(1, 16384 / c), tasks = (rows + per - 1) / per;
- fixed (float* xp0 = x, wp0 = w, bp0 = b, dp0 = dst)
- {
- float* xp = xp0, wp = wp0, bp = bp0, dp = dp0;
- Parallel.For(0, tasks, PO, t =>
- {
- for (int r = t * per, re = Math.Min(rows, r + per); r < re; r++)
- LayerNormRow(xp + (long)r * c, dp + (long)r * c, c, wp, bp, eps);
- });
- }
- }
-
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void LayerNormRow(float* xr, float* dr, int c, float* wp, float* bp, float eps)
- {
- V4 sum = V4.Zero, sq = V4.Zero;
- int i = 0;
- for (; i + 4 <= c; i += 4)
- {
- V4 v = AdvSimd.LoadVector128(xr + i);
- sum = AdvSimd.Add(sum, v);
- sq = AdvSimd.FusedMultiplyAdd(sq, v, v);
- }
- float s = Vector128.Sum(sum), ss = Vector128.Sum(sq);
- for (; i < c; i++) { float v = xr[i]; s += v; ss += v * v; }
- float mean = s / c;
- float inv = 1f / MathF.Sqrt(MathF.Max(ss / c - mean * mean, 0f) + eps);
- V4 vm = Vector128.Create(mean), vi = Vector128.Create(inv);
- for (i = 0; i + 4 <= c; i += 4)
- {
- V4 t = AdvSimd.Subtract(AdvSimd.LoadVector128(xr + i), vm);
- AdvSimd.Store(dr + i, AdvSimd.FusedMultiplyAdd(AdvSimd.LoadVector128(bp + i), t,
- AdvSimd.Multiply(vi, AdvSimd.LoadVector128(wp + i))));
- }
- for (; i < c; i++) dr[i] = (xr[i] - mean) * inv * wp[i] + bp[i];
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Avx512.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Avx512.cs
deleted file mode 100644
index c379e04..0000000
--- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Avx512.cs
+++ /dev/null
@@ -1,392 +0,0 @@
-using System.Runtime.CompilerServices;
-using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.X86;
-using Sdcb.SimdGcdrNet.Engine;
-
-using F32x8 = System.Runtime.Intrinsics.Vector256;
-using F32x16 = System.Runtime.Intrinsics.Vector512;
-
-namespace Sdcb.SimdGcdrNet.Kernels;
-
-///
-/// AVX-512 tier of the NHWC conv / Gemm micro-kernel: pixels × 32 oc
-/// (two packed 16-oc blocks, one zmm each) per pass, so every broadcast feeds two FMAs and
-/// the 32 zmm registers hold the whole accumulator tile. Per output element the reduction
-/// is the AVX2 one (bias-init, taps then channels ascending, one FMA each) and the
-/// epilogue reuses 's math, so results are bit-identical to AVX2.
-/// The k loops store to a scratch tile and never call out (calls make RyuJIT spill the
-/// accumulators); the residual/activation epilogue runs afterwards.
-///
-internal static unsafe partial class Nhwc
-{
- private const int Tile5 = 10;
-
- /// AVX-512 tier switch: Avx512F.IsSupported, off with GCDRNET_AVX512=0 or the
- /// .NET 8/9-style DOTNET_EnableAVX512F=0 (the .NET 10 JIT only honours DOTNET_EnableAVX512).
- /// Mutable for in-process A/B tests.
- internal static bool UseAvx512 = Avx512F.IsSupported
- && Environment.GetEnvironmentVariable("GCDRNET_AVX512") != "0"
- && Environment.GetEnvironmentVariable("DOTNET_EnableAVX512F") != "0";
-
- /// Tile5 rows × 32 oc: dst[r*ds + 0..15] ← block wb, dst[r*ds + 16..31] ← block wb + wStep.
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void Tile512x32(float** ptrs, int taps, int ic, float* wb, long wStep, float* bb, float* dst, int ds)
- {
- F32x16 a0 = Avx512F.LoadVector512(bb), b0 = Avx512F.LoadVector512(bb + Ob);
- F32x16 a1 = a0, a2 = a0, a3 = a0, a4 = a0, a5 = a0, a6 = a0, a7 = a0, a8 = a0, a9 = a0;
- F32x16 b1 = b0, b2 = b0, b3 = b0, b4 = b0, b5 = b0, b6 = b0, b7 = b0, b8 = b0, b9 = b0;
- for (int tap = 0; tap < taps; tap++)
- {
- float** pp = ptrs + tap * Tile5;
- float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3], p4 = pp[4], p5 = pp[5], p6 = pp[6], p7 = pp[7], p8 = pp[8], p9 = pp[9];
- float* w = wb + (long)tap * ic * Ob;
- float* x = w + wStep;
- for (int c = 0; c < ic; c++, w += Ob, x += Ob)
- {
- F32x16 wa = Avx512F.LoadVector512(w), wx = Avx512F.LoadVector512(x);
- F32x16 v = Vector512.Create(p0[c]);
- a0 = Avx512F.FusedMultiplyAdd(v, wa, a0); b0 = Avx512F.FusedMultiplyAdd(v, wx, b0);
- v = Vector512.Create(p1[c]);
- a1 = Avx512F.FusedMultiplyAdd(v, wa, a1); b1 = Avx512F.FusedMultiplyAdd(v, wx, b1);
- v = Vector512.Create(p2[c]);
- a2 = Avx512F.FusedMultiplyAdd(v, wa, a2); b2 = Avx512F.FusedMultiplyAdd(v, wx, b2);
- v = Vector512.Create(p3[c]);
- a3 = Avx512F.FusedMultiplyAdd(v, wa, a3); b3 = Avx512F.FusedMultiplyAdd(v, wx, b3);
- v = Vector512.Create(p4[c]);
- a4 = Avx512F.FusedMultiplyAdd(v, wa, a4); b4 = Avx512F.FusedMultiplyAdd(v, wx, b4);
- v = Vector512.Create(p5[c]);
- a5 = Avx512F.FusedMultiplyAdd(v, wa, a5); b5 = Avx512F.FusedMultiplyAdd(v, wx, b5);
- v = Vector512.Create(p6[c]);
- a6 = Avx512F.FusedMultiplyAdd(v, wa, a6); b6 = Avx512F.FusedMultiplyAdd(v, wx, b6);
- v = Vector512.Create(p7[c]);
- a7 = Avx512F.FusedMultiplyAdd(v, wa, a7); b7 = Avx512F.FusedMultiplyAdd(v, wx, b7);
- v = Vector512.Create(p8[c]);
- a8 = Avx512F.FusedMultiplyAdd(v, wa, a8); b8 = Avx512F.FusedMultiplyAdd(v, wx, b8);
- v = Vector512.Create(p9[c]);
- a9 = Avx512F.FusedMultiplyAdd(v, wa, a9); b9 = Avx512F.FusedMultiplyAdd(v, wx, b9);
- }
- }
- Avx512F.Store(dst, a0); Avx512F.Store(dst + Ob, b0); dst += ds;
- Avx512F.Store(dst, a1); Avx512F.Store(dst + Ob, b1); dst += ds;
- Avx512F.Store(dst, a2); Avx512F.Store(dst + Ob, b2); dst += ds;
- Avx512F.Store(dst, a3); Avx512F.Store(dst + Ob, b3); dst += ds;
- Avx512F.Store(dst, a4); Avx512F.Store(dst + Ob, b4); dst += ds;
- Avx512F.Store(dst, a5); Avx512F.Store(dst + Ob, b5); dst += ds;
- Avx512F.Store(dst, a6); Avx512F.Store(dst + Ob, b6); dst += ds;
- Avx512F.Store(dst, a7); Avx512F.Store(dst + Ob, b7); dst += ds;
- Avx512F.Store(dst, a8); Avx512F.Store(dst + Ob, b8); dst += ds;
- Avx512F.Store(dst, a9); Avx512F.Store(dst + Ob, b9);
- }
-
- /// Tile5 rows × 16 oc (odd block count tail): dst[r*ds + 0..15].
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void Tile512x16(float** ptrs, int taps, int ic, float* wb, float* bb, float* dst, int ds)
- {
- F32x16 a0 = Avx512F.LoadVector512(bb);
- F32x16 a1 = a0, a2 = a0, a3 = a0, a4 = a0, a5 = a0, a6 = a0, a7 = a0, a8 = a0, a9 = a0;
- for (int tap = 0; tap < taps; tap++)
- {
- float** pp = ptrs + tap * Tile5;
- float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3], p4 = pp[4], p5 = pp[5], p6 = pp[6], p7 = pp[7], p8 = pp[8], p9 = pp[9];
- float* w = wb + (long)tap * ic * Ob;
- for (int c = 0; c < ic; c++, w += Ob)
- {
- F32x16 wa = Avx512F.LoadVector512(w);
- a0 = Avx512F.FusedMultiplyAdd(Vector512.Create(p0[c]), wa, a0);
- a1 = Avx512F.FusedMultiplyAdd(Vector512.Create(p1[c]), wa, a1);
- a2 = Avx512F.FusedMultiplyAdd(Vector512.Create(p2[c]), wa, a2);
- a3 = Avx512F.FusedMultiplyAdd(Vector512.Create(p3[c]), wa, a3);
- a4 = Avx512F.FusedMultiplyAdd(Vector512.Create(p4[c]), wa, a4);
- a5 = Avx512F.FusedMultiplyAdd(Vector512.Create(p5[c]), wa, a5);
- a6 = Avx512F.FusedMultiplyAdd(Vector512.Create(p6[c]), wa, a6);
- a7 = Avx512F.FusedMultiplyAdd(Vector512.Create(p7[c]), wa, a7);
- a8 = Avx512F.FusedMultiplyAdd(Vector512.Create(p8[c]), wa, a8);
- a9 = Avx512F.FusedMultiplyAdd(Vector512.Create(p9[c]), wa, a9);
- }
- }
- Avx512F.Store(dst, a0); dst += ds;
- Avx512F.Store(dst, a1); dst += ds;
- Avx512F.Store(dst, a2); dst += ds;
- Avx512F.Store(dst, a3); dst += ds;
- Avx512F.Store(dst, a4); dst += ds;
- Avx512F.Store(dst, a5); dst += ds;
- Avx512F.Store(dst, a6); dst += ds;
- Avx512F.Store(dst, a7); dst += ds;
- Avx512F.Store(dst, a8); dst += ds;
- Avx512F.Store(dst, a9);
- }
-
- /// Runs the 32- or 16-wide kernel for one tile into ([Tile5][32]).
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void Tile512(float** ptrs, int nblk, int taps, int ic, float* wb, long wStep, float* bb, float* part)
- {
- if (nblk == 2) Tile512x32(ptrs, taps, ic, wb, wStep, bb, part, 2 * Ob);
- else Tile512x16(ptrs, taps, ic, wb, bb, part, 2 * Ob);
- }
-
- /// Epilogue of one tile: part ([Tile5][32]) → rows × (lanes0 + lanes1) outputs, + residual, act.
- [MethodImpl(MethodImplOptions.NoInlining)]
- private static void Epilogue512(float* part, int rows, int nblk, float* outp, float* addp, int oc,
- int lanes0, int lanes1, ActivationKind act)
- {
- bool simple = act is ActivationKind.None or ActivationKind.Relu;
- for (int r = 0; r < rows; r++, part += 2 * Ob)
- {
- float* o = outp + (long)r * oc;
- float* ad = addp == null ? null : addp + (long)r * oc;
- for (int h = 0; h < nblk; h++)
- {
- int lanes = h == 0 ? lanes0 : lanes1;
- float* s = part + h * Ob, oh = o + h * Ob, ah = ad == null ? null : ad + h * Ob;
- if (lanes == Ob && simple)
- {
- F32x16 v = Avx512F.LoadVector512(s);
- if (ah != null) v = Avx512F.Add(v, Avx512F.LoadVector512(ah));
- if (act == ActivationKind.Relu) v = Avx512F.Max(v, F32x16.Zero);
- Avx512F.Store(oh, v);
- }
- else
- Store16(oh, ah, Avx.LoadVector256(s), Avx.LoadVector256(s + 8), lanes, act);
- }
- }
- }
-
- private static void Dense512(ReadOnlySpan src, int iw, int ic, Span dst,
- int oh, int ow, int oc, int kh, int kw, int[] rm, int[] cm, float[] padVec,
- NhwcConvPlan kp, ReadOnlySpan add, ActivationKind act)
- {
- int taps = kh * kw, nb = kp.Blocks, np = (nb + 1) / 2;
- bool hasAdd = !add.IsEmpty;
- fixed (float* sp0 = src, dp0 = dst, ap0 = add, wp0 = kp.W, bp0 = kp.Bias, pv0 = padVec)
- fixed (int* rm0 = rm, cm0 = cm)
- {
- var a = new DenseArgs(sp0, wp0, bp0, pv0, rm0, cm0, iw, ic, ow, oc, kh, kw, nb, act, true);
- float* dp = dp0, ap = ap0;
- For(oh * np, (long)oh * ow * oc * taps * ic / 64, t =>
- {
- int y = t / np, blk = t % np * 2;
- long o = (long)y * ow * oc;
- Dense512Row(a, y, blk, dp + o, hasAdd ? ap + o : null);
- });
- }
- }
-
- /// All oc blocks of direct-conv output row (the row-level entry of
- /// the fused paths: conv+pool bands, UpHead skip recompute). AVX-512 tier: block pairs through
- /// (needs a padVec of ic*kw + 16 floats); else .
- private static void ConvRow(in DenseArgs a, int y, float* orow, float* arow)
- {
- if (UseAvx512)
- for (int blk = 0; blk < a.Nb; blk += 2) Dense512Row(a, y, blk, orow, arow);
- else
- for (int blk = 0; blk < a.Nb; blk++) DenseRow(a, y, blk, orow, arow);
- }
-
- /// Output row , oc blocks blk and blk+1 (if present) of a direct conv
- /// into (the row's first pixel; addend row likewise).
- private static void Dense512Row(in DenseArgs a, int y, int blk, float* orow, float* arow)
- {
- int kh = a.Kh, kw = a.Kw, taps = kh * kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, nblk = Math.Min(2, a.Nb - blk);
- long wStep = (long)taps * ic * Ob;
- float** ptrs = stackalloc float*[Tile5 * taps];
- float* part = stackalloc float[Tile5 * 2 * Ob];
- float* wb = a.W + blk * wStep;
- float* bb = a.Bias + blk * Ob;
- int lanes0 = Math.Min(Ob, oc - blk * Ob), lanes1 = nblk == 2 ? Math.Min(Ob, oc - (blk + 1) * Ob) : 0;
- int* rm = a.Rm, cm = a.Cm;
- for (int x0 = 0; x0 < ow; x0 += Tile5)
- {
- int rows = Math.Min(Tile5, ow - x0);
- // every column of the tile reads kw adjacent input pixels (no column pad/reflect): one
- // kernel row is then a single kw*ic float run in the order of the packed [tap][ic]
- // weights, so the tile walks kh "taps" of kw*ic channels (fewer pointers, longer k loops)
- bool merged = kw > 1;
- for (int i = 0; i < rows && merged; i++)
- {
- int* cx = cm + (x0 + i) * kw;
- merged = cx[0] >= 0;
- for (int kx = 1; kx < kw; kx++) merged &= cx[kx] == cx[0] + kx;
- }
- if (merged)
- {
- for (int ky = 0; ky < kh; ky++)
- {
- int r = rm[y * kh + ky];
- for (int i = 0; i < Tile5; i++)
- {
- int c = cm[(x0 + Math.Min(i, rows - 1)) * kw];
- ptrs[ky * Tile5 + i] = r < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic;
- }
- }
- Tile512(ptrs, nblk, kh, kw * ic, wb, wStep, bb, part);
- }
- else
- {
- for (int tap = 0; tap < taps; tap++)
- {
- int r = rm[y * kh + tap / kw];
- for (int i = 0; i < Tile5; i++)
- {
- int c = cm[(x0 + Math.Min(i, rows - 1)) * kw + tap % kw];
- ptrs[tap * Tile5 + i] = r < 0 || c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic;
- }
- }
- Tile512(ptrs, nblk, taps, ic, wb, wStep, bb, part);
- }
- long o = (long)x0 * oc + blk * Ob;
- Epilogue512(part, rows, nblk, orow + o, arow != null ? arow + o : null, oc, lanes0, lanes1, a.Act);
- }
- }
-
- private static void Gemm512(ReadOnlySpan a, int m, int k, Span dst, int n,
- NhwcConvPlan kp, ActivationKind act)
- {
- const int G = 8; // row tiles per task sharing one weight panel walk
- int nb = kp.Blocks, np = (nb + 1) / 2, tiles = (m + Tile5 - 1) / Tile5, groups = (tiles + G - 1) / G;
- long wStep = (long)k * Ob;
- fixed (float* ap0 = a, dp0 = dst, wp0 = kp.W, bp0 = kp.Bias)
- {
- float* ap = ap0, dp = dp0, wp = wp0, bp = bp0;
- For(groups * np, (long)m * n * k / 64, t =>
- {
- int g = t / np, blk = t % np * 2, nblk = Math.Min(2, nb - blk);
- float** ptrs = stackalloc float*[Tile5];
- float* part = stackalloc float[Tile5 * 2 * Ob];
- float* wb = wp + blk * wStep, bb = bp + blk * Ob;
- int lanes0 = Math.Min(Ob, n - blk * Ob), lanes1 = nblk == 2 ? Math.Min(Ob, n - (blk + 1) * Ob) : 0;
- for (int tile = g * G, te = Math.Min(tiles, tile + G); tile < te; tile++)
- {
- int m0 = tile * Tile5, rows = Math.Min(Tile5, m - m0);
- for (int i = 0; i < Tile5; i++) ptrs[i] = ap + (long)(m0 + Math.Min(i, rows - 1)) * k;
- Tile512(ptrs, nblk, 1, k, wb, wStep, bb, part);
- Epilogue512(part, rows, nblk, dp + (long)m0 * n + blk * Ob, null, n, lanes0, lanes1, act);
- }
- });
- }
- }
-
- // ======================== memory-bound NHWC ops (c % 16 == 0) ========================
-
- /// with 16-lane vectors: same per-lane ops (and IEEE Max) → same bits.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void ResizePixel512(float* a, float* b, float* e, float* f, F32x16 vX, F32x16 vY,
- int c, bool relu, float* add, bool addRelu, float* o)
- {
- for (int ch = 0; ch < c; ch += 16)
- {
- var va = Vector512.Load(a + ch); var ve = Vector512.Load(e + ch);
- var top = va + (Vector512.Load(b + ch) - va) * vX;
- var bot = ve + (Vector512.Load(f + ch) - ve) * vX;
- var v = top + (bot - top) * vY;
- if (relu) v = Vector512.Max(v, F32x16.Zero);
- if (add != null)
- {
- var sk = Vector512.Load(add + ch);
- v += addRelu ? Vector512.Max(sk, F32x16.Zero) : sk;
- }
- v.Store(o + ch);
- }
- }
-
- /// MaxPool, one output pixel per step: max over the valid taps in registers, one store
- /// (the generic path copies the first tap and read-modify-writes the output per tap).
- private static void MaxPool512(float* sp, int iw, int c, float* dp, int oh, int ow, int kh, int kw,
- int[] rm, int[] cm, bool relu)
- {
- For(oh, (long)oh * ow * c * kh * kw, y =>
- {
- float** taps = stackalloc float*[kh * kw];
- for (int x = 0; x < ow; x++)
- {
- int n = 0;
- for (int ky = 0; ky < kh; ky++)
- {
- int r = rm[y * kh + ky];
- if (r < 0) continue;
- for (int kx = 0; kx < kw; kx++)
- {
- int cc = cm[x * kw + kx];
- if (cc >= 0) taps[n++] = sp + ((long)r * iw + cc) * c;
- }
- }
- float* o = dp + ((long)y * ow + x) * c;
- if (n == 0) { Fill(o, c, 0f); continue; }
- for (int ch = 0; ch < c; ch += 16)
- {
- var v = Vector512.Load(taps[0] + ch);
- for (int k = 1; k < n; k++) v = Vector512.Max(v, Vector512.Load(taps[k] + ch));
- if (relu) v = Vector512.Max(v, F32x16.Zero);
- v.Store(o + ch);
- }
- }
- });
- }
-
- ///
- /// for the AVX-512 tier: 16-lane resize, and the c-long oc dot runs as four
- /// interleaved per-pixel FMA chains (one chain per pixel was FMA-latency bound). Each pixel keeps
- /// its bias-initialized, channel-ascending chain, so outputs match the AVX2 path bit for bit.
- ///
- private static void UpHead512(float* sp, int iw, int c, float* kpp, bool skipConv, DenseArgs ca, float* dp, int oh, int ow,
- int[] y0, int[] y1, int[] x0, int[] x1, float[] wy, float[] wx, bool relu, bool skipRelu,
- float* wp, float* bp, int oc, ActivationKind act)
- {
- bool v512 = c % 16 == 0;
- For(oh, (long)oh * ow * c * (oc + 4), y =>
- {
- float* s = stackalloc float[4 * c + 8];
- float* acc = stackalloc float[4 * 8];
- float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c;
- var vY = new System.Numerics.Vector(wy[y]);
- var vY16 = Vector512.Create(wy[y]);
- float wY = wy[y];
- float* orow = dp + (long)y * ow * oc;
- float* krow = kpp + (long)y * ow * c;
- float[]? band = skipConv ? BandBuf((long)ow * c) : null;
- fixed (float* sb = band)
- {
- if (skipConv)
- {
- ConvRow(ca, y, sb, null); // the skip row, recomputed by the same conv kernel
- krow = sb;
- }
- for (int x = 0; x < ow; x += 4)
- {
- int n = Math.Min(4, ow - x);
- for (int i = 0; i < n; i++)
- {
- int xx = x + i;
- float* a = r0 + (long)x0[xx] * c, b = r0 + (long)x1[xx] * c, e = r1 + (long)x0[xx] * c, f = r1 + (long)x1[xx] * c;
- if (v512) ResizePixel512(a, b, e, f, Vector512.Create(wx[xx]), vY16, c, relu, krow + (long)xx * c, skipRelu, s + i * c);
- else ResizePixel(a, b, e, f, wx[xx], wY, new System.Numerics.Vector(wx[xx]), vY, c, relu, krow + (long)xx * c, skipRelu, s + i * c);
- }
- UpHeadDot4(s, c, wp, bp, acc, n);
- for (int i = 0; i < n; i++)
- {
- float* o = orow + (long)(x + i) * oc;
- for (int l = 0; l < oc; l++) o[l] = acc[i * 8 + l];
- }
- }
- if (act != ActivationKind.None) ActRow(orow, ow * oc, act, true);
- }
- });
- }
-
- /// acc[i*8 + l] = bias[l] + Σ_ch s_i[ch]·w[ch][l] for 4 pixels (s_i = s + i*c), chains interleaved.
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void UpHeadDot4(float* s, int c, float* wp, float* bp, float* acc, int n)
- {
- float* s0 = s, s1 = n > 1 ? s + c : s, s2 = n > 2 ? s + 2 * c : s, s3 = n > 3 ? s + 3 * c : s;
- F32x8 a0 = Avx.LoadVector256(bp), a1 = a0, a2 = a0, a3 = a0;
- float* w = wp;
- for (int ch = 0; ch < c; ch++, w += 8)
- {
- F32x8 wv = Avx.LoadVector256(w);
- a0 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s0 + ch), wv, a0);
- a1 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s1 + ch), wv, a1);
- a2 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s2 + ch), wv, a2);
- a3 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s3 + ch), wv, a3);
- }
- Avx.Store(acc, a0); Avx.Store(acc + 8, a1); Avx.Store(acc + 16, a2); Avx.Store(acc + 24, a3);
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Neon.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Neon.cs
deleted file mode 100644
index 277f774..0000000
--- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Neon.cs
+++ /dev/null
@@ -1,373 +0,0 @@
-using System.Runtime.CompilerServices;
-using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.Arm;
-using Sdcb.SimdGcdrNet.Engine;
-
-using V4 = System.Runtime.Intrinsics.Vector128;
-
-namespace Sdcb.SimdGcdrNet.Kernels;
-
-///
-/// AArch64 AdvSimd (NEON) tier: the 6 × 16 conv / Gemm micro-kernel as fmla-by-element on
-/// 4-channel input loads, 24 register accumulators (of 32 vector registers), used wherever the
-/// x86 tiers take ; direct convs feed it kernel-row runs (kw·ic contiguous
-/// NHWC floats per tap row) so the 3- / 6-channel stems run the same loop.
-///
-internal static unsafe partial class Nhwc
-{
- /// NEON tier switch: AdvSimd.Arm64.IsSupported (DOTNET_EnableAdvSimd=0 drops to the
- /// scalar kernels). Settable only for the in-process tier comparison tests.
- internal static bool UseNeon = AdvSimd.Arm64.IsSupported;
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static V4 Ld4(float* p) => AdvSimd.LoadVector128(p);
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static V4 Fma4(V4 acc, V4 a, V4 b) => AdvSimd.FusedMultiplyAdd(acc, a, b);
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static V4 Act4(V4 v, ActivationKind act) => act switch
- {
- ActivationKind.Relu => AdvSimd.Max(v, V4.Zero),
- ActivationKind.Sigmoid => SimdMath.SigmoidCore(v),
- ActivationKind.Gelu => SimdMath.GeluCore(v),
- _ => v,
- };
-
- /// 6 × 16 tile: into a scratch tile, then the residual /
- /// activation epilogue per row.
- private static void TileNeon(float** ptrs, int rows, int taps, int ic, float* wb, float* bb,
- float* outp, float* addp, int oc, int lanes, ActivationKind act)
- {
- float* part = stackalloc float[Tile * Ob];
- TileNeonK(ptrs, taps, ic, wb, bb, part, Ob);
- TileNeonStore(part, rows, outp, addp, oc, lanes, act);
- }
-
- /// Residual / activation epilogue of a finished 6 × 16 scratch tile.
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void TileNeonStore(float* part, int rows, float* outp, float* addp, int oc, int lanes, ActivationKind act)
- {
- for (int r = 0; r < rows; r++, part += Ob)
- {
- float* o = outp + (long)r * oc;
- float* ad = addp == null ? null : addp + (long)r * oc;
- if (lanes == Ob)
- {
- for (int l = 0; l < Ob; l += 4)
- {
- V4 v = Ld4(part + l);
- if (ad != null) v = AdvSimd.Add(v, Ld4(ad + l));
- AdvSimd.Store(o + l, Act4(v, act));
- }
- continue;
- }
- for (int l = 0; l < lanes; l++)
- o[l] = ActS(ad != null ? part[l] + ad[l] : part[l], act);
- }
- }
-
- /// 6 × 16 k loop over all ic channels of every tap (weights [tap][ic][16]), bias-initialized.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void TileNeonK(float** ptrs, int taps, int ic, float* wb, float* bb, float* dst, int ds) =>
- TileNeonK(ptrs, taps, 0, ic, (long)ic * Ob, wb, bb, dst, ds);
-
- ///
- /// The 6 × 16 k loop: dst[r*ds + 0..15] = init + Σ x·w for all rows, where
- /// ptrs[tap*Tile + r] + coff = row r's ic inputs of tap and wb + tap·wTap its [ic][16] weights;
- /// init = bias , or (bb == null) the partial sums already in dst. Four
- /// channels per step (one 128-bit load per pixel, fmla by element; the next weight vector is
- /// loaded one group ahead for the in-order A53), ic % 4 leftovers by broadcast. Never calls
- /// out, so the 24 accumulators stay in registers.
- ///
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void TileNeonK(float** ptrs, int taps, int coff, int ic, long wTap, float* wb, float* bb, float* dst, int ds,
- bool relu = false)
- {
- V4 a0, a1, a2, a3, a4, a5, b0, b1, b2, b3, b4, b5, c0, c1, c2, c3, c4, c5, d0, d1, d2, d3, d4, d5;
- if (bb != null)
- {
- a0 = Ld4(bb); b0 = Ld4(bb + 4); c0 = Ld4(bb + 8); d0 = Ld4(bb + 12);
- a1 = a2 = a3 = a4 = a5 = a0;
- b1 = b2 = b3 = b4 = b5 = b0;
- c1 = c2 = c3 = c4 = c5 = c0;
- d1 = d2 = d3 = d4 = d5 = d0;
- }
- else
- {
- float* s = dst;
- a0 = Ld4(s); b0 = Ld4(s + 4); c0 = Ld4(s + 8); d0 = Ld4(s + 12); s += ds;
- a1 = Ld4(s); b1 = Ld4(s + 4); c1 = Ld4(s + 8); d1 = Ld4(s + 12); s += ds;
- a2 = Ld4(s); b2 = Ld4(s + 4); c2 = Ld4(s + 8); d2 = Ld4(s + 12); s += ds;
- a3 = Ld4(s); b3 = Ld4(s + 4); c3 = Ld4(s + 8); d3 = Ld4(s + 12); s += ds;
- a4 = Ld4(s); b4 = Ld4(s + 4); c4 = Ld4(s + 8); d4 = Ld4(s + 12); s += ds;
- a5 = Ld4(s); b5 = Ld4(s + 4); c5 = Ld4(s + 8); d5 = Ld4(s + 12);
- }
- int ic4 = ic & ~3;
- for (int tap = 0; tap < taps; tap++)
- {
- float** pp = ptrs + tap * Tile;
- float* p0 = pp[0] + coff, p1 = pp[1] + coff, p2 = pp[2] + coff, p3 = pp[3] + coff, p4 = pp[4] + coff, p5 = pp[5] + coff;
- float* w = wb + tap * wTap;
- int c = 0;
- for (; c < ic4; c += 4, w += 4 * Ob)
- {
- V4 x0 = Ld4(p0 + c), x1 = Ld4(p1 + c), x2 = Ld4(p2 + c);
- V4 x3 = Ld4(p3 + c), x4 = Ld4(p4 + c), x5 = Ld4(p5 + c);
- V4 wv0 = Ld4(w), wv1;
- wv1 = Ld4(w + 4);
- a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 0);
- a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 0);
- a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 0);
- a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 0);
- a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 0);
- a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 0);
- wv0 = Ld4(w + 8);
- b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 0);
- b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 0);
- b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 0);
- b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 0);
- b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 0);
- b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 0);
- wv1 = Ld4(w + 12);
- c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 0);
- c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 0);
- c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 0);
- c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 0);
- c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 0);
- c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 0);
- wv0 = Ld4(w + 16);
- d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 0);
- d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 0);
- d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 0);
- d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 0);
- d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 0);
- d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 0);
- wv1 = Ld4(w + 20);
- a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 1);
- a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 1);
- a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 1);
- a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 1);
- a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 1);
- a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 1);
- wv0 = Ld4(w + 24);
- b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 1);
- b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 1);
- b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 1);
- b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 1);
- b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 1);
- b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 1);
- wv1 = Ld4(w + 28);
- c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 1);
- c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 1);
- c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 1);
- c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 1);
- c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 1);
- c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 1);
- wv0 = Ld4(w + 32);
- d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 1);
- d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 1);
- d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 1);
- d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 1);
- d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 1);
- d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 1);
- wv1 = Ld4(w + 36);
- a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 2);
- a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 2);
- a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 2);
- a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 2);
- a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 2);
- a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 2);
- wv0 = Ld4(w + 40);
- b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 2);
- b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 2);
- b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 2);
- b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 2);
- b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 2);
- b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 2);
- wv1 = Ld4(w + 44);
- c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 2);
- c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 2);
- c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 2);
- c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 2);
- c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 2);
- c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 2);
- wv0 = Ld4(w + 48);
- d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 2);
- d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 2);
- d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 2);
- d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 2);
- d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 2);
- d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 2);
- wv1 = Ld4(w + 52);
- a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 3);
- a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 3);
- a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 3);
- a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 3);
- a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 3);
- a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 3);
- wv0 = Ld4(w + 56);
- b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 3);
- b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 3);
- b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 3);
- b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 3);
- b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 3);
- b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 3);
- wv1 = Ld4(w + 60);
- c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 3);
- c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 3);
- c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 3);
- c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 3);
- c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 3);
- c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 3);
- d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 3);
- d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 3);
- d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 3);
- d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 3);
- d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 3);
- d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 3);
- }
- for (; c < ic; c++, w += Ob)
- {
- V4 w0 = Ld4(w), w1 = Ld4(w + 4), w2 = Ld4(w + 8), w3 = Ld4(w + 12);
- V4 x = AdvSimd.LoadAndReplicateToVector128(p0 + c);
- a0 = Fma4(a0, w0, x); b0 = Fma4(b0, w1, x); c0 = Fma4(c0, w2, x); d0 = Fma4(d0, w3, x);
- x = AdvSimd.LoadAndReplicateToVector128(p1 + c);
- a1 = Fma4(a1, w0, x); b1 = Fma4(b1, w1, x); c1 = Fma4(c1, w2, x); d1 = Fma4(d1, w3, x);
- x = AdvSimd.LoadAndReplicateToVector128(p2 + c);
- a2 = Fma4(a2, w0, x); b2 = Fma4(b2, w1, x); c2 = Fma4(c2, w2, x); d2 = Fma4(d2, w3, x);
- x = AdvSimd.LoadAndReplicateToVector128(p3 + c);
- a3 = Fma4(a3, w0, x); b3 = Fma4(b3, w1, x); c3 = Fma4(c3, w2, x); d3 = Fma4(d3, w3, x);
- x = AdvSimd.LoadAndReplicateToVector128(p4 + c);
- a4 = Fma4(a4, w0, x); b4 = Fma4(b4, w1, x); c4 = Fma4(c4, w2, x); d4 = Fma4(d4, w3, x);
- x = AdvSimd.LoadAndReplicateToVector128(p5 + c);
- a5 = Fma4(a5, w0, x); b5 = Fma4(b5, w1, x); c5 = Fma4(c5, w2, x); d5 = Fma4(d5, w3, x);
- }
- }
- if (relu)
- {
- V4 z = V4.Zero;
- a0 = AdvSimd.Max(a0, z); b0 = AdvSimd.Max(b0, z); c0 = AdvSimd.Max(c0, z); d0 = AdvSimd.Max(d0, z);
- a1 = AdvSimd.Max(a1, z); b1 = AdvSimd.Max(b1, z); c1 = AdvSimd.Max(c1, z); d1 = AdvSimd.Max(d1, z);
- a2 = AdvSimd.Max(a2, z); b2 = AdvSimd.Max(b2, z); c2 = AdvSimd.Max(c2, z); d2 = AdvSimd.Max(d2, z);
- a3 = AdvSimd.Max(a3, z); b3 = AdvSimd.Max(b3, z); c3 = AdvSimd.Max(c3, z); d3 = AdvSimd.Max(d3, z);
- a4 = AdvSimd.Max(a4, z); b4 = AdvSimd.Max(b4, z); c4 = AdvSimd.Max(c4, z); d4 = AdvSimd.Max(d4, z);
- a5 = AdvSimd.Max(a5, z); b5 = AdvSimd.Max(b5, z); c5 = AdvSimd.Max(c5, z); d5 = AdvSimd.Max(d5, z);
- }
- StoreRow(dst, a0, b0, c0, d0); dst += ds;
- StoreRow(dst, a1, b1, c1, d1); dst += ds;
- StoreRow(dst, a2, b2, c2, d2); dst += ds;
- StoreRow(dst, a3, b3, c3, d3); dst += ds;
- StoreRow(dst, a4, b4, c4, d4); dst += ds;
- StoreRow(dst, a5, b5, c5, d5);
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void StoreRow(float* d, V4 a, V4 b, V4 c, V4 e)
- {
- AdvSimd.Store(d, a); AdvSimd.Store(d + 4, b); AdvSimd.Store(d + 8, c); AdvSimd.Store(d + 12, e);
- }
-
- [ThreadStatic] private static float[]? t_runs;
-
- ///
- /// One output row for oc <= 4, c % 4 == 0: per 4 channels the bilinear
- /// lerps (fma), [relu], + [relu](skip), then fmla by element into one 4-lane accumulator against
- /// the packed [c][8] weights — the resized pixel never leaves registers.
- ///
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void UpHeadRowNeon(float* r0, float* r1, int* x0, int* x1, float* wx, float wy, float* krow,
- int c, int ow, float* wp, float* bp, float* orow, int oc, bool relu, bool skipRelu, ActivationKind act)
- {
- V4 vY = Vector128.Create(wy), bias = Ld4(bp);
- for (int x = 0; x < ow; x++)
- {
- float* a = r0 + (long)x0[x] * c, b = r0 + (long)x1[x] * c, e = r1 + (long)x0[x] * c, f = r1 + (long)x1[x] * c;
- float* k = krow + (long)x * c;
- V4 vX = Vector128.Create(wx[x]), acc = bias;
- float* w = wp;
- for (int ch = 0; ch < c; ch += 4, w += 32)
- {
- V4 va = Ld4(a + ch), ve = Ld4(e + ch);
- V4 top = Fma4(va, AdvSimd.Subtract(Ld4(b + ch), va), vX);
- V4 bot = Fma4(ve, AdvSimd.Subtract(Ld4(f + ch), ve), vX);
- V4 v = Fma4(top, AdvSimd.Subtract(bot, top), vY);
- if (relu) v = AdvSimd.Max(v, V4.Zero);
- V4 s = Ld4(k + ch);
- v = AdvSimd.Add(v, skipRelu ? AdvSimd.Max(s, V4.Zero) : s);
- acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w), v, 0);
- acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 8), v, 1);
- acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 16), v, 2);
- acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 24), v, 3);
- }
- acc = Act4(acc, act);
- float* o = orow + (long)x * oc;
- o[0] = acc.ToScalar();
- if (oc > 1) o[1] = acc.GetElement(1);
- if (oc > 2) o[2] = acc.GetElement(2);
- if (oc > 3) o[3] = acc.GetElement(3);
- }
- }
-
- /// Thread-local pinned scratch (the pointer outlives any fixed scope).
- private static T* Pinned(ref T[]? slot, int n) where T : unmanaged
- {
- if (slot is not { } b || b.Length < n) slot = b = GC.AllocateUninitializedArray(n, pinned: true);
- return (T*)Unsafe.AsPointer(ref System.Runtime.InteropServices.MemoryMarshal.GetArrayDataReference(b));
- }
-
- ///
- /// on NEON: a k×k conv as kh taps over kernel-row runs. For each tap row
- /// the kw·ic inputs of a pixel are contiguous in NHWC (weights [ky][kx][ic][16] are already
- /// [ky][kx·ic][16]), so interior pixels point straight into the source; pixels whose run
- /// crosses a padded / reflected border are gathered into a thread-local buffer, constant-pad
- /// rows read .
- ///
- private static void DenseRowNeon(in DenseArgs a, int y, int blk, float* orow, float* arow)
- {
- int kh = a.Kh, kw = a.Kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, run = kw * ic;
- float** ptrs = stackalloc float*[Tile * kh];
- float* part = stackalloc float[Tile * Ob];
- float* wb = a.W + (long)blk * kh * run * Ob;
- float* bb = a.Bias + blk * Ob;
- int lanes = Math.Min(Ob, oc - blk * Ob);
- bool direct = lanes == Ob && arow == null && a.Act is ActivationKind.None or ActivationKind.Relu;
- float* gather = null;
- for (int x0 = 0; x0 < ow; x0 += Tile)
- {
- int rows = Math.Min(Tile, ow - x0);
- for (int ky = 0; ky < kh; ky++)
- {
- int r = a.Rm[y * kh + ky];
- for (int i = 0; i < Tile; i++)
- {
- int* cmx = a.Cm + (x0 + Math.Min(i, rows - 1)) * kw;
- int cx = cmx[0];
- float* p;
- if (r < 0) p = a.PadVec;
- else if (cx >= 0 && cmx[kw - 1] == cx + kw - 1) p = a.Src + ((long)r * a.Iw + cx) * ic;
- else
- {
- if (gather == null) gather = Pinned(ref t_runs, Tile * kh * run);
- p = gather + (ky * Tile + i) * run;
- for (int kx = 0; kx < kw; kx++)
- {
- int c = cmx[kx];
- Buffer.MemoryCopy(c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic, p + kx * ic, ic * 4L, ic * 4L);
- }
- }
- ptrs[ky * Tile + i] = p;
- }
- }
- long o = (long)x0 * oc + blk * Ob;
- if (direct && rows == Tile)
- {
- // full tile, no residual / transcendental epilogue: store straight from the accumulators
- TileNeonK(ptrs, kh, 0, run, (long)run * Ob, wb, bb, orow + o, oc, a.Act == ActivationKind.Relu);
- continue;
- }
- TileNeonK(ptrs, kh, run, wb, bb, part, Ob);
- TileNeonStore(part, rows, orow + o, arow != null ? arow + o : null, oc, lanes, a.Act);
- }
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.PackedSimd.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.PackedSimd.cs
deleted file mode 100644
index ba2568a..0000000
--- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.PackedSimd.cs
+++ /dev/null
@@ -1,251 +0,0 @@
-using System.Runtime.CompilerServices;
-using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.Wasm;
-using Sdcb.SimdGcdrNet.Engine;
-
-using V4 = System.Runtime.Intrinsics.Vector128;
-
-namespace Sdcb.SimdGcdrNet.Kernels;
-
-///
-/// WebAssembly SIMD128 (PackedSimd) tier, after SimdPaddleOCR's Nhwc.Conv.PackedSimd. WASM SIMD has
-/// no FMA (relaxed SIMD is off) and V8 maps v128 values onto at most 15 allocatable XMM registers
-/// on x64, so the conv / Gemm micro-kernel is pixels × 8 oc (8 accumulators +
-/// 2 weights + 1 splat), run twice per 16-oc packed block; each input float is broadcast straight
-/// from memory (v128.load32_splat). Every reduction keeps the Vector<float> tier's per-lane
-/// order (bias-init, taps then channels ascending, separate multiply then add), so conv sums are
-/// bit-identical to it; direct convs read NHWC kernel-row runs like the NEON tier (same k order).
-///
-internal static unsafe partial class Nhwc
-{
- /// PackedSimd tier switch: PackedSimd.IsSupported (browser-wasm with SIMD).
- /// Settable only for in-process tier comparisons (WasmBench --tiers / --vector).
- internal static bool UsePacked = PackedSimd.IsSupported;
-
- private const int PkTile = 4;
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static V4 LdP(float* p) => PackedSimd.LoadVector128(p);
-
- /// acc + a·b, rounded twice (no FMA): the Vector<float> tier's acc += a * b.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static V4 MadP(V4 acc, V4 a, V4 b) => PackedSimd.Add(acc, PackedSimd.Multiply(a, b));
-
- // pmax(0, v): one maxps in V8 (f32x4.max needs a NaN / -0 fix-up); equals Max(v, 0) for finite v
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static V4 ActP(V4 v, ActivationKind act) => act switch
- {
- ActivationKind.Relu => PackedSimd.PseudoMax(V4.Zero, v),
- ActivationKind.Sigmoid => SimdMath.SigmoidPk(v),
- ActivationKind.Gelu => SimdMath.GeluPk(v),
- _ => v,
- };
-
- ///
- /// rows × 16 oc: dst[r*ds + 0..15] = bias + Σ x·w, where ptrs[tap*ps + r] are
- /// row r's ic inputs of tap and wb + tap·wTap its [ic][16] weights. Two 8-oc passes; the k loop
- /// never calls out, so the accumulators stay in registers.
- ///
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void TilePkK(float** ptrs, int ps, int taps, int ic, long wTap, float* wb, float* bb, float* dst, int ds)
- {
- for (int h = 0; h < Ob; h += 8)
- {
- V4 a0 = LdP(bb + h), b0 = LdP(bb + h + 4);
- V4 a1 = a0, a2 = a0, a3 = a0, b1 = b0, b2 = b0, b3 = b0;
- for (int tap = 0; tap < taps; tap++)
- {
- float** pp = ptrs + tap * ps;
- float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3];
- float* w = wb + tap * wTap + h;
- for (int c = 0; c < ic; c++, w += Ob)
- {
- V4 w0 = LdP(w), w1 = LdP(w + 4);
- V4 v = PackedSimd.LoadScalarAndSplatVector128(p0 + c);
- a0 = MadP(a0, v, w0); b0 = MadP(b0, v, w1);
- v = PackedSimd.LoadScalarAndSplatVector128(p1 + c);
- a1 = MadP(a1, v, w0); b1 = MadP(b1, v, w1);
- v = PackedSimd.LoadScalarAndSplatVector128(p2 + c);
- a2 = MadP(a2, v, w0); b2 = MadP(b2, v, w1);
- v = PackedSimd.LoadScalarAndSplatVector128(p3 + c);
- a3 = MadP(a3, v, w0); b3 = MadP(b3, v, w1);
- }
- }
- float* d = dst + h;
- PackedSimd.Store(d, a0); PackedSimd.Store(d + 4, b0); d += ds;
- PackedSimd.Store(d, a1); PackedSimd.Store(d + 4, b1); d += ds;
- PackedSimd.Store(d, a2); PackedSimd.Store(d + 4, b2); d += ds;
- PackedSimd.Store(d, a3); PackedSimd.Store(d + 4, b3);
- }
- }
-
- /// Residual / activation epilogue of a finished [rows][16] scratch tile.
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void StorePk(float* part, int rows, float* outp, float* addp, int oc, int lanes, ActivationKind act)
- {
- for (int r = 0; r < rows; r++, part += Ob)
- {
- float* o = outp + (long)r * oc;
- float* ad = addp == null ? null : addp + (long)r * oc;
- if (lanes == Ob)
- {
- for (int l = 0; l < Ob; l += 4)
- {
- V4 v = LdP(part + l);
- if (ad != null) v = PackedSimd.Add(v, LdP(ad + l));
- PackedSimd.Store(o + l, ActP(v, act));
- }
- continue;
- }
- for (int l = 0; l < lanes; l++)
- o[l] = ActS(ad != null ? part[l] + ad[l] : part[l], act);
- }
- }
-
- /// on PackedSimd: kh taps over NHWC kernel-row runs (kw·ic contiguous
- /// floats, see ); border-crossing runs are gathered, constant-pad rows read PadVec.
- private static void DenseRowPk(in DenseArgs a, int y, int blk, float* orow, float* arow)
- {
- int kh = a.Kh, kw = a.Kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, run = kw * ic;
- float** ptrs = stackalloc float*[PkTile * kh];
- float* part = stackalloc float[PkTile * Ob];
- float* wb = a.W + (long)blk * kh * run * Ob;
- float* bb = a.Bias + blk * Ob;
- int lanes = Math.Min(Ob, oc - blk * Ob);
- float* gather = null;
- for (int x0 = 0; x0 < ow; x0 += PkTile)
- {
- int rows = Math.Min(PkTile, ow - x0);
- for (int ky = 0; ky < kh; ky++)
- {
- int r = a.Rm[y * kh + ky];
- for (int i = 0; i < PkTile; i++)
- {
- int* cmx = a.Cm + (x0 + Math.Min(i, rows - 1)) * kw;
- int cx = cmx[0];
- float* p;
- if (r < 0) p = a.PadVec;
- else if (cx >= 0 && cmx[kw - 1] == cx + kw - 1) p = a.Src + ((long)r * a.Iw + cx) * ic;
- else
- {
- if (gather == null) gather = Pinned(ref t_runs, PkTile * kh * run);
- p = gather + (ky * PkTile + i) * run;
- for (int kx = 0; kx < kw; kx++)
- {
- int c = cmx[kx];
- Buffer.MemoryCopy(c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic, p + kx * ic, ic * 4L, ic * 4L);
- }
- }
- ptrs[ky * PkTile + i] = p;
- }
- }
- long o = (long)x0 * oc + blk * Ob;
- TilePkK(ptrs, PkTile, kh, run, (long)run * Ob, wb, bb, part, Ob);
- StorePk(part, rows, orow + o, arow != null ? arow + o : null, oc, lanes, a.Act);
- }
- }
-
- /// on PackedSimd: -row tiles, 12 per task sharing one weight panel walk.
- private static void GemmPk(ReadOnlySpan a, int m, int k, Span dst, int n, NhwcConvPlan kp, ActivationKind act)
- {
- const int G = 12;
- int nb = kp.Blocks, tiles = (m + PkTile - 1) / PkTile, groups = (tiles + G - 1) / G;
- fixed (float* ap0 = a, dp0 = dst, wp0 = kp.W, bp0 = kp.Bias)
- {
- float* ap = ap0, dp = dp0, wp = wp0, bp = bp0;
- For(groups * nb, (long)m * n * k / 64, t =>
- {
- int g = t / nb, blk = t % nb;
- float** ptrs = stackalloc float*[PkTile];
- float* part = stackalloc float[PkTile * Ob];
- float* wb = wp + (long)blk * k * Ob, bb = bp + blk * Ob;
- int lanes = Math.Min(Ob, n - blk * Ob);
- for (int tile = g * G, te = Math.Min(tiles, tile + G); tile < te; tile++)
- {
- int m0 = tile * PkTile, rows = Math.Min(PkTile, m - m0);
- for (int i = 0; i < PkTile; i++) ptrs[i] = ap + (long)(m0 + Math.Min(i, rows - 1)) * k;
- TilePkK(ptrs, PkTile, 1, k, (long)k * Ob, wb, bb, part, Ob);
- StorePk(part, rows, dp + (long)m0 * n + blk * Ob, null, n, lanes, act);
- }
- });
- }
- }
-
- ///
- /// One depthwise output pixel on PackedSimd: 16 then 4 channels per step, bias-initialized, taps
- /// ascending, + residual, activation. Returns the first channel left for the scalar tail.
- ///
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static int DepthwisePixelPk(float** ptrs, int taps, int c, float* wp, float* bp, float* ad, float* o, ActivationKind act)
- {
- int ch = 0;
- for (; ch + 16 <= c; ch += 16)
- {
- V4 a0 = LdP(bp + ch), a1 = LdP(bp + ch + 4), a2 = LdP(bp + ch + 8), a3 = LdP(bp + ch + 12);
- float* w = wp + ch;
- for (int tap = 0; tap < taps; tap++, w += c)
- {
- float* s = ptrs[tap] + ch;
- a0 = MadP(a0, LdP(s), LdP(w));
- a1 = MadP(a1, LdP(s + 4), LdP(w + 4));
- a2 = MadP(a2, LdP(s + 8), LdP(w + 8));
- a3 = MadP(a3, LdP(s + 12), LdP(w + 12));
- }
- if (ad != null)
- {
- a0 = PackedSimd.Add(a0, LdP(ad + ch)); a1 = PackedSimd.Add(a1, LdP(ad + ch + 4));
- a2 = PackedSimd.Add(a2, LdP(ad + ch + 8)); a3 = PackedSimd.Add(a3, LdP(ad + ch + 12));
- }
- PackedSimd.Store(o + ch, ActP(a0, act)); PackedSimd.Store(o + ch + 4, ActP(a1, act));
- PackedSimd.Store(o + ch + 8, ActP(a2, act)); PackedSimd.Store(o + ch + 12, ActP(a3, act));
- }
- for (; ch + 4 <= c; ch += 4)
- {
- V4 acc = LdP(bp + ch);
- for (int tap = 0; tap < taps; tap++)
- acc = MadP(acc, LdP(ptrs[tap] + ch), LdP(wp + tap * c + ch));
- if (ad != null) acc = PackedSimd.Add(acc, LdP(ad + ch));
- PackedSimd.Store(o + ch, ActP(acc, act));
- }
- return ch;
- }
-
- ///
- /// One output row for oc <= 4, c % 4 == 0 (see ):
- /// per 4 channels the bilinear lerps, [relu], + [relu](skip), then each channel is splatted back
- /// from a 16-byte spill (WASM SIMD has no by-element multiply) into one 4-lane accumulator.
- /// Same per-element ops as the unfused Vector<float> path.
- ///
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void UpHeadRowPk(float* r0, float* r1, int* x0, int* x1, float* wx, float wy, float* krow,
- int c, int ow, float* wp, float* bp, float* orow, int oc, bool relu, bool skipRelu, ActivationKind act)
- {
- V4 vY = PackedSimd.Splat(wy), bias = LdP(bp);
- float* s = stackalloc float[4];
- for (int x = 0; x < ow; x++)
- {
- float* a = r0 + (long)x0[x] * c, b = r0 + (long)x1[x] * c, e = r1 + (long)x0[x] * c, f = r1 + (long)x1[x] * c;
- float* k = krow + (long)x * c;
- V4 vX = PackedSimd.Splat(wx[x]), acc = bias;
- float* w = wp;
- for (int ch = 0; ch < c; ch += 4, w += 32)
- {
- V4 va = LdP(a + ch), ve = LdP(e + ch);
- V4 top = MadP(va, PackedSimd.Subtract(LdP(b + ch), va), vX);
- V4 bot = MadP(ve, PackedSimd.Subtract(LdP(f + ch), ve), vX);
- V4 v = MadP(top, PackedSimd.Subtract(bot, top), vY);
- if (relu) v = PackedSimd.PseudoMax(V4.Zero, v);
- V4 sk = LdP(k + ch);
- v = PackedSimd.Add(v, skipRelu ? PackedSimd.PseudoMax(V4.Zero, sk) : sk);
- PackedSimd.Store(s, v);
- acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s), LdP(w));
- acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 1), LdP(w + 8));
- acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 2), LdP(w + 16));
- acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 3), LdP(w + 24));
- }
- PackedSimd.Store(s, ActP(acc, act));
- float* o = orow + (long)x * oc;
- for (int l = 0; l < oc; l++) o[l] = s[l];
- }
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.cs
deleted file mode 100644
index e3122af..0000000
--- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.cs
+++ /dev/null
@@ -1,904 +0,0 @@
-using System.Numerics;
-using System.Runtime.CompilerServices;
-using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.Arm;
-using System.Runtime.Intrinsics.Wasm;
-using System.Runtime.Intrinsics.X86;
-using Sdcb.SimdGcdrNet.Engine;
-
-using F32x8 = System.Runtime.Intrinsics.Vector256;
-using V128 = System.Runtime.Intrinsics.Vector128;
-using V256 = System.Runtime.Intrinsics.Vector256;
-
-namespace Sdcb.SimdGcdrNet.Kernels;
-
-/// Backend-private packed conv weights for the NHWC CPU kernels (lives in ).
-internal sealed class NhwcConvPlan
-{
- public required float[] W; // dense: [oc/16][tap][ic][16] (oc zero-padded); depthwise: [tap][c]
- public required float[] Bias; // dense: padded to blocks*16; depthwise: [c]
- public bool Depthwise;
- public int Blocks;
- public float[]? WinoU; // Winograd F(4x4,3x3) weights [36][oc/16][ic][16], null = direct conv
-}
-
-/// Packed weights: [c][8] oc lanes (zero-padded) + bias[8].
-internal sealed class UpHeadPlan
-{
- public required float[] W;
- public required float[] Bias;
- public int Oc, C;
-}
-
-///
-/// Channels-last (NHWC, N==1) kernels. Shapes are passed in storage order
-/// ([h,w,c]). Every pad mode goes through precomputed source row/col maps
-/// (-1 = constant pad → a pad-value pixel vector), so the inner loops never
-/// branch on borders. Dense/pointwise conv: register-tiled 6 pixels × 16 oc
-/// AVX2 micro-kernel (after SimdPaddleOCR's Nhwc.Conv) with bias-init and a
-/// fused residual + activation epilogue; AVX-512 widens it to 10 × 32 (Nhwc.Avx512.cs),
-/// NEON keeps 6 × 16 (Nhwc.Neon.cs), WASM SIMD128 narrows it to 4 × 8 (Nhwc.PackedSimd.cs);
-/// other tiers use Vector<float>.
-///
-internal static unsafe partial class Nhwc
-{
- private const int Ob = 16, Tile = 6;
- private static readonly ParallelOptions PO = new() { MaxDegreeOfParallelism = Avx2Kernels.Threads };
- private static bool UseAvx => Avx2.IsSupported && Fma.IsSupported;
-
- private static void For(int n, long work, Action body)
- {
- if (work < 32768 || n == 1) { for (int i = 0; i < n; i++) body(i); return; }
- Parallel.For(0, n, PO, body);
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static int Reflect(int i, int dim)
- {
- if (dim <= 1) return 0;
- while (i < 0 || i >= dim) i = i < 0 ? -i : 2 * dim - 2 - i;
- return i;
- }
-
- /// map[o*k+kk] = source index of output o, tap kk, or -1 for constant pad.
- private static int[] Map(int outLen, int k, int stride, int pad, int inLen, PadMode mode)
- {
- int[] m = new int[outLen * k];
- for (int o = 0; o < outLen; o++)
- for (int kk = 0; kk < k; kk++)
- {
- int s = o * stride + kk - pad;
- m[o * k + kk] = (uint)s < (uint)inLen ? s : mode == PadMode.Reflect ? Reflect(s, inLen) : -1;
- }
- return m;
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static float ActS(float v, ActivationKind act) =>
- act == ActivationKind.Relu ? MathF.Max(v, 0f) :
- act == ActivationKind.Sigmoid ? 1f / (1f + MathF.Exp(-v)) :
- act == ActivationKind.Gelu ? 0.5f * v * (1f + SimdMath.ErfScalar(v * 0.70710678f)) : v;
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static F32x8 Act8(F32x8 v, ActivationKind act) => act switch
- {
- ActivationKind.Relu => V256.Max(v, F32x8.Zero),
- ActivationKind.Sigmoid => SimdMath.SigmoidCore(v),
- ActivationKind.Gelu => SimdMath.GeluCore(v),
- _ => v,
- };
-
- // ======================== weight packing ========================
- public static NhwcConvPlan Pack(ReadOnlySpan w, int[] wShape, ReadOnlySpan bias, int group, int ic)
- {
- int oc = wShape[0], icg = wShape[1], taps = wShape[2] * wShape[3];
- if (group > 1)
- {
- if (group != ic || icg != 1 || oc != ic)
- throw new NotSupportedException($"NHWC grouped conv supports depthwise only (g={group}, ic={ic}, oc={oc}).");
- float[] pw = new float[taps * oc];
- for (int c = 0; c < oc; c++)
- for (int t = 0; t < taps; t++) pw[t * oc + c] = w[c * taps + t];
- float[] pb = new float[oc];
- if (!bias.IsEmpty) bias.CopyTo(pb);
- return new NhwcConvPlan { W = pw, Bias = pb, Depthwise = true };
- }
- int nb = (oc + Ob - 1) / Ob;
- float[] p = new float[checked(nb * taps * ic * Ob)];
- for (int o = 0; o < oc; o++)
- for (int c = 0; c < ic; c++)
- for (int t = 0; t < taps; t++)
- p[(((o / Ob) * taps + t) * ic + c) * Ob + o % Ob] = w[(o * ic + c) * taps + t];
- float[] b = new float[nb * Ob];
- if (!bias.IsEmpty) bias.CopyTo(b);
- return new NhwcConvPlan { W = p, Bias = b, Blocks = nb };
- }
-
- /// Packs Gemm B ([K,N], or [N,K] when transB) as [N/16][K][16] — a 1x1 conv panel.
- public static NhwcConvPlan PackGemm(ReadOnlySpan b, int k, int n, bool transB, ReadOnlySpan bias)
- {
- int nb = (n + Ob - 1) / Ob;
- float[] p = new float[checked(nb * k * Ob)];
- for (int o = 0; o < n; o++)
- for (int c = 0; c < k; c++)
- p[((o / Ob) * k + c) * Ob + o % Ob] = transB ? b[o * k + c] : b[c * n + o];
- float[] pb = new float[nb * Ob];
- if (!bias.IsEmpty) bias.CopyTo(pb);
- return new NhwcConvPlan { W = p, Bias = pb, Blocks = nb };
- }
-
- /// dst[M,N] = act(a[M,K]·B + bias) through the 6x16 conv micro-kernel.
- public static void Gemm(ReadOnlySpan a, int m, int k, Span dst, int n,
- NhwcConvPlan kp, ActivationKind act)
- {
- if (UseAvx512) { Gemm512(a, m, k, dst, n, kp, act); return; }
- if (UsePacked) { GemmPk(a, m, k, dst, n, kp, act); return; }
- const int G = 8; // row tiles per task sharing one weight panel walk
- int nb = kp.Blocks, tiles = (m + Tile - 1) / Tile, groups = (tiles + G - 1) / G;
- bool avx = UseAvx;
- fixed (float* ap0 = a, dp0 = dst, wp0 = kp.W, bp0 = kp.Bias)
- {
- float* ap = ap0, dp = dp0, wp = wp0, bp = bp0;
- For(groups * nb, (long)m * n * k / 64, t =>
- {
- int g = t / nb, blk = t % nb;
- float** ptrs = stackalloc float*[Tile];
- float* wb = wp + (long)blk * k * Ob, bb = bp + blk * Ob;
- int lanes = Math.Min(Ob, n - blk * Ob);
- for (int tile = g * G, te = Math.Min(tiles, tile + G); tile < te; tile++)
- {
- int m0 = tile * Tile, rows = Math.Min(Tile, m - m0);
- for (int i = 0; i < Tile; i++) ptrs[i] = ap + (long)(m0 + Math.Min(i, rows - 1)) * k;
- float* outp = dp + (long)m0 * n + blk * Ob;
- if (avx) TileAvx(ptrs, rows, 1, k, wb, bb, outp, null, n, lanes, act);
- else TileVec(ptrs, rows, 1, k, wb, bb, outp, null, n, lanes, act);
- }
- });
- }
- }
-
- // ======================== conv ========================
- public static void Conv(ReadOnlySpan src, int ih, int iw, int ic,
- Span dst, int oh, int ow, int oc, ConvP p, NhwcConvPlan kp, ReadOnlySpan add)
- {
- if (!add.IsEmpty && add.Length != dst.Length)
- throw new NotSupportedException($"conv residual addend has {add.Length} elems, expected {dst.Length}.");
- if (kp.WinoU is not null)
- {
- WinogradConv(src, ih, iw, ic, dst, oh, ow, oc, p, kp, add);
- return;
- }
- int kh = p.Kernel[0], kw = p.Kernel[1];
- int[] rm = Map(oh, kh, p.Strides[0], p.Pads[0], ih, p.PadMode);
- int[] cm = Map(ow, kw, p.Strides[1], p.Pads[1], iw, p.PadMode);
- float[] padVec = new float[ic * kw + Ob]; // one constant-pad kernel row (Dense512 row runs)
- Array.Fill(padVec, p.PadValue);
- if (kp.Depthwise)
- Depthwise(src, iw, ic, dst, oh, ow, kh, kw, rm, cm, padVec, kp, add, p.Act);
- else if (UseAvx512)
- Dense512(src, iw, ic, dst, oh, ow, oc, kh, kw, rm, cm, padVec, kp, add, p.Act);
- else
- Dense(src, iw, ic, dst, oh, ow, oc, kh, kw, rm, cm, padVec, kp, add, p.Act);
- }
-
- private static void Dense(ReadOnlySpan src, int iw, int ic, Span dst,
- int oh, int ow, int oc, int kh, int kw, int[] rm, int[] cm, float[] padVec,
- NhwcConvPlan kp, ReadOnlySpan add, ActivationKind act)
- {
- int taps = kh * kw, nb = kp.Blocks;
- bool hasAdd = !add.IsEmpty, avx = UseAvx;
- fixed (float* sp0 = src, dp0 = dst, ap0 = add, wp0 = kp.W, bp0 = kp.Bias, pv0 = padVec)
- fixed (int* rm0 = rm, cm0 = cm)
- {
- var a = new DenseArgs(sp0, wp0, bp0, pv0, rm0, cm0, iw, ic, ow, oc, kh, kw, nb, act, avx);
- float* dp = dp0, ap = ap0;
- For(oh * nb, (long)oh * ow * oc * taps * ic / 64, t =>
- {
- int y = t / nb, blk = t % nb;
- long o = (long)y * ow * oc;
- DenseRow(a, y, blk, dp + o, hasAdd ? ap + o : null);
- });
- }
- }
-
- /// Per-call direct-conv constants (pointers into pinned/fixed storage).
- private readonly unsafe struct DenseArgs(float* src, float* w, float* bias, float* padVec, int* rm, int* cm,
- int iw, int ic, int ow, int oc, int kh, int kw, int nb, ActivationKind act, bool avx)
- {
- public readonly float* Src = src, W = w, Bias = bias, PadVec = padVec;
- public readonly int* Rm = rm, Cm = cm;
- public readonly int Iw = iw, Ic = ic, Ow = ow, Oc = oc, Kh = kh, Kw = kw, Nb = nb;
- public readonly ActivationKind Act = act;
- public readonly bool Avx = avx;
- }
-
- /// Output row , oc block of a direct conv
- /// into (the row's first pixel; addend row likewise).
- private static void DenseRow(in DenseArgs a, int y, int blk, float* orow, float* arow)
- {
- if (UseNeon)
- {
- DenseRowNeon(a, y, blk, orow, arow);
- return;
- }
- if (UsePacked)
- {
- DenseRowPk(a, y, blk, orow, arow);
- return;
- }
- int taps = a.Kh * a.Kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, kw = a.Kw;
- float** ptrs = stackalloc float*[Tile * taps];
- float* wb = a.W + (long)blk * taps * ic * Ob;
- float* bb = a.Bias + blk * Ob;
- int lanes = Math.Min(Ob, oc - blk * Ob);
- for (int x0 = 0; x0 < ow; x0 += Tile)
- {
- int rows = Math.Min(Tile, ow - x0);
- for (int tap = 0; tap < taps; tap++)
- {
- int r = a.Rm[y * a.Kh + tap / kw];
- for (int i = 0; i < Tile; i++)
- {
- int c = a.Cm[(x0 + Math.Min(i, rows - 1)) * kw + tap % kw];
- ptrs[tap * Tile + i] = r < 0 || c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic;
- }
- }
- long o = (long)x0 * oc + blk * Ob;
- float* outp = orow + o;
- float* addp = arow != null ? arow + o : null;
- if (a.Avx) TileAvx(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, a.Act);
- else TileVec(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, a.Act);
- }
- }
-
- /// 6 × 16 tile: into a scratch tile, then the
- /// residual / activation epilogue () per row.
- private static void TileAvx(float** ptrs, int rows, int taps, int ic, float* wb, float* bb,
- float* outp, float* addp, int oc, int lanes, ActivationKind act)
- {
- float* part = stackalloc float[Tile * Ob];
- TileAvxK(ptrs, taps, ic, wb, bb, part, Ob);
- for (int r = 0; r < rows; r++, part += Ob)
- Store16(outp + (long)r * oc, addp == null ? null : addp + (long)r * oc,
- Avx.LoadVector256(part), Avx.LoadVector256(part + 8), lanes, act);
- }
-
- ///
- /// The 6 × 16 k loop: dst[r*ds + 0..15] = bias + Σ x·w for all rows. It never
- /// calls out — passing the accumulators to a helper (by reference under the x64 ABIs) makes
- /// RyuJIT keep all twelve on the stack, a load + store around every FMA.
- ///
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void TileAvxK(float** ptrs, int taps, int ic, float* wb, float* bb, float* dst, int ds)
- {
- F32x8 a0 = Avx.LoadVector256(bb), b0 = Avx.LoadVector256(bb + 8);
- F32x8 a1 = a0, a2 = a0, a3 = a0, a4 = a0, a5 = a0;
- F32x8 b1 = b0, b2 = b0, b3 = b0, b4 = b0, b5 = b0;
- for (int tap = 0; tap < taps; tap++)
- {
- float** pp = ptrs + tap * Tile;
- float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3], p4 = pp[4], p5 = pp[5];
- float* w = wb + (long)tap * ic * Ob;
- for (int c = 0; c < ic; c++, w += Ob)
- {
- F32x8 w0 = Avx.LoadVector256(w), w1 = Avx.LoadVector256(w + 8);
- F32x8 v0 = Avx.BroadcastScalarToVector256(p0 + c), v1 = Avx.BroadcastScalarToVector256(p1 + c);
- a0 = Fma.MultiplyAdd(v0, w0, a0); b0 = Fma.MultiplyAdd(v0, w1, b0);
- a1 = Fma.MultiplyAdd(v1, w0, a1); b1 = Fma.MultiplyAdd(v1, w1, b1);
- F32x8 v2 = Avx.BroadcastScalarToVector256(p2 + c), v3 = Avx.BroadcastScalarToVector256(p3 + c);
- a2 = Fma.MultiplyAdd(v2, w0, a2); b2 = Fma.MultiplyAdd(v2, w1, b2);
- a3 = Fma.MultiplyAdd(v3, w0, a3); b3 = Fma.MultiplyAdd(v3, w1, b3);
- F32x8 v4 = Avx.BroadcastScalarToVector256(p4 + c), v5 = Avx.BroadcastScalarToVector256(p5 + c);
- a4 = Fma.MultiplyAdd(v4, w0, a4); b4 = Fma.MultiplyAdd(v4, w1, b4);
- a5 = Fma.MultiplyAdd(v5, w0, a5); b5 = Fma.MultiplyAdd(v5, w1, b5);
- }
- }
- Avx.Store(dst, a0); Avx.Store(dst + 8, b0); dst += ds;
- Avx.Store(dst, a1); Avx.Store(dst + 8, b1); dst += ds;
- Avx.Store(dst, a2); Avx.Store(dst + 8, b2); dst += ds;
- Avx.Store(dst, a3); Avx.Store(dst + 8, b3); dst += ds;
- Avx.Store(dst, a4); Avx.Store(dst + 8, b4); dst += ds;
- Avx.Store(dst, a5); Avx.Store(dst + 8, b5);
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void Store16(float* o, float* add, F32x8 a, F32x8 b, int lanes, ActivationKind act)
- {
- if (lanes == Ob)
- {
- if (add != null) { a = Avx.Add(a, Avx.LoadVector256(add)); b = Avx.Add(b, Avx.LoadVector256(add + 8)); }
- Avx.Store(o, Act8(a, act)); Avx.Store(o + 8, Act8(b, act));
- return;
- }
- // channel tail (e.g. the final oc=3 conv): only the valid lanes touch memory
- float* t = stackalloc float[Ob];
- Avx.Store(t, a); Avx.Store(t + 8, b);
- for (int l = 0; l < lanes; l++)
- o[l] = ActS(add != null ? t[l] + add[l] : t[l], act);
- }
-
- private static void TileVec(float** ptrs, int rows, int taps, int ic, float* wb, float* bb,
- float* outp, float* addp, int oc, int lanes, ActivationKind act)
- {
- if (UseNeon)
- {
- TileNeon(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, act);
- return;
- }
- int VL = Vector.Count;
- // software Vector (DOTNET_EnableHWIntrinsic=0) is far slower than plain scalar
- bool vec = Vector.IsHardwareAccelerated && Ob % VL == 0;
- if (vec && VL == 4)
- {
- TileVec4(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, act);
- return;
- }
- float* acc = stackalloc float[Ob];
- for (int i = 0; i < rows; i++)
- {
- for (int l = 0; l < Ob; l++) acc[l] = bb[l];
- for (int tap = 0; tap < taps; tap++)
- {
- float* p = ptrs[tap * Tile + i];
- float* w = wb + (long)tap * ic * Ob;
- for (int c = 0; c < ic; c++, w += Ob)
- {
- float v = p[c];
- if (vec)
- {
- var vv = new Vector(v);
- for (int l = 0; l < Ob; l += VL)
- Simd.VectorStore(acc + l, Simd.VectorLoad(acc + l) + Simd.VectorLoad(w + l) * vv);
- }
- else
- for (int l = 0; l < Ob; l++) acc[l] += v * w[l];
- }
- }
- float* o = outp + (long)i * oc;
- float* ad = addp == null ? null : addp + (long)i * oc;
- for (int l = 0; l < lanes; l++)
- o[l] = ActS(ad != null ? acc[l] + ad[l] : acc[l], act);
- }
- }
-
- ///
- /// 128-bit Vector<float> tile (AVX2 disabled): two pixel rows × 16 oc in eight register
- /// accumulators (+4 weight vectors), instead of a stack accumulator round-tripped through
- /// memory on every multiply-add. Same per-lane op order (acc + w·x) as the generic loop.
- ///
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void TileVec4(float** ptrs, int rows, int taps, int ic, float* wb, float* bb,
- float* outp, float* addp, int oc, int lanes, ActivationKind act)
- {
- float* tmp = stackalloc float[2 * Ob];
- for (int i = 0; i < rows; i += 2)
- {
- Vector a0 = Simd.VectorLoad(bb), a1 = Simd.VectorLoad(bb + 4),
- a2 = Simd.VectorLoad(bb + 8), a3 = Simd.VectorLoad(bb + 12);
- Vector c0 = a0, c1 = a1, c2 = a2, c3 = a3;
- for (int tap = 0; tap < taps; tap++)
- {
- // ptrs holds Tile entries per tap (short tiles repeat their last pixel)
- float* p = ptrs[tap * Tile + i], q = ptrs[tap * Tile + i + 1];
- float* w = wb + (long)tap * ic * Ob;
- for (int c = 0; c < ic; c++, w += Ob)
- {
- Vector w0 = Simd.VectorLoad(w), w1 = Simd.VectorLoad(w + 4),
- w2 = Simd.VectorLoad(w + 8), w3 = Simd.VectorLoad(w + 12);
- var vp = new Vector(p[c]);
- a0 += w0 * vp; a1 += w1 * vp; a2 += w2 * vp; a3 += w3 * vp;
- var vq = new Vector(q[c]);
- c0 += w0 * vq; c1 += w1 * vq; c2 += w2 * vq; c3 += w3 * vq;
- }
- }
- Simd.VectorStore(tmp, a0); Simd.VectorStore(tmp + 4, a1);
- Simd.VectorStore(tmp + 8, a2); Simd.VectorStore(tmp + 12, a3);
- Simd.VectorStore(tmp + 16, c0); Simd.VectorStore(tmp + 20, c1);
- Simd.VectorStore(tmp + 24, c2); Simd.VectorStore(tmp + 28, c3);
- for (int r = 0; r < 2 && i + r < rows; r++)
- {
- float* o = outp + (long)(i + r) * oc;
- float* ad = addp == null ? null : addp + (long)(i + r) * oc;
- float* t = tmp + r * Ob;
- for (int l = 0; l < lanes; l++)
- o[l] = ActS(ad != null ? t[l] + ad[l] : t[l], act);
- }
- }
- }
-
- private static void Depthwise(ReadOnlySpan src, int iw, int c, Span dst,
- int oh, int ow, int kh, int kw, int[] rm, int[] cm, float[] padVec,
- NhwcConvPlan kp, ReadOnlySpan add, ActivationKind act)
- {
- int taps = kh * kw;
- bool hasAdd = !add.IsEmpty, avx = UseAvx;
- int VL = Vector.Count;
- fixed (float* sp0 = src, dp0 = dst, ap0 = add, wp0 = kp.W, bp0 = kp.Bias, pv0 = padVec)
- fixed (int* rm0 = rm, cm0 = cm)
- {
- float* sp = sp0, dp = dp0, ap = ap0, wp = wp0, bp = bp0, pv = pv0;
- int* rmp = rm0, cmp = cm0;
- For(oh, (long)oh * ow * c * taps, y =>
- {
- float** ptrs = stackalloc float*[taps];
- for (int x = 0; x < ow; x++)
- {
- for (int tap = 0; tap < taps; tap++)
- {
- int r = rmp[y * kh + tap / kw], cc = cmp[x * kw + tap % kw];
- ptrs[tap] = r < 0 || cc < 0 ? pv : sp + ((long)r * iw + cc) * c;
- }
- long ob = ((long)y * ow + x) * c;
- float* o = dp + ob;
- float* ad = hasAdd ? ap + ob : null;
- int ch = 0;
- if (UseNeon)
- {
- for (; ch + 4 <= c; ch += 4)
- {
- V128 acc = AdvSimd.LoadVector128(bp + ch);
- for (int tap = 0; tap < taps; tap++)
- acc = AdvSimd.FusedMultiplyAdd(acc, AdvSimd.LoadVector128(ptrs[tap] + ch), AdvSimd.LoadVector128(wp + tap * c + ch));
- if (ad != null) acc = AdvSimd.Add(acc, AdvSimd.LoadVector128(ad + ch));
- AdvSimd.Store(o + ch, Act4(acc, act));
- }
- }
- else if (avx)
- {
- for (; ch + 8 <= c; ch += 8)
- {
- F32x8 acc = Avx.LoadVector256(bp + ch);
- for (int tap = 0; tap < taps; tap++)
- acc = Fma.MultiplyAdd(Avx.LoadVector256(ptrs[tap] + ch), Avx.LoadVector256(wp + tap * c + ch), acc);
- if (ad != null) acc = Avx.Add(acc, Avx.LoadVector256(ad + ch));
- Avx.Store(o + ch, Act8(acc, act));
- }
- }
- else if (UsePacked)
- ch = DepthwisePixelPk(ptrs, taps, c, wp, bp, ad, o, act);
- else if (Vector.IsHardwareAccelerated && act is ActivationKind.None or ActivationKind.Relu)
- {
- for (; ch + VL <= c; ch += VL)
- {
- var acc = Simd.VectorLoad(bp + ch);
- for (int tap = 0; tap < taps; tap++)
- acc += Simd.VectorLoad(ptrs[tap] + ch) * Simd.VectorLoad(wp + tap * c + ch);
- if (ad != null) acc += Simd.VectorLoad(ad + ch);
- if (act == ActivationKind.Relu) acc = Vector.Max(acc, Vector.Zero);
- Simd.VectorStore(o + ch, acc);
- }
- }
- for (; ch < c; ch++)
- {
- float acc = bp[ch];
- for (int tap = 0; tap < taps; tap++) acc += ptrs[tap][ch] * wp[tap * c + ch];
- if (ad != null) acc += ad[ch];
- o[ch] = ActS(acc, act);
- }
- }
- });
- }
- }
-
- // ======================== pixel-vector helpers ========================
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void Copy(float* s, float* d, int n) =>
- Buffer.MemoryCopy(s, d, (long)n * 4, (long)n * 4);
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void Fill(float* d, int n, float v) => new Span(d, n).Fill(v);
-
- // ======================== conv + 2x2 max pool ========================
- [ThreadStatic] private static float[]? t_band;
-
- private static float[] BandBuf(long floats) =>
- t_band is { } b && b.Length >= floats ? b : (t_band = new float[checked((int)floats)]);
-
- ///
- /// Dense / Winograd conv with a fused 2x2 stride-2 max pool ():
- /// each task produces a band of conv rows (2 direct, 4 = one Winograd tile row) in a
- /// thread-local buffer and pools it straight into ([ph, pw, oc]),
- /// so the full-resolution conv output is never stored. Per-element math is the same as
- /// the separate Conv → MaxPool nodes.
- ///
- public static void ConvPool(ReadOnlySpan src, int ih, int iw, int ic,
- Span dst, int ph, int pw, int oc, ConvP p, NhwcConvPlan kp)
- {
- int oh = ph * 2, ow = pw * 2;
- if (kp.Depthwise || p.AddValue >= 0 || dst.Length != (long)ph * pw * oc)
- throw new NotSupportedException("ConvPool shape/kind mismatch.");
- bool relu = p.Pool!.FuseRelu, wino = kp.WinoU is not null;
- int band = wino ? 4 : 2, tasks = (oh + band - 1) / band;
- int kh = p.Kernel[0], kw = p.Kernel[1];
- float[] padVec = new float[ic * kw + Ob]; // one constant-pad kernel row (Dense512Row)
- Array.Fill(padVec, p.PadValue);
- int[] rm = Map(oh, kh, p.Strides[0], p.Pads[0], ih, p.PadMode);
- int[] cm = Map(ow, kw, p.Strides[1], p.Pads[1], iw, p.PadMode);
- fixed (float* sp = src, dp0 = dst, wp = kp.W, up = kp.WinoU, bp = kp.Bias, pv = padVec)
- fixed (int* rmp = rm, cmp = cm)
- {
- var da = new DenseArgs(sp, wp, bp, pv, rmp, cmp, iw, ic, ow, oc, kh, kw, kp.Blocks, p.Act, UseAvx);
- var wa = wino ? new WinoArgs(sp, up, bp, pv, ih, iw, ic, oh, ow, oc, kp.Blocks, p) : default;
- float* dp = dp0;
- For(tasks, (long)oh * ow * oc * kh * kw * ic / 64, t =>
- {
- int y0 = t * band, rows = Math.Min(band, oh - y0);
- float[] buf = BandBuf((long)band * ow * oc);
- fixed (float* sb = buf)
- {
- if (wino)
- for (int t0 = y0 / 4 * wa.Tw, te = t0 + wa.Tw; t0 < te; t0 += wa.T)
- WinoTiles(wa, t0, Math.Min(wa.T, te - t0), sb, null, y0);
- else
- for (int r = 0; r < rows; r++)
- ConvRow(da, y0 + r, sb + (long)r * ow * oc, null);
- PoolBand(sb, rows / 2, ow, oc, dp + (long)(y0 / 2) * pw * oc, pw, relu);
- }
- });
- }
- }
-
- /// 2x2/2 max over conv row pairs of a band; same op order as .
- private static void PoolBand(float* sb, int prows, int ow, int c, float* drow, int pw, bool relu)
- {
- int VL = Vector.Count;
- for (int r = 0; r < prows; r++)
- {
- float* top = sb + (long)(2 * r) * ow * c, bot = top + (long)ow * c;
- float* orow = drow + (long)r * pw * c;
- for (int x = 0; x < pw; x++)
- {
- float* o = orow + (long)x * c;
- float* s0 = top + (long)(2 * x) * c, s1 = s0 + c, s2 = bot + (long)(2 * x) * c, s3 = s2 + c;
- int ch = 0;
- for (; ch + VL <= c; ch += VL)
- {
- var v = Vector.Max(Simd.VectorLoad(s0 + ch), Simd.VectorLoad(s1 + ch));
- v = Vector.Max(v, Simd.VectorLoad(s2 + ch));
- Simd.VectorStore(o + ch, Vector.Max(v, Simd.VectorLoad(s3 + ch)));
- }
- for (; ch < c; ch++) o[ch] = MathF.Max(MathF.Max(MathF.Max(s0[ch], s1[ch]), s2[ch]), s3[ch]);
- if (relu)
- for (ch = 0; ch < c; ch++) o[ch] = MathF.Max(o[ch], 0f);
- }
- }
- }
-
- // ======================== maxpool ========================
- public static void MaxPool(ReadOnlySpan src, int ih, int iw, int c,
- Span dst, int oh, int ow, PoolP p)
- {
- int kh = p.Kernel[0], kw = p.Kernel[1];
- int[] rm = Map(oh, kh, p.Strides[0], p.Pads[0], ih, PadMode.Constant);
- int[] cm = Map(ow, kw, p.Strides[1], p.Pads[1], iw, PadMode.Constant);
- bool relu = p.FuseRelu;
- int VL = Vector.Count;
- fixed (float* sp0 = src, dp0 = dst)
- {
- float* sp = sp0, dp = dp0;
- if (UseAvx512 && c % 16 == 0) { MaxPool512(sp, iw, c, dp, oh, ow, kh, kw, rm, cm, relu); return; }
- For(oh, (long)oh * ow * c * kh * kw, y =>
- {
- for (int x = 0; x < ow; x++)
- {
- float* o = dp + ((long)y * ow + x) * c;
- bool any = false;
- for (int ky = 0; ky < kh; ky++)
- {
- int r = rm[y * kh + ky];
- if (r < 0) continue;
- for (int kx = 0; kx < kw; kx++)
- {
- int cc = cm[x * kw + kx];
- if (cc < 0) continue;
- float* s = sp + ((long)r * iw + cc) * c;
- if (!any) { Copy(s, o, c); any = true; continue; }
- int ch = 0;
- for (; ch + VL <= c; ch += VL)
- Simd.VectorStore(o + ch, Vector.Max(Simd.VectorLoad(o + ch), Simd.VectorLoad(s + ch)));
- for (; ch < c; ch++) o[ch] = MathF.Max(o[ch], s[ch]);
- }
- }
- if (!any) Fill(o, c, 0f);
- else if (relu)
- for (int ch = 0; ch < c; ch++) o[ch] = MathF.Max(o[ch], 0f);
- }
- });
- }
- }
-
- // ======================== bilinear resize (half-pixel) ========================
- private static void ResizeAxis(int outLen, int inLen, float scale, int[] i0, int[] i1, float[] wt)
- {
- for (int o = 0; o < outLen; o++)
- {
- float f = (o + 0.5f) / scale - 0.5f;
- float fl = MathF.Floor(f);
- i0[o] = Math.Clamp((int)fl, 0, inLen - 1);
- i1[o] = Math.Clamp((int)fl + 1, 0, inLen - 1);
- wt[o] = f - fl;
- }
- }
-
- /// One output pixel of the bilinear resize into (c channels):
- /// v = [relu](lerp) + [relu](add) — the same per-element ops as separate Resize/Relu/Add nodes.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void ResizePixel(float* a, float* b, float* e, float* f, float wX, float wY,
- Vector vX, Vector vY, int c, bool relu, float* add, bool addRelu, float* o)
- {
- int VL = Vector.Count, ch = 0;
- for (; ch + VL <= c; ch += VL)
- {
- var va = Simd.VectorLoad(a + ch); var ve = Simd.VectorLoad(e + ch);
- var top = va + (Simd.VectorLoad(b + ch) - va) * vX;
- var bot = ve + (Simd.VectorLoad(f + ch) - ve) * vX;
- var v = top + (bot - top) * vY;
- if (relu) v = Vector.Max(v, Vector.Zero);
- if (add != null)
- {
- var s = Simd.VectorLoad(add + ch);
- v += addRelu ? Vector.Max(s, Vector.Zero) : s;
- }
- Simd.VectorStore(o + ch, v);
- }
- for (; ch < c; ch++)
- {
- float top = a[ch] + (b[ch] - a[ch]) * wX;
- float bot = e[ch] + (f[ch] - e[ch]) * wX;
- float v = top + (bot - top) * wY;
- if (relu) v = MathF.Max(v, 0f);
- if (add != null) v += addRelu ? MathF.Max(add[ch], 0f) : add[ch];
- o[ch] = v;
- }
- }
-
- /// NHWC bilinear resize; optional same-shape addend (out = [relu](resize) + [relu](add)).
- public static void ResizeBilinear(ReadOnlySpan src, int ih, int iw, int c,
- Span dst, int oh, int ow, float sy, float sx, ActivationKind act,
- ReadOnlySpan add = default, bool addRelu = false)
- {
- if (!add.IsEmpty && add.Length != dst.Length)
- throw new NotSupportedException($"resize addend has {add.Length} elems, expected {dst.Length}.");
- int[] y0 = new int[oh], y1 = new int[oh], x0 = new int[ow], x1 = new int[ow];
- float[] wy = new float[oh], wx = new float[ow];
- ResizeAxis(oh, ih, sy, y0, y1, wy);
- ResizeAxis(ow, iw, sx, x0, x1, wx);
- bool relu = act == ActivationKind.Relu, hasAdd = !add.IsEmpty, v512 = UseAvx512 && c % 16 == 0;
- fixed (float* sp0 = src, dp0 = dst, ap0 = add)
- {
- float* sp = sp0, dp = dp0, ap = ap0;
- For(oh, (long)oh * ow * c, y =>
- {
- float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c;
- var vY = new Vector(wy[y]);
- float wY = wy[y];
- if (v512)
- {
- var vY16 = Vector512.Create(wY);
- for (int x = 0; x < ow; x++)
- {
- long ob = ((long)y * ow + x) * c;
- ResizePixel512(r0 + (long)x0[x] * c, r0 + (long)x1[x] * c, r1 + (long)x0[x] * c, r1 + (long)x1[x] * c,
- Vector512.Create(wx[x]), vY16, c, relu, hasAdd ? ap + ob : null, addRelu, dp + ob);
- }
- return;
- }
- for (int x = 0; x < ow; x++)
- {
- long ob = ((long)y * ow + x) * c;
- ResizePixel(r0 + (long)x0[x] * c, r0 + (long)x1[x] * c, r1 + (long)x0[x] * c, r1 + (long)x1[x] * c,
- wx[x], wY, new Vector(wx[x]), vY, c, relu, hasAdd ? ap + ob : null, addRelu, dp + ob);
- }
- });
- }
- }
-
- // ======================== fused upsample tail ========================
- /// Packs UpHead weights [oc, c, 1, 1] as [c][8] (oc lanes, zero-padded) + bias[8].
- public static UpHeadPlan PackUpHead(ReadOnlySpan w, int oc, int c, ReadOnlySpan bias)
- {
- if (oc > 8) throw new NotSupportedException($"UpHead supports oc <= 8 (got {oc}).");
- float[] pw = new float[c * 8], pb = new float[8];
- for (int o = 0; o < oc; o++)
- for (int ch = 0; ch < c; ch++) pw[ch * 8 + o] = w[o * c + ch];
- if (!bias.IsEmpty) bias.CopyTo(pb);
- return new UpHeadPlan { W = pw, Bias = pb, Oc = oc, C = c };
- }
-
- ///
- /// out[y,x,o] = act(bias[o] + Σ_c w[o,c]·s[c]), s = [relu](resize(src)) + [relu](skip): the
- /// full-resolution resize/add tensors live only in a per-pixel stack buffer. Per-element
- /// math follows the unfused nodes (bias-initialized, channel-ascending accumulation).
- ///
- public static void UpHead(ReadOnlySpan src, int ih, int iw, int c, ReadOnlySpan skip,
- Span dst, int oh, int ow, float sy, float sx, UpHeadP p, UpHeadPlan kp,
- int skipH = 0, int skipW = 0, int skipC = 0)
- {
- // SkipConv: `skip` is the conv's input [skipH, skipW, skipC]; each task recomputes
- // its skip row (same direct-conv kernel → same values) instead of reading a stored one
- ConvP? sc = p.SkipConv;
- var skp = sc?.KernelPlan as NhwcConvPlan;
- if (kp.C != c || dst.Length != (long)oh * ow * kp.Oc
- || (sc is null ? skip.Length != (long)oh * ow * c
- : skp is null || skp.Depthwise || skp.WinoU is not null || sc.AddValue >= 0 || skip.Length != (long)skipH * skipW * skipC))
- throw new NotSupportedException("UpHead shape mismatch.");
- int[] y0 = new int[oh], y1 = new int[oh], x0 = new int[ow], x1 = new int[ow];
- float[] wy = new float[oh], wx = new float[ow];
- ResizeAxis(oh, ih, sy, y0, y1, wy);
- ResizeAxis(ow, iw, sx, x0, x1, wx);
- bool relu = p.ResizeRelu, skipRelu = p.SkipRelu, avx = UseAvx;
- int oc = kp.Oc;
- bool neonUp = UseNeon && c % 4 == 0 && oc <= 4;
- bool pkUp = UsePacked && c % 4 == 0 && oc <= 4;
- ActivationKind act = p.Act;
- int[] rm = [], cm = [];
- float[] padVec = [];
- if (sc is not null)
- {
- rm = Map(oh, sc.Kernel[0], sc.Strides[0], sc.Pads[0], skipH, sc.PadMode);
- cm = Map(ow, sc.Kernel[1], sc.Strides[1], sc.Pads[1], skipW, sc.PadMode);
- padVec = new float[skipC * sc.Kernel[1] + Ob]; // one constant-pad kernel row (Dense512Row)
- Array.Fill(padVec, sc.PadValue);
- }
- fixed (float* sp0 = src, dp0 = dst, kp0 = skip, wp0 = kp.W, bp0 = kp.Bias)
- fixed (float* cw = skp?.W, cb = skp?.Bias, pv = padVec)
- fixed (int* rmp = rm, cmp = cm)
- {
- float* sp = sp0, dp = dp0, kpp = kp0, wp = wp0, bp = bp0;
- var ca = sc is null ? default
- : new DenseArgs(kp0, cw, cb, pv, rmp, cmp, skipW, skipC, ow, c, sc.Kernel[0], sc.Kernel[1], skp!.Blocks, sc.Act, avx);
- if (UseAvx512)
- {
- UpHead512(sp, iw, c, kpp, sc is not null, ca, dp, oh, ow, y0, y1, x0, x1, wy, wx, relu, skipRelu, wp, bp, oc, act);
- return;
- }
- For(oh, (long)oh * ow * c * (oc + 4), y =>
- {
- float* s = stackalloc float[c + 8];
- float* acc = stackalloc float[8];
- float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c;
- var vY = new Vector(wy[y]);
- float wY = wy[y];
- float* orow = dp + (long)y * ow * oc;
- float[]? band = sc is null ? null : BandBuf((long)ow * c);
- fixed (float* sb = band)
- {
- float* krow = kpp + (long)y * ow * c;
- if (sc is not null)
- {
- ConvRow(ca, y, sb, null);
- krow = sb;
- }
- if (neonUp)
- {
- fixed (int* xa = x0, xb = x1)
- fixed (float* wxp = wx)
- UpHeadRowNeon(r0, r1, xa, xb, wxp, wY, krow, c, ow, wp, bp, orow, oc, relu, skipRelu, act);
- return;
- }
- if (pkUp)
- {
- fixed (int* xa = x0, xb = x1)
- fixed (float* wxp = wx)
- UpHeadRowPk(r0, r1, xa, xb, wxp, wY, krow, c, ow, wp, bp, orow, oc, relu, skipRelu, act);
- return;
- }
- for (int x = 0; x < ow; x++)
- {
- ResizePixel(r0 + (long)x0[x] * c, r0 + (long)x1[x] * c, r1 + (long)x0[x] * c, r1 + (long)x1[x] * c,
- wx[x], wY, new Vector(wx[x]), vY, c, relu, krow + (long)x * c, skipRelu, s);
- float* o = orow + (long)x * oc;
- if (avx)
- {
- F32x8 a = Avx.LoadVector256(bp);
- float* w = wp;
- for (int ch = 0; ch < c; ch++, w += 8)
- a = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s + ch), Avx.LoadVector256(w), a);
- Avx.Store(acc, a);
- }
- else if (UseNeon)
- {
- V128 a = AdvSimd.LoadVector128(bp), b = AdvSimd.LoadVector128(bp + 4);
- float* w = wp;
- for (int ch = 0; ch < c; ch++, w += 8)
- {
- V128 v = AdvSimd.LoadAndReplicateToVector128(s + ch);
- a = AdvSimd.FusedMultiplyAdd(a, AdvSimd.LoadVector128(w), v);
- b = AdvSimd.FusedMultiplyAdd(b, AdvSimd.LoadVector128(w + 4), v);
- }
- AdvSimd.Store(acc, a); AdvSimd.Store(acc + 4, b);
- }
- else
- {
- for (int l = 0; l < 8; l++) acc[l] = bp[l];
- float* w = wp;
- for (int ch = 0; ch < c; ch++, w += 8)
- {
- float v = s[ch];
- for (int l = 0; l < oc; l++) acc[l] += v * w[l];
- }
- }
- for (int l = 0; l < oc; l++) o[l] = acc[l];
- }
- if (act != ActivationKind.None) ActRow(orow, ow * oc, act, avx);
- }
- });
- }
- }
-
- private static void ActRow(float* p, int n, ActivationKind act, bool avx)
- {
- int i = 0;
- if (avx)
- for (; i + 8 <= n; i += 8) Avx.Store(p + i, Act8(Avx.LoadVector256(p + i), act));
- else if (UseNeon)
- for (; i + 4 <= n; i += 4) AdvSimd.Store(p + i, Act4(AdvSimd.LoadVector128(p + i), act));
- else if (UsePacked)
- for (; i + 4 <= n; i += 4) PackedSimd.Store(p + i, ActP(LdP(p + i), act));
- else if (act == ActivationKind.Sigmoid)
- {
- var row = new Span(p + i, n - i);
- SimdMath.Sigmoid(row, row);
- return;
- }
- for (; i < n; i++) p[i] = ActS(p[i], act);
- }
-
- // ======================== pad (spatial only) ========================
- public static void Pad(ReadOnlySpan src, int ih, int iw, int c,
- Span dst, int oh, int ow, PadP p)
- {
- int rank = p.Pads.Length / 2;
- if (rank != 4) throw new NotSupportedException("NHWC Pad supports rank-4 only.");
- int[] rm = Map(oh, 1, 1, p.Pads[2], ih, p.Mode);
- int[] cm = Map(ow, 1, 1, p.Pads[3], iw, p.Mode);
- float val = p.Value;
- fixed (float* sp0 = src, dp0 = dst)
- {
- float* sp = sp0, dp = dp0;
- For(oh, (long)oh * ow * c, y =>
- {
- int r = rm[y];
- for (int x = 0; x < ow; x++)
- {
- float* o = dp + ((long)y * ow + x) * c;
- int cc = cm[x];
- if (r < 0 || cc < 0) Fill(o, c, val);
- else Copy(sp + ((long)r * iw + cc) * c, o, c);
- }
- });
- }
- }
-
- // ======================== channel shift ========================
- // out[y,x,c] = in[y+off_g, x, c] (logical axis 2) or in[y, x+off_g, c] (axis 3), zero-fill
- public static void ChannelShift(ReadOnlySpan src, int h, int w, int c,
- Span dst, ShiftP p)
- {
- int[] ends = p.GroupEnds, offs = p.Offsets;
- bool alongH = p.Axis == 2;
- fixed (float* sp0 = src, dp0 = dst)
- {
- float* sp = sp0, dp = dp0;
- For(h, (long)h * w * c, y =>
- {
- for (int x = 0; x < w; x++)
- {
- float* o = dp + ((long)y * w + x) * c;
- int g0 = 0;
- for (int g = 0; g < offs.Length; g++)
- {
- int g1 = ends[g];
- int sy = alongH ? y + offs[g] : y, sx = alongH ? x : x + offs[g];
- if ((uint)sy < (uint)h && (uint)sx < (uint)w)
- Copy(sp + ((long)sy * w + sx) * c + g0, o + g0, g1 - g0);
- else
- Fill(o + g0, g1 - g0, 0f);
- g0 = g1;
- }
- if (g0 < c) Copy(sp + ((long)y * w + x) * c + g0, o + g0, c - g0);
- }
- });
- }
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.Neon.cs b/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.Neon.cs
deleted file mode 100644
index d70409a..0000000
--- a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.Neon.cs
+++ /dev/null
@@ -1,75 +0,0 @@
-namespace Sdcb.SimdGcdrNet.Kernels;
-
-///
-/// NEON Winograd schedule for layers whose transformed weights U (36·oc·ic floats) outgrow the L2
-/// caches (the deep, low-resolution layers): the per-task tile blocks of stream
-/// every U panel once per block of tiles — DRAM-bound on the RK3399 (512 KB A53 / 1 MB A72 L2).
-/// Instead tiles are taken in large groups and each group runs three parallel phases: input
-/// transforms into V[36][Tg][ic], one GEMM task per (xi, oc block) that loads its U panel once for
-/// the whole group, then the output transforms with the usual epilogue. Same per-element math as
-/// (same transforms, same k loop), so the outputs are bit-identical.
-///
-internal static unsafe partial class Nhwc
-{
- /// U sizes (floats) above this take on NEON (512 KB).
- private const long NeonWinoUFloats = 128 * 1024;
-
- [ThreadStatic] private static float[]? t_wgV, t_wgM;
-
- private static void WinogradGrouped(in WinoArgs a, float* dp, float* ap)
- {
- int ic = a.Ic, nb = a.Nb, tw = a.Tw, oh = a.Oh, ow = a.Ow, oc = a.Oc;
- int nt = (oh + 3) / 4 * tw;
- // V group ~2 MB (spread over the L2s, re-read nb times), whole Tile-row GEMM tiles
- int tg = Math.Clamp(524288 / (36 * ic) / Tile * Tile, Tile, (nt + Tile - 1) / Tile * Tile);
- float* V = Pinned(ref t_wgV, 36 * tg * ic), M = Pinned(ref t_wgM, 36 * nb * tg * Ob);
- WinoArgs ca = a;
- for (int g0 = 0; g0 < nt; g0 += tg)
- {
- int tn = Math.Min(tg, nt - g0);
- Parallel.For(0, (tn + 7) / 8, PO, k =>
- {
- float** dp6 = stackalloc float*[36];
- for (int t = k * 8, te = Math.Min(tn, t + 8); t < te; t++)
- {
- int ty = (g0 + t) / tw, tx = (g0 + t) % tw;
- int iy0 = ty * 4 - ca.PadT, ix0 = tx * 4 - ca.PadL;
- for (int i = 0; i < 6; i++)
- {
- int r = SrcIdx(iy0 + i, ca.Ih, ca.Reflect);
- for (int j = 0; j < 6; j++)
- {
- int c = SrcIdx(ix0 + j, ca.Iw, ca.Reflect);
- dp6[i * 6 + j] = r < 0 || c < 0 ? ca.PadVec : ca.Src + ((long)r * ca.Iw + c) * ic;
- }
- }
- InputTransform(dp6, ic, V + (long)t * ic, (long)tg * ic);
- }
- });
- Parallel.For(0, 36 * nb, PO, j =>
- {
- int xi = j / nb, blk = j % nb;
- float** ptrs = stackalloc float*[Tile];
- float* zero = stackalloc float[Ob];
- new Span(zero, Ob).Clear();
- float* wb = ca.U + ((long)xi * nb + blk) * ic * Ob;
- float* vx = V + (long)xi * tg * ic, mx = M + ((long)xi * nb + blk) * tg * Ob;
- // all Tile rows land in M (tg is a multiple of Tile); rows past tn are never read
- for (int r0 = 0; r0 < tn; r0 += Tile)
- {
- int rows = Math.Min(Tile, tn - r0);
- for (int i = 0; i < Tile; i++) ptrs[i] = vx + (long)(r0 + Math.Min(i, rows - 1)) * ic;
- TileNeonK(ptrs, 1, ic, wb, zero, mx + (long)r0 * Ob, Ob);
- }
- });
- int chunks = (tn + 15) / 16;
- Parallel.For(0, chunks * nb, PO, j =>
- {
- int blk = j % nb, t0 = j / nb * 16;
- float* tmp = stackalloc float[16 * Ob];
- WinoOutput(M + ((long)blk * tg + t0) * Ob, (long)nb * tg * Ob, Ob, Math.Min(16, tn - t0), g0 + t0,
- tw, oh, ow, oc, blk * Ob, ca.Bias + blk * Ob, dp, ap, 0, ca.Act, tmp, false, false);
- });
- }
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.cs b/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.cs
deleted file mode 100644
index aea5029..0000000
--- a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.cs
+++ /dev/null
@@ -1,399 +0,0 @@
-using System.Numerics;
-using System.Runtime.CompilerServices;
-using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.Arm;
-using System.Runtime.Intrinsics.Wasm;
-using System.Runtime.Intrinsics.X86;
-using Sdcb.SimdGcdrNet.Engine;
-using F32x8 = System.Runtime.Intrinsics.Vector256;
-
-namespace Sdcb.SimdGcdrNet.Kernels;
-
-///
-/// Winograd F(4x4, 3x3) for dense 3x3 stride-1 NHWC convs: 36 multiplies per
-/// 4x4 output tile and (ic, oc) pair instead of 144. Tiles are processed in
-/// cache-sized blocks per task — input transform into a block-local
-/// V[36][T][ic], then per 16-oc block 36 small GEMMs through the 6x16 conv
-/// micro-kernel into M[36][T][16], then the output transform with the usual
-/// bias / residual / activation epilogue — so the 2.25x-expanded transform
-/// tensors never leave L2.
-///
-internal static unsafe partial class Nhwc
-{
- private static readonly bool s_winograd = Environment.GetEnvironmentVariable("GCDRNET_WINOGRAD") != "0";
-
- [ThreadStatic] private static float[]? t_wino;
-
- /// Dense 3x3 s1 convs with enough channels for the transforms to amortize.
- internal static bool WinogradEligible(ConvP p, int[] wShape, int ic) =>
- s_winograd && Vector.IsHardwareAccelerated && p.Group == 1
- && wShape[2] == 3 && wShape[3] == 3 && p.Strides[0] == 1 && p.Strides[1] == 1
- && p.Dilations[0] == 1 && p.Dilations[1] == 1
- && ic >= 16 && ic % Vector.Count == 0 && ic % 8 == 0;
-
- // G (6x3) of F(4,3): U = G g G^T
- private static readonly double[,] s_g =
- {
- { 1.0 / 4, 0, 0 },
- { -1.0 / 6, -1.0 / 6, -1.0 / 6 },
- { -1.0 / 6, 1.0 / 6, -1.0 / 6 },
- { 1.0 / 24, 1.0 / 12, 1.0 / 6 },
- { 1.0 / 24, -1.0 / 12, 1.0 / 6 },
- { 0, 0, 1 },
- };
-
- /// Transformed weights [36][oc/16][ic][16] (oc zero-padded), from ONNX [oc][ic][3][3].
- internal static float[] PackWinograd(ReadOnlySpan w, int oc, int ic)
- {
- int nb = (oc + Ob - 1) / Ob;
- float[] u = new float[checked(36 * nb * ic * Ob)];
- double* gg = stackalloc double[18];
- for (int o = 0; o < oc; o++)
- for (int c = 0; c < ic; c++)
- {
- ReadOnlySpan k = w.Slice((o * ic + c) * 9, 9);
- // gg = G·k (6x3)
- for (int i = 0; i < 6; i++)
- for (int j = 0; j < 3; j++)
- gg[i * 3 + j] = s_g[i, 0] * k[j] + s_g[i, 1] * k[3 + j] + s_g[i, 2] * k[6 + j];
- // U = gg·G^T (6x6)
- for (int i = 0; i < 6; i++)
- for (int j = 0; j < 6; j++)
- {
- double v = gg[i * 3] * s_g[j, 0] + gg[i * 3 + 1] * s_g[j, 1] + gg[i * 3 + 2] * s_g[j, 2];
- u[(((i * 6 + j) * nb + o / Ob) * ic + c) * Ob + o % Ob] = (float)v;
- }
- }
- return u;
- }
-
- /// Per-call Winograd constants shared by every tile block.
- private readonly unsafe struct WinoArgs
- {
- public readonly float* Src, U, Bias, PadVec;
- public readonly int Ih, Iw, Ic, Oh, Ow, Oc, Tw, T, Nb, PadT, PadL;
- public readonly bool Reflect, Avx, Avx512, Pk;
- public readonly ActivationKind Act;
-
- public WinoArgs(float* src, float* u, float* bias, float* padVec, int ih, int iw, int ic,
- int oh, int ow, int oc, int nb, ConvP p)
- {
- Src = src; U = u; Bias = bias; PadVec = padVec;
- Ih = ih; Iw = iw; Ic = ic; Oh = oh; Ow = ow; Oc = oc; Nb = nb;
- Tw = (ow + 3) / 4;
- Avx512 = UseAvx512;
- Avx = UseAvx;
- Pk = UsePacked;
- // V block ~256 KB (AVX2); the AVX-512 tier (Tile5-row x 32-oc GEMM tiles) measured
- // best at ~64 KB on Zen 5, NEON at ~16 KB (one Tile row for ic >= 32: four RK3399 A53s share 512 KB L2)
- int tr = Avx512 ? Tile5 : Pk ? PkTile : Tile, vBudget = Avx512 ? 16384 : UseNeon ? 4096 : 65536;
- T = Math.Clamp(vBudget / (36 * ic) / tr * tr, tr, 60 / tr * tr);
- PadT = p.Pads[0]; PadL = p.Pads[1];
- Reflect = p.PadMode == PadMode.Reflect;
- Act = p.Act;
- }
- }
-
- private static void WinogradConv(ReadOnlySpan src, int ih, int iw, int ic,
- Span dst, int oh, int ow, int oc, ConvP p, NhwcConvPlan kp, ReadOnlySpan add)
- {
- float[] padVec = new float[ic];
- Array.Fill(padVec, p.PadValue);
- bool hasAdd = !add.IsEmpty;
- fixed (float* sp = src, dp0 = dst, ap0 = add, up = kp.WinoU, bp = kp.Bias, pv = padVec)
- {
- var a = new WinoArgs(sp, up, bp, pv, ih, iw, ic, oh, ow, oc, kp.Blocks, p);
- int nt = (oh + 3) / 4 * a.Tw, chunks = (nt + a.T - 1) / a.T;
- float* dp = dp0, ap = hasAdd ? ap0 : null;
- if (UseNeon && 36L * a.Nb * ic * Ob > NeonWinoUFloats)
- {
- WinogradGrouped(a, dp, ap);
- return;
- }
- For(chunks, (long)oh * ow * oc * ic, chunk =>
- {
- int t0 = chunk * a.T;
- WinoTiles(a, t0, Math.Min(a.T, nt - t0), dp, ap, 0);
- });
- }
- }
-
- ///
- /// Tiles [t0, t0+tn) (raster order, tn <= T) of one Winograd conv, written to
- /// whose row 0 is output row (rows past
- /// Oh are skipped). Every tile's math is independent of how tiles are grouped.
- ///
- private static void WinoTiles(in WinoArgs a, int t0, int tn, float* dp, float* ap, int rowBase)
- {
- int ic = a.Ic, T = a.T, tw = a.Tw, nb = a.Nb, oc = a.Oc, ow = a.Ow, oh = a.Oh;
- bool avx = a.Avx, avx512 = a.Avx512, pk = a.Pk;
- // AVX-512: Tile5-row x 32-oc GEMM tiles over block pairs; M rows hold both blocks
- int tr = avx512 ? Tile5 : pk ? PkTile : Tile, step = avx512 ? 2 : 1, mRow = step * Ob;
- int vSize = 36 * T * ic, mSize = 36 * T * mRow;
- int need = vSize + mSize + 16 * Ob + 2 * Ob;
- float[] buf = t_wino is { } tb && tb.Length >= need ? tb : (t_wino = new float[need]);
- fixed (float* b0 = buf)
- {
- float* V = b0, M = b0 + vSize, tmp = M + mSize, zero = tmp + 16 * Ob;
- new Span(zero, 2 * Ob).Clear();
- float** dp6 = stackalloc float*[36];
- for (int t = 0; t < tn; t++)
- {
- int ty = (t0 + t) / tw, tx = (t0 + t) % tw;
- int iy0 = ty * 4 - a.PadT, ix0 = tx * 4 - a.PadL;
- for (int i = 0; i < 6; i++)
- {
- int r = SrcIdx(iy0 + i, a.Ih, a.Reflect);
- for (int j = 0; j < 6; j++)
- {
- int c = SrcIdx(ix0 + j, a.Iw, a.Reflect);
- dp6[i * 6 + j] = r < 0 || c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic;
- }
- }
- if (avx512 && ic % 16 == 0) InputTransform512(dp6, ic, V + (long)t * ic, (long)T * ic);
- else InputTransform(dp6, ic, V + (long)t * ic, (long)T * ic);
- }
-
- float** ptrs = stackalloc float*[Math.Max(Tile, Tile5)];
- for (int blk0 = 0; blk0 < nb; blk0 += step)
- {
- int nblk = Math.Min(step, nb - blk0);
- for (int xi = 0; xi < 36; xi++)
- {
- float* wb = a.U + ((long)xi * nb + blk0) * ic * Ob;
- float* vx = V + (long)xi * T * ic, mx = M + (long)xi * T * mRow;
- for (int r0 = 0; r0 < tn; r0 += tr)
- {
- int rows = Math.Min(tr, tn - r0);
- for (int i = 0; i < tr; i++) ptrs[i] = vx + (long)(r0 + Math.Min(i, rows - 1)) * ic;
- // AVX2 / NEON / PackedSimd write all tr rows into M (T is a multiple of tr); rows past tn are never read
- if (avx512) Tile512(ptrs, nblk, 1, ic, wb, (long)ic * Ob, zero, mx + (long)r0 * mRow);
- else if (avx) TileAvxK(ptrs, 1, ic, wb, zero, mx + (long)r0 * Ob, Ob);
- else if (UseNeon) TileNeonK(ptrs, 1, ic, wb, zero, mx + (long)r0 * Ob, Ob);
- else if (pk) TilePkK(ptrs, PkTile, 1, ic, (long)ic * Ob, wb, zero, mx + (long)r0 * Ob, Ob);
- else TileVec(ptrs, rows, 1, ic, wb, zero, mx + (long)r0 * Ob, null, Ob, Ob, ActivationKind.None);
- }
- }
- for (int h = 0; h < nblk; h++)
- {
- int blk = blk0 + h;
- WinoOutput(M + h * Ob, (long)T * mRow, mRow, tn, t0, tw, oh, ow, oc, blk * Ob, a.Bias + blk * Ob,
- dp, ap, rowBase, a.Act, tmp, avx, avx512);
- }
- }
- }
- }
-
- /// Output transform + bias/residual/activation epilogue of one 16-oc block for tn tiles;
- /// tile t's 36 products are m[xi*mStride + t*mRow + 0..15].
- private static void WinoOutput(float* m, long mStride, int mRow, int tn, int t0, int tw, int oh, int ow, int oc,
- int ocOff, float* bb, float* dp, float* ap, int rowBase, ActivationKind act, float* tmp, bool avx, bool avx512)
- {
- int lanes = Math.Min(Ob, oc - ocOff);
- bool fast512 = avx512 && lanes == Ob && act is ActivationKind.None or ActivationKind.Relu;
- bool fastNeon = UseNeon && lanes == Ob;
- bool fastPk = UsePacked && lanes == Ob;
- for (int t = 0; t < tn; t++)
- {
- int ty = (t0 + t) / tw, tx = (t0 + t) % tw;
- if (avx512) OutputTransform512(m + (long)t * mRow, mStride, tmp);
- else OutputTransform(m + (long)t * mRow, mStride, tmp);
- for (int py = 0; py < 4; py++)
- {
- int oy = ty * 4 + py;
- if (oy >= oh) break;
- for (int px = 0; px < 4; px++)
- {
- int ox = tx * 4 + px;
- if (ox >= ow) break;
- long o = ((long)(oy - rowBase) * ow + ox) * oc + ocOff;
- float* tp = tmp + (py * 4 + px) * Ob;
- if (fast512)
- {
- var v = Avx512F.Add(Avx512F.LoadVector512(tp), Avx512F.LoadVector512(bb));
- if (ap != null) v = Avx512F.Add(v, Avx512F.LoadVector512(ap + o));
- if (act == ActivationKind.Relu) v = Avx512F.Max(v, Vector512.Zero);
- Avx512F.Store(dp + o, v);
- }
- else if (fastNeon)
- for (int l = 0; l < Ob; l += 4)
- {
- var v = AdvSimd.Add(AdvSimd.LoadVector128(tp + l), AdvSimd.LoadVector128(bb + l));
- if (ap != null) v = AdvSimd.Add(v, AdvSimd.LoadVector128(ap + o + l));
- AdvSimd.Store(dp + o + l, Act4(v, act));
- }
- else if (fastPk)
- for (int l = 0; l < Ob; l += 4)
- {
- var v = PackedSimd.Add(LdP(tp + l), LdP(bb + l));
- if (ap != null) v = PackedSimd.Add(v, LdP(ap + o + l));
- PackedSimd.Store(dp + o + l, ActP(v, act));
- }
- else if (avx)
- Store16(dp + o, ap != null ? ap + o : null,
- Avx.Add(Avx.LoadVector256(tp), Avx.LoadVector256(bb)),
- Avx.Add(Avx.LoadVector256(tp + 8), Avx.LoadVector256(bb + 8)), lanes, act);
- else
- for (int l = 0; l < lanes; l++)
- {
- float v = tp[l] + bb[l];
- if (ap != null) v += ap[o + l];
- dp[o + l] = ActS(v, act);
- }
- }
- }
- }
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static int SrcIdx(int i, int n, bool reflect) =>
- (uint)i < (uint)n ? i : reflect ? Reflect(i, n) : -1;
-
- /// V[xi] = (B^T d B)[xi] for one tile, all channels; V row stride per xi = vStride.
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void InputTransform(float** d, int ic, float* v, long vStride)
- {
- int VL = Vector.Count;
- var t = stackalloc Vector[36];
- var four = new Vector(4f);
- var five = new Vector(5f);
- var two = new Vector(2f);
- for (int c = 0; c < ic; c += VL)
- {
- // columns: t = B^T d
- for (int j = 0; j < 6; j++)
- {
- var d0 = Simd.VectorLoad(d[j] + c); var d1 = Simd.VectorLoad(d[6 + j] + c);
- var d2 = Simd.VectorLoad(d[12 + j] + c); var d3 = Simd.VectorLoad(d[18 + j] + c);
- var d4 = Simd.VectorLoad(d[24 + j] + c); var d5 = Simd.VectorLoad(d[30 + j] + c);
- t[j] = four * d0 - five * d2 + d4;
- t[6 + j] = d3 + d4 - four * (d1 + d2);
- t[12 + j] = d4 - d3 + four * (d1 - d2);
- t[18 + j] = d4 - d2 + two * (d3 - d1);
- t[24 + j] = d4 - d2 - two * (d3 - d1);
- t[30 + j] = four * d1 - five * d3 + d5;
- }
- // rows: V = t B
- for (int i = 0; i < 6; i++)
- {
- var r0 = t[i * 6]; var r1 = t[i * 6 + 1]; var r2 = t[i * 6 + 2];
- var r3 = t[i * 6 + 3]; var r4 = t[i * 6 + 4]; var r5 = t[i * 6 + 5];
- float* o = v + c;
- long b = (long)(i * 6) * vStride;
- Simd.VectorStore(o + b, four * r0 - five * r2 + r4);
- Simd.VectorStore(o + b + vStride, r3 + r4 - four * (r1 + r2));
- Simd.VectorStore(o + b + 2 * vStride, r4 - r3 + four * (r1 - r2));
- Simd.VectorStore(o + b + 3 * vStride, r4 - r2 + two * (r3 - r1));
- Simd.VectorStore(o + b + 4 * vStride, r4 - r2 - two * (r3 - r1));
- Simd.VectorStore(o + b + 5 * vStride, four * r1 - five * r3 + r5);
- }
- }
- }
-
- /// tmp[(py*4+px)*16 + l] = (A^T m A)[py,px] for one tile's 16 oc lanes; m row stride per xi = mStride.
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void OutputTransform(float* m, long mStride, float* tmp)
- {
- int VL = Vector.Count;
- var s = stackalloc Vector[24];
- var two = new Vector(2f);
- var four = new Vector(4f);
- var eight = new Vector(8f);
- for (int l = 0; l < Ob; l += VL)
- {
- // columns: s = A^T m (4x6)
- for (int j = 0; j < 6; j++)
- {
- var m0 = Simd.VectorLoad(m + j * mStride + l); var m1 = Simd.VectorLoad(m + (6 + j) * mStride + l);
- var m2 = Simd.VectorLoad(m + (12 + j) * mStride + l); var m3 = Simd.VectorLoad(m + (18 + j) * mStride + l);
- var m4 = Simd.VectorLoad(m + (24 + j) * mStride + l); var m5 = Simd.VectorLoad(m + (30 + j) * mStride + l);
- var a = m1 + m2; var b = m1 - m2; var c = m3 + m4; var e = m3 - m4;
- s[j] = m0 + a + c;
- s[6 + j] = b + two * e;
- s[12 + j] = a + four * c;
- s[18 + j] = b + eight * e + m5;
- }
- // rows: y = s A (4x4)
- for (int i = 0; i < 4; i++)
- {
- var r0 = s[i * 6]; var r1 = s[i * 6 + 1]; var r2 = s[i * 6 + 2];
- var r3 = s[i * 6 + 3]; var r4 = s[i * 6 + 4]; var r5 = s[i * 6 + 5];
- var a = r1 + r2; var b = r1 - r2; var c = r3 + r4; var e = r3 - r4;
- float* o = tmp + i * 4 * Ob + l;
- Simd.VectorStore(o, r0 + a + c);
- Simd.VectorStore(o + Ob, b + two * e);
- Simd.VectorStore(o + 2 * Ob, a + four * c);
- Simd.VectorStore(o + 3 * Ob, b + eight * e + r5);
- }
- }
- }
-
- /// with 16 lanes (AVX-512; ic % 16 == 0). Same per-lane ops → same bits.
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void InputTransform512(float** d, int ic, float* v, long vStride)
- {
- var t = stackalloc Vector512[36];
- var four = Vector512.Create(4f);
- var five = Vector512.Create(5f);
- var two = Vector512.Create(2f);
- for (int c = 0; c < ic; c += 16)
- {
- for (int j = 0; j < 6; j++)
- {
- var d0 = Vector512.Load(d[j] + c); var d1 = Vector512.Load(d[6 + j] + c);
- var d2 = Vector512.Load(d[12 + j] + c); var d3 = Vector512.Load(d[18 + j] + c);
- var d4 = Vector512.Load(d[24 + j] + c); var d5 = Vector512.Load(d[30 + j] + c);
- t[j] = four * d0 - five * d2 + d4;
- t[6 + j] = d3 + d4 - four * (d1 + d2);
- t[12 + j] = d4 - d3 + four * (d1 - d2);
- t[18 + j] = d4 - d2 + two * (d3 - d1);
- t[24 + j] = d4 - d2 - two * (d3 - d1);
- t[30 + j] = four * d1 - five * d3 + d5;
- }
- for (int i = 0; i < 6; i++)
- {
- var r0 = t[i * 6]; var r1 = t[i * 6 + 1]; var r2 = t[i * 6 + 2];
- var r3 = t[i * 6 + 3]; var r4 = t[i * 6 + 4]; var r5 = t[i * 6 + 5];
- float* o = v + c + (long)(i * 6) * vStride;
- (four * r0 - five * r2 + r4).Store(o);
- (r3 + r4 - four * (r1 + r2)).Store(o + vStride);
- (r4 - r3 + four * (r1 - r2)).Store(o + 2 * vStride);
- (r4 - r2 + two * (r3 - r1)).Store(o + 3 * vStride);
- (r4 - r2 - two * (r3 - r1)).Store(o + 4 * vStride);
- (four * r1 - five * r3 + r5).Store(o + 5 * vStride);
- }
- }
- }
-
- /// with one 16-lane vector per element (AVX-512).
- [MethodImpl(MethodImplOptions.AggressiveOptimization)]
- private static void OutputTransform512(float* m, long mStride, float* tmp)
- {
- var s = stackalloc Vector512[24];
- var two = Vector512.Create(2f);
- var four = Vector512.Create(4f);
- var eight = Vector512.Create(8f);
- for (int j = 0; j < 6; j++)
- {
- var m0 = Vector512.Load(m + j * mStride); var m1 = Vector512.Load(m + (6 + j) * mStride);
- var m2 = Vector512.Load(m + (12 + j) * mStride); var m3 = Vector512.Load(m + (18 + j) * mStride);
- var m4 = Vector512.Load(m + (24 + j) * mStride); var m5 = Vector512.Load(m + (30 + j) * mStride);
- var a = m1 + m2; var b = m1 - m2; var c = m3 + m4; var e = m3 - m4;
- s[j] = m0 + a + c;
- s[6 + j] = b + two * e;
- s[12 + j] = a + four * c;
- s[18 + j] = b + eight * e + m5;
- }
- for (int i = 0; i < 4; i++)
- {
- var r0 = s[i * 6]; var r1 = s[i * 6 + 1]; var r2 = s[i * 6 + 2];
- var r3 = s[i * 6 + 3]; var r4 = s[i * 6 + 4]; var r5 = s[i * 6 + 5];
- var a = r1 + r2; var b = r1 - r2; var c = r3 + r4; var e = r3 - r4;
- float* o = tmp + i * 4 * Ob;
- (r0 + a + c).Store(o);
- (b + two * e).Store(o + Ob);
- (a + four * c).Store(o + 2 * Ob);
- (b + eight * e + r5).Store(o + 3 * Ob);
- }
- }
-}
diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Ops.cs b/src/Sdcb.SimdGcdrNet/Kernels/Ops.cs
deleted file mode 100644
index 3f1eabe..0000000
--- a/src/Sdcb.SimdGcdrNet/Kernels/Ops.cs
+++ /dev/null
@@ -1,1005 +0,0 @@
-using System.Numerics;
-using System.Runtime.CompilerServices;
-using System.Runtime.Intrinsics.X86;
-using Sdcb.SimdGcdrNet.Engine;
-
-namespace Sdcb.SimdGcdrNet.Kernels;
-
-///
-/// Flat-float NCHW kernel library. Dispatch is a plain IsSupported chain:
-/// AVX2+FMA hot paths, then the portable Vector<float> tier, else scalar
-/// (the reference). Runtime knobs: DOTNET_EnableAVX2 / DOTNET_EnableAVX /
-/// DOTNET_EnableHWIntrinsic.
-///
-internal static unsafe class Ops
-{
- // ======================== Conv ========================
- // src [1,IC,H,W]; w [OC,IC/g,kh,kw]; dst [1,OC,OH,OW]. N==1 assumed.
- // `add` is an optional same-shape residual addend, applied before the activation.
- public static void Conv(ReadOnlySpan src, int[] inShape,
- ReadOnlySpan w, int[] wShape, ReadOnlySpan bias,
- Span dst, int[] outShape, ConvP p, ReadOnlySpan add = default)
- {
- if (!add.IsEmpty && add.Length != dst.Length)
- throw new NotSupportedException(
- $"conv residual addend has {add.Length} elems, expected {dst.Length}.");
- if (!add.IsEmpty)
- {
- // activation must run after the residual add: conv without act, then fused add+act
- var q = new ConvP
- {
- Pads = p.Pads, Strides = p.Strides, Dilations = p.Dilations, Kernel = p.Kernel,
- Group = p.Group, Act = ActivationKind.None, PadMode = p.PadMode,
- PadValue = p.PadValue, BiasValue = p.BiasValue, KernelPlan = p.KernelPlan,
- };
- Conv(src, inShape, w, wShape, bias, dst, outShape, q);
- AddInPlace(dst, add, p.Act);
- return;
- }
- int ic = inShape[1], ih = inShape[2], iw = inShape[3];
- int oc = wShape[0], kh = wShape[2], kw = wShape[3];
- int oh = outShape[2], ow = outShape[3];
- int sh = p.Strides[0], sw = p.Strides[1];
- int pt = p.Pads[0], pl = p.Pads[1], pb = p.Pads[2], pr = p.Pads[3];
- int group = p.Group;
-
- bool symmetricPad = pt == pb && pl == pr;
- if (Avx2.IsSupported && Fma.IsSupported)
- {
- if (group > 1 && group == ic)
- {
- Avx2Kernels.Depthwise(src, ic, ih, iw, w, bias, dst, oh, ow,
- kh, kw, pt, pl, p.PadMode, p.PadValue, p.Act);
- return;
- }
- if (group == 1 && kh == 3 && kw == 3 && oc % 4 == 0 && symmetricPad)
- {
- if (sh == 1 && sw == 1)
- {
- Avx2Kernels.Conv3x3S1(src, ic, ih, iw, w, bias, dst, oc, oh, ow,
- pt, pl, p.PadMode, p.PadValue, p.Act);
- return;
- }
- if (sh == 2 && sw == 2)
- {
- Avx2Kernels.Conv3x3S2(src, ic, ih, iw, w, bias, dst, oc, oh, ow,
- pt, pl, p.PadMode, p.PadValue, p.Act);
- return;
- }
- }
- if (group == 1 && kh == 1 && kw == 1 && sh == 1 && sw == 1 && pt == 0 && pl == 0 && pb == 0 && pr == 0)
- {
- Avx2Kernels.Conv1x1(src, ic, ih * iw, w, bias, dst, oc, p.Act);
- return;
- }
- // unspecialized shape: fall through to the Vector tier
- }
- else if (!Vector.IsHardwareAccelerated)
- {
- ConvScalar(src, ic, ih, iw, w, bias, dst, oc, oh, ow, kh, kw, sh, sw,
- pt, pl, group, p.PadMode, p.PadValue, p.Act);
- return;
- }
-
- // Vector tier
- if (group > 1)
- {
- if (group != ic) // grouped (non-depthwise) conv: scalar reference handles groups correctly
- {
- ConvScalar(src, ic, ih, iw, w, bias, dst, oc, oh, ow, kh, kw, sh, sw,
- pt, pl, group, p.PadMode, p.PadValue, p.Act);
- return;
- }
- ConvDepthwiseVec(src, ic, ih, iw, w, bias, dst, oh, ow, kh, kw,
- pt, pl, p.PadMode, p.PadValue, p.Act);
- return;
- }
- if (kh == 1 && kw == 1 && sh == 1 && sw == 1 && pt == 0 && pl == 0)
- {
- Conv1x1Vec(src, ic, ih, iw, w, bias, dst, oc, oh, ow, p.Act);
- return;
- }
- ConvVecW(src, ic, ih, iw, w, bias, dst, oc, oh, ow, kh, kw, sh, sw,
- pt, pl, group, p.PadMode, p.PadValue, p.Act);
- }
-
- /// dst = act(dst + add) — residual epilogue for fused Conv+Add.
- private static void AddInPlace(Span dst, ReadOnlySpan add, ActivationKind act)
- {
- int n = dst.Length, i = 0, VL = Vector.Count;
- var zero = Vector.Zero;
- for (; i + VL <= n; i += VL)
- {
- var v = Simd.VectorLoad(dst, i) + Simd.VectorLoad(add, i);
- if (act == ActivationKind.Relu) v = Vector.Max(v, zero);
- Simd.VectorStore(dst, i, v);
- }
- for (; i < n; i++)
- {
- float v = dst[i] + add[i];
- dst[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v;
- }
- if (act != ActivationKind.None && act != ActivationKind.Relu)
- ApplyActVec(dst, act);
- }
-
- // ---- scalar reference conv (N==1, dense or grouped) ----
- private static void ConvScalar(ReadOnlySpan src, int ic, int ih, int iw,
- ReadOnlySpan w, ReadOnlySpan bias,
- Span dst, int oc, int oh, int ow, int kh, int kw,
- int sh, int sw, int pt, int pl, int group,
- PadMode padMode, float padValue, ActivationKind act)
- {
- int icg = ic / group, ocg = oc / group;
- for (int g = 0; g < group; g++)
- for (int o = g * ocg; o < (g + 1) * ocg; o++)
- for (int y = 0; y < oh; y++)
- for (int x = 0; x < ow; x++)
- {
- float acc = bias.IsEmpty ? 0f : bias[o];
- for (int c = g * icg; c < (g + 1) * icg; c++)
- for (int ky = 0; ky < kh; ky++)
- {
- int sy = y * sh + ky - pt;
- if ((uint)sy >= (uint)ih)
- {
- if (padMode == PadMode.Reflect) sy = Reflect(sy, ih);
- else continue;
- }
- for (int kx = 0; kx < kw; kx++)
- {
- int sx = x * sw + kx - pl;
- if ((uint)sx >= (uint)iw)
- {
- if (padMode == PadMode.Reflect) sx = Reflect(sx, iw);
- else continue;
- }
- acc += src[(c * ih + sy) * iw + sx] * w[((o * icg + c - g * icg) * kh + ky) * kw + kx];
- }
- }
- dst[(o * oh + y) * ow + x] = ApplyAct(acc, act);
- }
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static int Reflect(int i, int dim)
- {
- // ONNX reflect (half-sample symmetric, no edge repeat): -1→1, dim→dim-2
- if (dim <= 1) return 0; // reflect is undefined on a singleton axis; don't spin
- while (i < 0 || i >= dim) i = i < 0 ? -i : 2 * dim - 2 - i;
- return i;
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static float ApplyAct(float v, ActivationKind act) =>
- act == ActivationKind.Relu ? MathF.Max(v, 0f) :
- act == ActivationKind.Sigmoid ? 1f / (1f + MathF.Exp(-v)) :
- act == ActivationKind.Gelu ? 0.5f * v * (1f + SimdMath.ErfScalar(v * 0.70710678f)) : v;
-
- // ---- vector-over-output-W conv: one output row vector per (oc,y) ----
- private static void ConvVecW(ReadOnlySpan src, int ic, int ih, int iw,
- ReadOnlySpan w, ReadOnlySpan bias,
- Span dst, int oc, int oh, int ow, int kh, int kw,
- int sh, int sw, int pt, int pl, int group,
- PadMode padMode, float padValue, ActivationKind act)
- {
- int icg = ic / group;
- int VL = Vector.Count;
- // interior x-range where all taps in-bounds (no pad needed)
- for (int o = 0; o < oc; o++)
- {
- int g = o / (oc / group); // output channel -> group index
- for (int y = 0; y < oh; y++)
- {
- Span drow = dst.Slice((o * oh + y) * ow, ow);
- if (!bias.IsEmpty) drow.Fill(bias[o]); else drow.Clear();
- for (int c = g * icg; c < (g + 1) * icg; c++)
- {
- ReadOnlySpan sbase = src.Slice(c * ih * iw, ih * iw);
- ReadOnlySpan wk = w.Slice((o * icg + (c - g * icg)) * kh * kw, kh * kw);
- for (int ky = 0; ky < kh; ky++)
- {
- int sy = y * sh + ky - pt;
- bool yOk = (uint)sy < (uint)ih;
- int syR = yOk ? sy : (padMode == PadMode.Reflect ? Reflect(sy, ih) : -1);
- for (int kx = 0; kx < kw; kx++)
- {
- float wv = wk[ky * kw + kx];
- if (wv == 0f) continue;
- if (sw == 1)
- {
- // contiguous reads: dst[x] += wv * src[sy][x + kx - pl]
- int xBegin = Math.Max(0, pl - kx);
- int xEnd = Math.Min(ow, iw + pl - kx);
- if (yOk || padMode == PadMode.Reflect)
- {
- VecAccMul(drow.Slice(xBegin, xEnd - xBegin),
- sbase.Slice(syR * iw + xBegin + kx - pl, xEnd - xBegin), wv);
- }
- else if (padValue != 0f)
- {
- for (int x = xBegin; x < xEnd; x++) drow[x] += wv * padValue;
- }
- // pad regions (constant → add wv*padValue; reflect → gather)
- for (int x = 0; x < xBegin; x++) drow[x] += wv * PadFetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue, syR, sy);
- for (int x = xEnd; x < ow; x++) drow[x] += wv * PadFetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue, syR, sy);
- }
- else
- {
- for (int x = 0; x < ow; x++)
- drow[x] += wv * PadFetch(sbase, sy, x * sw + kx - pl, ih, iw, padMode, padValue, syR, sy);
- }
- }
- }
- }
- ApplyActVec(drow, act);
- }
- }
-
- static float PadFetch(ReadOnlySpan sbase, int sy, int sx, int ih, int iw,
- PadMode mode, float padValue, int syR, int syOrig)
- {
- int yy = syOrig, xx = sx;
- if ((uint)yy >= (uint)ih || (uint)xx >= (uint)iw)
- {
- if (mode == PadMode.Constant) return padValue;
- yy = Reflect(yy, ih); xx = Reflect(xx, iw);
- }
- return sbase[yy * iw + xx];
- }
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void VecAccMul(Span dst, ReadOnlySpan src, float w)
- {
- int i = 0, VL = Vector.Count;
- var vw = new Vector(w);
- for (; i + VL <= dst.Length; i += VL)
- Simd.VectorStore(dst, i, Simd.VectorAddMul(Simd.VectorLoad(dst, i), Simd.VectorLoad(src, i), vw));
- for (; i < dst.Length; i++) dst[i] += src[i] * w;
- }
-
- private static void ApplyActVec(Span row, ActivationKind act)
- {
- if (act == ActivationKind.None) return;
- if (act == ActivationKind.Relu) { Relu(row, row); return; }
- if (act == ActivationKind.Gelu) { SimdMath.Gelu(row, row); return; }
- if (act == ActivationKind.Sigmoid) { SimdMath.Sigmoid(row, row); return; }
- for (int i = 0; i < row.Length; i++) row[i] = ApplyAct(row[i], act);
- }
-
- // ---- 1x1 conv = GEMM over pixels: dst[oc, HW] = W[oc, ic] * src[ic, HW] ----
- private static void Conv1x1Vec(ReadOnlySpan src, int ic, int ih, int iw,
- ReadOnlySpan w, ReadOnlySpan bias,
- Span dst, int oc, int oh, int ow, ActivationKind act)
- {
- int hw = ih * iw;
- int VL = Vector.Count;
- for (int o = 0; o < oc; o++)
- {
- Span drow = dst.Slice(o * hw, hw);
- float b = bias.IsEmpty ? 0f : bias[o];
- for (int i = 0; i < hw; i++) drow[i] = b;
- ReadOnlySpan wo = w.Slice(o * ic, ic);
- for (int c = 0; c < ic; c++)
- {
- float wv = wo[c];
- if (wv == 0f) continue;
- VecAccMul(drow, src.Slice(c * hw, hw), wv);
- }
- ApplyActVec(drow, act);
- }
- }
-
- // ---- depthwise conv (group == C): dst[c,y,x] ----
- private static void ConvDepthwiseVec(ReadOnlySpan src, int ic, int ih, int iw,
- ReadOnlySpan w, ReadOnlySpan bias,
- Span dst, int oh, int ow, int kh, int kw,
- int pt, int pl, PadMode padMode, float padValue, ActivationKind act)
- {
- int VL = Vector.Count;
- for (int c = 0; c < ic; c++)
- {
- ReadOnlySpan sbase = src.Slice(c * ih * iw, ih * iw);
- ReadOnlySpan wk = w.Slice(c * kh * kw, kh * kw);
- for (int y = 0; y < oh; y++)
- {
- Span drow = dst.Slice((c * oh + y) * ow, ow);
- float b = bias.IsEmpty ? 0f : bias[c];
- for (int i = 0; i < ow; i++) drow[i] = b;
- for (int ky = 0; ky < kh; ky++)
- {
- int sy = y + ky - pt;
- for (int kx = 0; kx < kw; kx++)
- {
- float wv = wk[ky * kw + kx];
- if (wv == 0f) continue;
- int xBegin = Math.Max(0, pl - kx);
- int xEnd = Math.Min(ow, iw + pl - kx);
- if ((uint)sy < (uint)ih)
- {
- VecAccMul(drow.Slice(xBegin, xEnd - xBegin),
- sbase.Slice(sy * iw + xBegin + kx - pl, xEnd - xBegin), wv);
- }
- for (int x = 0; x < xBegin; x++) drow[x] += wv * Fetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue);
- for (int x = xEnd; x < ow; x++) drow[x] += wv * Fetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue);
- if ((uint)sy >= (uint)ih && padMode == PadMode.Constant && padValue != 0f)
- for (int x = xBegin; x < xEnd; x++) drow[x] += wv * padValue;
- else if ((uint)sy >= (uint)ih && padMode == PadMode.Reflect)
- for (int x = xBegin; x < xEnd; x++) drow[x] += wv * sbase[Reflect(sy, ih) * iw + x + kx - pl];
- }
- }
- ApplyActVec(drow, act);
- }
- }
-
- static float Fetch(ReadOnlySpan s, int y, int x, int ih, int iw, PadMode m, float pv)
- {
- if ((uint)y >= (uint)ih || (uint)x >= (uint)iw)
- {
- if (m == PadMode.Constant) return pv;
- y = Reflect(y, ih); x = Reflect(x, iw);
- }
- return s[y * iw + x];
- }
- }
-
- // ======================== MaxPool ========================
- public static void MaxPool(ReadOnlySpan src, int[] inShape, Span dst, int[] outShape,
- PoolP p)
- {
- int c = inShape[1], ih = inShape[2], iw = inShape[3];
- int oh = outShape[2], ow = outShape[3];
- int kh = p.Kernel[0], kw = p.Kernel[1], sh = p.Strides[0], sw = p.Strides[1];
- int pt = p.Pads[0], pl = p.Pads[1];
- if (Avx2.IsSupported && Fma.IsSupported &&
- kh == 2 && kw == 2 && sh == 2 && sw == 2 && pt == 0 && pl == 0)
- {
- Avx2Kernels.MaxPool2x2(src, c, ih, iw, dst, oh, ow, p.FuseRelu);
- return;
- }
- for (int ch = 0; ch < c; ch++)
- {
- ReadOnlySpan s = src.Slice(ch * ih * iw, ih * iw);
- Span d = dst.Slice(ch * oh * ow, oh * ow);
- for (int y = 0; y < oh; y++)
- for (int x = 0; x < ow; x++)
- {
- float m = float.NegativeInfinity;
- for (int ky = 0; ky < kh; ky++)
- {
- int sy = y * sh + ky - pt;
- if ((uint)sy >= (uint)ih) continue;
- for (int kx = 0; kx < kw; kx++)
- {
- int sx = x * sw + kx - pl;
- if ((uint)sx >= (uint)iw) continue;
- float v = s[sy * iw + sx];
- if (v > m) m = v;
- }
- }
- if (m == float.NegativeInfinity) m = 0f;
- d[y * ow + x] = p.FuseRelu && m < 0f ? 0f : m;
- }
- }
- }
-
- // ======================== elementwise ========================
- public static void Elementwise(ReadOnlySpan a, int[] aShape,
- ReadOnlySpan b, int[] bShape, Span