From 223fc69cd5c494fcb54405e5fdcb617de147078c Mon Sep 17 00:00:00 2001 From: sdcb Date: Tue, 13 Oct 2026 23:15:40 +0800 Subject: [PATCH 1/9] Run on Sdcb.SimdOnnx: engine removed, UpHead/ChannelShift as engine extensions (2.0.0) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Directory.Build.props | 6 +- bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs | 2 +- bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs | 86 +- bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs | 12 +- src/Sdcb.SimdGcdrNet/Backends/Metal/Metal.cs | 337 ---- .../Backends/Metal/MetalBackend.cs | 86 - .../Backends/Metal/MtlGraph.cs | 798 --------- .../Backends/Metal/MtlPipeline.cs | 16 +- .../Backends/Metal/MtlSession.cs | 250 --- src/Sdcb.SimdGcdrNet/Backends/Metal/ObjC.cs | 229 --- .../Backends/Metal/Shaders/common.metal | 43 - .../Backends/Metal/Shaders/conv.metal | 333 ---- .../Backends/Metal/Shaders/conv_mm.metal | 229 --- .../Backends/Metal/Shaders/glue.metal | 332 +--- .../Backends/Metal/Shaders/shift.metal | 38 + .../Backends/Vulkan/Shaders/bilinear.comp | 44 - .../Backends/Vulkan/Shaders/bilinear.spv | Bin 3600 -> 0 bytes .../Backends/Vulkan/Shaders/bilinear4.comp | 44 - .../Backends/Vulkan/Shaders/bilinear4.spv | Bin 3720 -> 0 bytes .../Backends/Vulkan/Shaders/bilinear4_h.spv | Bin 3852 -> 0 bytes .../Backends/Vulkan/Shaders/bilinear_h.spv | Bin 3716 -> 0 bytes .../Backends/Vulkan/Shaders/concat_ax.comp | 27 - .../Backends/Vulkan/Shaders/concat_ax.spv | Bin 1612 -> 0 bytes .../Backends/Vulkan/Shaders/concat_ax_h.spv | Bin 1620 -> 0 bytes .../Backends/Vulkan/Shaders/conv_cm.comp | 358 ---- .../Backends/Vulkan/Shaders/conv_cm_h.spv | Bin 29444 -> 0 bytes .../Backends/Vulkan/Shaders/conv_cm_l_h.spv | Bin 15856 -> 0 bytes .../Vulkan/Shaders/conv_cm_l_n32_h.spv | Bin 17980 -> 0 bytes .../Vulkan/Shaders/conv_cm_l_n32s_h.spv | Bin 23940 -> 0 bytes .../Vulkan/Shaders/conv_cm_l_n64_h.spv | Bin 16564 -> 0 bytes .../Backends/Vulkan/Shaders/conv_cm_n32_h.spv | Bin 30676 -> 0 bytes .../Vulkan/Shaders/conv_cm_n32s_h.spv | Bin 32888 -> 0 bytes .../Backends/Vulkan/Shaders/conv_cm_n64_h.spv | Bin 33428 -> 0 bytes .../Vulkan/Shaders/conv_cm_sg16_h.spv | Bin 15648 -> 0 bytes .../Vulkan/Shaders/conv_cm_sg16_n32_h.spv | Bin 17656 -> 0 bytes .../Vulkan/Shaders/conv_cm_sg16_n32s_h.spv | Bin 23940 -> 0 bytes .../Vulkan/Shaders/conv_cm_sg16_n64_h.spv | Bin 16372 -> 0 bytes .../Backends/Vulkan/Shaders/conv_dot.comp | 146 -- .../Vulkan/Shaders/conv_dot_s1_4x1_h.spv | Bin 23000 -> 0 bytes .../Vulkan/Shaders/conv_dot_s1_4x2_h.spv | Bin 37212 -> 0 bytes .../Vulkan/Shaders/conv_dot_s2_2x1_h.spv | Bin 16092 -> 0 bytes .../Vulkan/Shaders/conv_dot_s2_2x2_h.spv | Bin 23856 -> 0 bytes .../Backends/Vulkan/Shaders/conv_gen.spv | Bin 36836 -> 0 bytes .../Backends/Vulkan/Shaders/conv_gen_h.spv | Bin 37080 -> 0 bytes .../Backends/Vulkan/Shaders/conv_k11s1.spv | Bin 36516 -> 0 bytes .../Backends/Vulkan/Shaders/conv_k11s1_h.spv | Bin 36760 -> 0 bytes .../Backends/Vulkan/Shaders/conv_k33s1.spv | Bin 36616 -> 0 bytes .../Backends/Vulkan/Shaders/conv_k33s1_h.spv | Bin 36860 -> 0 bytes .../Backends/Vulkan/Shaders/conv_k33s2.spv | Bin 36656 -> 0 bytes .../Backends/Vulkan/Shaders/conv_k33s2_h.spv | Bin 36900 -> 0 bytes .../Backends/Vulkan/Shaders/conv_tile.comp | 166 -- .../Backends/Vulkan/Shaders/convpool.comp | 135 -- .../Backends/Vulkan/Shaders/convpool.spv | Bin 8404 -> 0 bytes .../Backends/Vulkan/Shaders/convpool_h.spv | Bin 8600 -> 0 bytes .../Backends/Vulkan/Shaders/convpool_k3.comp | 104 -- .../Backends/Vulkan/Shaders/convpool_k3.spv | Bin 21556 -> 0 bytes .../Backends/Vulkan/Shaders/convpool_k3_h.spv | Bin 22248 -> 0 bytes .../Backends/Vulkan/Shaders/convpool_pi_h.spv | Bin 11732 -> 0 bytes .../Backends/Vulkan/Shaders/depthwise.comp | 57 - .../Backends/Vulkan/Shaders/depthwise.spv | Bin 6284 -> 0 bytes .../Backends/Vulkan/Shaders/depthwise4.comp | 58 - .../Backends/Vulkan/Shaders/depthwise4.spv | Bin 6600 -> 0 bytes .../Backends/Vulkan/Shaders/depthwise4_h.spv | Bin 6700 -> 0 bytes .../Backends/Vulkan/Shaders/depthwise_h.spv | Bin 6368 -> 0 bytes .../Backends/Vulkan/Shaders/ew2.comp | 33 - .../Backends/Vulkan/Shaders/ew2.spv | Bin 3500 -> 0 bytes .../Backends/Vulkan/Shaders/ew2_h.spv | Bin 3568 -> 0 bytes .../Backends/Vulkan/Shaders/ew2v.comp | 26 - .../Backends/Vulkan/Shaders/ew2v.spv | Bin 2336 -> 0 bytes .../Backends/Vulkan/Shaders/ew2v_h.spv | Bin 2420 -> 0 bytes .../Backends/Vulkan/Shaders/gemm.comp | 88 - .../Backends/Vulkan/Shaders/gemm.spv | Bin 7040 -> 0 bytes .../Backends/Vulkan/Shaders/gemm_dot.comp | 89 - .../Backends/Vulkan/Shaders/gemm_dot_h.spv | Bin 26404 -> 0 bytes .../Backends/Vulkan/Shaders/gemm_h.spv | Bin 7132 -> 0 bytes .../Backends/Vulkan/Shaders/layernorm.comp | 41 - .../Backends/Vulkan/Shaders/layernorm.spv | Bin 2932 -> 0 bytes .../Backends/Vulkan/Shaders/layernorm_h.spv | Bin 3016 -> 0 bytes .../Backends/Vulkan/Shaders/maxpool.comp | 37 - .../Backends/Vulkan/Shaders/maxpool.spv | Bin 3116 -> 0 bytes .../Backends/Vulkan/Shaders/maxpool4.comp | 39 - .../Backends/Vulkan/Shaders/maxpool4.spv | Bin 3332 -> 0 bytes .../Backends/Vulkan/Shaders/maxpool4_h.spv | Bin 3400 -> 0 bytes .../Backends/Vulkan/Shaders/maxpool_h.spv | Bin 3168 -> 0 bytes .../Backends/Vulkan/Shaders/pad.comp | 47 - .../Backends/Vulkan/Shaders/pad.spv | Bin 6260 -> 0 bytes .../Backends/Vulkan/Shaders/pad_h.spv | Bin 6296 -> 0 bytes .../Backends/Vulkan/Shaders/sgprobe.comp | 13 - .../Backends/Vulkan/Shaders/sgprobe.spv | Bin 1080 -> 0 bytes .../Backends/Vulkan/Shaders/slice.comp | 27 - .../Backends/Vulkan/Shaders/slice.spv | Bin 2668 -> 0 bytes .../Backends/Vulkan/Shaders/slice_h.spv | Bin 2676 -> 0 bytes .../Backends/Vulkan/Shaders/transpose.comp | 24 - .../Backends/Vulkan/Shaders/transpose.spv | Bin 1900 -> 0 bytes .../Backends/Vulkan/Shaders/transpose_h.spv | Bin 1908 -> 0 bytes .../Backends/Vulkan/Shaders/unary.comp | 44 - .../Backends/Vulkan/Shaders/unary.spv | Bin 4712 -> 0 bytes .../Backends/Vulkan/Shaders/unary_h.spv | Bin 4764 -> 0 bytes src/Sdcb.SimdGcdrNet/Backends/Vulkan/Vk.cs | 589 ------- .../Backends/Vulkan/VkBackend.cs | 91 - .../Backends/Vulkan/VkConstStore.cs | 91 - .../Backends/Vulkan/VkDevice.cs | 938 ---------- .../Backends/Vulkan/VkGraph.cs | 738 -------- .../Backends/Vulkan/VkPipeline2.cs | 16 +- .../Backends/Vulkan/VkSession.cs | 394 ----- .../Backends/Vulkan/VkShaders.cs | 21 - .../Backends/Vulkan/tools/build-shaders.bat | 56 +- src/Sdcb.SimdGcdrNet/Engine/CompiledModel.cs | 30 - src/Sdcb.SimdGcdrNet/Engine/CpuKernels.cs | 76 - src/Sdcb.SimdGcdrNet/Engine/ExecPlan.cs | 208 --- src/Sdcb.SimdGcdrNet/Engine/GraphCompiler.cs | 1529 ----------------- src/Sdcb.SimdGcdrNet/Engine/Layout.cs | 111 -- src/Sdcb.SimdGcdrNet/Engine/LayoutPlanner.cs | 243 --- src/Sdcb.SimdGcdrNet/Engine/Session.cs | 327 ---- .../Extensions/ChannelShiftOp.cs | 189 ++ .../Extensions/GcdrExtensions.cs | 16 + .../Extensions/GcdrKernels.Avx512.cs | 88 + .../Extensions/GcdrKernels.Neon.cs | 59 + .../Extensions/GcdrKernels.PackedSimd.cs | 59 + .../Extensions/GcdrKernels.cs | 312 ++++ src/Sdcb.SimdGcdrNet/Extensions/UpHeadOp.cs | 279 +++ src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs | 21 +- src/Sdcb.SimdGcdrNet/Kernels/Avx2Kernels.cs | 970 ----------- src/Sdcb.SimdGcdrNet/Kernels/NeonKernels.cs | 54 - src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Avx512.cs | 392 ----- src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Neon.cs | 373 ---- .../Kernels/Nhwc.PackedSimd.cs | 251 --- src/Sdcb.SimdGcdrNet/Kernels/Nhwc.cs | 904 ---------- .../Kernels/NhwcWinograd.Neon.cs | 75 - src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.cs | 399 ----- src/Sdcb.SimdGcdrNet/Kernels/Ops.cs | 1005 ----------- src/Sdcb.SimdGcdrNet/Kernels/PackedKernels.cs | 55 - src/Sdcb.SimdGcdrNet/Kernels/PixelOps.cs | 1 + src/Sdcb.SimdGcdrNet/Kernels/Simd.cs | 33 - src/Sdcb.SimdGcdrNet/Kernels/SimdMath.cs | 270 --- src/Sdcb.SimdGcdrNet/Onnx/OnnxModel.cs | 49 - src/Sdcb.SimdGcdrNet/Onnx/OnnxModelData.cs | 195 --- src/Sdcb.SimdGcdrNet/Onnx/OnnxProtoReader.cs | 514 ------ src/Sdcb.SimdGcdrNet/Sdcb.SimdGcdrNet.csproj | 6 + .../Avx512KernelTests.cs | 163 +- tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs | 132 +- tests/Sdcb.SimdGcdrNet.Tests/GoldenTests.cs | 5 +- tests/Sdcb.SimdGcdrNet.Tests/MetalTests.cs | 10 +- .../Sdcb.SimdGcdrNet.Tests/MtlKernelTests.cs | 221 --- .../Sdcb.SimdGcdrNet.Tests/NeonKernelTests.cs | 143 +- .../PixelFormatTests.cs | 2 + tests/Sdcb.SimdGcdrNet.Tests/VkDebugTests.cs | 86 - tests/Sdcb.SimdGcdrNet.Tests/VkKernelTests.cs | 216 +-- tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs | 14 +- 149 files changed, 1201 insertions(+), 16262 deletions(-) delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Metal/Metal.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Metal/MetalBackend.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Metal/MtlGraph.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Metal/MtlSession.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Metal/ObjC.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/common.metal delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv.metal delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv_mm.metal create mode 100644 src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/shift.metal delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32s_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n64_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32s_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n64_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32s_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n64_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x1_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x2_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x1_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x2_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_tile.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_pi_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.comp delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary_h.spv delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/Vk.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkBackend.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkConstStore.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkDevice.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkGraph.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkSession.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkShaders.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Engine/CompiledModel.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Engine/CpuKernels.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Engine/ExecPlan.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Engine/GraphCompiler.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Engine/Layout.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Engine/LayoutPlanner.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Engine/Session.cs create mode 100644 src/Sdcb.SimdGcdrNet/Extensions/ChannelShiftOp.cs create mode 100644 src/Sdcb.SimdGcdrNet/Extensions/GcdrExtensions.cs create mode 100644 src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Avx512.cs create mode 100644 src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Neon.cs create mode 100644 src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.PackedSimd.cs create mode 100644 src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.cs create mode 100644 src/Sdcb.SimdGcdrNet/Extensions/UpHeadOp.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/Avx2Kernels.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/NeonKernels.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Avx512.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Neon.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/Nhwc.PackedSimd.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/Nhwc.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.Neon.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/Ops.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/PackedKernels.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/Simd.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Kernels/SimdMath.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Onnx/OnnxModel.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Onnx/OnnxModelData.cs delete mode 100644 src/Sdcb.SimdGcdrNet/Onnx/OnnxProtoReader.cs delete mode 100644 tests/Sdcb.SimdGcdrNet.Tests/MtlKernelTests.cs delete mode 100644 tests/Sdcb.SimdGcdrNet.Tests/VkDebugTests.cs diff --git a/Directory.Build.props b/Directory.Build.props index d0b3d96..7e66b66 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -4,8 +4,12 @@ never force a models release). Edit here and keep build/00-common.linq in sync. --> - 0.1.0 + 2.0.0 1.0.0 + + 0.1.0-preview.1 + $(MSBuildThisFileDirectory)..\Sdcb.SimdOnnx\src\Sdcb.SimdOnnx\Sdcb.SimdOnnx.csproj diff --git a/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs b/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs index ad03c02..6aad87e 100644 --- a/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs +++ b/bench/Sdcb.SimdGcdrNet.Benchmark/CiRunner.cs @@ -195,7 +195,7 @@ private static JsonObject HostInfo() string isa = RuntimeInformation.ProcessArchitecture switch { Architecture.X64 or Architecture.X86 => - Kernels.Nhwc.UseAvx512 ? "avx512" : Avx2.IsSupported ? "avx2" : Avx.IsSupported ? "avx" : + Sdcb.SimdOnnx.Kernels.Nhwc.UseAvx512 ? "avx512" : Avx2.IsSupported ? "avx2" : Avx.IsSupported ? "avx" : System.Numerics.Vector.IsHardwareAccelerated ? "sse" : "scalar", Architecture.Arm64 => AdvSimd.IsSupported ? "neon" : "scalar", _ => System.Numerics.Vector.IsHardwareAccelerated ? "vector" : "scalar", diff --git a/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs b/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs index 100c298..1f3caa5 100644 --- a/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs +++ b/bench/Sdcb.SimdGcdrNet.Benchmark/Program.cs @@ -1,7 +1,8 @@ using System.Diagnostics; using System.Runtime.InteropServices; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Onnx; namespace Sdcb.SimdGcdrNet.Benchmark; @@ -44,7 +45,7 @@ public static int Main(string[] args) var m = OnnxModel.Load(mpath); int c = which == "gc" ? 3 : 6; // run the SAME (gpu-layout) plan on CPU and Metal so node indices align - var cpuPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdGcdrNet.Backends.Metal.MetalBackend.PlanOptions); + var cpuPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdOnnx.Backends.Metal.MetalBackend.PlanOptions with { Extensions = GcdrExtensions.Instance }); using var cpu = new InferenceSession(cpuPlan); var rng = new Random(0); if (args.Length > 4) @@ -56,9 +57,9 @@ public static int Main(string[] args) } else for (int i = 0; i < cpu.Input.Length; i++) cpu.Input[i] = rng.NextSingle(); - var mbe = Sdcb.SimdGcdrNet.Backends.Metal.MetalBackend.Default; - var mPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdGcdrNet.Backends.Metal.MetalBackend.PlanOptions); - using var mtl = mbe.NewSession(m, [1, c, h, w]); + var mbe = Sdcb.SimdOnnx.Backends.Metal.MetalBackend.Default; + var mPlan = GraphCompiler.Compile(m, [1, c, h, w], Sdcb.SimdOnnx.Backends.Metal.MetalBackend.PlanOptions with { Extensions = GcdrExtensions.Instance }); + using var mtl = mbe.NewSession(m, [1, c, h, w], GcdrExtensions.Instance); cpu.Input.CopyTo(mtl.Input); mtl.DebugUploadInput(); if (Environment.GetEnvironmentVariable("MTRACE_NODES") == "1") @@ -106,9 +107,9 @@ public static int Main(string[] args) var exp = new float[wsrc.Length]; for (int o2 = 0; o2 < oc_; o2++) for (int c2 = 0; c2 < ic_; c2++) for (int t = 0; t < taps; t++) exp[(t * ic_ + c2) * oc_ + o2] = wsrc[(o2 * ic_ + c2) * taps + t]; - var mw = mtl.DebugConst(nd_.In[1], 2, wsrc.Length); + var mw = mtl.DebugConst(nd_.In[1], ConstKind.ConvW, wsrc.Length); int wb = -1; for (int k = 0; k < exp.Length; k++) if (Math.Abs(mw[k] - exp[k]) > 1e-3) { wb = k; break; } - Console.WriteLine($" w-check: pack len {exp.Length} off {mtl.DebugConstOff(nd_.In[1], 2)} first-diff {wb} head {string.Join(' ', mw.Take(8).Select(x => $"{x:F3}"))} vs exp {string.Join(' ', exp.Take(8).Select(x => $"{x:F3}"))}"); + Console.WriteLine($" w-check: pack len {exp.Length} off {mtl.DebugConstOff(nd_.In[1], ConstKind.ConvW)} first-diff {wb} head {string.Join(' ', mw.Take(8).Select(x => $"{x:F3}"))} vs exp {string.Join(' ', exp.Take(8).Select(x => $"{x:F3}"))}"); if (wb >= 0) Console.WriteLine($" @{wb}: got {mw[wb]:F4} exp {exp[wb]:F4}; around got {string.Join(' ', mw.Skip(wb).Take(6).Select(x => $"{x:F3}"))} exp {string.Join(' ', exp.Skip(wb).Take(6).Select(x => $"{x:F3}"))}"); } if (bad < 0 && (rmse > 1e-2 || nan > 0 || nanw > 0)) @@ -177,7 +178,7 @@ public static int Main(string[] args) // --node gc|dr h w idx reps : time one node in isolation (after a full warmup run) bool isDr = args[1] == "dr"; int nh = int.Parse(args[2]), nw = int.Parse(args[3]), idx = int.Parse(args[4]), nreps = int.Parse(args[5]); - var cm = CompiledModel.Compile(OnnxModel.Load(isDr ? drPath : gcPath), [1, isDr ? 6 : 3, nh, nw]); + var cm = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(isDr ? drPath : gcPath), [1, isDr ? 6 : 3, nh, nw], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); var ss = cm.CreateSession(); new Random(0).NextBytes(MemoryMarshal.AsBytes(ss.Input)); if (args.Length > 6 && args[6] == "u") { var rr = new Random(0); var inp = ss.Input; for (int i = 0; i < inp.Length; i++) inp[i] = rr.NextSingle(); } @@ -202,8 +203,8 @@ public static int Main(string[] args) foreach ((string tag, string mp, int cin) in new[] { ("gc", gcPath, 3), ("dr", drPath, 6) }) { var m = OnnxModel.Load(mp); - var cpu = CompiledModel.Compile(m, [1, cin, ch, cw]).CreateSession(); - using var vk = Backends.Vulkan.VkBackend.Default.NewSession(m, [1, cin, ch, cw]); + var cpu = new CompiledModel(GraphCompiler.Compile(m, [1, cin, ch, cw], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })).CreateSession(); + using var vk = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.NewSession(m, [1, cin, ch, cw], GcdrExtensions.Instance); var rng = new Random(7); for (int i = 0; i < cpu.Input.Length; i++) cpu.Input[i] = vk.Input[i] = rng.NextSingle(); cpu.Run(); vk.Run(); @@ -227,14 +228,14 @@ public static int Main(string[] args) int nh = args.Length > 2 ? int.Parse(args[2]) : 128, nw = args.Length > 3 ? int.Parse(args[3]) : 192; var m = OnnxModel.Load(isDr ? drPath : gcPath); int[] shape = [1, isDr ? 6 : 3, nh, nw]; - var plan = GraphCompiler.Compile(m, shape, Backends.Vulkan.VkBackend.PlanOptions); + var plan = GraphCompiler.Compile(m, shape, Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.PlanOptions with { Extensions = GcdrExtensions.Instance }); using var cpu = new InferenceSession(plan); - using var gpu = Backends.Vulkan.VkBackend.Default.NewSession(m, shape); + using var gpu = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); var rng = new Random(7); for (int i = 0; i < cpu.Input.Length; i++) cpu.Input[i] = gpu.Input[i] = rng.NextSingle(); gpu.DebugUploadInput(); - var ranges = Backends.Vulkan.VkBackend.Default.GraphFor(m, shape).NodeRanges; - var gd = Backends.Vulkan.VkBackend.Default.GraphFor(m, shape).Dispatches; + var ranges = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.GraphFor(m, shape, ext: GcdrExtensions.Instance).NodeRanges; + var gd = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default.GraphFor(m, shape, ext: GcdrExtensions.Instance).Dispatches; for (int i = 0; i < plan.Nodes.Length; i++) { var nd = plan.Nodes[i]; @@ -304,11 +305,11 @@ public static int Main(string[] args) { int h = args.Length > 1 ? int.Parse(args[1]) : 1152; int w = args.Length > 2 ? int.Parse(args[2]) : 1600; - var be = Backends.Metal.MetalBackend.Default; + var be = Sdcb.SimdOnnx.Backends.Metal.MetalBackend.Default; foreach ((string tag, string mp, int cin) in new[] { ("gc", gcPath, 3), ("dr", drPath, 6) }) { var m = OnnxModel.Load(mp); - using var s = be.NewSession(m, [1, cin, h, w]); + using var s = be.NewSession(m, [1, cin, h, w], GcdrExtensions.Instance); s.Run(); // warm var agg = s.DebugProfile(); Console.WriteLine($"--- {tag} metal per-shader (total {agg.Values.Sum():F1} ms) ---"); @@ -387,7 +388,7 @@ public static int Main(string[] args) // destroy, recreate — any hash change = stale-read/nondeterminism at session level. private static int VkDup(string gcPath, string drPath, int h, int w, bool useDr) { - var be = Backends.Vulkan.VkBackend.Default; + var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default; string path = useDr ? drPath : gcPath; int c = useDr ? 6 : 3; var m = OnnxModel.Load(path); @@ -399,7 +400,7 @@ static ulong Fhash(ReadOnlySpan d) } for (int round = 0; round < 4; round++) { - using var s = be.NewSession(m, [1, c, h, w]); + using var s = be.NewSession(m, [1, c, h, w], GcdrExtensions.Instance); var rng = new Random(42); for (int i = 0; i < s.Input.Length; i++) s.Input[i] = rng.NextSingle(); s.Run(); @@ -409,7 +410,7 @@ static ulong Fhash(ReadOnlySpan d) // interleave: run smaller sessions to churn memory foreach (int cs in new[] { 1024, 1312, 1152 }) { - using var s2 = be.NewSession(m, [1, c, cs, cs]); + using var s2 = be.NewSession(m, [1, c, cs, cs], GcdrExtensions.Instance); new Random(0).NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(s2.Input)); s2.Run(); } @@ -423,7 +424,7 @@ private static void Analyze(string gcPath, string drPath, int h, int w) { foreach (var (path, c) in new[] { (gcPath, 3), (drPath, 6) }) { - var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w]); + var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }); var written = new List<(long s, long e)>(); bool Covered(long s, long e) => written.Any(r => r.s <= s && r.e >= e); @@ -463,7 +464,7 @@ bool Covered(long s, long e) => // stage first diverges. private static int Probe(string gcPath, string drPath, string imgPath, int[] churnPx) { - var be = Backends.Vulkan.VkBackend.Default; + var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default; var gc = OnnxModel.Load(gcPath); var dr = OnnxModel.Load(drPath); int iw, ih; byte[] bgr; (iw, ih) = ImageSize(imgPath); @@ -488,10 +489,10 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag) { p.Enhance(bgr, iw * 3, dst, iw * 3); p.DebugIntermediates(gcI, gcO, drI); - Console.WriteLine($"{tag}: dst={Fb(dst):x16} gcIn={Ff(gcI):x16} gcOut={Ff(gcO):x16} drIn={Ff(drI):x16} live={Backends.Vulkan.VkDevice.LiveDeviceBytes >> 20}MB"); + Console.WriteLine($"{tag}: dst={Fb(dst):x16} gcIn={Ff(gcI):x16} gcOut={Ff(gcO):x16} drIn={Ff(drI):x16} live={Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.LiveDeviceBytes >> 20}MB"); } - var p1 = be.NewPipe(gc, dr, ih, iw); + var p1 = NewVkPipe(be, gc, dr, ih, iw); Run(p1, "p1-run1"); Run(p1, "p1-run2"); var churn = new List(); @@ -499,7 +500,7 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag) { int cw = Math.Max(64, (int)Math.Sqrt(px * 16.0 / 9) / 32 * 32); int chh = Math.Max(64, px / Math.Max(cw, 1) / 32 * 32); - var cp = be.NewPipe(gc, dr, chh, cw); + var cp = NewVkPipe(be, gc, dr, chh, cw); byte[] cb = new byte[chh * cw * 3]; new Random(px).NextBytes(cb); byte[] cd = new byte[chh * cw * 3]; cp.Enhance(cb, cw * 3, cd, cw * 3); @@ -507,7 +508,7 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag) if (churn.Count > 1) { churn[0].Dispose(); churn.RemoveAt(0); } } foreach (var c in churn) c.Dispose(); - var p2 = be.NewPipe(gc, dr, ih, iw); + var p2 = NewVkPipe(be, gc, dr, ih, iw); Run(p2, "p2-after-churn"); p2.Dispose(); p1.Dispose(); return 0; @@ -516,14 +517,14 @@ void Run(Backends.Vulkan.VkGcdrPipeline p, string tag) // bisect which plan value first diverges between two pipeline instances. private static int Bisect(string gcPath, string drPath, string imgPath, int[] churnPx) { - var be = Backends.Vulkan.VkBackend.Default; + var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default; var gc = OnnxModel.Load(gcPath); var dr = OnnxModel.Load(drPath); int iw, ih; (iw, ih) = ImageSize(imgPath); byte[] bgr = DecodeBgr(imgPath); byte[] dst = new byte[bgr.Length]; - var p1 = be.NewPipe(gc, dr, ih, iw); + var p1 = NewVkPipe(be, gc, dr, ih, iw); p1.Enhance(bgr, iw * 3, dst, iw * 3); // replicate corpus allocation history: run every corpus image sorted by px // (up to `churnPx[0]` predecessors, or all when empty) with real (h,w) sizes @@ -537,7 +538,7 @@ private static int Bisect(string gcPath, string drPath, string imgPath, int[] ch var alive = new List(); foreach (var e in seq.Take(take)) { - var cp = be.NewPipe(gc, dr, e.h, e.w); + var cp = NewVkPipe(be, gc, dr, e.h, e.w); byte[] cb = new byte[e.h * e.w * 3]; new Random(e.w * e.h).NextBytes(cb); byte[] cd = new byte[e.h * e.w * 3]; cp.Enhance(cb, e.w * 3, cd, e.w * 3); @@ -545,7 +546,7 @@ private static int Bisect(string gcPath, string drPath, string imgPath, int[] ch if (alive.Count > 1) { alive[0].Dispose(); alive.RemoveAt(0); } } foreach (var c in alive) c.Dispose(); - var p2 = be.NewPipe(gc, dr, ih, iw); + var p2 = NewVkPipe(be, gc, dr, ih, iw); p2.Enhance(bgr, iw * 3, dst, iw * 3); // dump BOTH whole gc arenas in one shot each, diff per plan value on host @@ -749,7 +750,7 @@ static ulong Ff(Span d) }; } - Backends.Vulkan.VkDevice.ResetPeakDeviceBytes(); + Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.ResetPeakDeviceBytes(); long gc0 = GC.GetTotalMemory(true); double wsLast = wsLoaded; var hashes = new Dictionary(); @@ -824,8 +825,8 @@ static ulong Ff(Span d) Console.WriteLine($" ws: init {wsInit0:F0} -> loaded {wsLoaded:F0} MB ({wsLoaded - wsInit0:+0;-0}), " + $"loaded -> last {wsLast:F0} MB (delta {wsLast - wsLoaded:+0;-0} MB), peak {wsPeak:F0} MB"); Console.WriteLine($" mem: managed {gcD / 1048576.0:+0;-0} MB (after forced GC), " + - $"vkDevice live {Backends.Vulkan.VkDevice.LiveDeviceBytes / 1048576.0:F0} MB " + - $"(peak {Backends.Vulkan.VkDevice.PeakDeviceBytes / 1048576.0:F0} MB)"); + $"vkDevice live {Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.LiveDeviceBytes / 1048576.0:F0} MB " + + $"(peak {Sdcb.SimdOnnx.Backends.Vulkan.VkDevice.PeakDeviceBytes / 1048576.0:F0} MB)"); return 0; } @@ -948,7 +949,7 @@ private static int Trace(string gcPath, string drPath, int h, int w) { foreach (var (path, c) in new[] { (gcPath, 3), (drPath, 6) }) { - var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w]); + var plan = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }); long maxT = plan.Values.Max(v => v.Shape.Aggregate(1L, (a, x) => a * x)); Console.WriteLine($"max tensor {maxT * 4 / 1048576.0:F0} MB, arena {plan.ArenaFloats * 4L / 1048576.0:F0} MB"); using var s = new InferenceSession(plan); @@ -972,8 +973,8 @@ private static int Trace(string gcPath, string drPath, int h, int w) private static int Bench(string gcPath, string drPath, int h, int w, int reps) { - var gc = CompiledModel.Compile(OnnxModel.Load(gcPath), [1, 3, h, w]); - var dr = CompiledModel.Compile(OnnxModel.Load(drPath), [1, 6, h, w]); + var gc = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(gcPath), [1, 3, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); + var dr = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(drPath), [1, 6, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); var s1 = gc.CreateSession(); var s2 = dr.CreateSession(); new Random(0).NextBytes(MemoryMarshal.AsBytes(s1.Input)); new Random(1).NextBytes(MemoryMarshal.AsBytes(s2.Input)); @@ -1033,11 +1034,11 @@ private static int BenchE2e(string gcPath, string drPath, int h, int w, int reps private static int VkProf(string gcPath, string drPath, int h, int w) { - var be = Backends.Vulkan.VkBackend.Default; + var be = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend.Default; foreach ((string tag, string mp, int cin) in new[] { ("gc", gcPath, 3), ("dr", drPath, 6) }) { var m = OnnxModel.Load(mp); - using var s = be.NewSession(m, [1, cin, h, w]); + using var s = be.NewSession(m, [1, cin, h, w], GcdrExtensions.Instance); s.Run(); // warm var agg = s.DebugProfile(); double tot = agg.Values.Sum(); @@ -1078,4 +1079,11 @@ private static string FindTestdata() } throw new DirectoryNotFoundException("testdata not found"); } -} + + private static Backends.Vulkan.VkGcdrPipeline NewVkPipe(Sdcb.SimdOnnx.Backends.Vulkan.VkBackend be, OnnxModel gc, OnnxModel dr, int ih, int iw) + { + var p = new Backends.Vulkan.VkGcdrPipeline(be, gc, dr); + try { p.Bind(ih, iw); return p; } + catch { p.Dispose(); throw; } + } +} \ No newline at end of file diff --git a/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs b/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs index 0362e48..af12d38 100644 --- a/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs +++ b/bench/Sdcb.SimdGcdrNet.WasmBench/Program.cs @@ -5,9 +5,11 @@ using System.Runtime.Intrinsics.Wasm; using System.Text.Json; using System.Text.Json.Nodes; -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Kernels; using Sdcb.SimdGcdrNet.Kernels; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Onnx; using Sdcb.SimdGcdrNet.Testing; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; @@ -274,8 +276,8 @@ private static async Task Corpus(string backend, int count, JsonObject doc) private static async Task Bench(int h, int w, int reps, JsonObject doc) { var (gcBytes, drBytes) = await Models(); - var gc = CompiledModel.Compile(OnnxModel.Load(gcBytes), [1, 3, h, w]); - var dr = CompiledModel.Compile(OnnxModel.Load(drBytes), [1, 6, h, w]); + var gc = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(gcBytes), [1, 3, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); + var dr = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(drBytes), [1, 6, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); using var s1 = gc.CreateSession(); using var s2 = dr.CreateSession(); new Random(0).NextBytes(MemoryMarshal.AsBytes(s1.Input)); @@ -381,7 +383,7 @@ private static async Task TiersCore(int h, int w, JsonObject doc) bool ok = true; foreach (var (tag, model, c) in new[] { ("gc", gcBytes, 3), ("dr", drBytes, 6) }) { - var cm = CompiledModel.Compile(OnnxModel.Load(model), [1, c, h, w]); + var cm = new CompiledModel(GraphCompiler.Compile(OnnxModel.Load(model), [1, c, h, w], CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); var outs = new float[2][]; for (int pass = 0; pass < 2; pass++) { diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Metal.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/Metal.cs deleted file mode 100644 index 0263c36..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Metal.cs +++ /dev/null @@ -1,337 +0,0 @@ -using System.Runtime.InteropServices; - -namespace Sdcb.SimdGcdrNet.Backends.Metal; - -// Thin Metal wrappers over ObjC msgSend. +1-owned objects are released on -// Dispose; autoreleased objects (commandBuffer, computeCommandEncoder, …) are -// only valid inside the current NSAutoreleasePool — all public entry points -// that create them take their own pool. - -// MTLResourceOptions: StorageModeShared = 0 << 4 = 0 (the zero value), which -// is what the whole backend uses — unified memory means the same allocation -// is CPU- and GPU-visible without didModifyRange/coherence calls. -internal static class MtlStorageMode { public const nuint Shared = 0; } - -// MTLDataType values (MTLDataType.h) — only what NewPipeline's intConstants -// path uses. -internal static class MtlDataType -{ - public const nuint Int = 29; -} - -internal sealed class MtlDevice : IDisposable -{ - public readonly IntPtr H; - private readonly IntPtr _queue; - // shader library + PSO cache, shared by every MetalGraphModel on this - // device (each MtlGcdrPipeline carries gc+dr models — per-model - // libraries would recompile ~2.5k lines of MSL three times and leak the - // whole set on every release). Disposed with the device. - private MtlLibrary? _lib; - private readonly Dictionary _pipes = new(); - - private MtlDevice(IntPtr h) - { - H = h; - // newCommandQueue is +1 — owned, released in Dispose. - _queue = ObjC.Send0(h, ObjC.S.NewCommandQueue); - if (_queue == 0) throw new InvalidOperationException("Metal: newCommandQueue returned nil"); - } - - public IntPtr Queue => _queue; - - public string Name => ObjC.NsToString(ObjC.Send0(H, ObjC.S.Name)) ?? "?"; - - // MTLGPUFamily enum values (MTLGPUFamily.h) - public bool SupportsFamily(int family) => ObjC.SendBool1L(H, ObjC.S.SupportsFamily, family) != 0; - public bool SupportsApple1 => SupportsFamily(1001); - public bool SupportsApple2 => SupportsFamily(1002); - public bool SupportsApple3 => SupportsFamily(1003); - public bool SupportsApple4 => SupportsFamily(1004); - public bool SupportsApple5 => SupportsFamily(1005); - public bool SupportsApple6 => SupportsFamily(1006); - public bool SupportsApple7 => SupportsFamily(1007); - public bool SupportsApple8 => SupportsFamily(1008); - public bool SupportsApple9 => SupportsFamily(1009); - public bool SupportsMetal3 => SupportsFamily(5001); - public bool SupportsMetal4 => SupportsFamily(5002); - - public nuint MaxThreadgroupMemoryLength => ObjC.SendNuint(H, ObjC.S.MaxThreadgroupMemoryLength); - - /// Defensive check for optional selectors — an unrecognized - /// selector on this paravirt driver raises an ObjC exception that kills - /// the process, so never call a maybe-absent method unconditionally. - public bool RespondsTo(IntPtr sel) => ObjC.SendBool1P(H, ObjC.S.RespondsToSelector, sel) != 0; - public bool HasUnifiedMemory => ObjC.SendBool(H, ObjC.S.HasUnifiedMemory) != 0; - public ulong RecommendedMaxWorkingSetSize => (ulong)ObjC.SendNuint(H, ObjC.S.RecommendedMaxWorkingSetSize); - public nuint CurrentAllocatedSize => ObjC.SendNuint(H, ObjC.S.CurrentAllocatedSize); - public ulong RegistryID => (ulong)ObjC.SendNuint(H, ObjC.S.RegistryID); - - public static MtlDevice? Probe() - { - if (!OperatingSystem.IsMacOS()) return null; - using var pool = AutoReleasePool.Create(); - // MTLCreateSystemDefaultDevice follows the Create rule: already +1. - IntPtr h = ObjC.CreateSystemDefaultDevice(); - return h == 0 ? null : new MtlDevice(h); - } - - /// Pipeline for a kernel in the embedded shader library, - /// compiled once and cached per device. - internal MtlPipeline Pipe(string name) - { - lock (_pipes) - { - if (!_pipes.TryGetValue(name, out MtlPipeline? p)) - { - p = NewPipeline(ShaderLibrary(), name); - // every kernel is dispatched at 256 threads/threadgroup - if (p.MaxTotalThreadsPerThreadgroup < 256) - { - p.Dispose(); - throw new NotSupportedException( - $"Metal: {name} pso supports {p.MaxTotalThreadsPerThreadgroup} threads/group"); - } - _pipes[name] = p; - } - return p; - } - } - - private MtlLibrary ShaderLibrary() - { - if (_lib is not null) return _lib; - var asm = typeof(MtlDevice).Assembly; - var names = asm.GetManifestResourceNames() - .Where(n => n.StartsWith("Sdcb.SimdGcdrNet.Backends.Metal.Shaders.") - && n.EndsWith(".metal")) - .OrderBy(n => n).ToArray(); - if (names.Length == 0) - throw new FileNotFoundException("Metal: no embedded .metal shaders"); - var sb = new System.Text.StringBuilder(); - foreach (var n in names) - { - using Stream s = asm.GetManifestResourceStream(n)!; - using var r = new StreamReader(s); - sb.AppendLine(r.ReadToEnd()); - } - return _lib = NewLibrary(sb.ToString()); - } - - public MtlBuffer NewBuffer(nuint bytes) - { - // MTLResourceStorageModeShared = 0 options. - IntPtr b = ObjC.Send1N1N(H, ObjC.S.NewBufferWithLengthOptions, bytes, 0); - if (b == 0) throw new InvalidOperationException($"Metal: newBufferWithLength({bytes}) failed"); - return new MtlBuffer(b, bytes); - } - - public unsafe MtlBuffer NewBuffer(void* src, nuint bytes) - { - IntPtr b = ObjC.Send1P2N(H, ObjC.S.NewBufferWithBytesLengthOptions, (IntPtr)src, bytes, 0); - if (b == 0) throw new InvalidOperationException($"Metal: newBufferWithBytes({bytes}) failed"); - return new MtlBuffer(b, bytes); - } - - /// Compile MSL source into a library. Throws with the NSError - /// localizedDescription on failure — MSL compile errors must never be - /// silently swallowed. - public unsafe MtlLibrary NewLibrary(string source, bool fastMath = false) - { - using var pool = AutoReleasePool.Create(); - IntPtr src = ObjC.NsStr(source); - IntPtr opts = MakeCompileOptions(fastMath); - IntPtr err = 0; - IntPtr lib; - lib = ObjC.Send3P(H, ObjC.S.NewLibraryWithSourceOptionsError, src, opts, (IntPtr)(&err)); - ObjC.Release(src); - ObjC.Release(opts); - if (lib == 0) throw new InvalidOperationException("MSL compile failed: " + DescribeError(err)); - return new MtlLibrary(lib); - } - - private static IntPtr MakeCompileOptions(bool fastMath) - { - // MTLCompileOptions: mathMode is a private-ish key; the public knob is - // fastMathEnabled (deprecated in MSL 3.x but still honored) plus - // languageVersion. MSL 3.1 = 0x30001 << 16? No — MTLLanguageVersion is - // (major<<16)+minor: 2.4=0x20004, 3.1=0x30001. We request 3.1 when - // supported for simdgroup_matrix; harmless if clamped. - IntPtr o = ObjC.Send0(ObjC.Send0(ObjC.ClassMTLCompileOptions, ObjC.S.Alloc), ObjC.S.Init); - ObjC.SendV1N(o, ObjC.S.SetLanguageVersion, (nuint)0x00030001); // MSL 3.1 - ObjC.SendV1B(o, ObjC.S.SetFastMathEnabled, (byte)(fastMath ? 1 : 0)); - return o; - } - - public unsafe MtlPipeline NewPipeline(MtlLibrary lib, string fnName, int[]? intConstants = null) - { - using var pool = AutoReleasePool.Create(); - IntPtr name = ObjC.NsStr(fnName); - IntPtr fn; - if (intConstants is { Length: > 0 }) - { - IntPtr fc = ObjC.Send0(ObjC.Send0(ObjC.ClassMTLFunctionConstantValues, ObjC.S.Alloc), ObjC.S.Init); - for (int i = 0; i < intConstants.Length; ++i) - { - int v = intConstants[i]; - // setConstantValue:type:atIndex: — (const void*, MTLDataType, NSUInteger). - ObjC.Send3P(fc, ObjC.S.SetConstantValueTypeAtIndex, (IntPtr)(&v), (IntPtr)(nint)(long)MtlDataType.Int, (IntPtr)(nint)i); - } - IntPtr err = 0; - fn = ObjC.Send3P(lib.H, ObjC.S.NewFunctionWithNameConstantValuesError, name, fc, (IntPtr)(&err)); - ObjC.Release(fc); - if (fn == 0) { ObjC.Release(name); throw new InvalidOperationException($"Metal: function '{fnName}' failed: {DescribeError(err)}"); } - } - else - { - fn = ObjC.Send1P(lib.H, ObjC.S.NewFunctionWithName, name); - } - ObjC.Release(name); - if (fn == 0) throw new InvalidOperationException($"Metal: no function '{fnName}' in library"); - - IntPtr err2 = 0; - IntPtr pso = ObjC.Send2P(H, ObjC.S.NewComputePipelineStateWithFunctionError, fn, (IntPtr)(&err2)); - ObjC.Release(fn); - if (pso == 0) throw new InvalidOperationException($"Metal: PSO '{fnName}' failed: {DescribeError(err2)}"); - return new MtlPipeline(pso, fnName); - } - - internal static unsafe string DescribeError(IntPtr err) - { - if (err == 0) return "(no NSError)"; - IntPtr desc = ObjC.Send0(err, ObjC.S.LocalizedDescription); - return ObjC.NsToString(desc) ?? "(unreadable NSError)"; - } - - /// Begin a command buffer + compute encoder pair. Both are - /// autoreleased — caller must hold an AutoReleasePool open. - public MtlEncoder BeginCommands() - { - IntPtr cmd = ObjC.Send0(_queue, ObjC.S.CommandBuffer); - IntPtr enc = ObjC.Send0(cmd, ObjC.S.ComputeCommandEncoder); - return new MtlEncoder(cmd, enc); - } - - /// Begin encoder with a specific dispatch type - /// (MTLDispatchType: 0=Serial, 1=Concurrent — concurrent allows the GPU to - /// overlap dispatches that don't depend on each other; serial inserts an - /// implicit barrier between dispatches which is what the schedule assumes). - public MtlEncoder BeginCommands(bool concurrent) - { - IntPtr cmd = ObjC.Send0(_queue, ObjC.S.CommandBuffer); - IntPtr enc = concurrent - ? ObjC.Send1N(cmd, ObjC.S.ComputeCommandEncoderWithDispatchType, 1) - : ObjC.Send0(cmd, ObjC.S.ComputeCommandEncoder); - return new MtlEncoder(cmd, enc); - } - - public void Dispose() - { - lock (_pipes) - { - foreach (MtlPipeline p in _pipes.Values) p.Dispose(); - _pipes.Clear(); - } - _lib?.Dispose(); - ObjC.Release(_queue); - ObjC.Release(H); - } -} - -internal sealed class MtlBuffer : IDisposable -{ - public IntPtr H; - public readonly nuint Length; - public MtlBuffer(IntPtr h, nuint length) { H = h; Length = length; } - public IntPtr Contents => ObjC.Send0(H, ObjC.S.Contents); - public void Dispose() { ObjC.Release(H); H = 0; } -} - -internal sealed class MtlLibrary : IDisposable -{ - public readonly IntPtr H; - public MtlLibrary(IntPtr h) => H = h; - public void Dispose() => ObjC.Release(H); -} - -internal sealed class MtlPipeline : IDisposable -{ - public readonly IntPtr Pso; - public readonly string Name; - public readonly nuint MaxTotalThreadsPerThreadgroup; - public readonly nuint ThreadExecutionWidth; - public MtlPipeline(IntPtr pso, string name) - { - Pso = pso; - Name = name; - MaxTotalThreadsPerThreadgroup = ObjC.SendNuint(pso, ObjC.S.MaxTotalThreadsPerThreadgroup); - ThreadExecutionWidth = ObjC.SendNuint(pso, ObjC.S.ThreadExecutionWidth); - } - public void Dispose() => ObjC.Release(Pso); -} - -/// One-shot command-buffer + compute-encoder pair. The objects are -/// autoreleased; keep the enclosing AutoReleasePool alive until Commit has -/// been issued (and waitUntilCompleted returned if you read results). -internal readonly struct MtlEncoder -{ - public readonly IntPtr Cmd; - public readonly IntPtr Enc; - - public MtlEncoder(IntPtr cmd, IntPtr enc) { Cmd = cmd; Enc = enc; } - - public void SetPipeline(MtlPipeline p) => ObjC.Send1P(Enc, ObjC.S.SetComputePipelineState, p.Pso); - - public void SetBuffer(MtlBuffer buf, ulong byteOffset, int index) => - ObjC.SendV1P2N(Enc, ObjC.S.SetBufferOffsetAtIndex, buf.H, (nuint)byteOffset, (nuint)index); - - public unsafe void SetPc(in T value, int index) where T : unmanaged - { - fixed (T* p = &value) - ObjC.SendV1P2N(Enc, ObjC.S.SetBytesLengthAtIndex, (IntPtr)p, (nuint)sizeof(T), (nuint)index); - } - - public unsafe void SetPc(void* data, nuint bytes, int index) => - ObjC.SendV1P2N(Enc, ObjC.S.SetBytesLengthAtIndex, (IntPtr)data, bytes, (nuint)index); - - public void SetThreadgroupMemory(nuint bytes, int index) => - ObjC.SendV1N1N(Enc, ObjC.S.SetThreadgroupMemoryLengthAtIndex, bytes, (nuint)index); - - public void Dispatch(nuint gx, nuint gy, nuint gz, nuint tx, nuint ty, nuint tz) => - ObjC.SendV2Size(Enc, ObjC.S.DispatchThreadgroupsThreadsPerThreadgroup, - new ObjC.MTLSize(gx, gy, gz), new ObjC.MTLSize(tx, ty, tz)); - - /// MTLBarrierScope buffers(=1): visibility barrier between - /// dispatches — needed when a later dispatch consumes a buffer written by - /// an earlier one and we used a Concurrent encoder. Serial encoders imply - /// this already. - public void BarrierBuffers() => ObjC.SendV1N(Enc, ObjC.S.MemoryBarrierWithScope, 1); - - public void End() => ObjC.Send0(Enc, ObjC.S.EndEncoding); - - public void Commit() => ObjC.Send0(Cmd, ObjC.S.Commit); - - /// Wait for completion and check status; throws with NSError on - /// any non-completed terminal state (incl. GPU watchdog kills). - public void CommitAndWait() - { - Commit(); - ObjC.Send0(Cmd, ObjC.S.WaitUntilCompleted); - long status = (long)ObjC.SendLong(Cmd, ObjC.S.Status); - // MTLCommandBufferStatus: 4=Completed; 5=Error; others shouldn't occur post-wait. - if (status != 4) - { - IntPtr err = ObjC.Send0(Cmd, ObjC.S.Error); - throw new InvalidOperationException($"Metal command buffer failed (status={status}): {MtlDevice.DescribeError(err)}"); - } - } - - /// GPU-side wall time of the buffer (seconds). 0 when timestamps - /// are unavailable (paravirt drivers often return 0). - public double GpuDurationSeconds() - { - double s = ObjC.SendDouble(Cmd, ObjC.S.GpuStartTime); - double e = ObjC.SendDouble(Cmd, ObjC.S.GpuEndTime); - return e > s ? e - s : 0; - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MetalBackend.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MetalBackend.cs deleted file mode 100644 index 735e478..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MetalBackend.cs +++ /dev/null @@ -1,86 +0,0 @@ -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; - -namespace Sdcb.SimdGcdrNet.Backends.Metal; - -/// -/// Metal backend root: the device, the per-(model, shape) graph cache, and -/// factories for inference sessions and fused enhancer pipelines. Thread-safe; -/// dispose with the process or when the backend is torn down. -/// -internal sealed class MetalBackend : IDisposable -{ - internal readonly MtlDevice Dev; - private readonly Dictionary<(OnnxModel m, string shape, bool physIo), MtlGraphModel> _graphs = new(); - private readonly object _gate = new(); - - private static readonly Lazy s_default = new(() => new MetalBackend()); - internal static MetalBackend Default => s_default.Value; - - internal static CompileOptions PlanOptions => new(Nhwc: true, PhysicalIo: false, Gpu: true); - internal static CompileOptions PipeOptions => new(Nhwc: true, PhysicalIo: true, Gpu: true); - - internal MetalBackend() - { - Dev = MtlDevice.Probe() ?? throw new PlatformNotSupportedException("Metal: no device"); - if (Environment.GetEnvironmentVariable("GCDRNET_MTL_VERBOSE") == "1") - Console.Error.WriteLine($"[metal] device: {Dev.Name}"); - } - - /// New inference session for a model and input shape (non-physical I/O plan). - public MtlInferenceSession NewSession(OnnxModel m, int[] shape) - { - MtlGraphModel g = AcquireGraph(m, shape, physIo: false); // caller ref: later evictions can't free it under us - try { return new MtlInferenceSession(Dev, g.Plan, g); } - finally { g.Release(); } - } - - /// The cached graph for (model, shape); acquires a reference for the caller. - internal MtlGraphModel GraphFor(OnnxModel m, int[] shape, bool physIo = false) - { - var key = (m, string.Join('x', shape), physIo); - lock (_gate) - { - if (_graphs.TryGetValue(key, out var gg)) - { - gg.Acquire(); - return gg; - } - } - ExecPlan plan = GraphCompiler.Compile(m, shape, physIo ? PipeOptions : PlanOptions); - var g = new MtlGraphModel(Dev, plan, constKey: m); - lock (_gate) - { - if (_graphs.TryGetValue(key, out var won)) { g.Release(); won.Acquire(); return won; } - _graphs[key] = g; - // bound the cache: graphs share the model's const store, so eviction only - // frees the small per-size plan+dispatch list - while (_graphs.Count > MaxGraphs) - { - var victim = _graphs.Keys.First(k => !k.Equals(key)); - _graphs.Remove(victim, out var old); - old?.Release(); - } - g.Acquire(); - return g; - } - } - private const int MaxGraphs = 32; - - /// Alias: acquires the graph for the caller (caller must Release). - internal MtlGraphModel AcquireGraph(OnnxModel m, int[] shape, bool physIo) - => GraphFor(m, shape, physIo); - - /// New fused enhancer worker (size-agnostic; binds per frame). - internal MtlGcdrPipeline NewWorker(OnnxModel gc, OnnxModel dr) => new(this, gc, dr); - - /// New fused enhancer worker pre-bound to a size. - internal MtlGcdrPipeline NewPipe(OnnxModel gc, OnnxModel dr, int ih, int iw) - { - var p = new MtlGcdrPipeline(this, gc, dr); - p.Bind(ih, iw); - return p; - } - - public void Dispose() => Dev.Dispose(); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlGraph.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlGraph.cs deleted file mode 100644 index e06e77c..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlGraph.cs +++ /dev/null @@ -1,798 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using Sdcb.SimdGcdrNet.Engine; - -namespace Sdcb.SimdGcdrNet.Backends.Metal; - -/// Binding slots resolved per-session: which underlying buffer. -internal enum MtlSlot : int { Arena = 0, Consts = 1, Dummy = 2, Direct = 3 } - -internal readonly struct MtlBind -{ - public readonly MtlSlot Slot; - public readonly MtlBuffer? Buf; // when Slot == Direct - public readonly ulong Offset, Size; - public MtlBind(MtlSlot s, ulong off, ulong size) { Slot = s; Offset = off; Size = size; } - public MtlBind(MtlBuffer b, ulong off, ulong size) { Slot = MtlSlot.Direct; Buf = b; Offset = off; Size = size; } -} - -/// One recorded compute dispatch. -internal sealed class MtlDispatch -{ - public required MtlPipeline Pipe; - public required MtlBind[] Binds; - public required byte[] Pc; - public uint Gx, Gy = 1, Gz = 1; - public string Name = ""; -} - -/// -/// Device-side const tensors (backend-packed weights) plus a 64B zero buffer, -/// shared by every of one model: per-size plans copy -/// the same initializers, so instead of re-packing and re-uploading the weights -/// per input size, a new graph whose const list is element-for-element identical -/// (same consumer packing, same contents) reuses the existing buffer. A graph -/// whose consts differ gets a private, unshared store. Ref-counted; the -/// buffers are freed when the last graph releases. -/// -internal sealed class MtlConstStore -{ - private static readonly Dictionary<(MtlDevice dev, object key), MtlConstStore> s_shared = new(); - - private readonly (MtlDevice, object)? _key; - private readonly float[][] _src; - private readonly int[] _kinds; - private int _refs = 1; - internal readonly MtlBuffer Consts, Dummy; - internal readonly ulong[] Offsets; - internal readonly ulong Bytes; - - private MtlConstStore(MtlDevice dev, (MtlDevice, object)? key, float[][] src, int[] kinds, - ulong[] offsets, byte[] host) - { - _key = key; - _src = src; - _kinds = kinds; - Offsets = offsets; - Bytes = (ulong)host.LongLength; - unsafe - { - fixed (byte* hp = host) Consts = dev.NewBuffer(hp, (nuint)host.Length); - byte* z = stackalloc byte[64]; - new Span(z, 64).Clear(); - Dummy = dev.NewBuffer(z, 64); - } - } - - /// - /// Returns a store holding (kinds = consumer packing ids), - /// shared under when an identical list was uploaded before. - /// (offsets, host bytes) only runs when a new store is built. - /// - internal static MtlConstStore Acquire(MtlDevice dev, object? key, float[][] src, int[] kinds, - Func<(ulong[] offsets, byte[] host)> pack) - { - if (key is null) - { - var (o, h) = pack(); - return new MtlConstStore(dev, null, src, kinds, o, h); - } - var k = (dev, key); - lock (s_shared) - { - if (s_shared.TryGetValue(k, out MtlConstStore? s) && s.Matches(src, kinds)) - { - s._refs++; - return s; - } - var (offs, host) = pack(); - bool share = s is null; // keep the first registered store on a mismatch - var ns = new MtlConstStore(dev, share ? k : null, src, kinds, offs, host); - if (share) s_shared[k] = ns; - return ns; - } - } - - private bool Matches(float[][] src, int[] kinds) - { - if (src.Length != _src.Length || !kinds.AsSpan().SequenceEqual(_kinds)) return false; - for (int i = 0; i < src.Length; i++) - if (!ReferenceEquals(src[i], _src[i]) && !src[i].AsSpan().SequenceEqual(_src[i])) return false; - return true; - } - - internal void Release() - { - lock (s_shared) - { - if (--_refs > 0) return; - if (_key is { } k && s_shared.TryGetValue(k, out MtlConstStore? cur) && cur == this) - s_shared.Remove(k); - } - Consts.Dispose(); - Dummy.Dispose(); - } -} - -/// -/// Per-(device, ExecPlan) half of the Metal backend: backend-private constant -/// packing/upload and the node list compiled to -/// records. Sessions own the activation arena and resolve slot ids to live -/// buffers. Activations and packed weights are stored as fp16; all shader math -/// is fp32; conv bias and LayerNorm params stay fp32. -/// -internal sealed class MtlGraphModel : IDisposable -{ - private readonly MtlDevice _dev; - private readonly ExecPlan _plan; - private int _refs = 1; // the graph cache holds the first reference - private readonly MtlConstStore _store; - internal readonly MtlBuffer Consts; // all const tensors, backend-packed (shared store) - private readonly Dictionary<(int v, CK k), ulong> _constOff = new(); - internal readonly MtlBuffer Dummy; // 64B zero buffer for unused bindings - internal readonly MtlDispatch[] Dispatches; - internal readonly ulong ConstBytes; - /// Activations + weights stored as float16 (fp32 math in every shader). - internal const bool F16 = true; - /// Bytes per stored activation element. - internal int Es => 2; - - /// How a const is laid out on the device (decided by its consumer). - private enum CK { Raw, F32, ConvW, DwW, ConvWT, GemmNK } - - /// Metal device the graph is lowered for. - /// Compiled execution plan to lower. - /// Sharing key for the const store (the source model): graphs of the - /// same model at other sizes reuse its uploaded weights. Null = private store. - internal MtlGraphModel(MtlDevice dev, ExecPlan plan, object? constKey = null) - { - _dev = dev; - _plan = plan; - if (!plan.Nhwc) throw new NotSupportedException("Metal backend requires a layout-planned (NHWC) plan."); - - // ---- consts: per-consumer packing ---- - var need = new List<(int v, CK k)>(); - void Need(int v, CK k) - { - if (v < 0 || plan.Values[v].Floats is null) return; - if (!need.Contains((v, k))) need.Add((v, k)); - } - foreach (ExecNode n in plan.Nodes) - { - switch (n.P) - { - case ConvP cp: - Need(n.In[0], CK.Raw); - Need(n.In[1], cp.Group > 1 ? CK.DwW : UseMmConv(plan, n, cp) ? CK.ConvWT : CK.ConvW); - Need(cp.BiasValue, CK.F32); - Need(cp.AddValue, CK.Raw); - break; - case GemmP gp: - Need(n.In[0], CK.Raw); Need(n.In[1], UseMmGemm(plan, n, gp) ? CK.GemmNK : CK.Raw); - Need(gp.BiasValue, CK.F32); - break; - case LnP: - Need(n.In[0], CK.Raw); Need(n.In[1], CK.F32); Need(n.In[2], CK.F32); - break; - case UpHeadP up: - Need(n.In[2], CK.Raw); - if (n.In.Length > 3) Need(n.In[3], CK.F32); - if (up.SkipConv is not null) { Need(up.SkipW, CK.ConvW); Need(up.SkipB, CK.F32); } - break; - default: - foreach (int iv in n.In) Need(iv, CK.Raw); - break; - } - } - foreach (var (v, k) in need) - { - CValue cv = plan.Values[v]; - if (k == CK.Raw && cv.Layout is { } l && !Engine.Layout.IsIdentity(l)) - throw new NotSupportedException($"Metal: const '{cv.Name}' with non-identity layout"); - } - float[][] srcs = need.Select(x => plan.Values[x.v].Floats!).ToArray(); - int[] kinds = need.Select(x => (int)x.k).ToArray(); - _store = MtlConstStore.Acquire(dev, constKey, srcs, kinds, () => - { - ulong total = 0; - var offs = new ulong[need.Count]; - var items = new List<(ulong off, float[] data, bool half)>(); - for (int i = 0; i < need.Count; i++) - { - var (v, k) = need[i]; - CValue cv = plan.Values[v]; - float[] data = k switch - { - CK.ConvW => PackConv(cv), - CK.ConvWT => PackConvT(cv), - CK.GemmNK => PackGemmNK(cv, plan.Nodes.First(m => m.P is GemmP && m.In[1] == v)), - CK.DwW => PackDw(cv), - _ => cv.Floats!, - }; - bool half = k != CK.F32; - total = (total + 63) & ~63UL; - offs[i] = total; - items.Add((total, data, half)); - total += (ulong)data.Length * (half ? 2UL : 4UL); - } - byte[] host = new byte[Math.Max((total + 63) & ~63UL, 64)]; - foreach (var (off, data, half) in items) - { - if (half) - { - Span dst = MemoryMarshal.Cast(host.AsSpan(checked((int)off), data.Length * 2)); - for (int i = 0; i < data.Length; i++) dst[i] = (Half)data[i]; - } - else - MemoryMarshal.AsBytes(data.AsSpan()).CopyTo(host.AsSpan(checked((int)off))); - } - return (offs, host); - }); - for (int i = 0; i < need.Count; i++) _constOff[need[i]] = _store.Offsets[i]; - Consts = _store.Consts; - Dummy = _store.Dummy; - ConstBytes = _store.Bytes; - - var disps = new List(); - NodeRanges = new (int start, int count)[plan.Nodes.Length]; - for (int i = 0; i < plan.Nodes.Length; i++) - { - int start = disps.Count; - disps.AddRange(Emit(plan.Nodes[i])); - NodeRanges[i] = (start, disps.Count - start); - } - Dispatches = disps.ToArray(); - } - - // ONNX [OC][IC][KH][KW] -> [K][OC], k = (ky*KW + kx)*IC + ic - private static float[] PackConv(CValue w) - { - int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3]; - float[] src = w.Floats!, dst = new float[src.Length]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < taps; t++) - dst[(t * ic + c) * oc + o] = src[(o * ic + c) * taps + t]; - return dst; - } - - // ONNX [OC][IC][KH][KW] -> [OC][Kpad], k = (ky*KW + kx)*IC + ic, Kpad = K rounded up to 8 - private static float[] PackConvT(CValue w) - { - int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3], k = (taps * ic + 7) & ~7; - float[] src = w.Floats!, dst = new float[oc * k]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < taps; t++) - dst[o * k + t * ic + c] = src[(o * ic + c) * taps + t]; - return dst; - } - - /// simdgroup-MMA conv eligibility (conv_mm): dense, output channels in - /// 16 B quads, an epilogue it implements, M within the grid limit. cin%4!=0 uses - /// the scalar-gather variant; constant pad must be zero. - private static bool UseMmConv(ExecPlan plan, ExecNode n, ConvP cp) - { - if (cp.Group != 1 || cp.Pool is not null || cp.Dilations.Any(d => d != 1)) return false; - if (cp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false; - if (cp.PadMode == PadMode.Constant && cp.PadValue != 0f) return false; - int[] so = plan.Values[n.Out[0]].PhysShape; // [1,h,w,c] - long m = (long)so[1] * so[2]; - return so[3] % 8 == 0 && m / 64 <= int.MaxValue; - } - - // Gemm B -> [N][K] (k fastest), the conv_mm weight layout - private static float[] PackGemmNK(CValue b, ExecNode n) - { - var gp = (GemmP)n.P!; - if (gp.TransB) return b.Floats!; // already [N][K] - int k = b.Shape[0], nn = b.Shape[1]; - float[] src = b.Floats!, dst = new float[src.Length]; - for (int i = 0; i < k; i++) - for (int j = 0; j < nn; j++) dst[j * k + i] = src[i * nn + j]; - return dst; - } - - /// MMA Gemm (a 1x1 conv over M "pixels" through conv_mm): const B, - /// K/N in 16 B quads, relu/gelu/none epilogue. - private static bool UseMmGemm(ExecPlan plan, ExecNode n, GemmP gp) - { - if (plan.Values[n.In[1]].Floats is null || plan.Values[n.In[1]].Shape.Length != 2) return false; - // B packed per value: only when no other node reads it (its transB could differ) - if (plan.Nodes.Count(m => m.In.Contains(n.In[1])) != 1) return false; - if (gp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false; - CValue a = plan.Values[n.In[0]]; - int k = a.Shape[^1], nn = plan.Values[n.Out[0]].Shape[^1]; - if (a.PhysShape[^1] != k) return false; - long m = a.ElementCount / k; - return k % 8 == 0 && nn % 8 == 0 && m / 64 <= int.MaxValue; - } - - // ONNX [C][1][KH][KW] -> [tap][C] - private static float[] PackDw(CValue w) - { - int c = w.Shape[0], taps = w.Shape[2] * w.Shape[3]; - float[] src = w.Floats!, dst = new float[src.Length]; - for (int ch = 0; ch < c; ch++) - for (int t = 0; t < taps; t++) dst[t * c + ch] = src[ch * taps + t]; - return dst; - } - - /// Dispatch index range per source ExecNode (for per-node debug runs). - internal readonly (int start, int count)[] NodeRanges; - - // ---- value → bind resolution (byte offsets/sizes: all long math) ---- - private MtlBind Bind(int valueId, CK kind = CK.Raw) - { - CValue v = _plan.Values[valueId]; - if (v.Buf >= 0) - return new MtlBind(MtlSlot.Arena, (ulong)_plan.BufOffset[v.Buf] * (ulong)Es, - (ulong)Math.Max(v.ElementCount, 1) * (ulong)Es); - if (_constOff.TryGetValue((valueId, kind), out ulong off)) - { - ulong es = kind != CK.F32 ? 2UL : 4UL; - // ConvWT rows are K-padded to 8: bind the packed length, not the logical one - long elems = kind == CK.ConvWT - ? (long)v.Shape[0] * ((v.Shape[1] * v.Shape[2] * v.Shape[3] + 7) & ~7) - : Math.Max(v.ElementCount, 1); - return new MtlBind(MtlSlot.Consts, off, (ulong)elems * es); - } - if (v.Floats != null) - throw new InvalidOperationException($"const value '{v.Name}' not packed as {kind}"); - throw new InvalidOperationException($"value '{v.Name}' has no GPU storage (ints must be folded)"); - } - - private static byte[] PcOf(params uint[] ints) - { - byte[] b = new byte[ints.Length * 4]; - Buffer.BlockCopy(ints, 0, b, 0, b.Length); - return b; - } - - private static byte[] PcOfF(uint[] ints, params float[] floats) - { - byte[] b = new byte[(ints.Length + floats.Length) * 4]; - Buffer.BlockCopy(ints, 0, b, 0, ints.Length * 4); - Buffer.BlockCopy(floats, 0, b, ints.Length * 4, floats.Length * 4); - return b; - } - - private static uint[] U(params int[] v) { uint[] u = new uint[v.Length]; for (int i = 0; i < v.Length; i++) u[i] = (uint)v[i]; return u; } - - private static uint G(uint n) => (n + 255) / 256; - - /// - /// Broadcast strides (padded rank-4, storage order of the output) of operand - /// onto output . Output physical dim i - /// holds logical dim o.Layout[i]; the operand's logical dim is that minus the - /// numpy right-alignment offset. - /// - private static int[] PhysBroadcastStrides(CValue a, CValue o) - { - int rank = o.Shape.Length, off = rank - a.Shape.Length; - if (rank > 4) throw new NotSupportedException("Metal elementwise: rank > 4"); - int[] lo = o.Layout ?? Engine.Layout.Identity(rank); - int[] la = a.Layout ?? Engine.Layout.Identity(a.Shape.Length); - int[] ap = a.PhysShape; - int[] aStr = new int[ap.Length]; - for (int i = ap.Length - 1, acc = 1; i >= 0; i--) { aStr[i] = acc; acc *= ap[i]; } - int[] r4 = new int[4]; - for (int i = 0; i < rank; i++) - { - int d = lo[i] - off; - r4[4 - rank + i] = d < 0 || a.Shape[d] == 1 ? 0 : aStr[Array.IndexOf(la, d)]; - } - return r4; - } - - private IEnumerable Emit(ExecNode n) - { - CValue[] V = _plan.Values; - // all addressing on storage-order shapes - int[] shapeIn(int k) => V[n.In[k]].PhysShape; - int[] shapeOut() => V[n.Out[0]].PhysShape; - int nOut() => checked((int)V[n.Out[0]].ElementCount); - - switch (n.Op) - { - case ExecOp.Copy: - yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)nOut(), 4), G((uint)nOut())); - break; - - case ExecOp.Relu or ExecOp.Sigmoid or ExecOp.Erf or ExecOp.Gelu: - { - uint mode = n.Op switch - { - ExecOp.Relu => 0, ExecOp.Sigmoid => 1, ExecOp.Erf => 2, _ => 3, - }; - yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)nOut(), mode), G((uint)nOut())); - break; - } - - case ExecOp.Elementwise: - { - var p = (EwP)n.P!; - CValue a = V[n.In[0]], bv = V[n.In[1]], o = V[n.Out[0]]; - int[] os = Pad4(shapeOut()); - int[] ostr = Strides(os); - int[] astr = PhysBroadcastStrides(a, o); - int[] bstr = PhysBroadcastStrides(bv, o); - uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, _ => 0u }; - // same-storage fast path: vec4 (n % 4 guard — no OOB tail writes) - bool dense = astr.SequenceEqual(ostr) && bstr.SequenceEqual(ostr); - if (nOut() % 4 == 0 && dense) - { - yield return ND("ew2v", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])], - PcOf((uint)(nOut() / 4), (uint)p.Op, act), G((uint)(nOut() / 4))); - break; - } - yield return ND("ew2", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - astr[0], astr[1], astr[2], astr[3], - bstr[0], bstr[1], bstr[2], bstr[3], (int)p.Op, (int)act)), - G((uint)nOut())); - break; - } - - case ExecOp.Conv: - { - var p = (ConvP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - if (p.Pool is { } pool) - { - // small-ic stem conv + 2x2/2 max pool (GraphCompiler.GpuSmallConv) - int[] ci = shapeIn(0), co = shapeOut(), cw = V[n.In[1]].Shape; - if (!GraphCompiler.GpuSmallConv(p, cw, ci[3]) || p.AddValue >= 0 || ci[1] != co[1] * 2 || ci[2] != co[2] * 2) - throw new NotSupportedException("Metal convpool: unsupported conv"); - uint pf = (p.BiasValue >= 0 ? 1u : 0u) | (p.PadMode == PadMode.Reflect ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) | (pool.FuseRelu ? 8u : 0u); - bool k3 = p.Kernel[0] == 3 && p.Kernel[1] == 3; - yield return ND(k3 ? "convpool_k3" : "convpool", - [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOfF(U(ci[3], ci[1], ci[2], cw[0], co[1], co[2], p.Kernel[0], p.Kernel[1], p.Pads[0], p.Pads[1], (int)pf), - p.PadMode == PadMode.Constant ? p.PadValue : 0f), - G((uint)co[1] * (uint)co[2] * (uint)(cw[0] / 8)), name: "convpool"); - break; - } - if (p.AddValue >= 0) RequireNhwc(p.AddValue); - int[] si = shapeIn(0), sw_ = V[n.In[1]].Shape, so = shapeOut(); // [1,h,w,c] - uint ih = (uint)si[1], iw = (uint)si[2], ic = (uint)si[3]; - uint oc = (uint)sw_[0], oh = (uint)so[1], ow = (uint)so[2]; - if (p.PadMode == PadMode.Constant && p.PadValue != 0f) - throw new NotSupportedException("Metal conv: nonzero constant pad"); - if (p.Dilations.Any(d => d != 1)) - throw new NotSupportedException("Metal conv: dilation"); - if (p.Act == ActivationKind.Sigmoid) - throw new NotSupportedException("Metal conv: sigmoid epilogue"); - uint flags = (p.BiasValue >= 0 ? 1u : 0u) - | (p.PadMode == PadMode.Reflect ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) - | (p.AddValue >= 0 ? 8u : 0u) - | (p.Act == ActivationKind.Gelu ? 16u : 0u); - if (p.Group > 1) - { - if (p.Group != ic || oc != ic) throw new NotSupportedException("grouped non-depthwise conv"); - if (p.AddValue >= 0) throw new NotSupportedException("depthwise conv + residual"); - bool v4 = ic % 4 == 0; - yield return ND(v4 ? "depthwise4" : "depthwise", - [Bind(n.In[0]), Bind(n.In[1], CK.DwW), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOfF(U((int)ic, (int)ih, (int)iw, (int)oh, (int)ow, p.Kernel[0], p.Kernel[1], - p.Strides[0], p.Strides[1], p.Pads[0], p.Pads[1], (int)(flags & ~8u)), p.PadValue), - G(ic * oh * ow / (v4 ? 4u : 1u)), name: "depthwise"); - break; - } - if (UseMmConv(_plan, n, p)) - { - // BM = 64 rows (pixels); BN by oc divisibility; SCK when cin%4!=0 - uint sck = ic % 4 != 0 ? 1u : 0u; - (string cm, uint bn) = oc % 128 == 0 ? ("conv_mm_128", 128u) - : oc % 64 == 0 ? ("conv_mm_64", 64u) : ("conv_mm_32", 32u); - cm += "_" + sck; - uint m = oh * ow, kr = (uint)(p.Kernel[0] * p.Kernel[1]) * ic, kd = (kr + 7u) & ~7u; - uint cflags = (p.BiasValue >= 0 ? 1u : 0u) | (p.AddValue >= 0 ? 2u : 0u) - | (p.PadMode == PadMode.Reflect ? 4u : 0u) - | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4); - yield return ND(cm, - [Bind(n.In[0]), Bind(n.In[1], CK.ConvWT), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])], - PcOf(m, oc, kd, cflags, ow, iw, ih, ic, (uint)p.Kernel[1], - (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], kr), - (m + 63) / 64, (oc + bn - 1) / bn, name: "conv_mm"); - break; - } - yield return ND("conv_tile", - [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])], - PcOf(ic, ih, iw, oc, oh, ow, (uint)p.Kernel[0], (uint)p.Kernel[1], - (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], - flags, 0), - (ow + 15) / 16, (oh + 7) / 8, (oc + 63) / 64, name: "conv"); - break; - } - - case ExecOp.MaxPool: - { - var p = (PoolP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0), so = shapeOut(); - bool v4 = si[3] % 4 == 0; - yield return ND(v4 ? "maxpool4" : "maxpool", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], - (uint)p.Kernel[0], (uint)p.Kernel[1], (uint)p.Strides[0], (uint)p.Strides[1], - (uint)p.Pads[0], (uint)p.Pads[1], p.FuseRelu ? 1u : 0u), - G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "maxpool"); - break; - } - - case ExecOp.LayerNorm: - { - var p = (LnP)n.P!; - uint len = (uint)V[n.In[0]].Shape[^1]; - if ((uint)shapeIn(0)[^1] != len) throw new NotSupportedException("Metal LayerNorm: last dim not innermost"); - uint rows = (uint)(V[n.In[0]].ElementCount / len); - yield return ND("layernorm", - [Bind(n.In[0]), Bind(n.In[1], CK.F32), Bind(n.In[2], CK.F32), Bind(n.Out[0])], - PcOfF([len, rows], p.Eps), (rows + 7) / 8, name: "layernorm"); - break; - } - - case ExecOp.Gemm: - { - var p = (GemmP)n.P!; - uint k = (uint)V[n.In[0]].Shape[^1]; - if ((uint)shapeIn(0)[^1] != k) throw new NotSupportedException("Metal Gemm: K not innermost"); - uint m = (uint)(V[n.In[0]].ElementCount / k); - uint nn = (uint)V[n.Out[0]].Shape[^1]; - if (UseMmGemm(_plan, n, p)) - { - (string cm, uint bn) = nn % 128 == 0 ? ("conv_mm_128", 128u) - : nn % 64 == 0 ? ("conv_mm_64", 64u) : ("conv_mm_32", 32u); - cm += "_0"; - uint cflags = (p.BiasValue >= 0 ? 1u : 0u) - | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4); - // [M,K] x [N,K]^T as a 1x1 conv over a 1 x M image - yield return ND(cm, - [Bind(n.In[0]), Bind(n.In[1], CK.GemmNK), B(p.BiasValue, CK.F32), B(-1), Bind(n.Out[0])], - PcOf(m, nn, k, cflags, m, m, 1u, k, 1u, 1u, 1u, 0u, 0u, k), - (m + 63) / 64, (nn + bn - 1) / bn, name: "gemm_mm"); - break; - } - uint gflags = (p.TransB ? 1u : 0u) | (p.BiasValue >= 0 ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) - | (p.Act == ActivationKind.Gelu ? 8u : 0u); - yield return ND("gemm", - [Bind(n.In[0]), Bind(n.In[1]), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOf(m, nn, k, gflags), - (nn + 63) / 64, (m + 63) / 64, name: "gemm"); - break; - } - - case ExecOp.ResizeBilinear: - { - var p = (ResizeP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - bool hasAdd = n.In.Length > 1; - if (hasAdd) RequireNhwc(n.In[1]); - int[] si = shapeIn(0), so = shapeOut(); - uint flags = (p.FuseRelu ? 1u : 0u) | (hasAdd ? 2u : 0u) | (p.AddRelu ? 4u : 0u); - bool v4 = si[3] % 4 == 0; - yield return ND(v4 ? "bilinear4" : "bilinear", [Bind(n.In[0]), hasAdd ? Bind(n.In[1]) : B(-1), Bind(n.Out[0])], - PcOfF([(uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], flags], - p.Scales[2], p.Scales[3]), - G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "bilinear"); - break; - } - - case ExecOp.UpHead: - { - var p = (UpHeadP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.In[1]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0), so = shapeOut(); - uint c = (uint)si[3], oc = (uint)so[3]; - if (oc > 8 || c > 128 || c % 4 != 0) throw new NotSupportedException($"Metal UpHead: c={c} oc={oc}"); - bool hasBias = n.In.Length > 3; - uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, ActivationKind.None => 0u, - _ => throw new NotSupportedException($"Metal UpHead: {p.Act}") }; - uint flags = (p.ResizeRelu ? 1u : 0u) | (p.SkipRelu ? 2u : 0u) | (hasBias ? 4u : 0u) | (act << 3); - if (p.SkipConv is { } sc) - { - // skip recomputed from the stem conv's input (GraphCompiler.GpuSmallConv, c <= 32) - int[] xi = shapeIn(1), cw = V[p.SkipW].Shape; - if (!GraphCompiler.GpuSmallConv(sc, cw, xi[3]) || cw[0] != c || c > 32 || xi[1] != so[1] || xi[2] != so[2]) - throw new NotSupportedException("Metal UpHead: unsupported skip conv"); - uint cf = (sc.BiasValue >= 0 ? 1u : 0u) | (sc.PadMode == PadMode.Reflect ? 2u : 0u) - | (sc.Act == ActivationKind.Relu ? 4u : 0u); - yield return ND("uphead_sc", - [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0]), - Bind(p.SkipW, CK.ConvW), B(p.SkipB, CK.F32)], - PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], - p.Scales[2], p.Scales[3]).Concat(PcOfF(U(xi[3], sc.Kernel[0], sc.Kernel[1], sc.Pads[0], sc.Pads[1], (int)cf), - sc.PadMode == PadMode.Constant ? sc.PadValue : 0f)).ToArray(), - G((uint)so[1] * (uint)so[2]), name: "uphead_sc"); - break; - } - yield return ND("uphead", - [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0])], - PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], - p.Scales[2], p.Scales[3]), - G((uint)so[1] * (uint)so[2])); - break; - } - - case ExecOp.Pad: - { - // generic rank-4 pad in storage order (pads permuted to physical dims) - var p = (PadP)n.P!; - CValue x = V[n.In[0]]; - if (x.Shape.Length != 4) throw new NotSupportedException("Metal pad: rank != 4"); - int[] l = x.Layout ?? Engine.Layout.Identity(4); - int[] id4 = shapeIn(0), od4 = shapeOut(); - int[] pb4 = new int[4]; - for (int d = 0; d < 4; d++) pb4[d] = p.Pads[l[d]]; - yield return ND("pad", [Bind(n.In[0]), Bind(n.Out[0])], - PcOfF(U(id4[0], id4[1], id4[2], id4[3], od4[0], od4[1], od4[2], od4[3], - pb4[0], pb4[1], pb4[2], pb4[3], p.Mode == PadMode.Reflect ? 1 : 0), - p.Value), - G((uint)V[n.Out[0]].ElementCount)); - break; - } - - case ExecOp.Transpose: - { - // only planner relayouts reach here; the perm is physical - var p = (TransposeP)n.P!; - int[] is4 = Pad4(shapeIn(0)); - int[] os4 = Pad4(shapeOut()); - int[] ostr = Strides(os4), istr = Strides(is4); - int[] perm4 = PermuteTo4(p.Perm, shapeIn(0).Length); - yield return ND("transpose", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - istr[perm4[0]], istr[perm4[1]], istr[perm4[2]], istr[perm4[3]])), - G((uint)nOut())); - break; - } - - case ExecOp.Slice: - { - // SliceP.Axes are physical after the layout pass - var p = (SliceP)n.P!; - int[] is4 = Pad4(shapeIn(0)), os4 = Pad4(shapeOut()); - int[] ostr = Strides(os4), istr = Strides(is4); - uint[] st = [0, 0, 0, 0], sp = [1, 1, 1, 1]; - int shift = 4 - shapeIn(0).Length; - for (int i = 0; i < p.Axes.Length; i++) - { - st[p.Axes[i] + shift] = (uint)p.Starts[i]; - sp[p.Axes[i] + shift] = (uint)p.Steps[i]; - } - yield return ND("slice", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - (int)st[0], (int)st[1], (int)st[2], (int)st[3], - (int)sp[0], (int)sp[1], (int)sp[2], (int)sp[3], - istr[0], istr[1], istr[2], istr[3])), - G((uint)nOut())); - break; - } - - case ExecOp.Concat: - { - // ConcatP.Axis is physical after the layout pass - var p = (ConcatP)n.P!; - int[] os = shapeOut(); - int axis = p.Axis; - int inner = 1, dstAxis = os[axis]; - for (int d = axis + 1; d < os.Length; d++) inner *= os[d]; - int axisOff = 0; - foreach (int iv in n.In) - { - int[] ishape = V[iv].PhysShape; - int cnt = checked((int)V[iv].ElementCount); - if (cnt > 0) - { - yield return ND("concat_ax", [Bind(iv), Bind(n.Out[0])], - PcOf((uint)cnt, (uint)inner, (uint)ishape[axis], (uint)dstAxis, (uint)axisOff), - G((uint)cnt)); - } - axisOff += ishape[axis]; - } - break; - } - - case ExecOp.ChannelShift: - { - var p = (ShiftP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0); // [1,h,w,c] - var ge = p.GroupEnds; var of = p.Offsets; - if (ge.Length > 5) throw new NotSupportedException("Metal shift: > 5 groups"); - var pc = new List { (uint)si[3], (uint)si[1], (uint)si[2], (uint)p.Axis }; - for (int i = 0; i < 5; i++) pc.Add((uint)(i < ge.Length ? ge[i] : si[3])); - for (int i = 0; i < 5; i++) pc.Add((uint)(i < of.Length ? of[i] : 0)); - bool v4 = si[3] % 4 == 0 && ge.All(e => e % 4 == 0); - yield return ND(v4 ? "shift4" : "shift", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(pc.ToArray()), G((uint)nOut() / (v4 ? 4u : 1u)), name: "shift"); - break; - } - - default: - throw new NotSupportedException($"Metal: ExecOp {n.Op} not implemented"); - } - - MtlDispatch ND(string shader, MtlBind[] binds, byte[] pc, uint gx, uint gy = 1, uint gz = 1, string? name = null) - { - return new MtlDispatch - { - Pipe = _dev.Pipe(shader), - Pc = pc, Binds = binds, Gx = gx, Gy = gy, Gz = gz, - Name = $"{shader}({n.Name})", - }; - } - - MtlBind B(int valueId, CK kind = CK.Raw) => valueId >= 0 ? Bind(valueId, kind) : new MtlBind(MtlSlot.Dummy, 0, 64); - } - - private void RequireNhwc(int v) - { - CValue cv = _plan.Values[v]; - if (cv.Shape.Length != 4 || !Engine.Layout.Equivalent(cv.Shape, cv.Layout ?? Engine.Layout.Identity(4), Engine.Layout.Nhwc)) - throw new NotSupportedException($"Metal: '{cv.Name}' expected NHWC storage, got {Engine.Layout.Str(cv.Layout)}"); - } - - // pad a shape to rank 4 with leading 1s - private static int[] Pad4(int[] s) - { - if (s.Length == 4) return s; - int[] r = [1, 1, 1, 1]; - s.CopyTo(r, 4 - s.Length); - return r; - } - - private static int[] Strides(int[] shape4) - { - int[] s = new int[4]; s[3] = 1; - for (int i = 2; i >= 0; i--) s[i] = s[i + 1] * shape4[i + 1]; - return s; - } - - // lift an original-rank perm into padded rank-4 (leading dims map to themselves) - private static int[] PermuteTo4(int[] perm, int rank) - { - int shift = 4 - rank; - int[] p4 = [0, 1, 2, 3]; - for (int i = 0; i < rank; i++) p4[i + shift] = perm[i] + shift; - return p4; - } - - /// Sessions hold a reference so cache eviction can't free live const/dummy buffers. - public void Acquire() - { - if (Interlocked.Increment(ref _refs) <= 1) - throw new ObjectDisposedException(nameof(MtlGraphModel), "const buffers already freed"); - } - /// Debug/test: const buffer + byte offset of (value, packing-kind) for diagnostics. - internal (MtlBuffer buf, ulong off, bool half) DebugConst(int valueId, int kind) - { - return _constOff.TryGetValue((valueId, (CK)kind), out ulong o) - ? (Consts, o, (CK)kind != CK.F32) - : (Dummy, ulong.MaxValue, false); - } - - /// Acquire unless already freed (no resurrection; for racing cache readers). - public bool TryAcquire() - { - while (true) - { - int r = Volatile.Read(ref _refs); - if (r <= 0) return false; - if (Interlocked.CompareExchange(ref _refs, r + 1, r) == r) return true; - } - } - public void Release() - { - if (Interlocked.Decrement(ref _refs) == 0) _store.Release(); - } - - /// The plan this graph was compiled from (arena offsets, I/O values). - internal ExecPlan Plan => _plan; - public void Dispose() => Release(); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs index b55f67c..6e49615 100644 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs +++ b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlPipeline.cs @@ -1,5 +1,7 @@ -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Backends.Metal; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Onnx; namespace Sdcb.SimdGcdrNet.Backends.Metal; @@ -56,9 +58,9 @@ internal void Bind(int ih, int iw) if (_gcG is null || h != _h || w != _w) { // a new size compiles two plans + graphs: build them side by side - var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], physIo: true)); + var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], physIo: true, GcdrExtensions.Instance)); MtlGraphModel gd; - try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], physIo: true); } + try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], physIo: true, GcdrExtensions.Instance); } catch { try { tg.GetAwaiter().GetResult().Release(); } catch { } throw; } MtlGraphModel gg; try { gg = tg.GetAwaiter().GetResult(); } @@ -109,7 +111,7 @@ private void RecordAll(MtlEncoder enc, int srcStride) uint h = (uint)_h, w = (uint)_w; // bgr2nhwc -> gc arena input - enc.SetPipeline(_dev.Pipe("bgr2nhwc")); + enc.SetPipeline(_dev.Pipe("bgr2nhwc", GcdrExtensions.Owner)); enc.SetBuffer(_bgrIn!, 0, 0); enc.SetBuffer(_arena!, _gcBase + InOff(gcPlan), 1); var pc = stackalloc uint[7] { h, w, (uint)_ih, (uint)_iw, (uint)_pt, (uint)_pl, (uint)srcStride }; @@ -121,7 +123,7 @@ private void RecordAll(MtlEncoder enc, int srcStride) // div6: gc input + gc output -> dr arena input uint hw = h * w; - enc.SetPipeline(_dev.Pipe("div6")); + enc.SetPipeline(_dev.Pipe("div6", GcdrExtensions.Owner)); enc.SetBuffer(_arena!, _gcBase + InOff(gcPlan), 0); enc.SetBuffer(_arena!, _gcBase + OutOff(gcPlan), 1); enc.SetBuffer(_arena!, InOff(drPlan), 2); @@ -134,7 +136,7 @@ private void RecordAll(MtlEncoder enc, int srcStride) // nhwc2bgr -> bgrOut uint nwords = ((uint)_ih * (uint)_iw * 3u + 3u) / 4u; - enc.SetPipeline(_dev.Pipe("nhwc2bgr")); + enc.SetPipeline(_dev.Pipe("nhwc2bgr", GcdrExtensions.Owner)); enc.SetBuffer(_arena!, OutOff(drPlan), 0); enc.SetBuffer(_bgrOut!, 0, 1); var pc3 = stackalloc uint[7] { h, w, (uint)_ih, (uint)_iw, (uint)_pt, (uint)_pl, nwords }; diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlSession.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlSession.cs deleted file mode 100644 index ceaab1e..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/MtlSession.cs +++ /dev/null @@ -1,250 +0,0 @@ -using Sdcb.SimdGcdrNet.Engine; - -namespace Sdcb.SimdGcdrNet.Backends.Metal; - -/// -/// GPU counterpart of : one activation arena in -/// shared (CPU+GPU-visible) memory; Run() writes the input window straight -/// into the arena, records the graph's dispatches into one command buffer and -/// waits once — no staging copies (unified memory). Not thread-safe. -/// -public sealed unsafe class MtlInferenceSession : IDisposable -{ - private readonly MtlDevice _dev; - private readonly MtlGraphModel _graph; - private readonly ExecPlan _plan; - - internal readonly MtlBuffer Arena; // shared-memory activation arena - private readonly float[] _inHost, _outHost; - private readonly ulong _inOff, _outOff; - private readonly int _inFloats, _outFloats; - internal int Es => _graph.Es; // bytes per stored element (2 = fp16) - - internal MtlInferenceSession(MtlDevice dev, ExecPlan plan, MtlGraphModel graph) - { - _dev = dev; - _plan = plan; - _graph = graph; - _graph.Acquire(); - - Arena = dev.NewBuffer((nuint)((ulong)plan.ArenaFloats * (ulong)Es + 64)); - _inFloats = checked((int)plan.Values[plan.InputValue].ElementCount); - _outFloats = checked((int)plan.Values[plan.OutputValue].ElementCount); - _inOff = (ulong)plan.BufOffset[plan.Values[plan.InputValue].Buf] * (ulong)Es; - _outOff = (ulong)plan.BufOffset[plan.Values[plan.OutputValue].Buf] * (ulong)Es; - _inHost = new float[_inFloats]; - _outHost = new float[_outFloats]; - } - - internal MtlBuffer ResolveBuf(MtlBind b) => Resolve(b).buf; - internal ulong ResolveOff(MtlBind b) => b.Offset; - internal (MtlBuffer buf, ulong off, ulong size) Resolve(MtlBind b) => b.Slot switch - { - MtlSlot.Arena => (Arena, b.Offset, b.Size), - MtlSlot.Consts => (_graph.Consts, b.Offset, b.Size), - MtlSlot.Direct => (b.Buf!, b.Offset, b.Size), - _ => (_graph.Dummy, b.Offset, b.Size), - }; - - /// Input tensor shape (e.g. [1,3,H,W]). - public int[] InputShape => _plan.Values[_plan.InputValue].Shape; - /// Output tensor shape (e.g. [1,3,H,W]). - public int[] OutputShape => _plan.Values[_plan.OutputValue].Shape; - - /// The writable input window (staged into the arena on ). - public Span Input => _inHost; - /// The output tensor after . - public ReadOnlySpan Output => _outHost; - - internal ulong InputOffset => _inOff; - internal ulong OutputOffset => _outOff; - internal ulong InputBytes => (ulong)_inFloats * (ulong)Es; - internal ulong OutputBytes => (ulong)_outFloats * (ulong)Es; - - /// Host float -> stored elements (fp16 narrowing). - internal static void ToStorage(ReadOnlySpan src, void* dst) - { - var d = new Span(dst, src.Length); - for (int i = 0; i < src.Length; i++) d[i] = (Half)src[i]; - } - - /// Stored elements -> host float (fp16 widening). - internal static void FromStorage(void* src, Span dst) - { - var sp = new ReadOnlySpan(src, dst.Length); - for (int i = 0; i < dst.Length; i++) dst[i] = (float)sp[i]; - } - - /// Downloads dst.Length stored elements at byte offset . - internal void DownloadElems(MtlBuffer buf, ulong off, Span dst) - => FromStorage((byte*)buf.Contents + off, dst); - - /// Record one dispatch on an open encoder. - internal static void RecordOne(MtlEncoder enc, MtlGraphModel g, - MtlDispatch d, MtlBuffer arena, ulong arenaBase = 0) - { - enc.SetPipeline(d.Pipe); - for (int b = 0; b < d.Binds.Length; b++) - { - MtlBind mb = d.Binds[b]; - (MtlBuffer buf, ulong off, _) = mb.Slot switch - { - MtlSlot.Arena => (arena, arenaBase + mb.Offset, mb.Size), - MtlSlot.Consts => (g.Consts, mb.Offset, mb.Size), - MtlSlot.Direct => (mb.Buf!, mb.Offset, mb.Size), - _ => (g.Dummy, mb.Offset, mb.Size), - }; - enc.SetBuffer(buf, off, b); - } - if (d.Pc.Length > 0) - fixed (byte* pc = d.Pc) - enc.SetPc(pc, (nuint)d.Pc.Length, d.Binds.Length); - enc.Dispatch(d.Gx, d.Gy, d.Gz, 256, 1, 1); - } - - /// Record all dispatches on an open encoder (serial dispatch order). - internal void RecordCompute(MtlEncoder enc) - { - foreach (var d in _graph.Dispatches) - RecordOne(enc, _graph, d, Arena); - } - - /// Debug/test: run the dispatches of ExecNode i (sequential state persists). - internal void DebugRunNode(int i) - { - var (start, count) = _graph.NodeRanges[i]; - if (count > 0) DebugRunRange(start, count); - } - - /// Debug/test: a value's arena window as host floats. - internal float[] DebugValue(int valueId) - { - var dst = new float[checked((int)_plan.Values[valueId].ElementCount)]; - DebugDownload(valueId, dst); - return dst; - } - - /// Debug/test: run a single dispatch-list slice on its own command buffer. - internal void DebugRunRange(int dStart, int dCount) - { - using var pool = AutoReleasePool.Create(); - var enc = _dev.BeginCommands(); - for (int i = dStart; i < dStart + dCount; i++) - RecordOne(enc, _graph, _graph.Dispatches[i], Arena); - enc.End(); - enc.CommitAndWait(); - } - - /// - /// Debug/test: time each dispatch by running it alone (commit + wait) and - /// wall-timing; includes ~45us launch overhead per dispatch — rank only. - /// Returns name→best-of-2 ms. - /// - internal Dictionary DebugProfile() - { - int n = _graph.Dispatches.Length; - var best = new double[n]; - Array.Fill(best, double.MaxValue); - // warm pass to keep state identical to production (input already in arena) - using (var pool = AutoReleasePool.Create()) - { - var enc = _dev.BeginCommands(); - foreach (var d in _graph.Dispatches) RecordOne(enc, _graph, d, Arena); - enc.End(); - enc.CommitAndWait(); - } - var sw = new System.Diagnostics.Stopwatch(); - for (int rep = 0; rep < 2; rep++) - for (int i = 0; i < n; i++) - { - var d = _graph.Dispatches[i]; - using var pool = AutoReleasePool.Create(); - var enc = _dev.BeginCommands(); - RecordOne(enc, _graph, d, Arena); - enc.End(); - sw.Restart(); - enc.CommitAndWait(); - double ms = sw.Elapsed.TotalMilliseconds; - if (ms < best[i]) best[i] = ms; - } - var agg = new Dictionary(); - PerDispatch = new (string, double)[n]; - for (int i = 0; i < n; i++) - { - string name = _graph.Dispatches[i].Name ?? $"d{i}"; - PerDispatch[i] = (name, best[i]); - name = name[..(name.IndexOf('(') is int cut && cut > 0 ? cut : name.Length)]; - agg[name] = agg.TryGetValue(name, out double v) ? v + best[i] : best[i]; - } - return agg; - } - - /// Per-dispatch timings from the last call. - internal (string Name, double Ms)[]? PerDispatch; - - /// Debug: the compiled plan backing this session. - internal ExecPlan DebugPlan => _plan; - - /// Debug/test: const offset (ulong.MaxValue when not packed). - internal ulong DebugConstOff(int valueId, int kind) => _graph.DebugConst(valueId, kind).off; - - /// Debug/test: const storage of a value (kind 0 raw,1 f32,2 convw,3 dww,4 convwt,5 gemmnk). - internal float[] DebugConst(int valueId, int kind, int len) - { - var (buf, off, half) = _graph.DebugConst(valueId, kind); - var dst = new float[len]; - if (half) - { - var h = new ReadOnlySpan((byte*)buf.Contents + off, len); - for (int i = 0; i < len; i++) dst[i] = (float)h[i]; - } - else - new ReadOnlySpan((byte*)buf.Contents + off, len).CopyTo(dst); - return dst; - } - - /// Debug/test: dispatch names for ExecNode i. - internal string[] DebugNodeDispatches(int i) - { - var (st, cnt) = _graph.NodeRanges[i]; - return _graph.Dispatches.Skip(st).Take(cnt).Select(d => d.Name ?? "").ToArray(); - } - - /// Debug: read the whole arena (ArenaFloats floats) into dst. - internal void DebugArena(Span dst) - => FromStorage(Arena.Contents.ToPointer(), dst.Slice(0, _plan.ArenaFloats)); - - /// Debug: graph consts buffer + the session's arena buffer. - internal (MtlBuffer consts, MtlBuffer arena) DebugBuffers => (_graph.Consts, Arena); - - /// Debug/test: download an activation value's region into dst. - internal void DebugDownload(int valueId, float[] dst) - { - CValue v = _plan.Values[valueId]; - FromStorage((byte*)Arena.Contents + (ulong)_plan.BufOffset[v.Buf] * (ulong)Es, - dst.AsSpan(0, checked((int)v.ElementCount))); - } - - /// Debug/test: copy host input into the arena's input region. - internal void DebugUploadInput() - => ToStorage(_inHost, (byte*)Arena.Contents + _inOff); - - /// Write input into the arena and run all dispatches. - public void Run() - { - ToStorage(_inHost, (byte*)Arena.Contents + _inOff); - using var pool = AutoReleasePool.Create(); - var enc = _dev.BeginCommands(); - RecordCompute(enc); - enc.End(); - enc.CommitAndWait(); - FromStorage((byte*)Arena.Contents + _outOff, _outHost); - } - - /// Releases the session's device arena and its reference to the shared graph. - public void Dispose() - { - Arena.Dispose(); - _graph.Release(); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/ObjC.cs b/src/Sdcb.SimdGcdrNet/Backends/Metal/ObjC.cs deleted file mode 100644 index d0852a9..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/ObjC.cs +++ /dev/null @@ -1,229 +0,0 @@ -using System.Runtime.InteropServices; - -namespace Sdcb.SimdGcdrNet.Backends.Metal; - -// Minimal libobjc P/Invoke layer — pure net10.0 C#, no macOS workload. -// Ownership follows Cocoa rules: alloc/init/newXxx/copy return +1 (caller -// releases); everything else is autoreleased within the enclosing -// NSAutoreleasePool. Selectors and class handles are cached in static -// readonly fields — a dispatch already costs enough objc_msgSend calls that -// re-registering selector strings per call is wasted marshaling. -internal static unsafe partial class ObjC -{ - private const string LibObjC = "libobjc.dylib"; - private const string LibMetal = "/System/Library/Frameworks/Metal.framework/Metal"; - - [LibraryImport(LibObjC, EntryPoint = "objc_getClass")] - private static partial IntPtr GetClassRaw(IntPtr name); - - [LibraryImport(LibObjC, EntryPoint = "sel_registerName")] - private static partial IntPtr RegSelRaw(IntPtr name); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send0(IntPtr recv, IntPtr sel); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send1P(IntPtr recv, IntPtr sel, IntPtr a); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send2P(IntPtr recv, IntPtr sel, IntPtr a, IntPtr b); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send3P(IntPtr recv, IntPtr sel, IntPtr a, IntPtr b, IntPtr c); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send1N(IntPtr recv, IntPtr sel, nuint a); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send1P1N(IntPtr recv, IntPtr sel, IntPtr a, nuint b); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send2P1N(IntPtr recv, IntPtr sel, IntPtr a, IntPtr b, nuint c); - - // newBufferWithBytes:length:options: / newBufferWithLength:options: - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send1P2N(IntPtr recv, IntPtr sel, IntPtr a, nuint b, nuint c); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial IntPtr Send1N1N(IntPtr recv, IntPtr sel, nuint a, nuint b); - - // setBuffer:offset:atIndex: / setBytes:length:atIndex: - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial void SendV1P2N(IntPtr recv, IntPtr sel, IntPtr a, nuint b, nuint c); - - // dispatchThreadgroups:threadsPerThreadgroup: — MTLSize by value (24B) - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial void SendV2Size(IntPtr recv, IntPtr sel, MTLSize a, MTLSize b); - - // setThreadgroupMemoryLength:atIndex: - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial void SendV1N1N(IntPtr recv, IntPtr sel, nuint a, nuint b); - - // single-NSUInteger void calls (setLanguageVersion:, memoryBarrierWithScope:, …) - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial void SendV1N(IntPtr recv, IntPtr sel, nuint a); - - // single-BOOL void calls (setFastMathEnabled:…) — BOOL is 1 byte on arm64. - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial void SendV1B(IntPtr recv, IntPtr sel, byte a); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial byte SendBool(IntPtr recv, IntPtr sel); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial byte SendBool1P(IntPtr recv, IntPtr sel, IntPtr a); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial byte SendBool1N(IntPtr recv, IntPtr sel, nuint a); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial nuint SendNuint(IntPtr recv, IntPtr sel); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial double SendDouble(IntPtr recv, IntPtr sel); - - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial long SendLong(IntPtr recv, IntPtr sel); - - // supportsFamily: — (id,SEL,NSInteger) - [LibraryImport(LibObjC, EntryPoint = "objc_msgSend")] - public static partial byte SendBool1L(IntPtr recv, IntPtr sel, nint a); - - [LibraryImport(LibMetal, EntryPoint = "MTLCreateSystemDefaultDevice")] - public static partial IntPtr CreateSystemDefaultDevice(); - - [StructLayout(LayoutKind.Sequential)] - public readonly struct MTLSize - { - public readonly nuint X, Y, Z; - public MTLSize(nuint x, nuint y, nuint z) { X = x; Y = y; Z = z; } - } - - [StructLayout(LayoutKind.Sequential)] - public readonly struct NSRange - { - public readonly nuint Location, Length; - public NSRange(nuint location, nuint length) { Location = location; Length = length; } - } - - // ---- cached class/selector tables ------------------------------------ - // sel_registerName returns the same SEL for the same string, so register - // once. Classes likewise are process-wide stable. - - private static readonly IntPtr s_classNsString = Cls("NSString"); - private static readonly IntPtr s_classNsAutoreleasePool = Cls("NSAutoreleasePool"); - private static readonly IntPtr s_classMtlCompileOptions = Cls("MTLCompileOptions"); - private static readonly IntPtr s_classMtlFunctionConstantValues = Cls("MTLFunctionConstantValues"); - - public static IntPtr ClassNSString => s_classNsString; - public static IntPtr ClassNSAutoreleasePool => s_classNsAutoreleasePool; - public static IntPtr ClassMTLCompileOptions => s_classMtlCompileOptions; - public static IntPtr ClassMTLFunctionConstantValues => s_classMtlFunctionConstantValues; - - public static IntPtr Cls(string name) - { - IntPtr n = Marshal.StringToHGlobalAnsi(name); - try { return GetClassRaw(n); } finally { Marshal.FreeHGlobal(n); } - } - - public static IntPtr Sel(string name) - { - IntPtr n = Marshal.StringToHGlobalAnsi(name); - try { return RegSelRaw(n); } finally { Marshal.FreeHGlobal(n); } - } - - public static class S - { - public static readonly IntPtr Alloc = Sel("alloc"); - public static readonly IntPtr Init = Sel("init"); - public static readonly IntPtr Release = Sel("release"); - public static readonly IntPtr Retain = Sel("retain"); - public static readonly IntPtr Drain = Sel("drain"); - public static readonly IntPtr InitWithUTF8String = Sel("initWithUTF8String:"); - public static readonly IntPtr Utf8String = Sel("UTF8String"); - public static readonly IntPtr LocalizedDescription = Sel("localizedDescription"); - public static readonly IntPtr Length = Sel("length"); - - public static readonly IntPtr Name = Sel("name"); - public static readonly IntPtr RespondsToSelector = Sel("respondsToSelector:"); - public static readonly IntPtr SupportsFamily = Sel("supportsFamily:"); - public static readonly IntPtr MaxThreadgroupMemoryLength = Sel("maxThreadgroupMemoryLength"); - public static readonly IntPtr MaxThreadsPerThreadgroup = Sel("maxThreadsPerThreadgroup"); - public static readonly IntPtr MaxTotalThreadsPerThreadgroup = Sel("maxTotalThreadsPerThreadgroup"); - public static readonly IntPtr ThreadExecutionWidth = Sel("threadExecutionWidth"); - public static readonly IntPtr HasUnifiedMemory = Sel("hasUnifiedMemory"); - public static readonly IntPtr RecommendedMaxWorkingSetSize = Sel("recommendedMaxWorkingSetSize"); - public static readonly IntPtr CurrentAllocatedSize = Sel("currentAllocatedSize"); - public static readonly IntPtr RegistryID = Sel("registryID"); - - public static readonly IntPtr NewCommandQueue = Sel("newCommandQueue"); - public static readonly IntPtr NewBufferWithBytesLengthOptions = Sel("newBufferWithBytes:length:options:"); - public static readonly IntPtr NewBufferWithLengthOptions = Sel("newBufferWithLength:options:"); - public static readonly IntPtr Contents = Sel("contents"); - public static readonly IntPtr NewLibraryWithSourceOptionsError = Sel("newLibraryWithSource:options:error:"); - public static readonly IntPtr NewFunctionWithName = Sel("newFunctionWithName:"); - public static readonly IntPtr NewFunctionWithNameConstantValuesError = Sel("newFunctionWithName:constantValues:error:"); - public static readonly IntPtr NewComputePipelineStateWithFunctionError = Sel("newComputePipelineStateWithFunction:error:"); - public static readonly IntPtr FunctionName = Sel("functionName"); - public static readonly IntPtr FunctionConstantsDictionary = Sel("functionConstantsDictionary"); - public static readonly IntPtr SetConstantValueTypeAtIndex = Sel("setConstantValue:type:atIndex:"); - - public static readonly IntPtr SetFastMathEnabled = Sel("setFastMathEnabled:"); - public static readonly IntPtr SetMathMode = Sel("setMathMode:"); - public static readonly IntPtr SetLanguageVersion = Sel("setLanguageVersion:"); - - public static readonly IntPtr CommandBuffer = Sel("commandBuffer"); - public static readonly IntPtr ComputeCommandEncoder = Sel("computeCommandEncoder"); - public static readonly IntPtr ComputeCommandEncoderWithDispatchType = Sel("computeCommandEncoderWithDispatchType:"); - public static readonly IntPtr SetComputePipelineState = Sel("setComputePipelineState:"); - public static readonly IntPtr SetBufferOffsetAtIndex = Sel("setBuffer:offset:atIndex:"); - public static readonly IntPtr SetBytesLengthAtIndex = Sel("setBytes:length:atIndex:"); - public static readonly IntPtr DispatchThreadgroupsThreadsPerThreadgroup = Sel("dispatchThreadgroups:threadsPerThreadgroup:"); - public static readonly IntPtr DispatchThreadsThreadsPerThreadgroup = Sel("dispatchThreads:threadsPerThreadgroup:"); - public static readonly IntPtr SetThreadgroupMemoryLengthAtIndex = Sel("setThreadgroupMemoryLength:atIndex:"); - public static readonly IntPtr MemoryBarrierWithScope = Sel("memoryBarrierWithScope:"); - public static readonly IntPtr EndEncoding = Sel("endEncoding"); - public static readonly IntPtr Commit = Sel("commit"); - public static readonly IntPtr WaitUntilCompleted = Sel("waitUntilCompleted"); - public static readonly IntPtr Status = Sel("status"); - public static readonly IntPtr Error = Sel("error"); - public static readonly IntPtr GpuStartTime = Sel("GPUStartTime"); - public static readonly IntPtr GpuEndTime = Sel("GPUEndTime"); - public static readonly IntPtr Label = Sel("label"); - public static readonly IntPtr SetLabel = Sel("setLabel:"); - } - - // NSString* with UTF-8 — caller owns (+1 via alloc/init); pass owns:false - // to consume within the current autorelease pool. - public static IntPtr NsStr(string s) - { - IntPtr obj = Send0(s_classNsString, S.Alloc); - IntPtr cstr = Marshal.StringToHGlobalAnsi(s); - try { return Send1P(obj, S.InitWithUTF8String, cstr); } - finally { Marshal.FreeHGlobal(cstr); } - } - - public static string? NsToString(IntPtr nsString) - { - if (nsString == 0) return null; - IntPtr utf8 = Send0(nsString, S.Utf8String); - return utf8 == 0 ? null : Marshal.PtrToStringUTF8(utf8); - } - - public static void Release(IntPtr obj) { if (obj != 0) Send0(obj, S.Release); } - public static IntPtr Retain(IntPtr obj) => Send0(obj, S.Retain); -} - -/// Scoped NSAutoreleasePool — .NET worker threads have none, so every -/// public entry that allocates autoreleased ObjC objects wraps itself in one. -/// Drain at Dispose releases the pool itself. -internal readonly struct AutoReleasePool : IDisposable -{ - private readonly IntPtr _pool; - private AutoReleasePool(IntPtr pool) => _pool = pool; - - public static AutoReleasePool Create() => - new(ObjC.Send0(ObjC.Send0(ObjC.ClassNSAutoreleasePool, ObjC.S.Alloc), ObjC.S.Init)); - - public void Dispose() => ObjC.Send0(_pool, ObjC.S.Drain); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/common.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/common.metal deleted file mode 100644 index b8b9f95..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/common.metal +++ /dev/null @@ -1,43 +0,0 @@ -// Common helpers for the GCDRNet Metal kernels (fp16 storage, fp32 math — -// same contract as the Vulkan *_h shaders: half is storage-only, every load -// widens to float and all arithmetic runs in fp32). -#include -using namespace metal; - -static inline float ferf1(float v) -{ - float s = v < 0.0f ? -1.0f : 1.0f; - float ax = fabs(v); - float t = 1.0f / (1.0f + 0.3275911f * ax); - float poly = ((((1.061405429f * t - 1.453152027f) * t + 1.421413741f) * t - - 0.284496736f) * t + 0.254829592f) * t; - return (1.0f - poly * exp(-ax * ax)) * s; -} - -static inline float fexp1(float v) -{ - v = clamp(v, -87.3f, 88.7f); - float fx = v * 1.4426950408889634f + 0.5f; - float nf = floor(fx); - float r = v - nf * 0.693359375f - nf * -0.00021219444005469058f; - float poly = (((((1.9875691500e-4f * r + 1.3981999507e-3f) * r + 8.3334519073e-3f) * r - + 4.1665795894e-2f) * r + 1.6666665459e-1f) * r + 5.0000001201e-1f) * r * r + r + 1.0f; - return poly * as_type((uint(int(nf)) + 127u) << 23); -} - -// Conv epilogue activation: bits 4-6 (relu=1, gelu=2, sigmoid=3). -static inline float cact(float v, uint flags) -{ - uint a = (flags >> 4) & 7u; - if (a == 1u) return max(v, 0.0f); - if (a == 2u) return 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f)); - if (a == 3u) return 1.0f / (1.0f + fexp1(-v)); - return v; -} - -static inline int mirror1(int i, int d) -{ - if (d <= 1) return 0; - while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; - return i; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv.metal deleted file mode 100644 index 9cb02a7..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv.metal +++ /dev/null @@ -1,333 +0,0 @@ -// Conv kernels: conv_tile (implicit-GEMM tiled fallback, conv_gen/k33/k11 -// equivalent), depthwise(4), convpool(_k3) fused stem+pool. fp16 storage, fp32 math. - -struct PcConv { - uint ic, ih, iw; - uint oc, oh, ow; - uint kh, kw, sh, sw; - uint pt, pl; - uint flags; // bit0 bias, bit1 reflectPad, bit2 relu, bit3 hasAdd, bit4 gelu - uint pad0; -}; - -// conv_tile: O[px,oc] = sum_k X[px,k]*W[k,oc], k = (ky*kw+kx)*ic+ci (coalesced). -// W backend-packed [K][OC] oc-fastest. 3D dispatch: gx=x-tile(16), gy=y-tile(8), -// gz=oc-tile(64); 256 threads, each 4oc x 8px. K-chunk 32 staged. -kernel void conv_tile(device const half* x [[buffer(0)]], - device const half* w [[buffer(1)]], - device const float* b [[buffer(2)]], - device const half* r [[buffer(3)]], - device half* o [[buffer(4)]], - constant PcConv& p [[buffer(5)]], - uint3 wg [[threadgroup_position_in_grid]], - uint3 tp [[thread_position_in_threadgroup]]) -{ - const uint tid = tp.x; - const uint BM = 64u, BH = 8u, BW = 16u, BN = BH * BW, BK = 32u, BKP = 36u; - threadgroup float4 Asv[BK * BM / 4]; - threadgroup float4 Bsv[BN * BKP / 4]; - uint oc0 = wg.z * BM; - uint oy0 = wg.y * BH; - uint ox0 = wg.x * BW; - uint K = p.kh * p.kw * p.ic; - uint tm = tid & 15u, tn = tid >> 4; - float4 acc[8] = { float4(0.0f), float4(0.0f), float4(0.0f), float4(0.0f), - float4(0.0f), float4(0.0f), float4(0.0f), float4(0.0f) }; - bool reflect = (p.flags & 2u) != 0u; - uint bk = tid & 31u; - - for (uint k0 = 0u; k0 < K; k0 += BK) - { - for (uint e = tid; e < BK * BM; e += 256u) - { - uint k = e >> 6, m = e & 63u; - uint gm = oc0 + m, gk = k0 + k; - ((threadgroup float*)Asv)[e] = (gm < p.oc && gk < K) ? float(w[gk * p.oc + gm]) : 0.0f; - } - uint gk = k0 + bk; - bool kok = gk < K; - uint tap = kok ? gk / p.ic : 0u; - uint ch = gk - tap * p.ic; - uint ky = tap / p.kw, kx = tap - ky * p.kw; - for (uint n = tid >> 5; n < BN; n += 8u) - { - float v = 0.0f; - uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u); - if (kok && oy < p.oh && ox < p.ow) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - int sx = int(ox * p.sw + kx) - int(p.pl); - if (reflect) - { - if (uint(sy) >= p.ih) sy = mirror1(sy, int(p.ih)); - if (uint(sx) >= p.iw) sx = mirror1(sx, int(p.iw)); - v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]); - } - else if (uint(sy) < p.ih && uint(sx) < p.iw) - v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]); - } - uint si = n * BKP + bk; - ((threadgroup float*)Bsv)[si] = v; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint k = 0u; k < BK; k += 4u) - { - float4 a0 = Asv[k * 16u + tm]; - float4 a1 = Asv[(k + 1u) * 16u + tm]; - float4 a2 = Asv[(k + 2u) * 16u + tm]; - float4 a3 = Asv[(k + 3u) * 16u + tm]; - for (int j = 0; j < 8; j++) - { - float4 bv = Bsv[(tn * 8u + uint(j)) * (BKP / 4u) + (k >> 2)]; - acc[j] += a0 * bv.x + a1 * bv.y + a2 * bv.z + a3 * bv.w; - } - } - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - bool hasBias = (p.flags & 1u) != 0u; - uint gm0 = oc0 + tm * 4u; - float4 bias = float4( - hasBias && gm0 < p.oc ? b[gm0] : 0.0f, - hasBias && gm0 + 1u < p.oc ? b[gm0 + 1u] : 0.0f, - hasBias && gm0 + 2u < p.oc ? b[gm0 + 2u] : 0.0f, - hasBias && gm0 + 3u < p.oc ? b[gm0 + 3u] : 0.0f); - for (int j = 0; j < 8; j++) - { - uint n = tn * 8u + uint(j); - uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u); - if (oy >= p.oh || ox >= p.ow || gm0 >= p.oc) continue; - uint base = (oy * p.ow + ox) * p.oc + gm0; - uint full = gm0 + 3u < p.oc ? 4u : p.oc - gm0; - for (uint l = 0u; l < 4u && l < full; l++) - { - float v = acc[j][l] + bias[l]; - if ((p.flags & 8u) != 0u) v += float(r[base + l]); - if ((p.flags & 4u) != 0u) v = max(v, 0.0f); - if ((p.flags & 16u) != 0u) v = 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f)); - o[base + l] = half(v); - } - } -} - -// ---- depthwise / depthwise4: out[oy,ox,c] = bias[c] + sum w[tap,c]*x[sy,sx,c] ---- -struct PcDw { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl; - uint flags; // bit0 hasBias, bit1 reflect, bit2 relu, bit4 gelu - float padVal; -}; -kernel void depthwise(device const half* x [[buffer(0)]], - device const half* w [[buffer(1)]], - device const float* b [[buffer(2)]], - device half* o [[buffer(3)]], - constant PcDw& p [[buffer(4)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.c * p.oh * p.ow) return; - uint ch = gid % p.c, px = gid / p.c; - uint ox = px % p.ow, oy = px / p.ow; - bool refl = (p.flags & 2u) != 0u; - float acc = ((p.flags & 1u) != 0u) ? b[ch] : 0.0f; - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - float wv = float(w[(ky * p.kw + kx) * p.c + ch]); - int yy = sy, xx = sx; - if (uint(sy) >= p.ih || uint(sx) >= p.iw) - { - if (!refl) { acc += wv * p.padVal; continue; } - if (uint(sy) >= p.ih) yy = mirror1(sy, int(p.ih)); - if (uint(sx) >= p.iw) xx = mirror1(sx, int(p.iw)); - } - acc += wv * float(x[(uint(yy) * p.iw + uint(xx)) * p.c + ch]); - } - } - if ((p.flags & 4u) != 0u) acc = max(acc, 0.0f); - if ((p.flags & 16u) != 0u) acc = 0.5f * acc * (1.0f + ferf1(acc * 0.7071067811865475f)); - o[gid] = half(acc); -} -kernel void depthwise4(device const half4* x [[buffer(0)]], - device const half4* w [[buffer(1)]], - device const float4* b [[buffer(2)]], - device half4* o [[buffer(3)]], - constant PcDw& p [[buffer(4)]], - uint gid [[thread_position_in_grid]]) -{ - uint c4 = p.c >> 2; - if (gid >= c4 * p.oh * p.ow) return; - uint ch = gid % c4, px = gid / c4; - uint ox = px % p.ow, oy = px / p.ow; - bool refl = (p.flags & 2u) != 0u; - float4 acc = ((p.flags & 1u) != 0u) ? b[ch] : float4(0.0f); - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - float4 wv = float4(w[(ky * p.kw + kx) * c4 + ch]); - int yy = sy, xx = sx; - if (uint(sy) >= p.ih || uint(sx) >= p.iw) - { - if (!refl) { acc += wv * p.padVal; continue; } - if (uint(sy) >= p.ih) yy = mirror1(sy, int(p.ih)); - if (uint(sx) >= p.iw) xx = mirror1(sx, int(p.iw)); - } - acc += wv * float4(x[(uint(yy) * p.iw + uint(xx)) * c4 + ch]); - } - } - if ((p.flags & 4u) != 0u) acc = max(acc, float4(0.0f)); - if ((p.flags & 16u) != 0u) - for (int l = 0; l < 4; l++) acc[l] = 0.5f * acc[l] * (1.0f + ferf1(acc[l] * 0.7071067811865475f)); - o[gid] = half4(acc); -} - -// ---- convpool / convpool_k3: small-ic stem conv + fused 2x2/2 maxpool ---- -struct PcConvPool { - uint ic, ih, iw, oc, ph, pw, kh, kw, pt, pl, flags; - float padv; -}; -kernel void convpool(device const half* x [[buffer(0)]], - device const half* w [[buffer(1)]], - device const float* b [[buffer(2)]], - device half* o [[buffer(3)]], - constant PcConvPool& p [[buffer(4)]], - uint gid [[thread_position_in_grid]], - uint tid [[thread_position_in_threadgroup]]) -{ - threadgroup float4 ws[1024]; - uint K = p.kh * p.kw * p.ic, oc4 = p.oc >> 2; - for (uint e = tid; e < K * p.oc; e += 256u) ((threadgroup float*)ws)[e] = float(w[e]); - threadgroup_barrier(mem_flags::mem_threadgroup); - uint groups = p.oc >> 3; - if (gid >= p.ph * p.pw * groups) return; - uint g = gid % groups, pp = gid / groups; - uint pxx = pp % p.pw, pyy = pp / p.pw; - bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u; - uint n0 = g * 8u; - float4 b0 = float4(0.0f), b1 = float4(0.0f); - if ((p.flags & 1u) != 0u) - { - b0 = float4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]); - b1 = float4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]); - } - float4 m0 = float4(-3.4e38f), m1 = float4(-3.4e38f); - for (uint d = 0u; d < 4u; d++) - { - uint oy = pyy * 2u + (d >> 1), ox = pxx * 2u + (d & 1u); - float4 a0 = b0, a1 = b1; - for (uint ky = 0u; ky < p.kh; ky++) - { - int iy = int(oy + ky) - int(p.pt); - bool vy = iy >= 0 && iy < int(p.ih); - if (!vy && reflect) { iy = mirror1(iy, int(p.ih)); vy = true; } - for (uint kx = 0u; kx < p.kw; kx++) - { - int ix = int(ox + kx) - int(p.pl); - bool vx = ix >= 0 && ix < int(p.iw); - if (!vx && reflect) { ix = mirror1(ix, int(p.iw)); vx = true; } - bool valid = vy && vx; - uint base = valid ? (uint(iy) * p.iw + uint(ix)) * p.ic : 0u; - uint kb = (ky * p.kw + kx) * p.ic; - for (uint ci = 0u; ci < p.ic; ci++) - { - float xv = valid ? float(x[base + ci]) : p.padv; - uint wr = (kb + ci) * oc4 + g * 2u; - a0 += xv * ws[wr]; - a1 += xv * ws[wr + 1u]; - } - } - } - if (crelu) { a0 = max(a0, float4(0.0f)); a1 = max(a1, float4(0.0f)); } - m0 = max(m0, a0); m1 = max(m1, a1); - } - if ((p.flags & 8u) != 0u) { m0 = max(m0, float4(0.0f)); m1 = max(m1, float4(0.0f)); } - uint ob = pp * p.oc + n0; - for (uint l = 0u; l < 4u; l++) { o[ob + l] = half(m0[l]); o[ob + 4u + l] = half(m1[l]); } -} - -kernel void convpool_k3(device const half* x [[buffer(0)]], - device const half* w [[buffer(1)]], - device const float* b [[buffer(2)]], - device half* o [[buffer(3)]], - constant PcConvPool& p [[buffer(4)]], - uint gid [[thread_position_in_grid]], - uint tid [[thread_position_in_threadgroup]]) -{ - threadgroup float4 ws[1024]; - uint K = 9u * p.ic, oc4 = p.oc >> 2; - for (uint e = tid; e < K * p.oc; e += 256u) ((threadgroup float*)ws)[e] = float(w[e]); - threadgroup_barrier(mem_flags::mem_threadgroup); - uint groups = p.oc >> 3; - if (gid >= p.ph * p.pw * groups) return; - uint g = gid % groups, pp = gid / groups; - uint pxx = pp % p.pw, pyy = pp / p.pw; - bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u; - int ro[4], co[4]; - bool rv[4], cv[4]; - for (uint r = 0u; r < 4u; r++) - { - int iy = int(pyy * 2u + r) - int(p.pt); - bool vy = iy >= 0 && iy < int(p.ih); - if (!vy && reflect) { iy = mirror1(iy, int(p.ih)); vy = true; } - rv[r] = vy; ro[r] = vy ? iy * int(p.iw) : 0; - int ix = int(pxx * 2u + r) - int(p.pl); - bool vx = ix >= 0 && ix < int(p.iw); - if (!vx && reflect) { ix = mirror1(ix, int(p.iw)); vx = true; } - cv[r] = vx; co[r] = vx ? ix : 0; - } - uint n0 = g * 8u; - float4 b0 = float4(0.0f), b1 = float4(0.0f); - if ((p.flags & 1u) != 0u) - { - b0 = float4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]); - b1 = float4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]); - } - float4 a0[4], a1[4]; - for (uint d = 0u; d < 4u; d++) { a0[d] = b0; a1[d] = b1; } - for (uint r = 0u; r < 4u; r++) - { - float xr[4][8]; - for (uint c = 0u; c < 4u; c++) - { - bool v = rv[r] && cv[c]; - uint base = uint(ro[r] + co[c]) * p.ic; - for (uint ci = 0u; ci < 8u; ci++) - xr[c][ci] = ci < p.ic ? (v ? float(x[base + ci]) : p.padv) : 0.0f; - } - for (uint dy = 0u; dy < 2u; dy++) - { - if (r < dy || r - dy > 2u) continue; - uint ky = r - dy; - for (uint kx = 0u; kx < 3u; kx++) - { - uint kb = (ky * 3u + kx) * p.ic; - for (uint ci = 0u; ci < 8u; ci++) - { - if (ci >= p.ic) break; - uint wr = (kb + ci) * oc4 + g * 2u; - float4 w0 = ws[wr], w1 = ws[wr + 1u]; - for (uint dx = 0u; dx < 2u; dx++) - { - float xv = xr[kx + dx][ci]; - a0[dy * 2u + dx] += xv * w0; - a1[dy * 2u + dx] += xv * w1; - } - } - } - } - } - float4 m0 = float4(-3.4e38f), m1 = float4(-3.4e38f); - for (uint d = 0u; d < 4u; d++) - { - if (crelu) { a0[d] = max(a0[d], float4(0.0f)); a1[d] = max(a1[d], float4(0.0f)); } - m0 = max(m0, a0[d]); m1 = max(m1, a1[d]); - } - if ((p.flags & 8u) != 0u) { m0 = max(m0, float4(0.0f)); m1 = max(m1, float4(0.0f)); } - uint ob = pp * p.oc + n0; - for (uint l = 0u; l < 4u; l++) { o[ob + l] = half(m0[l]); o[ob + 4u + l] = half(m1[l]); } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv_mm.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv_mm.metal deleted file mode 100644 index f784851..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/conv_mm.metal +++ /dev/null @@ -1,229 +0,0 @@ -// conv_mm: simdgroup-MMA implicit-GEMM NHWC conv (Metal port of conv_cm.comp). -// A gathers input pixels through tap-major im2col addressing — no im2col -// materialization; W is backend-packed [oc][Kd] tap-major, Kd = K rounded up -// to 8 (zero-filled rows make pad k-slices contribute 0 for any gathered -// pixel). BM = 64 output pixels x BN output channels, 8 simdgroups -// (256 threads), fp16 storage + simdgroup fp16xfp32 MMA. -// cflags: bit0 bias, bit1 residual, bit2 reflect pad, bits4-6 act (1 relu, 2 gelu). -struct PcConvMm { - uint mDim, nDim, kDim, flags; - uint outW, inW, inH, cin, kW; - uint sH, sW, pT, pL, kReal; -}; - -template -static inline void conv_mm_body(device const half* x, - device const half* w, - device const float* bias, - device const half* res, - device half* o, - constant PcConvMm& p, - uint2 wg, uint tid, - threadgroup half* tA, threadgroup half* tB, - threadgroup float* outst) -{ - const uint JN = BN / 16u; // acc columns per simdgroup row-half - uint sgid = tid >> 5; - uint sm = (sgid & 3u) * 16u; // 4 rows of 16 -> 64 M - uint sn = (sgid >> 2) * (BN / 2u); // 2 cols of BN/2 - uint m0 = wg.x * 64u + sm; // m tiles on gx (grid allows 2^31 in x) - uint n0 = wg.y * BN + sn; - - simdgroup_float8x8 acc[2][8]; - for (uint i = 0u; i < 2u; i++) for (uint j = 0u; j < 8u; j++) - acc[i][j] = simdgroup_float8x8(0.0f); - - bool reflect = (p.flags & 4u) != 0u; - - for (uint kb = 0u; kb < p.kDim; kb += 16u) - { - if (SCK == 0) - { - // vec4 gather: 64 rows x 16 k as one half4 per thread - uint am = tid >> 2, ak = (tid & 3u) * 4u; - uint gm = wg.x * 64u + am; - uint gk = kb + ak; - half4 av = half4(0.0h); - if (gm < p.mDim && gk < p.kReal) - { - uint tap = gk / p.cin, ci = gk - tap * p.cin; - uint dy = tap / p.kW, dx = tap - dy * p.kW; - uint oy = gm / p.outW, ox = gm - oy * p.outW; - int iy = int(oy * p.sH + dy) - int(p.pT); - int ix = int(ox * p.sW + dx) - int(p.pL); - if (reflect) - { - if (uint(iy) >= p.inH) iy = mirror1(iy, int(p.inH)); - if (uint(ix) >= p.inW) ix = mirror1(ix, int(p.inW)); - av = *(device const half4*)(x + (((uint(iy) * p.inW + uint(ix)) * (p.cin >> 2)) + (ci >> 2)) * 4u); - } - else if (iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW)) - av = *(device const half4*)(x + (((uint(iy) * p.inW + uint(ix)) * (p.cin >> 2)) + (ci >> 2)) * 4u); - } - *(threadgroup half4*)(tA + am * 20 + ak) = av; - } - else - { - // scalar gather for cin % 4 != 0 (RGB stems): 64x16 one half per thread x4 - for (uint e = tid; e < 64u * 16u; e += 256u) - { - uint am = e >> 4, ak = e & 15u; - uint gm = wg.x * 64u + am; - uint gk = kb + ak; - half av = 0.0h; - if (gm < p.mDim && gk < p.kReal) - { - uint tap = gk / p.cin, ci = gk - tap * p.cin; - uint dy = tap / p.kW, dx = tap - dy * p.kW; - uint oy = gm / p.outW, ox = gm - oy * p.outW; - int iy = int(oy * p.sH + dy) - int(p.pT); - int ix = int(ox * p.sW + dx) - int(p.pL); - if (reflect) - { - if (uint(iy) >= p.inH) iy = mirror1(iy, int(p.inH)); - if (uint(ix) >= p.inW) ix = mirror1(ix, int(p.inW)); - av = x[(uint(iy) * p.inW + uint(ix)) * p.cin + ci]; - } - else if (iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW)) - av = x[(uint(iy) * p.inW + uint(ix)) * p.cin + ci]; - } - tA[am * 20 + ak] = av; - } - } - // B stage: [N][Kd] rows, one half4 per thread covers 64x16; BN>64 loops - for (uint rr = tid >> 2; rr < uint(BN); rr += 64u) - { - uint bk = (tid & 3u) * 4u; - uint gn = wg.y * BN + rr; - uint gkB = kb + bk; - half4 wv = half4(0.0h); - if (gn < p.nDim && gkB < p.kDim) - wv = *(device const half4*)(w + (ulong)gn * p.kDim + gkB); - *(threadgroup half4*)(tB + rr * 20 + bk) = wv; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint kk = 0u; kk < 16u; kk += 8u) - { - simdgroup_half8x8 a0, a1; - simdgroup_load(a0, tA + sm * 20 + kk, 20); - simdgroup_load(a1, tA + (sm + 8u) * 20 + kk, 20); - simdgroup_half8x8 bfr[8]; - for (uint j = 0u; j < JN; j++) - simdgroup_load(bfr[j], tB + (sn + j * 8u) * 20 + kk, 20, ulong2(0, 0), true); - for (uint j = 0u; j < JN; j++) - { - simdgroup_multiply_accumulate(acc[0][j], a0, bfr[j], acc[0][j]); - simdgroup_multiply_accumulate(acc[1][j], a1, bfr[j], acc[1][j]); - } - } - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - uint lane = tid & 31u; - for (uint i = 0u; i < 2u; i++) - for (uint j = 0u; j < JN; j++) - { - threadgroup_barrier(mem_flags::mem_threadgroup); - simdgroup_store(acc[i][j], outst + sgid * 64, 8); - threadgroup_barrier(mem_flags::mem_threadgroup); - uint r = lane >> 2, c2 = (lane & 3u) * 2u; - for (uint e = 0u; e < 2u; e++) - { - uint mm = m0 + i * 8u + r; - uint nn = n0 + j * 8u + c2 + e; - if (mm >= p.mDim || nn >= p.nDim) continue; - float v = outst[sgid * 64 + r * 8 + c2 + e]; - if ((p.flags & 1u) != 0u) v += bias[nn]; - if ((p.flags & 2u) != 0u) v += float(res[mm * p.nDim + nn]); - o[mm * p.nDim + nn] = half(cact(v, p.flags)); - } - } -} - -#define CONV_MM_INST(BN, SCK) \ -kernel void conv_mm_##BN##_##SCK(device const half* x [[buffer(0)]], \ - device const half* w [[buffer(1)]], device const float* bias [[buffer(2)]], \ - device const half* res [[buffer(3)]], device half* o [[buffer(4)]], \ - constant PcConvMm& p [[buffer(5)]], uint2 wg [[threadgroup_position_in_grid]], \ - uint2 tp [[thread_position_in_threadgroup]]) \ -{ \ - threadgroup half tA[64 * 20]; \ - threadgroup half tB[BN * 20]; \ - threadgroup float outst[8 * 64]; \ - conv_mm_body(x, w, bias, res, o, p, wg, tp.x, tA, tB, outst); \ -} - -CONV_MM_INST(128, 0) -CONV_MM_INST(64, 0) -CONV_MM_INST(32, 0) -CONV_MM_INST(128, 1) -CONV_MM_INST(64, 1) -CONV_MM_INST(32, 1) - -// ---- gemm: C[M,N] = A[M,K]*B + bias[N]; B [K,N] or [N,K] (transB) ---- -// tiled 64x64x16, 256 threads -> 4x4 each (fallback; MMA path uses conv_mm). -struct PcGemm { uint m, n, k, flags; }; // bit0 transB bit1 bias bit2 relu bit3 gelu -kernel void gemm(device const half* a [[buffer(0)]], - device const half* b [[buffer(1)]], - device const float* bs [[buffer(2)]], - device half* o [[buffer(3)]], - constant PcGemm& p [[buffer(4)]], - uint2 wg [[threadgroup_position_in_grid]], - uint2 tp [[thread_position_in_threadgroup]]) -{ - const uint tid = tp.x; - const uint BM = 64u, BN = 64u, BK = 16u; - threadgroup float As[BK * BM]; - threadgroup float Bs[BK * BN]; - uint m0 = wg.y * BM; - uint n0 = wg.x * BN; - uint ty = tid >> 4, tx = tid & 15u; - bool transB = (p.flags & 1u) != 0u; - float acc[4][4]; - for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) acc[i][j] = 0.0f; - for (uint k0 = 0u; k0 < p.k; k0 += BK) - { - for (uint e = tid; e < BK * BM; e += 256u) - { - uint kk = e >> 6, mm = e & 63u; - uint gm = m0 + mm, gk = k0 + kk; - As[e] = (gm < p.m && gk < p.k) ? float(a[gm * p.k + gk]) : 0.0f; - } - for (uint e = tid; e < BK * BN; e += 256u) - { - uint kk = e >> 6, nn = e & 63u; - uint gn = n0 + nn, gk = k0 + kk; - float v = 0.0f; - if (gn < p.n && gk < p.k) - v = float(transB ? b[gn * p.k + gk] : b[gk * p.n + gn]); - Bs[e] = v; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint kk = 0u; kk < BK; kk++) - { - float av[4], bv[4]; - for (int i = 0; i < 4; i++) av[i] = As[kk * BM + ty * 4u + uint(i)]; - for (int j = 0; j < 4; j++) bv[j] = Bs[kk * BN + tx * 4u + uint(j)]; - for (int i = 0; i < 4; i++) - for (int j = 0; j < 4; j++) - acc[i][j] += av[i] * bv[j]; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - } - bool hasBias = (p.flags & 2u) != 0u; - for (int i = 0; i < 4; i++) - { - uint gm = m0 + ty * 4u + uint(i); - if (gm >= p.m) continue; - for (int j = 0; j < 4; j++) - { - uint gn = n0 + tx * 4u + uint(j); - if (gn >= p.n) continue; - float v = acc[i][j]; - if (hasBias) v += bs[gn]; - if ((p.flags & 4u) != 0u) v = max(v, 0.0f); - if ((p.flags & 8u) != 0u) v = 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f)); - o[gm * p.n + gn] = half(v); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal index 02142af..e0bd563 100644 --- a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal +++ b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/glue.metal @@ -1,4 +1,4 @@ -// Pipeline glue + elementwise/shape kernels. All activation storage is fp16 +// GCDR pipeline glue kernels. All activation storage is fp16 // (half/half4); math fp32 — see common.metal. // ---- bgr2nhwc: BGR u8 HWC -> NHWC [h][w][3] /255, top/left replicate pad ---- @@ -67,333 +67,3 @@ kernel void nhwc2bgr(device const half* x [[buffer(0)]], o[gid] = packed; } -// ---- unary: mode 0 relu, 1 sigmoid, 2 erf, 3 gelu, 4 copy ---- -struct PcUnary { uint n, mode; }; -kernel void unary(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcUnary& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.n) return; - float v = float(x[gid]); - float r; - if (p.mode == 0u) r = max(v, 0.0f); - else if (p.mode == 1u) r = 1.0f / (1.0f + fexp1(-v)); - else if (p.mode == 2u) r = ferf1(v); - else if (p.mode == 3u) r = 0.5f * v * (1.0f + ferf1(v * 0.7071067811865475f)); - else r = v; - o[gid] = half(r); -} - -// ---- ew2: elementwise binary + act, rank<=4 broadcast ---- -struct PcEw2 { - uint n, s0, s1, s2; - uint as0, as1, as2, as3; - uint bs0, bs1, bs2, bs3; - uint op, act; -}; -kernel void ew2(device const half* a [[buffer(0)]], - device const half* b [[buffer(1)]], - device half* o [[buffer(2)]], - constant PcEw2& p [[buffer(3)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.n) return; - uint d0 = gid / p.s0, r0 = gid % p.s0; - uint d1 = r0 / p.s1, r1 = r0 % p.s1; - uint d2 = r1 / p.s2, d3 = r1 % p.s2; - float va = float(a[d0 * p.as0 + d1 * p.as1 + d2 * p.as2 + d3 * p.as3]); - float vb = float(b[d0 * p.bs0 + d1 * p.bs1 + d2 * p.bs2 + d3 * p.bs3]); - float v = p.op == 0u ? va + vb : p.op == 1u ? va - vb : p.op == 2u ? va * vb : va / vb; - if (p.act == 1u) v = max(v, 0.0f); - else if (p.act == 2u) v = 1.0f / (1.0f + exp(-v)); - o[gid] = half(v); -} - -// ---- ew2v: same-shape vec4 fast path ---- -struct PcEw2v { uint nv4, op, act; }; -kernel void ew2v(device const half4* a [[buffer(0)]], - device const half4* b [[buffer(1)]], - device half4* o [[buffer(2)]], - constant PcEw2v& p [[buffer(3)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.nv4) return; - float4 va = float4(a[gid]), vb = float4(b[gid]), v; - if (p.op == 0u) v = va + vb; - else if (p.op == 1u) v = va - vb; - else if (p.op == 2u) v = va * vb; - else v = va / vb; - if (p.act == 1u) v = max(v, float4(0.0f)); - else if (p.act == 2u) v = 1.0f / (1.0f + exp(-v)); - o[gid] = half4(v); -} - -// ---- transpose: rank<=4 permute ---- -struct PcTranspose { uint n, os0, os1, os2, ps0, ps1, ps2, ps3; }; -kernel void transpose(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcTranspose& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.n) return; - uint d0 = gid / p.os0, r0 = gid % p.os0; - uint d1 = r0 / p.os1, r1 = r0 % p.os1; - uint d2 = r1 / p.os2, d3 = r1 % p.os2; - o[gid] = x[d0 * p.ps0 + d1 * p.ps1 + d2 * p.ps2 + d3 * p.ps3]; -} - -// ---- slice: rank<=4 strided slice ---- -struct PcSlice { - uint n, os0, os1, os2; - uint st0, st1, st2, st3; - uint sp0, sp1, sp2, sp3; - uint is0, is1, is2, is3; -}; -kernel void slice(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcSlice& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.n) return; - uint d0 = gid / p.os0, r0 = gid % p.os0; - uint d1 = r0 / p.os1, r1 = r0 % p.os1; - uint d2 = r1 / p.os2, d3 = r1 % p.os2; - o[gid] = x[(p.st0 + d0 * p.sp0) * p.is0 + (p.st1 + d1 * p.sp1) * p.is1 - + (p.st2 + d2 * p.sp2) * p.is2 + (p.st3 + d3 * p.sp3) * p.is3]; -} - -// ---- concat_ax: copy one input into dst at axis offset ---- -struct PcConcat { uint n, inner, srcAxis, dstAxis, axisOff; }; -kernel void concat_ax(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcConcat& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.n) return; - uint ii = gid % p.inner; - uint a = (gid / p.inner) % p.srcAxis; - uint oi = gid / (p.inner * p.srcAxis); - o[(oi * p.dstAxis + a + p.axisOff) * p.inner + ii] = x[gid]; -} - -// ---- pad: rank-4 all-dims pad, constant or reflect ---- -struct PcPad { - uint id0, id1, id2, id3; - uint od0, od1, od2, od3; - uint pb0, pb1, pb2, pb3; - uint mode; // 0 const 1 reflect - float value; -}; -static inline int padcoord(uint oidx, uint pb, uint idim, uint mode, thread bool& miss) -{ - int s = int(oidx) - int(pb); - if (uint(s) >= idim) - { - if (mode == 0u) { miss = true; return 0; } - s = mirror1(s, int(idim)); - } - miss = false; return s; -} -kernel void pad(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcPad& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - uint n = p.od0 * p.od1 * p.od2 * p.od3; - if (gid >= n) return; - uint s2 = p.od3, s1 = p.od2 * s2, s0 = p.od1 * s1; - uint d0 = gid / s0, r0 = gid % s0; - uint d1 = r0 / s1, r1 = r0 % s1; - uint d2 = r1 / s2, d3 = r1 % s2; - bool m0, m1, m2, m3; - int c0 = padcoord(d0, p.pb0, p.id0, p.mode, m0); - int c1 = padcoord(d1, p.pb1, p.id1, p.mode, m1); - int c2 = padcoord(d2, p.pb2, p.id2, p.mode, m2); - int c3 = padcoord(d3, p.pb3, p.id3, p.mode, m3); - if (m0 || m1 || m2 || m3) { o[gid] = half(p.value); return; } - uint si2 = p.id3, si1 = p.id2 * si2, si0 = p.id1 * si1; - o[gid] = x[uint(c0) * si0 + uint(c1) * si1 + uint(c2) * si2 + uint(c3)]; -} - -// ---- shift: NHWC channel-shift (fused shift-MLP) ---- -struct PcShift { - uint c, h, w, axis; - uint ge0, ge1, ge2, ge3, ge4; - int of0, of1, of2, of3, of4; -}; -static inline int shiftoff(uint ch, constant PcShift& p) -{ - return ch < p.ge0 ? p.of0 : ch < p.ge1 ? p.of1 : ch < p.ge2 ? p.of2 : ch < p.ge3 ? p.of3 : ch < p.ge4 ? p.of4 : 0; -} -kernel void shift(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcShift& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.c * p.h * p.w) return; - uint ch = gid % p.c, px = gid / p.c; - uint ox = px % p.w, oy = px / p.w; - int off = shiftoff(ch, p); - int sy = int(oy), sx = int(ox); - if (p.axis == 2u) sy += off; else sx += off; - o[gid] = half((uint(sy) < p.h && uint(sx) < p.w) ? float(x[(uint(sy) * p.w + uint(sx)) * p.c + ch]) : 0.0f); -} -kernel void shift4(device const half4* x [[buffer(0)]], - device half4* o [[buffer(1)]], - constant PcShift& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - uint c4 = p.c >> 2; - if (gid >= c4 * p.h * p.w) return; - uint q = gid % c4, px = gid / c4, ch = q * 4u; - uint ox = px % p.w, oy = px / p.w; - int off = shiftoff(ch, p); - int sy = int(oy), sx = int(ox); - if (p.axis == 2u) sy += off; else sx += off; - o[gid] = (uint(sy) < p.h && uint(sx) < p.w) ? x[(uint(sy) * p.w + uint(sx)) * c4 + q] : half4(0.0h); -} - -// ---- bilinear / bilinear4: NHWC half-pixel bilinear + fused add ---- -struct PcBilinear { uint c, ih, iw, oh, ow, flags; float shy, shx; }; -kernel void bilinear(device const half* x [[buffer(0)]], - device const half* a [[buffer(1)]], - device half* o [[buffer(2)]], - constant PcBilinear& p [[buffer(3)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.c * p.oh * p.ow) return; - uint ch = gid % p.c, px = gid / p.c; - uint ox = px % p.ow, oy = px / p.ow; - float fy = (float(oy) + 0.5f) / p.shy - 0.5f; - float fx = (float(ox) + 0.5f) / p.shx - 0.5f; - int fy0 = int(floor(fy)), fx0 = int(floor(fx)); - float dy = fy - float(fy0), dx = fx - float(fx0); - uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1)); - uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1)); - float v00 = float(x[(y0 * p.iw + x0) * p.c + ch]); - float v01 = float(x[(y0 * p.iw + x1) * p.c + ch]); - float v10 = float(x[(y1 * p.iw + x0) * p.c + ch]); - float v11 = float(x[(y1 * p.iw + x1) * p.c + ch]); - float v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy); - if ((p.flags & 1u) != 0u) v = max(v, 0.0f); - if ((p.flags & 2u) != 0u) - { - float s = float(a[gid]); - v += (p.flags & 4u) != 0u ? max(s, 0.0f) : s; - } - o[gid] = half(v); -} -kernel void bilinear4(device const half4* x [[buffer(0)]], - device const half4* a [[buffer(1)]], - device half4* o [[buffer(2)]], - constant PcBilinear& p [[buffer(3)]], - uint gid [[thread_position_in_grid]]) -{ - uint c4 = p.c >> 2; - if (gid >= c4 * p.oh * p.ow) return; - uint ch = gid % c4, px = gid / c4; - uint ox = px % p.ow, oy = px / p.ow; - float fy = (float(oy) + 0.5f) / p.shy - 0.5f; - float fx = (float(ox) + 0.5f) / p.shx - 0.5f; - int fy0 = int(floor(fy)), fx0 = int(floor(fx)); - float dy = fy - float(fy0), dx = fx - float(fx0); - uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1)); - uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1)); - float4 v00 = float4(x[(y0 * p.iw + x0) * c4 + ch]); - float4 v01 = float4(x[(y0 * p.iw + x1) * c4 + ch]); - float4 v10 = float4(x[(y1 * p.iw + x0) * c4 + ch]); - float4 v11 = float4(x[(y1 * p.iw + x1) * c4 + ch]); - float4 v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy); - if ((p.flags & 1u) != 0u) v = max(v, float4(0.0f)); - if ((p.flags & 2u) != 0u) - { - float4 s = float4(a[gid]); - v += (p.flags & 4u) != 0u ? max(s, float4(0.0f)) : s; - } - o[gid] = half4(v); -} - -// ---- maxpool / maxpool4: NHWC max pool + optional relu ---- -struct PcPool { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl, relu; -}; -kernel void maxpool(device const half* x [[buffer(0)]], - device half* o [[buffer(1)]], - constant PcPool& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= p.c * p.oh * p.ow) return; - uint ch = gid % p.c, px = gid / p.c; - uint ox = px % p.ow, oy = px / p.ow; - float m = -3.4e38f; - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - if (uint(sy) >= p.ih) continue; - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - if (uint(sx) >= p.iw) continue; - m = max(m, float(x[(uint(sy) * p.iw + uint(sx)) * p.c + ch])); - } - } - if (m == -3.4e38f) m = 0.0f; - if (p.relu != 0u && m < 0.0f) m = 0.0f; - o[gid] = half(m); -} -kernel void maxpool4(device const half4* x [[buffer(0)]], - device half4* o [[buffer(1)]], - constant PcPool& p [[buffer(2)]], - uint gid [[thread_position_in_grid]]) -{ - uint c4 = p.c >> 2; - if (gid >= c4 * p.oh * p.ow) return; - uint ch = gid % c4, px = gid / c4; - uint ox = px % p.ow, oy = px / p.ow; - float4 m = float4(-3.4e38f); - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - if (uint(sy) >= p.ih) continue; - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - if (uint(sx) >= p.iw) continue; - m = max(m, float4(x[(uint(sy) * p.iw + uint(sx)) * c4 + ch])); - } - } - if (m.x == -3.4e38f) m = float4(0.0f); - if (p.relu != 0u) m = select(m, float4(0.0f), m < float4(0.0f)); - o[gid] = half4(m); -} - -// ---- layernorm: one workgroup (256 threads) per row ---- -struct PcLn { uint len; uint rows; float eps; }; -kernel void layernorm(device const half* x [[buffer(0)]], - device const float* w [[buffer(1)]], - device const float* b [[buffer(2)]], - device half* o [[buffer(3)]], - constant PcLn& p [[buffer(4)]], - uint wg [[threadgroup_position_in_grid]], - uint tid [[thread_position_in_threadgroup]]) -{ - // one simdgroup (32 lanes) per row, 8 rows per threadgroup: lane-strided - // accumulate + simd_sum only, no threadgroup barrier at all. - uint lane = tid & 31u; - uint row = wg * 8u + (tid >> 5); - if (row >= p.rows) return; - uint base = row * p.len; - float s = 0.0f, s2 = 0.0f; - for (uint i = lane; i < p.len; i += 32u) - { - float v = float(x[base + i]); - s += v; s2 += v * v; - } - s = simd_sum(s); s2 = simd_sum(s2); - float mean = s / float(p.len); - float inv = rsqrt(s2 / float(p.len) - mean * mean + p.eps); - for (uint i = lane; i < p.len; i += 32u) - o[base + i] = half((float(x[base + i]) - mean) * inv * w[i] + b[i]); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/shift.metal b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/shift.metal new file mode 100644 index 0000000..a6509c3 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders/shift.metal @@ -0,0 +1,38 @@ +// ---- shift: NHWC channel-shift (fused shift-MLP) ---- +struct PcShift { + uint c, h, w, axis; + uint ge0, ge1, ge2, ge3, ge4; + int of0, of1, of2, of3, of4; +}; +static inline int shiftoff(uint ch, constant PcShift& p) +{ + return ch < p.ge0 ? p.of0 : ch < p.ge1 ? p.of1 : ch < p.ge2 ? p.of2 : ch < p.ge3 ? p.of3 : ch < p.ge4 ? p.of4 : 0; +} +kernel void shift(device const half* x [[buffer(0)]], + device half* o [[buffer(1)]], + constant PcShift& p [[buffer(2)]], + uint gid [[thread_position_in_grid]]) +{ + if (gid >= p.c * p.h * p.w) return; + uint ch = gid % p.c, px = gid / p.c; + uint ox = px % p.w, oy = px / p.w; + int off = shiftoff(ch, p); + int sy = int(oy), sx = int(ox); + if (p.axis == 2u) sy += off; else sx += off; + o[gid] = half((uint(sy) < p.h && uint(sx) < p.w) ? float(x[(uint(sy) * p.w + uint(sx)) * p.c + ch]) : 0.0f); +} +kernel void shift4(device const half4* x [[buffer(0)]], + device half4* o [[buffer(1)]], + constant PcShift& p [[buffer(2)]], + uint gid [[thread_position_in_grid]]) +{ + uint c4 = p.c >> 2; + if (gid >= c4 * p.h * p.w) return; + uint q = gid % c4, px = gid / c4, ch = q * 4u; + uint ox = px % p.w, oy = px / p.w; + int off = shiftoff(ch, p); + int sy = int(oy), sx = int(ox); + if (p.axis == 2u) sy += off; else sx += off; + o[gid] = (uint(sy) < p.h && uint(sx) < p.w) ? x[(uint(sy) * p.w + uint(sx)) * c4 + q] : half4(0.0h); +} + diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.comp deleted file mode 100644 index 28ef98b..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.comp +++ /dev/null @@ -1,44 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// NHWC bilinear resize, half-pixel centers: src_coord = (dst + 0.5) / scale - 0.5. -// flags: bit0 relu on the resized value, bit1 add same-shape A, bit2 relu on A. -// out = [relu](resize(x)) + [relu](a) -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) readonly buffer A { ET a[]; }; // addend (dummy if none) -layout(set = 0, binding = 2) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint c, ih, iw, oh, ow, flags; - float shy, shx; // scales -} p; - -void main() -{ - uint id = gl_GlobalInvocationID.x; - if (id >= p.c * p.oh * p.ow) return; - uint ch = id % p.c, px = id / p.c; - uint ox = px % p.ow, oy = px / p.ow; - - float fy = (float(oy) + 0.5) / p.shy - 0.5; - float fx = (float(ox) + 0.5) / p.shx - 0.5; - int fy0 = int(floor(fy)), fx0 = int(floor(fx)); - float dy = fy - float(fy0), dx = fx - float(fx0); - uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1)); - uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1)); - - float v00 = float(x[(y0 * p.iw + x0) * p.c + ch]); - float v01 = float(x[(y0 * p.iw + x1) * p.c + ch]); - float v10 = float(x[(y1 * p.iw + x0) * p.c + ch]); - float v11 = float(x[(y1 * p.iw + x1) * p.c + ch]); - float v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy); - if ((p.flags & 1u) != 0u) v = max(v, 0.0); - if ((p.flags & 2u) != 0u) - { - float s = float(a[id]); - v += (p.flags & 4u) != 0u ? max(s, 0.0) : s; - } - o[id] = ET(v); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear.spv deleted file mode 100644 index a252fefc93bb79092aa4ac12bce2b8aec61ed257..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3600 zcmZveTZq+F6vqE^aVDLkY|M0Ur(YS5&>qA0=}U4FlF{$FgQ4Ik@U-&%XEwfEWQ%ye~* z^;TUSRbMr~>g%XlYoO|^qPXHGvkbFkgu;q@D* z4fKJ9=xPHBB$_KGQH_fWT7Fk`HntPoNF}*ztZn|&+^c5IB7nvQNvl9z_l2EE>_K)CjLAuWm@{xwQo(e<@J8wfXn>(SYy^0 zqKfab35KcUn^XTknAgtjX4ZB;^X;kc`*AjBUW9D|`#6LC5O6ueVr8L3cxUSNUj;PDo4|2i>#o6C*Sy5n!hO5(H^9re{Gxi-OX=>LcJDBJ2CHw~XR+R) zn&;K`&8ivSUGuW<0kk>5zRveKNMG{~0$1XP;F;qvJaZg@>)YoCtUcA7L*MtSmO97a z-kay~EvJreRo^mc8C z8gAWAeB;^)vV4aNfN$0KHB|DR`hmJRzLO=ux8?guj^7vcr>wx;s zdf)Z%)j<8JTDt+RZmsmY3GQs^cQahA>^FsPzqKI!Zh@<(->q=>s;A#=aCK`n_ynDE z9pD;Uf9ITMn)&^0xg8iw-W~AdStogS!i{aJ&wUr%z3F*3+`8(?Sr2z#a_)g!Q$6qK zKDcMx0P>FRhpTU^XM6yz_9XY|%o~B4dm5Zb?Lpw&mjdVV&6{V99_&NFob=rUx9>Da z-_3CK^nC=bmcEa|)ylqG@YC0`+Sk1F-3m8ndp-MOaOW#)Y{O3t<7dwPIGQ=Bu^n!& zvc@y`sbT!gId-6#lketpaPP$PS=T#y9;iEW@?NNQzh}vN5w7lcaDtt?u`hvLz`n-w zUSEbA%icY3dpJ*eyaL|?${w%ctCu}q!%q+6>G3+;Sn9nA_s+_*?ZsCw&-NC+dfE4F zeD$*LJNW5qJbmAV8_U_=gWID#+xz(HWseW=)yp0q;-`o4^!NyF%y&M3-4Fbn$ANe2 z+5KK6{xLjp=W~A<|D=dp%Y47P`Cfbq9|Q-0GxT6T1M2(gck)H8`M2y1v{CGrK)nI{ zUH=;E@^{_XS3u3*ae0IPEIiLQzR3$AfLrgaAWEDGu)oXKzjZHS5MF1;4XU_`xX2F z>?zMZ{0`4O*eCbkeskU9y~Nk%UV7Q+e(Sl{-27cR4)@;7asMp7^|aYPkC-{$wfWZ0 cUHlFA|C`*!N> 2; - if (id >= c4 * p.oh * p.ow) return; - uint ch = id % c4, px = id / c4; - uint ox = px % p.ow, oy = px / p.ow; - - float fy = (float(oy) + 0.5) / p.shy - 0.5; - float fx = (float(ox) + 0.5) / p.shx - 0.5; - int fy0 = int(floor(fy)), fx0 = int(floor(fx)); - float dy = fy - float(fy0), dx = fx - float(fx0); - uint y0 = uint(clamp(fy0, 0, int(p.ih) - 1)), x0 = uint(clamp(fx0, 0, int(p.iw) - 1)); - uint y1 = uint(clamp(fy0 + 1, 0, int(p.ih) - 1)), x1 = uint(clamp(fx0 + 1, 0, int(p.iw) - 1)); - - vec4 v00 = vec4(x[(y0 * p.iw + x0) * c4 + ch]); - vec4 v01 = vec4(x[(y0 * p.iw + x1) * c4 + ch]); - vec4 v10 = vec4(x[(y1 * p.iw + x0) * c4 + ch]); - vec4 v11 = vec4(x[(y1 * p.iw + x1) * c4 + ch]); - vec4 v = mix(mix(v00, v01, dx), mix(v10, v11, dx), dy); - if ((p.flags & 1u) != 0u) v = max(v, vec4(0.0)); - if ((p.flags & 2u) != 0u) - { - vec4 s = vec4(a[id]); - v += (p.flags & 4u) != 0u ? max(s, vec4(0.0)) : s; - } - o[id] = ET4(v); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear4.spv deleted file mode 100644 index 274db151cfc0e15283ccf304a61d810e16f08cb6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3720 zcmZvdTa2Ak7>55jwbj`((=u%xYmZ@6)I`K7aq2it6N8G7a6!Z*RkTVgae*jOBj|;Q zD0d<>5kzn<~CuD0PTRT0~7qN=O}AZ9LE3|0^pKEBUhC@BI$%x7MtGc30={?5eAy z>aF^!{*J1(=2xASu(z7k+Q%oZofuuWesc88vradmzv^n`nKM964|_MZ`j%B|4D@32 z;N|8^kZ7(p4r^RHw&izKhqHHL%jqO`4Y#-dIBVtXnLKOc<`kTh$u$bj8o7A|=ge}0 z1vjhajx4y@HP@cCyLtlOJbR6E4^OhUXYX!d+$#0PTwN<*^ZVG1*<&GH ze3wmZF`ayK^8aV%wR5|fwXJ8qn)<#UXLIHS>`hD^XV71Wxtw7Ud(M&g(QuwW{$j9a zjPJeqeh0}Ksd?ua!&nz3?-Cs6SLaf8b<9irO0aJ^{z|ZOdbd;T=BjfX>b`Ng#bDpC zzI~r!_fF+JzrJr+&iLjUTN|D2V81ne=i7m)tFPwQm}@E1S+^6DD=oFV;Jqtn^xMdc zd%*h6@FTnP$T_3_bj_vDUi_TZcb`7KYkm71WVer;ee`|Ta_Q6QL9nHq%=`9T%Xvoa zY)o+E{aE6kPJofs=QJQc?qPrHXuHQm*?*_|9_a3lZbk~99O5OGN(e@KQJ(EReECx2 zeSG;+;{*I?7>~wQu(1{OZ{s%bu zwV%MJRvmkppYQ2Y@F2Dwb9Ud;XPEr9`h9&~b352yg&Sf20+VlG{vLeA?(+A**q4}` z|Nq5}`X0UkJI@l#*>+<7RwUPbU!B_^(Hwt2tg}yM+6{KjU07!N7Q7qFOy7fD&SdO6 zOwO6a`38OhPh)-q16clsKZA`$^B1t1d$DN#3YL%N?_igj#(u;8z|<7y9`=EA59;I| ztT)#h?Z{}E^^SYaKdL`E6Bm&8gZep?c+<3g6VGJu-UT=+M}PRYT{TdzOJStBz*8v9pMog6>4t-nwR; zj&5W&yc}gJix{hn!0J~{Y}y^wTy#6Kj7nnHU`rk4%$2id;>?ljDmW*T>nS*MM3mF>^Z_YJdJM6-r2;MRr2+@?Bg7<801658XiQ;|$vK5tlP8KxZHE9}nlA zWA}sIV{FgO``bs4n z+TQuH&J1ifvxJ@bExhw`?om09ojb3+Tz>PvVLXq6SUu*m{t=^lvGuEmiRy)$iF70S zFQ$@bI17AeOTSUDT-om$Z2K)o((hWZeEO{bn=7AwE5Y*C>fy^f=k-V%(uFwB zI_CH9)(wch#N7x^oOKd+6IkEf^}cTgo131ufUPT^nA^bSC1w@an)10vw}ai|Y9#k) z4Osqxdd54za*uGH&U_~#XHE}0lDi9W?lTeR@*Nmw4e$P1#F+G554NvwA${)w%ct+X zV7c@i1Iv|tH(;l)d$q4|>3bj8n1}1x?*}_yS>r+M)X+b9_J`n%NsWzQdzCddVW)=v z$#XmoXH4GavtZA}{aM#DdJd6y=EOZ;^S-mhy#SW?4GxpJ1^ptj8L_W^&vXL)5~6Pm zdmR08&85~W*!Fb3^xO)TFMGaj(4#$hyKj*9#~)Icpog6y?+3+jn*x(%P_XpH&UzdHHz>KM=7cx$`o33L~>{+!ixu>XtYtS;f( R%NbYx&*1+KN7&H?$bX1(K)C<_ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/bilinear_h.spv deleted file mode 100644 index c7258e945ce539aac320d2247d7e51a1e2bf0d3f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3716 zcmZveTZoob6o$V!IOF6v2OGyuPXEj#1lG_+Mii-Ihq0WBFsO^VFtV~}v_|O0z)mt_ zUPKUC-38GJiaMP%6l13QpO1s0YTR?qX#_r^y0;AOq*U2Ctk_Wt)b-?X)k zc2;dI)q-kawWy_PKK)f|CDq>0UlHqQ`V*7uC @-adBWiVIBasoI+P=Jt{^58Dpz z-LPrOzyi>Ru11;4BIb$-RO8~Trr%aA!nT67RFeCQ&YAx-*Q!}Fd9G2LUuaII)?H|> zQR^!-XI2|1w2oRktcs+AKyMc=I_E9_Z;NIz$gC5 zoZIw&fS0`v;m@5@dnNZtulQxJ!*$%+a(m@B&@X%ajbHZq2S2^!KBoiEK7QG&g~G<| zCAXLBs+rTkKLtyfrhaMdTT^X$z1}4G`#e3NRLsat4ssA6$o6Buy*13M>+f(2B<802n7~25$aR&WC;NuKSu-QlA zXP~*K_(O2->i{R=nW@`<70@Ja9LIUBy9R4r^AcYN_wL5u2ruXIMfI#biy5r%#!~n! zR(}Y-6RU5J$FZKLntRvx{;C<@U31r_=RUaanZC2`2bsZH4*;LU55k?vIlTLs;}Bfm zK0jjZspcH|-etAaIRf_#t>e8;9q+BabsWSsYSz*B-j;Pb@YT;_`Yx#VR?U5gi`bp> z+RMk^@feDG{sV8m>v{fyTel0}xOSW@?_>|~-WtD*N}f|MP&dar<2&v>^DZUFcSk)r zXTr_#J|)M$S?Z5c$GS^_>(sOEY`E*ZcUjkft7qLX+_eqx8ns5Sqrf?hxz}Y_pFi1y zv2%c$`;{NZdLH`fv-M2N;ns1k9ku31CR*}Vps6=N@-8W~E%fzHtpw^1QA>X{P+JAu z^BU~sz&(!x_iWx3Kz*iOcO`r^P=BP>u7ayuEB&s9J6rl)16M2iP2$^c9Z0`x;p*vk z9o)6*>32O`-CEuJg3h@faE#4wp7TsGzkg?L1jdqg6Fhm=N#4zHW1H)J-vW1Sdfp1R zu6lAdz+IP|+u+tz&po;W?jARS+@m|;>f7oW?}Dp6%y~NVCZOh;Zg!-0H*oIrfpdB1 z&9g=a_8wqP`fi5XcM7EM7Pxx)-UnAp-}~WeW#4K1^mVWHH7|X)!p+%U&;9`1`N|sG z@KeM1$+JI*W=?8shuf>H@fdz;7(aQA9cbp{-FyP>nYcgedPYwIb!SfAQ?>4Umb|Co z>b`^HtlW)#2J8a%HSU@E_w!j`Y#QHt_*|`}*7NxGbiVX_0j^&5d=X#0?D-OYdKyp9 zm*K|J^A)&S>c0;6Ov}B$fv;ZfeGk5RImes$>g617;b#uxnd5D^vCQ!fTrGQl7jDmT z@9*KOmp$LdS1)^hfS;bm({nG}SbBa4SMxpb{67NTlQH0L;o10JCjK!zap!e?8ULh+ zTg!ak{k&J7!Uw=U;7lFZ&w%>g`ksAWYx}XUqK#m`0P5Yq@4{DDAHNI6z65H12jt!S zm*RfD2F`mfaJB=$Z$@&pN9)`^63tl#TxXrk^bOoO4}#3}EnGb_eGm6>CS%_LHD{9N zUHA!p7z~15kl*HKxUu{;zrfWp!>@2>I07=mZ*cX@@H^bc8H^nT$AL4*b1ui=IT!on zTwHIiYdoj;+MH7l3tewL*P5H})CsufXpZZ%e{I(H6EnxNH{aUsc?R2#Zail-5AOd@ YgTy?mmHhT<=9S-m{|7k1jxGlO0#sB~S diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.comp deleted file mode 100644 index 8b1f173..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.comp +++ /dev/null @@ -1,27 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Concat along an axis: one dispatch per input. Copies this input's slice into -// dst at axisOffset. dst[o, a, i] = src[o, a - axisOff, i]. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint n; // src element count - uint inner; // product of dims after axis - uint srcAxis; // src extent on axis - uint dstAxis; // dst extent on axis - uint axisOff; // dst offset of this input on axis -} p; - -void main() -{ - uint i = gl_GlobalInvocationID.x; - if (i >= p.n) return; - uint ii = i % p.inner; - uint a = (i / p.inner) % p.srcAxis; - uint oi = i / (p.inner * p.srcAxis); - o[(oi * p.dstAxis + a + p.axisOff) * p.inner + ii] = x[i]; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax.spv deleted file mode 100644 index efde93bf177938478a41d91b63f2097fbd8357b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1612 zcmZvbOHUL*6oqT15zvu013pO)vjHK5L_i2a7&RFmLr`A2C2`|IHuwWv7!&@Cf6rG; z{Jx&Pp<}G%q|QBc>)fio)mgngl(JfCq>=P4^~Xf2ry8`828+4dJL;`me0smKw!Uh{ zXv&I2bH>QI#ykK%o_#oXp#jFx)g}~3G*@&{T^IL?I4g4M@aH%c`S3Pygyvo~JwtPk z+HghlWNJ4nntRm7f%P|-2Z4H*yF6m<*B>Z2VY&^h zPELG-*}JH3m$9B|OXYsQt;75gX4mwXOG8Efy}#!cXG z{|s}ScbXe{PtWwfeT%Gl=NWFc&YXk$aC>h&2e-GzdT%m&FE!6*Y%kTK#x~sV6mbXc z8`9T4hp&A!*0;y3hnl{|_EF9CgOa;9zV8t{)vn^>ewdc|s^0$un4>N>cst(Jvp78C z4;25)??k7HjNfqUz6Gv3cgV7bqrg79zJrtf8w2X*`0Zzb{c3{9@$IUABgcN+0q#?` zw*R=}7oM)&1!~rp|77-j#_Cb41-DiXqSicI{gPUquMMoF{Gcq`I>VWWi z3RjPP&)|DNJ^Z_Hb{n;jwOlzlyg?i2wiq diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/concat_ax_h.spv deleted file mode 100644 index 3fdf16a841deade2223bb1254fa54325c154d1e6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1620 zcmZvb%T8255Qcl^AgIVq81PDFm<`mgR&eTsT}C>6CdoJP|7)L#>+o@zk--b5ldSjOGnesAgXm(!)? zl}A>NrlQyh_tkt@(=CyleT$D9+@HR?iJ-Cgzf^)B& zn!&k8?rMeeWO6quoO|TPG5fFL2Ql||xyzG`x5+uKPR^?(wwEj7YOh0hzpH1Pq>Fu8oVUP>8)$fzCgz$U zzJ`~xrb&Dq?_K2AbF8M^Lf-GUby+`xH>bvI8Y<@}{@1L%39qg*R@-o#IB`(=+{V-ve^qd77K8;GO+_u(LOw1v^_~wb$_8OU|irKfYvi?6-i~+m zEH2OZ8^-_g7txU<>jG@wW6Zq1LzOce#hkPG4V0YU7$$Fx-+l&jUJWd2e7o{Lsc{~6 zF!#yZ+kf13j;6W0n4JB^XL!$NEFZmEV0$&O=rsqH|3NR$*T(E6|2=c_6)vp57h z4i;K3!1A&0C3p*y5B)A!-t$J?E3mxZslylB#lOXTf6mZ**u%HChnb7IcVOSYb?R&~ QOF8SRGjn#AJk~Yr4;%MN>Hq)$ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm.comp deleted file mode 100644 index c843790..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm.comp +++ /dev/null @@ -1,358 +0,0 @@ -#version 450 - -#extension GL_KHR_cooperative_matrix : require -#extension GL_KHR_memory_scope_semantics : require -#extension GL_KHR_shader_subgroup_basic : require -#extension GL_EXT_shader_16bit_storage : require -#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require -#extension GL_EXT_control_flow_attributes : require - -// Tensor-core implicit-GEMM dense KxK conv on NHWC fp16 activations -// (after SimdPaddleOCR's convk_cm_sg32): -// OUT[m = oy*outW + ox, n] = act(sum_k A[m, k] * W[n, k] + bias[n] + res[m, n]) -// k = (dy*kW + dx)*cin + ci (tap-major), A gathered from the input (no im2col), -// zero padding or reflect padding (flag), fp16 in memory, fp32 accumulation. -// SGX*SGY 32-lane subgroups, each owning a TSM*16 x TSN*16 strip of 16x16 -// accumulators; A/B k-slices of 32 halves are staged through double-buffered -// shared memory with coalesced 16 B loads. Requires cin % 8 == 0 (a 16 B quad -// never straddles a tap), nDim % 8 == 0 and 16 B aligned bindings. W is the -// [N][kDim] tap-major repack (kDim = kH*kW*cin rounded up to 8, zero-filled). -// SCALARK: any cin (the RGB/6-channel stems) — each k element of a quad is -// gathered individually, k >= kReal reads as zero. -// LITE (wave64-capable parts / 32 KB shared memory, after SimdPaddleOCR's sg32l): single- -// buffered staging, and the epilogue stage aliases the staging memory (one uvec4 array; the -// coopmat load/store element type may differ from the matrix type), so a 512-thread / 16- -// subgroup tile fits in 20 KB; scalar epilogue; workgroups rastered in groups of RASTER -// N-columns so the tiles in flight share A strips in L2. Same k order and epilogue math as -// the default build. -// SG16 (with LITE; Intel Xe2 XMX, after SimdPaddleOCR's conv1x1_cm): 16-lane subgroups -// (requiredSubgroupSize = 16) and the 8x16x16 shape — A and accumulator fragments are 8 -// rows, so a TSM*8 x TSN*16 strip per subgroup; staging, k order and epilogue as LITE. -// flags: bit0 bias, bit1 residual, bit2 reflect pad, bits[4:6] act (0 none, 1 relu, 2 gelu) - -#ifndef TSM -#define TSM 4u -#endif -#ifndef TSN -#define TSN 2u -#endif -#ifndef SGX -#define SGX 4u -#endif -#ifndef SGY -#define SGY 2u -#endif -#ifndef NTHR -#define NTHR 256 -#endif -#ifdef SG16 -#ifndef LITE -#error SG16 requires LITE -#endif -#define SGSZ 16u -#define AM 8 // A / accumulator fragment rows -#define AMU 8u -#else -#define SGSZ 32u -#define AM 16 -#define AMU 16u -#endif -#define RASTER 4u - -layout(local_size_x = NTHR) in; - -const uint STRM = TSM * AMU; -const uint STRN = TSN * 16u; -const uint BM = SGY * STRM; -const uint BN = SGX * STRN; -const uint BKV = 8u; // k-slice in vec4 units (32 halves) -const uint LDS = BKV + 2u; // padded smem row stride (vec4 units) -const uint BKQ = BKV / 2u; // 16 B quads per slice row -const uint RSTEP = NTHR / BKQ; -const uint LA = BM / RSTEP; -const uint LB = BN / RSTEP; - -layout(set = 0, binding = 0) readonly buffer X8 { uvec4 x8[]; }; -layout(set = 0, binding = 0) readonly buffer X1 { float16_t x1[]; }; -layout(set = 0, binding = 1) readonly buffer W8 { uvec4 w8[]; }; -layout(set = 0, binding = 2) readonly buffer Bias { float bias[]; }; -layout(set = 0, binding = 3) readonly buffer Res8 { uvec4 res8[]; }; -layout(set = 0, binding = 4) writeonly buffer O8 { uvec4 o8[]; }; -#ifdef LITE -layout(set = 0, binding = 3) readonly buffer Res1 { float16_t res1[]; }; -layout(set = 0, binding = 4) writeonly buffer O1 { float16_t o1[]; }; -#endif - -layout(push_constant) uniform PC { - uint mDim; uint nDim; uint kDim; uint flags; - uint outW; uint inW; uint inH; uint cin; uint kW; - uint sH; uint sW; uint pT; uint pL; uint kReal; -} p; - -#ifdef LITE -const uint LDS4 = LDS / 2u; // row stride in 16 B units -const uint SM_AB = (BM + BN) * LDS4; // A rows, then B rows (single buffer) -const uint SM_ST = NTHR / SGSZ * (AMU * 4u); // one AMx16 fp32 tile per subgroup (epilogue) -shared uvec4 smem[SM_AB > SM_ST ? SM_AB : SM_ST]; -#else -shared f16vec4 sA[2u * BM * LDS]; -shared f16vec4 sB[2u * BN * LDS]; -shared float stage[NTHR / SGSZ * 16 * 16]; -#endif - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -float actf(float v, uint act) -{ - if (act == 1u) return max(v, 0.0); - if (act == 2u) return 0.5 * v * (1.0 + ferf(v * 0.7071067811865475)); - return v; -} - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -int rowY[LA], rowX[LA]; -bool rowOk[LA]; - -void loadSlice(out uvec4 ra[LA], out uvec4 rb[LB], out uint va, out uint vb, - uint kt, uint n0, uint r0, uint kq, bool reflect) -{ - uint q = kt * BKQ + kq; // global 16 B quad index along k - uint kv2 = p.kDim >> 3; - bool kok = q < kv2; - uint qc = min(q, kv2 - 1u); - va = 0u; vb = 0u; - uint k8 = qc * 8u; -#ifdef SCALARK -#ifdef LITE - // the k -> (tap, ci) split does not depend on the row: once per element, not per row - uint edy[8], edx[8], eci[8]; - [[unroll]] for (uint e = 0u; e < 8u; e++) { - uint k = k8 + e; - uint tap = k / p.cin; - eci[e] = k - tap * p.cin; - edy[e] = tap / p.kW; - edx[e] = tap - edy[e] * p.kW; - } -#endif - [[unroll]] for (uint i = 0u; i < LA; i++) { - float16_t h[8]; - [[unroll]] for (uint e = 0u; e < 8u; e++) { - uint k = k8 + e; -#ifdef LITE - uint ci = eci[e], dy = edy[e], dx = edx[e]; -#else - uint tap = k / p.cin, ci = k - tap * p.cin; - uint dy = tap / p.kW, dx = tap - dy * p.kW; -#endif - int iy = rowY[i] + int(dy), ix = rowX[i] + int(dx); - bool inb = iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW); - if (reflect && rowOk[i] && !inb) { iy = mirror(iy, int(p.inH)); ix = mirror(ix, int(p.inW)); inb = true; } - bool ok = kok && rowOk[i] && inb && k < p.kReal; - h[e] = x1[ok ? (uint(iy) * p.inW + uint(ix)) * p.cin + ci : 0u]; - h[e] = ok ? h[e] : float16_t(0.0); - } - ra[i] = uvec4(packFloat2x16(f16vec2(h[0], h[1])), packFloat2x16(f16vec2(h[2], h[3])), - packFloat2x16(f16vec2(h[4], h[5])), packFloat2x16(f16vec2(h[6], h[7]))); - va |= kok && rowOk[i] ? 1u << i : 0u; - } -#else - uint tap = k8 / p.cin, ci = k8 - tap * p.cin; - uint dy = tap / p.kW, dx = tap - dy * p.kW; - uint cq = p.cin >> 3; - [[unroll]] for (uint i = 0u; i < LA; i++) { - int iy = rowY[i] + int(dy), ix = rowX[i] + int(dx); - bool inb = iy >= 0 && iy < int(p.inH) && ix >= 0 && ix < int(p.inW); - if (reflect && rowOk[i] && !inb) { iy = mirror(iy, int(p.inH)); ix = mirror(ix, int(p.inW)); inb = true; } - bool ok = kok && rowOk[i] && inb; - uint px = ok ? uint(iy) * p.inW + uint(ix) : 0u; - ra[i] = x8[px * cq + (ci >> 3)]; - va |= ok ? 1u << i : 0u; - } -#endif - [[unroll]] for (uint i = 0u; i < LB; i++) { - uint gn = n0 + r0 + i * RSTEP; - bool rok = gn < p.nDim; - gn = min(gn, p.nDim - 1u); - rb[i] = w8[gn * kv2 + qc]; - vb |= rok && kok ? 1u << i : 0u; - } -} - -void storeSlice(uvec4 ra[LA], uvec4 rb[LB], uint va, uint vb, uint buf, uint r0, uint kq) -{ -#ifdef LITE - [[unroll]] for (uint i = 0u; i < LA; i++) - smem[(r0 + i * RSTEP) * LDS4 + kq] = (va & (1u << i)) != 0u ? ra[i] : uvec4(0u); - [[unroll]] for (uint i = 0u; i < LB; i++) - smem[(BM + r0 + i * RSTEP) * LDS4 + kq] = (vb & (1u << i)) != 0u ? rb[i] : uvec4(0u); -#else - [[unroll]] for (uint i = 0u; i < LA; i++) { - uint o = buf * BM * LDS + (r0 + i * RSTEP) * LDS + kq * 2u; - bool ok = (va & (1u << i)) != 0u; - sA[o] = ok ? f16vec4(unpackFloat2x16(ra[i].x), unpackFloat2x16(ra[i].y)) : f16vec4(0.0hf); - sA[o + 1u] = ok ? f16vec4(unpackFloat2x16(ra[i].z), unpackFloat2x16(ra[i].w)) : f16vec4(0.0hf); - } - [[unroll]] for (uint i = 0u; i < LB; i++) { - uint o = buf * BN * LDS + (r0 + i * RSTEP) * LDS + kq * 2u; - bool ok = (vb & (1u << i)) != 0u; - sB[o] = ok ? f16vec4(unpackFloat2x16(rb[i].x), unpackFloat2x16(rb[i].y)) : f16vec4(0.0hf); - sB[o + 1u] = ok ? f16vec4(unpackFloat2x16(rb[i].z), unpackFloat2x16(rb[i].w)) : f16vec4(0.0hf); - } -#endif -} - -void main() -{ - uint tid = gl_LocalInvocationID.x; - uint sg = gl_SubgroupID; - uint lane = gl_SubgroupInvocationID; - uint sgx = sg & (SGX - 1u), sgy = sg / SGX; -#ifdef LITE - // walk RASTER N-columns at a time, M fastest within the group - uint gmt = gl_NumWorkGroups.x; - uint lin = gl_WorkGroupID.y * gmt + gl_WorkGroupID.x; - uint grp = lin / (RASTER * gmt); - uint gw = min(gl_NumWorkGroups.y - grp * RASTER, RASTER); - uint gr = lin - grp * RASTER * gmt; - uint m0 = gr / gw * BM; - uint n0 = (grp * RASTER + gr % gw) * BN; -#else - uint m0 = gl_WorkGroupID.x * BM; - uint n0 = gl_WorkGroupID.y * BN; -#endif - - uint act = (p.flags >> 4) & 7u; - bool hasBias = (p.flags & 1u) != 0u; - bool hasRes = (p.flags & 2u) != 0u; - bool reflect = (p.flags & 4u) != 0u; - - coopmat acc[TSM][TSN]; - [[unroll]] for (uint i = 0u; i < TSM; i++) - [[unroll]] for (uint j = 0u; j < TSN; j++) - acc[i][j] = coopmat(0.0f); - - uint kq = tid % BKQ; - uint r0 = tid / BKQ; - [[unroll]] for (uint i = 0u; i < LA; i++) { - uint gm = m0 + r0 + i * RSTEP; - uint oy = gm / p.outW, ox = gm - oy * p.outW; - rowOk[i] = gm < p.mDim; - rowY[i] = int(oy * p.sH) - int(p.pT); - rowX[i] = int(ox * p.sW) - int(p.pL); - } - uint nk = (p.kDim + 31u) / 32u; - - uvec4 ra[LA], rb[LB]; - uint va, vb; - loadSlice(ra, rb, va, vb, 0u, n0, r0, kq, reflect); - storeSlice(ra, rb, va, vb, 0u, r0, kq); - barrier(); - - coopmat ma[TSM]; - coopmat mb[TSN]; -#ifdef LITE - uint ab = sgy * STRM * LDS4, bb = (BM + sgx * STRN) * LDS4; - for (uint kt = 0u; kt < nk; kt++) { - bool more = kt + 1u < nk; - if (more) loadSlice(ra, rb, va, vb, kt + 1u, n0, r0, kq, reflect); - [[unroll]] for (uint kk = 0u; kk < BKV / 2u; kk += 2u) { - [[unroll]] for (uint i = 0u; i < TSM; i++) - coopMatLoad(ma[i], smem, ab + i * AMU * LDS4 + kk, LDS4, gl_CooperativeMatrixLayoutRowMajor); - [[unroll]] for (uint j = 0u; j < TSN; j++) - coopMatLoad(mb[j], smem, bb + j * 16u * LDS4 + kk, LDS4, gl_CooperativeMatrixLayoutColumnMajor); - [[unroll]] for (uint i = 0u; i < TSM; i++) - [[unroll]] for (uint j = 0u; j < TSN; j++) - acc[i][j] = coopMatMulAdd(ma[i], mb[j], acc[i][j]); - } - barrier(); // one buffer: every read done before the refill - if (more) storeSlice(ra, rb, va, vb, 0u, r0, kq); - barrier(); - } -#else - uint aw = sgy * STRM * LDS, bw = sgx * STRN * LDS; - for (uint kt = 0u; kt < nk; kt++) { - uint cur = kt & 1u; - bool more = kt + 1u < nk; - if (more) loadSlice(ra, rb, va, vb, kt + 1u, n0, r0, kq, reflect); - uint ab = cur * BM * LDS + aw, bb = cur * BN * LDS + bw; - [[unroll]] for (uint kk = 0u; kk < BKV; kk += 4u) { - [[unroll]] for (uint i = 0u; i < TSM; i++) - coopMatLoad(ma[i], sA, ab + i * 16u * LDS + kk, LDS, gl_CooperativeMatrixLayoutRowMajor); - [[unroll]] for (uint j = 0u; j < TSN; j++) - coopMatLoad(mb[j], sB, bb + j * 16u * LDS + kk, LDS, gl_CooperativeMatrixLayoutColumnMajor); - [[unroll]] for (uint i = 0u; i < TSM; i++) - [[unroll]] for (uint j = 0u; j < TSN; j++) - acc[i][j] = coopMatMulAdd(ma[i], mb[j], acc[i][j]); - } - if (more) storeSlice(ra, rb, va, vb, cur ^ 1u, r0, kq); - barrier(); - } -#endif - -#ifdef LITE - // lane owns elements lane + SGSZ*l of each AMx16 tile: scalar fp16 stores, coalesced rows - [[unroll]] for (uint i = 0u; i < TSM; i++) { - [[unroll]] for (uint j = 0u; j < TSN; j++) { - coopMatStore(acc[i][j], smem, sg * (AMU * 4u), 4u, gl_CooperativeMatrixLayoutRowMajor); - subgroupMemoryBarrierShared(); - subgroupBarrier(); - uint fr = m0 + sgy * STRM + i * AMU, cn = n0 + sgx * STRN + j * 16u; - for (uint l = 0u; l < AMU * 16u / SGSZ; l++) { - uint e = l * SGSZ + lane; - uint gmr = fr + (e >> 4), gn = cn + (e & 15u); - if (gmr < p.mDim && gn < p.nDim) { - float v = uintBitsToFloat(smem[sg * (AMU * 4u) + (e >> 2)][e & 3u]); - if (hasBias) v += bias[gn]; - uint oe = gmr * p.nDim + gn; - if (hasRes) v += float(res1[oe]); - o1[oe] = float16_t(actf(v, act)); - } - } - subgroupMemoryBarrierShared(); - subgroupBarrier(); - } - } -#else - // epilogue through a per-subgroup row-major stage: each lane owns 8 - // consecutive columns of one row, so bias / residual / outputs move as 16 B - uint mm = lane >> 1, nb = (lane & 1u) * 8u; - [[unroll]] for (uint i = 0u; i < TSM; i++) { - [[unroll]] for (uint j = 0u; j < TSN; j++) { - coopMatStore(acc[i][j], stage, sg * 256u, 16u, gl_CooperativeMatrixLayoutRowMajor); - subgroupMemoryBarrierShared(); - subgroupBarrier(); - uint gmr = m0 + sgy * STRM + i * 16u + mm; - uint gnc = n0 + sgx * STRN + j * 16u + nb; - uint sb = sg * 256u + mm * 16u + nb; - if (gmr < p.mDim && gnc < p.nDim) { - float v[8]; - [[unroll]] for (uint l = 0u; l < 8u; l++) v[l] = stage[sb + l]; - if (hasBias) - [[unroll]] for (uint l = 0u; l < 8u; l++) v[l] += bias[gnc + l]; - uint oi = (gmr * p.nDim + gnc) >> 3; - if (hasRes) { - uvec4 rq = res8[oi]; - [[unroll]] for (uint l = 0u; l < 8u; l += 2u) { - vec2 r2 = vec2(unpackFloat2x16(rq[l >> 1])); - v[l] += r2.x; v[l + 1u] += r2.y; - } - } - uvec4 oq; - [[unroll]] for (uint l = 0u; l < 8u; l += 2u) - oq[l >> 1] = packFloat2x16(f16vec2(actf(v[l], act), actf(v[l + 1u], act))); - o8[oi] = oq; - } - subgroupMemoryBarrierShared(); - subgroupBarrier(); - } - } -#endif -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_h.spv deleted file mode 100644 index 8b3f3b89e2efe4b031dd1f90a85ddf2f9d66eb43..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29444 zcmbWNYLFx-NkOvEDk>uQlpG`nK@bcm7nIw8B3@9;N-K@|niFP3P*B1( zp(099%!mO41`J^0DuN)0$eX{@d-kn&#^e3*_88SQ*Ic2hR;}9K{z9K+mfmFfnPrxm zS!rgKnft6ZGkdK*v-HeTLHc`5{SH%q(CUl%U#+pYKW*)qnH6VNm^t{sLyy>Zug4$p z^b<~a&e10x`P^eqKKh8`k9_Wl$G+hI)SvvkXFc=CXCHC=(Z`=~;tP*B{)A^7{j7{V zZ+5&r_d9sMhn@7?XFP1{ZMHO#RcDsb&Hj7u%w)OXWz&v7^4MqNcUXF6X4kac(+)^m zEpg~G#TfyuLEAGC&P^YanY|JPc5Sk_Tf(u$iBUT~xD^J@tZ*w293Pw%*4KP-N!1c= z-o&jlaLZ5JdIM)Xz8emly})fca4SyS{DE6(;>!CPUK|WNv~$ea%SG@$Hf<>d(f=B%*-BxPL0vsf3kg0Hr*QRy*LjJV7^xv z*wr;u9@nr%Y`O;68u*(Bd1C=`&CVAj1siLbG_lF!SZ|Cx zj`gO<9Se5dFHId)9>;o1Z0N;?jm53d69uNT>wV}si;@&nTI$FC8q^(TIVi6@I= zPLED=Ro-@1I^|21w_7alRC$-k%YFXr9V_n{dE=kGYKeZI$iJES{1W*9@=c#!B0oOz ze}3%EOZ*RwJnx0Cn3b(@L7F@?{)xf4miV6>;g)5$nU&>dVOs6Ui_%>D$)~5u6IuMT z(&Tad+#$|QkNf{=^8E4mx}S-&-6Vfv;^@ztu=t(Ng~95g^SLYuG=%`QxmS z?7oPjf5L?MlAjpae97j(?xb)#r#H^R;Ev;-Napv-^lWj)A-f0S8s{UCPZ{L1BYW*| zzAg%O&v#xv7fc?{%jY9AmWM9{v%wkr;`D6s#wLGp;>5c$SiIH|{QWf77Fn%WKbSat zo7mSx7Mo4i@ta^Z(6LrdU^d3uF4&ss`EMUN=CeF|veyC}{f-mnLr?bFFzYkPUJoks z_ZmQMjXWvxn5(BmZq14NlE`v<%<9X}yfj#SSp!~EPxAV*VEW2055_rj=6Xu7xcDt1 zvpqGy)?jO@Nw{YH_;d~O_>LG_zN{@`H$z{uAP@ivQm=Ahp^@?!`6{K&tJ z{HO$#mn>`cvaPp=F_+mKWG!pQ&icsKdQ0#q|8bB{Nw9@!)q8FK$sBo|3_`z4Msfak zO4}h#?e35<EWwfpyQ_|fjT$xeQ1_snp3cDtkk-Q!M6v!^(1 z$DBo+dtq&Uvg-fXm>(P5XmkB!L*F>K(dNd%rg3ni&7G5t^)(J|v{^d!ZO&{O2RGWR zKG~?5#=(s?>kKxHgJbiWY5(33tX}1@wfClhJ7uzaZ#cC@r>@S6{TadP!``<(J{*j1 ztTTi8;#+Iy1mkh9nBt!s%#N<{&kGiRQCj1l9}a&({M66|!Tj+2y#8Dm?A$jmWOj}F z>0oia-Zk!Lg7KZ>XM@cV|B{?z&wdvL;~VGm!Qy-_edBx~7+;@@gK^d=#utOd!HeU) z{w2ZgL!B>~UB|gB*f^J_?>LtS<2&Cgg5?n3vA+_GZ=5TG#rbOb#`#(>zCK?M#&y2m z2o?w5`F=CleYEo>vumw zCbMfkE(;cW+`r2s-U0jSEn_vKM2Nq*19b9ezhPf>OMmJWyS_{#+2E}X_%HrQBrd(yaic{iVV!N!#{ z_Qvd7SBPv4i(+^76!Tu;_|TdEYUy3(Pq$(k&ivudf8}8F_ujtqUnLyA^ItU>*ZJQ& z*!Xz!Z(M!nzk0B7t%!SF7tY3&fk3_-FRyWH;xnh#hn%bx4sXo*=sV^* z!D5;RTk$@caroXp*qoZLb%WXC&1>`YF7u*WFAZm2aP#$m;JN#~emH#dwLvgW+%wa= z?hJOuPKzw2{*%+*y*ilB!sz+DHrQn@bgxOn@q?{jTr!^gx=Gf{hx4*u-t4L~Hv0#E zD1GyILdXMm8UPYvdEbd|D&t2fOS6 z-Fay^ez19qOU9Egm}Gs|aY*LwEVFq+@K@8fMm`x#zbNChMm{rf=F%GZY%p6oHF9No zaa`s{e^DCVd|>OoB7JT6e2gWl#d*c|_dW@?db7@WF=33uahfcY(AI7Zh=8?tH*LFel_QCy(-x%kp$j&jH zx}*D8-v7C;t(*_j){X45SNAu0eQ0`@+HPuINp2mAv zu$bn~R=nO*w+yB?C-c}o*kvAcTcvH6W*)G!;F)aeU~BC8*(UPb^Yf^1_&)#quV8-o z#(7MzJbIlMciYJ9@YZ8K#x{ph##0~zYxqo_B^SO8A{nD%n zzG9lwzQLY(#js9(Y}k%6zUzKw*2Pa8>t_3vDc*s>o)OvbKPa+z`mXulVB;-}FPlDR zdiJZ$;lVzKc~e^V`p9thnqQwkjtYnGbLP{7`Qn{r`?O$^$7lOv!X<^X&-U!ubnl)O zOz-_b=kx5yc+dRZrxPNdnjni3hravtoM5`v`g4QL%UN(1&1sd?J>Hz;u;aWl?}N<8 z`^B#5^l*4<>U)#-NACRT#`%9BGW|IJ4@Sn1^Z!uf&Y!;XKO>l~^FK4#9LD*tn)&1V z*-iKVoN(4gFW!1t+wZc6>hz%OhdO>+_8#Xd_x6dirtZV9vtIk}`<8Oohx1;{xXsxO z;n>n$Iqmr$gVmw@_A{|Rg~Q9YGjdn3%Ne2jbJ{=C)GKUm_Uf;}=InC+@V(5<;qbM& zb+R%4+T0cn&&KEbe-Gy8DtG(D@hf*nIKHlO|Cl&-2j>3cd!0MOi75sjHqMiES%bBV zGTs^HyJOBLf9v!e5?klY_eOUI`yNVOcML~nKR;uSd0%SYVjDS@&-+AnHJ{5wUOmlR zU~7@jdBO6j1e&Ym!{KYQ;$-7y%Hn&F31y@+n{PNoGGcpU=qIQSZ)v>;33(cxRyX zeoSQiSntP0R`2$w_5RFo_}2RigI)HA?)bE4rCFQa+Srikx)&z|JD;xZ#dE^pYjfgc zlW8uoIVl{zIe%_2KUcZuO&q^+Cx_$fD);<}W7qw9K{zqR;KOEIgSB*jUKGsUy6lhj zc7I+RY=7j{{*c+v-Jh3)W7B$nZLq8Pd|Bj|rpce)T-cE5tVR8w5-gu%{VxoMugxnb zo3Z{+4To<|76tQjl{;-c_^){cc z50+2)l20=G=Ck+chvpod6Mno$KP()+HM?`LOOELtk+xHsoWjl5*1=wPU9Hh=!r^Q4 z=*h;KYx9_J_~vihV1BN0+f5w5a@&XF>ngXy#Ib9Q?ifx?G5D~NQ|qz@YZ+yHbG&Wt z5&W%Fj@dTHy9WFIzV~P{`{vmF_zB@$&D|c64@omFY~Jc|?_jwb>v5lO_}c70*^Kpg zKsbEY|F~d&u5t%X+?czA!tovL9zSto?hXzorWkzK$dz?jgSCt@zPZ~zx!`Y|a>us0 zJ1ki4dOs$!Z|+*3k4bH+&+(qRZDf4wbFW~R9Mf%|wq2U}>CKM~nXdWVG1zmKtF^dO zWPELQnQY{wHoJzyH@~|D^K+GZ?8J>Z-#r}P(Qc248*{#AI5EZG!)9EAwKT_j2eY>> zIkw*Bc%NW7J|?|;0GWMrtUjL<+11?bA9=qtxzZb#4VkWad0epEjrDq9WPEKNKiQ1+ zdT=;=^K?isKUcXYOx&2eL&NbM?GBr`F?UZ4C#D#D*oQ4xs6n{ zRvWmEai4H_W2o~rgI(&JZjCgY_;B~&1-TFLJ1e%0|Dtes?|B;kl*oAR@A`i5)W~Xa zvoyN#yQWz`{kOs7wFbZYAT z>8)Acn!0k@D|uLY`8+?TRd>apTW-=d2OmlddS}2s&YSddAkXw?q~VPTJ72zIJ~P<& zN&LM>^*QlJ!r|R#$!7;!k9{D2G?)#(an1?8Wlo%P!{Hm}6T$M(IOhek!S{303xdU8 zlvwKJv%xOwpt~>)XC1Kd&x`((!E?V~_*6K2$NzLN?!x%^Oztzm#=_f^#?{L^n~Q>t zD`)JD*|~l`*c#0B>?!6KBJ-g$|I5<5%%AS!G@SXvo&P1lb3eQLQaF6)|K(s@=YMIi z@$u&0xcbij@?hgy6MJKt*Vof`UcUSJN@SdQ!RF=rq$`7c*4g@yldpxt8?!$8j`@vX zG0lUmc>O-I;2lHMg>bl*wCnHSuAT@^fczrPy}-+Wyij1%|J)V{0V z6+C)63;KRnux&7(g|Xwa{p4dVblauj%muc7amje{4wI~x59ej&JWr`JHmi6ROW!@* zIhbAzwXSxXICJTD1&)1H&Y25ya<5NsaSz=xh)YvjaWHDXS!hm*pUFZaA~__?vfY(C_`c*bXM z9OGJ#arI*^t%=-R>lSFMPJ*^qqh(4XZ*%E8%K7I>C_!vzfZb%KI3)2 z@_mx~o4o!qee>!$<=Qly`NO@({ycc@J@yyj@YYlxy?MF^Ul%Nld z{5iPup!;pwf2WxT>@3Va(*|2(&(H57&pkiC4~Oq_+8=^%NW(YIpMvGF&uuqGW{0;P z^D(wLj56N(%w>N3`I(>RSGFDVmSFnv`=ndLy(w+{KIyi|Z0Y(v#ovSbeG+{?=f6Fe z&f0ySbVsn?^m^U%eG+-rCiw1Pmop`wcc%S|K5K%nNDOPZD|kVAF|3mx8@8j2@46q7 zb@3C&y4m*o1kZ$?5!vwfd?=p2xn3^VcG+(rt3JH2cI6_ah%mN z4&HOFahjKhhBF6x@z#rOv+SMyJ1BeMeb+mZ6P&BuJ0~vNJ)5h(57Bo&ytngS&wC%^ z$YXWh+j*}y`@SfBb;k4F&U-yNIUIG~+j*}?XWgUDdpqy-=*)T4d2i>v9-X<<^-Mf8 zocCmYQ{VhQJTl(<-g4VaocFkD-FF;^1pADV{jsaBF#CTn4iCnA&GC8rmg(iaaUPv{ zv-g|*#@RNUICPD(T`>Nh;_MNfINobF&R*ffp=+GIgYowi=aA^cdEgM|&~W0=HO^td z__=X9_VW_M=MD12$NxEbX&QdvJ>=r3#A83^;u+z@p=+F@gYk3YsM(js_C1MlTKZ?M zK5tfjZ)E#me7gQd9bcal6YnQEC-pfgG9P2n^*8JI`YZ~*e(*UhvVAfR9Upppecl>; zeSQ|1k8$Yu(BoV4HwCM)vr-4Gm%oNnV|1;tn}hLlYpikZ zicTE=pQUmB6;2$w#<@Efe@}5%TxD?_{|}~dRthH$UE{1AjK8NiYe#23{{KwltP@Tg zy2iO*F#ew6JTN-vV9d{E;hY0Hah#{kgYgUR;XG{>oj7BD9vMy?y2jZ$7(X{o_jvc{ z#`CmCWP4+LI(tlyug}53<9RwHG9P2n@uA1p=a}H}JRKX^-Wi9E4?VsG_fQ z7>AAzJ-$A#3m(tY>m%D+xdmuD!S1AZh8=UD~*>huit(O|D@o-O)X_r|>L;Wtja zo~gHo!~5++xpzgzkKYZyH?r651&P=1hTj(s@0r5f&JT8(8{G%f&Py{l*z-fSYUabi zUawuAef<{V%y9VHoITk@wZ!J5;qYwy7UE;U{9NVEnK*vs&JD-cRqo>x$FASAeIlHg zV(?+(S}Vv50sjr>`cHCW3i&Behu%8akm=;b-uyJ!^QWt| zer-5>ZGJx4$X#uI5f0z{{xX=KtK4-H$FJP4!tr&L`}M@J>)!k(oS0(pVKc75TDlk4 z2eY>>dttrZiyMONg}mAeGW)rE@w;$r7Gw@;`_^Dr^ZAF!zfY4ty}7U<(^-pr{wY{K z$6CKJ9KJSxo@~Zi|4TT0b8=HKKUcZGPTZK!o5S%P?QWU4F`s`6C#D#D*oRkr#$>Jb?rTte70t<3x{vb-W=?bW4hm@{WVQa;pXdx zVDGzJt-s%e!`J4IlZ`dk=1<}9&EJi|{9NV!d*b+&`*S$Hu5y2wICibko5G1H1|K$Z zYF*Y~Eu)NYj&II-_*}Bn z+>Q14&v5wK+&$Th^*9qBeAmBJWPYx4OHbUGyJf=hb=7X!i5qjbTsSes;KN3)tjij# zWt8#F-QR-wTc_NyZSL+BEO)*4lG)GCeB`e6xkB!_>T|rO-YXow^|^MiOOEMQN?S3_ z{PgC>hD_Idt`h9C09R{q)o}RQtTx%mNo`gShi~5Q6U@(5?!FT@=6sECd`G)ACvMF7 zTH(YLgAbc=4c5{euM^DPy5!h;o8$Wh%W?0iWcJOm`kWux)!eNYdEGR*(i@iznXY+R zKUnU@dfgx#zBU_8He-c*^)`184wk##KgsNyJ2CsbtiLX`r$#lugP@-mUf%S5Ztxob-`g)_7hQkb z;5+`yrhCB`g7-)aHa!QIOx%KSeGdOpIK1!D#kw@uczSDeU%4=2IVa-3d)a(8nZDm! zUl~mA{i5$HE{{wvpX!1BvNXK7uyMXV$@-4-U(+~j-kEW}mA>O#6-+NT#`$Js`i?{Y zjWoP*V4sWnI|1Jd_J8ipQ;fdf`F?o1=HjQpyT#7m(Xfa9e#p<#{EZR1<5pjOR_5pZ z^GjkoKfO8qD(Are*SBx>5!U>@k?z}XgZ)2yzia83H-s}Uv1<3baBS(y{UI1{U(5YD z*#9ZW&`aXo6l{%SyuXHHOIP1pg7NmhbGbcO{L%N0$kt9*-#dfxe7iq)2b-h+$L*f5 z)UugZE;T@xZkDq*aQ&7GHkYIFo=;uyW209Wz9*R%EQUR0&)*m;z`2aUX8AN6f4FR?;TPT`Ph5re&O(QKNDSd#l?2cNd4Cf$DgkL4+zH3_1`RRuO2qYe!A4d zk?~(YGEP0f?gv|@e?*$U=i>~kJ@*4OvDKton6ppbCbBxgxAu?8n0yx3$KrU$hHE_f z#yc*UuJMi!en1+&@lFmFZ*jdWj`#djF9@bR^K2ZP01TwjaheJEVx z(Kp^1!E}xHkzjR)Z@lw@#amo&i{qUiuJPy_?}A{u#`|QjI>a~L#lhk&uD`|cz8J3Y z=o{~nV7kV;G+15Y-OE~gdiOFlb!N^58}}#n>aw~IM`u2C<~Hiih|b#SFn>Q`#^N+ho4#+b?=W(t z*FK-@92xJsUT1f=V3)J|?>W1>M#ee2aKGc+CwQ+kXIBh+;k&whgX#L*_~c-mtIym| ziHtufKK}OB;b}NN_U4F*n!d*mVxy4u|g? zz7vdV?5l!tuFm0mk?j>-?S44fjdS>MIQvc4Is7Kr9Fo!{bNFpIeCKd|Fs`w02*$ZO zhd)GCXLPl@ak3ld@Rx9Em9BI6N3c0$`ixWUO4BB&N*xy>3i0$i=NE};n=IO>V6#FICRDwbytPAK00e1bzcuJM|AQv>Mjd! zZ|LmPsJkS*{iUWWT{jk=G7SDSQ4O*&_3vpoN~`WvYGMs|+$=cLv>Q>__% zc#6X77@zx1KF%NAT4^|bu;1l6BkKgq_53vB`x%e#_-x^j7=lEj6t_v z8jc^lXgY`M2m5=a;+NYn9NFJ3y=Yp)!^7dt!S5K%Z_CK|ZG*Q}Y=9FwQJ2kf;y{l*8kjUyyPygge-?~36GL8-0x_?rzoXk&iCR_L7v*pveKRkG> z`=cYfj6wI*G#o#;`8Xn2-HTuD$Z+Jb?w=D5Ki2(!M8?mp`;)?%x4P%kxX%mTB^-VC z=;UB~MAsUB{>0IhdttD;$F&Y#6pSD1{w0y|Vswm`2II%Ne|cm)yT&^u7(dqiD%z&Ex~H%0>)|x|&hHJuay6@)&i)(2;q67& z`}E0gBx=|N=TKBKXCbK&*^YK0AYa^eMhWGs8 zvz*rj*|<++HL0gxYtr{@tPvT<2JYEdGg#i{r+F>w*$|&CpPr4igU7S6 zRb-bj=+;TY@q?SQ`vp52;+MOBIP!Ql)(wXr&&GO@@pI3{1Hzei(lP7Pxa)`CB^-VC zaf4v{P1kd<;l$CE+bG!Cz_nI34#tmXW0S~uF*?Sk!T9lPJTNkzUE^&Qj33X&=8^GZ zy!nyw_Q*L2Q8Ye-FuGQMYH>tHpgCi$xyb-|}+W1C=hv-kY;uGalyBC9t&{SK49 zb-!I?92>ZGzkP7)-t%4S{>hUqpVs}3!DHP&DYDBLbUUTti~%MXqdkJ{5nXG1&xxZew^y*b$F&aj4#tmlzfWYm z7#(9@Hfdws?-v=*uJQH{#*cM>KxF(F?{SgwW8EJZ8PBeDe^9WscKweJF4ww0I9%(V zzP5)1i%H-4Jt0`G=xTRpFy3Bty@v&}qpRH$gY7k)I&TfBiBZP4?w=B@2Gt~gb)zo$ zwC)cNX4lt*rv^I<_`W7QEf}x%Uzgt1vvFKxHL0h6_N4FGcvfT_8@OlV_~4!m?~8gi z#AnN=XXAw6@oc;{vdb8B&q>4agPXJe2zEBaFLz=%@_05*VwX0ajps(j&pjK@3)i#3 zr*TgXb~fm{kIxUbk90i;FPJ#GaxV;aHgK(#7X{y^*?4JWJiEqw zSulP)8!wNHALE@889$zlg^}^>dNy7WY^`1YD}&4RY@8adXM?`Bi-N_Z@BB^+mMgm2 zy($=QFS_1W2eYHA-D`sFHJ!828d4LZjPKcaeXtr-ll;|iWOZyIS{e zj;!AF^lzW^t^2n|#<780_iqbs-TSPdbuT_!KCSzA1dnxpVPuyv=-!!z;|Dh%?+RA; z;+K1OIPzHcr-#Fjb^o5o__=le-f*pZK8^dnV0BO5J$iqzJ)&!ke_-P1%6%|c-Q!vZ z9}32gb$>==yciwh!@>Bm?$3;jXV-Wi3C53ge^zAt81L-J__6Lk8X3>7b^ozoYwh~a z2`<;VKQ~x9&d~ ztOnI2e|4iS__Xdn70j-$37-yj7Vv#d_)IWf?f)jdt7qe~$ZAqg|J6y~vvEaa92>Z2 z<14{E8$M&}*$|&CpPr2?gU7S+>&Px+(0wfp#}97Kz8>ssh+pm-;mG6J_+~i#cs9Nj z89(=Id^=pv2A{_LPO!5<-+jC)*gn$r9DH};=*nFk>}=p#E8h#ok7wiik?~@5j2{H! z$FuRn$ar>*cTF&UJR3iXj349uI5K`b8$XGRXV;4bn)tjFFztg)~_kW6vV*|JDZwzkT``o#8FFsp7t@}R*k9B{qX%5Dq z`%4;*AKZN06s+#WFZb7QD>{+7u2xpn`yaIJejjeBdbx~K0R-4<+*=vw1{ zpE$a5w+E|xTqiqhs6|j34X%pONwG8t<-P{8;z@ii{uQ-5nV})_pc@ z2G6c_zf?GD?fRFFT&{J$3@+o5=fT{wuAv`17Qv z#qKj{^!=Xh?fFpUEouFp&i@_2@iEW-PDB6y`t1x`zw=S=`hKVA_d$N=vM}>t<99@C f{Qif0#U$(N8*2!;^?#k{xI_-Z0x}820 diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_h.spv deleted file mode 100644 index 9cc7c44d0cf90829090549955a00eaf1585967e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15856 zcmbW;d-ShknFjDL+upWQh{7a?5JinzX<7_}wPtLQK|`8aZIUK(8YIWsd$W;pDCwY* z3MCYoWT>={j zIlIo?`=Emk-sh0L4%_$eedZpp*I|e3d&K|LAO4>G-@Vs?a}U_(fP)S>a_#{Kyb(8J#G%GYeRsX(4LnW38c|BC5lX7Exe2kf=)f%sn}$cIvXmGWfD zA5&I}FE$BtlCY=zhMsH}FBdMNCUfEmcCELlTFkM<@lm_Dnc`-TIFYeie#B{mTWQ3J z8JBb|_BU(bRv&Ro58Rp~PJi02Gvdq#Zv7Fr?7(d_;+7k@O-7tpwUykCZH0k*P3rA3 z>K&2*@fm}h?HJkk=R^m)8h2$rUF>&?4Lx7*x>K6}mR~ZG=SM&8YeD4p1-sf;<#Auf z$EJP3>&9L)eUrz1Su1k#qCs!nkmXF=){skEnC@j$@{Gx^M>^|_uKikbhbE%5up$Q={x>X<5z$8<+*>Kk4+_S%b!JRZ|Sk>ziZe#YeC$n6VuwXe$SEut5V zIX3OySnlgi{!tlyiMepSAKD;%G*VL z$z3Nd*1taTeUmE}%R7-D{rX~g*T|<{@ZrVncaJ>l$OTifF*+#ato{!P&b7Gx;SnBR zYO^U>e&(mh1AFqq6j$Z5g6og}^Hbz;t(+JarpEn$iacMGw$3+xwix7#2af(L1LnW+ zTpDa$kn!J4aaF!L*g0DJYlF>YN&i)0#=AQk_5(T_z zxEBXIN3Ef?f}N9OV|YVqapH`T?3~2W?>1m<$$LZ=FIf!i4h=UiwSMLYw;yLDS-ay> zv&HF$>`cVf&le&ajIo>>%nwfc^HQ5vy!Pbt2d?cej4T(MBtmVkiY))*!q>Nkc5Ljw zpGOvld85BCa{im*^+o>0z|r47U~TBh&hPxUSaxQU8{4B1UCt zNkYdxE0{bUx4D*+4gVKi_mMtNkA6-bHrmJKkyi|F%-;)^dv?}z@1gdNdnB3Nk5aS2 znMbmFB98vf0h=3s+#kiukNY2)-6N^B!SO?O|Er(h3|L!!-23Xsy^YMzlc}}A@k4fR z!*!gq3@rKFBrC?ccW=YVjl$3RSrFEQ^M|~_AUj{`=hQ5Ktw-`{k#~-L^`}QZXC!|l z@}nd9jL7a4wf|`3&yVCYBmXY4{dr-qvrOnO&-mdh!@V}UJwut|Q$;hGJRQavVEp1< zEioDk*|;7L9?K7pWP9NaDb>5ro;OXtZ;-c1FZTT}Q=ARf-InP?t{)iq$`40&K2-Mn z=6tvzKG;7N%*GjU!odB0;M_0T*Hht~8{@v7j*PF(Gr`UhbJ_m>6d8Z~;QK#=aoV=s zv%%VpeM>HW-q3D|$lBuTYi6*sgs;XvD;!(8`kx&v_HkUxgu~a@a>3g2w_xaV`N-_( zme0({*9yVHcxehf`}$aAuoX{ztja!xt}(72jOVLu*9gYDpM7BHf9=S%YaZ4KCmy=C zT{jr7tucExSTC~Y1{t|cR@~tDTbM&ro=98}TabU3dkn7IJLBaU?IyhKc zeCPJ?U_7pC>b=42=<5HdVEz}T)c?`p@Rtm8c1*B#ch^P5Fuw7g9xR9W_Wj{teEpme%#Y`X`uS)uzHQD7#x>rL1@nV%ydMvC z?l)dCyY};`VEvqx(tbW24&U{3b}-JfOV`Rd!FbOv-Ott{|d z*`DdSwteI*gQ;_kJ)7ohhhX~N51cP=h%D}FQyTY9;qZ;|?}N3)*VjJ;<8h5=w_tX3 z&C8pD`M+i4e~)nZ`hRmUU-24Zy&G|bc)64IZ*$+1ldinihWcq(f!Rr(5df7MF`qo}v?SuWo z;k)kk4>m^YgZ_YEHu(BEDA>9i`#CrqzJ3k~mYe!HG?)$ktMToee@`&~&XuQf{vH$T zG7h@KQ*g!scm9qDHh&gX*T<3J@a_MoVBDocUq=V)3vW*9S8Yzjb8N7F<&3>P8|(Xn zjX|tm9DL4;tPP#`PfYC+Ki&KkocQ6!e_XKmmmkG{d^mjLUl@#Q{3irY^`Bb&^{a0D zCk5-*nAq!6T&JdXiR-o5e+IAnanYTTf)f|qe0?-{`h1@m4&QuzEEvb{i)^Y~&jj0Je-6*5 z`m$l}o(Kb`TWbvrGM%EA3rfcM-LmP8Iw?PU{J6PQO zlJVpX2U*>5{99tTm)YD9yk+XHk(US4FHCG*Bd-`Zv2=~RG8|hvYhE?oI?n}@?s_r+)PAqV=? zKYRV?*Ld`+9_P}S$j!C7zHS_HXFl{<+ny`x+Wt6t^WZ$wZtUmw$o4Uvbw_tW-W{B4 z7v$O9dF30V^G#mQP3@9<&qe2?;KUF2jQxD@^fUGg;qb=PHfnL|@4R3>#m<($o~d69 zrWcbqE(>;vgYJTquce3swimo>UKnhQ-9Hyap1yxB4u|ja&sT!A!`IIx!Sd)n&+ny? z+2M^xJo*;HSjHQlHg~4IcH(nhvu&SO1k-z_x@PF(%HZ8nJYTint0J?d^Zw@h?l*&d z^IaG_dd~{JjeaYb&e->e{@cOl<(PNh^BzcMBcIo$b~T^hiF|E}F~QdjFj!@~1baqgqrK-t{?r}wO~Lw`pSEoF9&9}O)#kQf@9=L; z>0I9t&RlEP=cgZs!}mG!&R}iv3x=5P8RYS^{ZGRsg;US=?Adhg?hB^hHl^|0AB^8V zrStU5;N#=OJkfWa9tfuET7M*1T=s&!XzhGC>mDyAIn;K|)R$$v=5&|DZq7GMo^YJ6|r;1RsPq=zE5_%UkTQ3Tsq#bMn`WB$KyRVeHpJfjhD=}>+9x> zVQ%p4Bi&mfTN8BuIMSKx-6I<}-EPx#yF|7I=yo3I%+uzP%?sV(`RJ>^?#DZV>CZ{+ zet%nPdY7|_{y$Ri#sPQyKMB^>8mPa!!{PaJ&fS~Z#TVUAQ*eC2&D+_TFYSD%s{eDs z;jOv)KR+_w-tTvSFGo(gr_KtxInmYcMUm-!2k1NaC6VzvjqJW28IS9^bopRM*S1#< zc6*H4UL6jPYuoP(c63wcbZTS1E_K(z4#T|2!$!06Z;&a~%^B%79(2vY4e>!Q@8)ci zK`#gLOn+kvUZ1dY!MCWJg1xJ2?@aRyesef{pJ%@tY&_sf#zc9Y6m%D>q#zFU^6r6Fuz1RFWc>25Bo#F89|0lt?J_Fws ztS`Jdsb96cJDcta)~}qg*JoqBH`o{!#;*5{pGDS&PW%t1c8Q_?_qleYcoG~+Pr;eBNn>7QgC8{jh|mKp1k)UtL4LfS&+SD zov}GC_{h|q!*>SLFN|H+)xHBKmcEDW7mh8RJ$_Vbeq7?Ce|HLAJg_mmH+5~aKQQH= zQ`o?L4?8$m9LAsxJv-;7F&r9vR%$Uh({P7{D_`#LaQNxI_-sDpK!5sYuOIyykABtT zTpAO(xmMTL@k8#+hd$YMjr=jOPTy0{gu{1@{3#eGSD#B;SJ%jKk;S9#8d(^uP1neY zLmP8IcR~tIJ6PQOlJVq|23g&4oSfM0Wi}rSK09^S$OnVz7bdo@ky8gwEL|g~X`4c4 zjeIsWKQ8gnpPqsj4{Y3@PF)-A&q(=L3LCg<qA5C%!fW}+jB)-+s{OA9-L>|js1Kg zvVBZv-O+gtbtYbt=Qrn-=UC^Pyv~f?CHJ0FmWYfKKio5RM)34AcByc9V`>||lB^iDT7ZmpJH_O<5&H9I(CMxp4VlW9pS3!gZr*d-*>>Zg6WLiyZYL}_vIeuzUN(?%tk&pNbPDq*Nwb>iZQ|04Kb}3 z?3s)YP2KT(|JgIedj-GZuFm_9_aQoYsE%*%Ki-Gv#5~q{|M5OVCk8t6?7eu^y#IJl z^p3ah%Cjea6SZaveqsE2Z`(BW_#OFevlrR>%})L78IB*i`q?WO|6G3dk50dS>r+1m zhU15>ehv!8KbN1QqT|PJgzD#*aQx8K&#}Sy>3-T~vmqDWS+(&Suq{*Y^QYy)9B&`H zw?wyV>UU>zliwOy{@9B4zQ4L!bl*ucM`0+cf`nfwC zKXmnTPcZ(u{5%vLKYrg;KM#lFhpv7e3C2H{pC_Z^$M3@G=c#b~(ACe=!T9I$vsBt! zALIPY495>0Ki18xVEp{&SU0OhH(ocZN0w9V=(=w3ozG2z$Lr=5k>yVtx~?01+iVj& zUN_rDmMd-Ox^D1ovwQG(-MlHXyl6w$b%US&9m)OSof*H&{lPbmeInzWb?^G!Y8qN8`l`To^!d-#pw!#uUWAH;|Ab==<%!{NvM z{Y%>7$EE#!Jvw^lZTmZR=466zf6n%fS3NyP^u8&{hq`;lwfRH!h`rd^%*=i1=;-=gP=SM&F&J>wGX_}fF`Xf`0N)ZdJpLv6p2^XF5`hq`&*Ds9ZC94(PNpO?CM{!%c#Z+G&1Ze)6w zy+;3q6g;Dq^jn7a9Dd87>ob#gf*d82Tf@_d z@AlL#bLKagw?)Q@57vj@e*Yxpjuh)zF6<4y?izISbB>bliOfG-CY6EiGkzcc@Bir=1#nGZ3~O#NsI-J!!6 z9virh;qk~~#2Z6>t2>5mqGw}V?DZ?QXACe`zgs*QSzPKbCr8%G!@(OSZfBG>zZu#X z58WdvIPGBH(o;=-o&RN;r_K`n`@075>65KC?c>p4eYof7;}4Nt`k;F(1*aXnaG1Zx zgZ&*7|K*+tNA~wjR}5oVDl%RS?ni->nc?u82XB$G{J<|APTZN&sW$byO!#fW(Kq+Y z25U=q(h$>f14mbGh2ZUChdXYtdtNa9_2JvcN|Et=w2vIm6Z}piyHz6N+11~w!Fa#z zufG>Y#*h8278&pF!)muiWIVg~59@5rU}LS#TEXR-tF^<)m9;@%+jWBZq;Gud2Fn#) z?eZ7L6TG?Tc%RDTu%oNp)8WiD-FOXI6Jy!hdLn-zlWMYU+FOIxr1sX0b)k*DWN-av z3cK#XXYGrW?m^FMIClEEKD9NdCZ9LR>h6E{9`}p+iGfY;Rp-X$|GJ;DJ#O%)u0PKV l?uYd^Z?IXAS{u(LVp6w_=Nb3kwmD+3@x1cq%-HKv{tL(IG*AEl diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32_h.spv deleted file mode 100644 index 53a037b0504b3732386bcc07d0c5da3474b4cfee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17980 zcmbW;clez}l?U)k5=f*3B25rt5~PZV3YKMQ>a+AFM8RhzAh2|iUWEuhAgqcFWyL~S zC7>d*=nA-$AczXPDt5#|FNQ892=0DAH*?~6pJo5p`!LM+e9xJ4&YYQd-uJ%um+|A) zm^c`}*kGx_GJ^?A4Thg7gK>k!g7hzI{idzoWy(VSt`!#cXRJIJEIF7o*mdW3&Dw73 zch1`9u){vE-w}Hsb>K1k%{p}NqmDT6xWCmObMzqx@BRK+hwgXiVMiQ4>(ImY-S3dZ zUSb$;tL=B)euE>A+INGEHr`MolLzBw$Z9$1T~obvn*cvRl979IFVY;qsVUQv@*HobjARx^3qLq5jBbn6b~@q@=Zoqa}EaqT(sszciegI|tcI4667 z-(N+pIl(?Pr;__Q-4UDe4Ns4~@$w?~bNXXs^;@VPKbRl6V!=KYtK>Bn(!<;yjZMXZ z#WGARav$rN$bGD5BUdcgXO%Sjs^m3>?7})Mhz&j8uviOgv{3G2O^QNI*)N|c;!ICw z!oVv9$z@vFit%D!!#?EY(@#rVY}ij2UVE-D{L0Z=D`UieO&f1tkXOt9U4v|J(;}~) z-gtc3PYvdOg=926Jv(z?M`kC-P11Zyc8(4Pv#&3CvxV~JCBGx`;@8aGtmJJY&$#o< zMfx2g-!r&ok-Q7}@oz1XXGT8z(oZZJzenUHjz4uMoAcpmYFPe{4$f!M_~RoyHDTkS ztX^Yj=E9H7ABMRgC{nQ@^igtn-bZO`3dp8W0&QW`4 zYOr&XYz{l6mnY5~$<9d}{kt1Bmi(T`@+HfG-O=Hur5DE;T5+6_WaCax&lV>R*_ntd zKVObKEyXjJi-Y+Ya*^joc0J&YCtuRI8h=@2wOBtH8hdSI_3xWpu5aV|Ioujqf49 zBF?sv`+c@uWcSDerc4}cA8e1X!E3HFb=@JDerl6<493;jvQw})`1NDYcGooLhPm$6 z^!Up4v|!gs$#Wt*FH8P-WX}X8e>!r%?iWOURny~di99LAU{C&G z^mjIFE%8#*j}JO|Gjj&H(c_k5!VRxslED z>EK?T-^up=&S|B0pPf4*-`C_V6UBA@P@1#AzS}f0)cWDZm;6X%=RwJy-<$`x#0UGQ zg4tM)GaC0y<6O5DYe6{YMjz|>$oR5(G1ytcSH=5FWc*n3t!Ox7YusYtjO%@m3x}WG z#*Gg)7GJ(520J_Ws@x}qV@p^5mkgGBpVw01@a1cnU}O0^wZ)tqnH}BISsC?OHaM;Z zYo*b%FCQy5TltiamBPt~u5w&87|&OYePuA-^>J2E8|Qi1;Mml9{9~}|OWw7XvpTNU=-_bHldjg|@L=nq*0ml-1mnxsk-^5|Yi*AY z#^Y*Foe<29uKa&6n19ct<^SYx_;0i|J0;jSym2#PJ3H7pRdXh@E5BpG{GOJ+{GJ|+ zuN=<^mLvY8oI{=)&kV+wpR}?S z-~IZ%Au_(^_NHK5zZP$fY%R(!nO*t)yI?u?b>1X0zWi<;jO%N*MPxNAzhrh*vu%Ub z?1yPpvmL_W`8Q<4zmu5#-HQTLmea&WuQ!{7A@Z5>auI9FTFyDR6_K1wHxxF_S z*Vk;%$ZA%8$?VGS-obM0Yxcg#`11Qt!MMI=`$tx@@=In{H9I_5%{)t0&5jDj_cc2< zGQO|bam|jdYIb7d`kI{-PR*R5YDQ*Pb2}xN@4jZUBjamsW5Kw-W~WD1v+_%3SAORN z%dxN7S&{MO_oKnMzGi1fR;sx7c&oMfbaEn2Am%bUv<4O7!e^1pj9|K1mr|2@Lt%l~_V`NEg~S;2T*`QIy; z9bMIGpI|wt)q*_q?HlabTK{x=4trZ3l&_24^Y-_{|De-t-sx^_x~lp9k?GyTpBYRT z+}8B!`+Q{j1Jdwf!gZD(6l@k$`bF@F`-yh5dUw%FiY~S^M4iASf zKSu`dl~#U^3TA`9Jie!A{4v4&yXQZf>+OTVKITDpY#Poy;99@qgRS4VOuE+ZgmCzZ ze_}B1iWcjnV6pJlr2OiwiF{5D7FW&Ki&?qO4mJn5&TT$F6xkR$`Oiu3BY(P6({S>K zEB~=z`FsCV`JWyRU-_RAjH~?53>F_R|MIJ^{Lcy&*PPgkDX)*E_mS6IbNzidGEQEw zykEO}z{!BQ0)$80~9KWl%sPg%9uxsou;rY~G+14)Iz?iY8TACBNYOOC{cedJD4>8%+9yu?$j$Tur3x}^g^7&w#T5XxJ zKD9^Ih%6s{?U6NujTwubF>ALm)_`uUG@Nm;y!j>L$?G&(U-P&-xx1Fx+!MT6`r0F} z38wG&$Oes*OYM=@g=0%+k8GZvA0PSAZYb@^eD>mqYd+%Y`&yb4xoWL1U-!1!Sr0MG zw$2rO**+b;b#R^;*ZX-qvg?@6zN5P|-|L-g_vabQdFA&3=bO5IIlYhCdoH>#4JUuN z&e(qq9zA2f6b^4rHAXK_@h%SLQ|@f}t26bIV0t;p>HVlXTP%fS+MtOv(jpEPwtg{|Ce2`~3fy zxcI(R{_~@wSMNUm$6_ykapg~DJDUG<;pOjrN#(yF9KO&0`N;S_{}&?5zkgTzVmN%C z|09Wu?_1^nXms@UMxXzLaPk*d{$#eJ`7aYr{@z_w{*#08eg4Zv#`pP8i7fwlUHxAk z4&UehSY&+PD*wmB)7!^={;PzOzqs-zvmMR<;Or~!_06I3KP()+&;Rhqc=_*|>(KT2 z-|;s!eaU{@l(*dIkB)tR{j!@LezDlOzm_~Mvfr)hdU+y#jq6*@_sQt!oge*tPfjfJ zm8bcV+17mjkl6OQc1t(cTZ8?k)bGFB8h3rhS51Bt&hJs%rLSx6$H8hxS9U)M#;cuk z`=`OqZGGJvPRd$2+iuGoP7MAu5!t-y4j<7S8rc~_cgTqDfXJ>- zy8TCV`$V?)>E1V@+cUC#MfcuLXKlBNY^~{z%i(Dbb$_`dnBKc4&#ymEPw(SiOaHSp zyxPE3)87UgYhRVW--pBV=e~V+dLO>%?n=Y)1=m`fpFLol^P&7-7!GeAm;bqu@$O}H z@B3=xEaC9pN4HIM<@d74^z)*xcS=`8#_!bG{d;6QuI{tnZgzAv_B+jPk8bP@;qbT` zJFnT%4WB#GoAb@-Yj5q|)qhL1p^7E5m@1A-;w}-=*pPvP*L;3l6 zFdKY*@BU>l|8wHYe)(Onk9pAjDh+2Ia6LQS5j^@X;y2;&75~m)+!dL}^!WU3uvmC& zQhxR7&gS>Q;;I>YF)P=*g3Up$^(=IEWMk;$|4@1#`P1E#hLb;B`QH~j`ka4%IDF;* zKrpWIe=t~ly!^|rzVd%KSX^^rFQ&X6PhWX?Pc%O=PF}FQX2jspVBd>sUDf2VaCk9m zjJ{$%5zMDN*z#BJo}LPpQ`PHF!R+z!%I#Ft6t9pkFNK#;qXU$ptole#v<9zD?GvhwJ5wxwh;xHs=SQ zkiOP%|6qE1sP@%Cjgw2gS2#EvTRP|RiRtY)bWiO*gfam`0ueP2s+B3G^T z&`K%`EYaSOT_u*L*Zf@}T>1&UCB$&S6BOhy=TxySeJRDm(d*p)j{P@U^ z{u60<`M~D=h4f`({5fgorm=x*k9;Or9_C;SJ-gZ?pAEK0^wz<7W?b**n~`0|boL#c zXIp3DQ+XzIUU@clzNzaH(fg>q=ag}gabm)C#!d(xJ!2OShc~AhqnD?66NCAbJ6rzh zOnq4}y`1E+Vz7@q=$1-bAx$2zYr%8jGQs9p*U#k0qu0-};qdi5wOp`q`114eV0EnL zw&f$U!<&zM#Fj%Z#41UdGqFuZ^v7{Fpb}diS($uxCUz#(O^GPhWFhKUlo6jAi5bl8tA-ve_#5-U$nz zgR3rEhpTrz^i`K_g6V2+?GWr;kG)sl75|idhBsF^*SnslQa?4^CH3$f=yk~p=TqE< zjZ4WFaj$P2)3SR*{Cq7K zKj$U?Fr)RL}hIXM_VXQVFWXO-yqxhek3&#K}0 zp({VH4935dpS7an$KN)WpS8pBLsx#*3C6#apEpLw&-Xe%8-?SCuKa8qjDIOVTSjMp z^!0gLIDY8(v2WfUjDM+pvs-lizL^-{5P^$-(`; zIVG}M8ADh5247=772NNeb0VvYF?6+W@T2d_+^-JE{C(W7ymQ$KL8NP4zj_;Z1=$)J1@p+%OSMC+~b>d_A4zI<#B>9Smt9X}& z!}sx?%~<^3v^@AYyPu1W-dSAnysyTO#=ADWc>O!)>%!stcndNX-?y6I^U={e-z%Q? z%lOfFx3_rny7~Py9KMhD=ZwYot>V299ld)-#q&NGKN{~BSrhlbsa?EZhQs&qUd&j0 z-zwf;qN8_DtavA9{qPmf{kQgyess^#duM2W>FW&gu5qjQbq%tq_p9C|l22%RQeXG& ztz$2DHWMf0Z)l>c_g=Gu>Fd4MNs;NZ?S^}u{=_uATwrldZL+@NY}4YfSv+x0PhWB7 z1kIkHn#O*>$ddESz(U z?w~0X2V~>sBoA}0^|&k7xW6^@ZrobHntu;ewLdr5-;R4{UNJu#PG0?c@$$ z2m7tEe^-8CIP>YpUKEZkojQLhy?W@Y&f8^-^;Dzrsq@_QRp+k+)6Z*lzBn?ykF$&Z z%V~Ii;hM{3!N&S6t@6G+9Nxc8xv`x&S4PIKo%xjAHzS)jU7dwj2eZ2>t?aG|Hb=V3 z`MO}^{k^KQ_J-ifm+tyBoP6OLcT+Hbe#cd-veB#6^fZhWON7I38oXKB(v6=KPTpD4;h6HfWcYW4 zqp#X86>KcsoR-tljiW1Wa`3jX!=2XbmJP=HH{lf{NAUp9N5yz~Fn*WLZiUErcI9uy zVEoL^Zso}M-rp-CJaxc}Ce6Pk_Zl^3&OZ%O+f-k32f diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32s_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_l_n32s_h.spv deleted file mode 100644 index 756df1d9af1e1105876c10f62809cb0c476ecd65..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23940 zcmbW_uX&Ud!K#I{mw_N zvhr4|&#bc2%-S>S%v`(9%1?ktE`hBK;(fkMbyEc5VzhL8;nYCutm^u2$ zXDxo(p+_zL+fzIpc&gk6*lW$r-1e@XG&Ff9A_iI&sO#i z7ao67Vy`wE?~uceK5U26&v@YuyX?A?MAn^IMK}B3^%7;(;CX3Fmz;1i{>YVQW}cb$ zoU|or$EU3yUu=@*%sFZ5&<=^052TOD%%SlF`)u`K_YlVx$4Bjq!L2cHGQ&M=;Echo zKX7u!Wx5{nx7x&QGH|O;+-3tOp0QgDob|wMJ#cGH+_nR^_QY*JaB?-)e zkW_tUCP02_HfOJh{NS9;o5i7Lnm3(~@jmR&3P;Zuyl__YU-FiNe0TI>nJ;617P+xt zpT?>@#`;BU8Vg<+d&9&gkFo3(bF%5Aw{OViOl*6|$5@!|F|+b2Gsg@%`;4w}?K$!z zW^L!q?3+!XCVPS3{Q?X>Gz{$1nktVq?H`-^hSg-eyvSQ}kPrE&$m3ca9l5b!pT?>@ z#(HLK8VeR{l{CJ|W31;#9%KDY)1{yNRRsyxP892=2FAvV= zq48%$xM$w3v$FYGmZnbj1}~iKACoNM7lfj2STW2=?f?#Jd zncXYXs|RQP$j)RO{UsANmV9}7e#vsuzctOrb@I+&T=$FGKN#6{V*ICqjd!gWFV1(u zZIND{&U-p}I?u_*{X9KeoIJ_S@8)^$gh#*YCW$Sw(B@~oV7gf@^7@hGgy)Yuf8v_g zMv<+{_Q}}TZ6ez%W0Tvq)3`DAF5%9JtvNY7Sf2JA`y(PJ-C162vU?AXe$j-Dp(nc+ z(N}i=scgJ^&M3R*kXy?Yk^@_jvo* zJ)F$$N$J_(tR>k!8%KZOgv~!c?%(p~$Nibi?irIEjvuo7apSuecbt1I*|-->HaO$R z?zP5kpH;V4-CK7_Z!R?B4#|OLU7cs-LnqmJ)%mzTi+E1PReoOP>e!5_{QSs0_r&{~ z$p1L#Ul94oL0%mBi-W&oBOf*Bmqb2y&>t81j>xXL^MV(~Hl@rwHggBRE!=LoqGK}4 zr;BE0<_}?9=jKCQPfks0C98Yk9C?#LcK#ocqH0e*a*n)cke$oN4l?`8ljJ!5iaGM@ z2if?y53=j+V`&}lxnsQ)HOhxg^8TsPS#ngG^Tu_sPinLlM^Aj^XGM1ARQ7!9%&Cv# zf}J^|kH3wKug!~sokR9e{k%9bepzx6_k>`au^so)VB<#LCq~AfGmSeb*jRjhy)4*$ zmahvYyHkVN($)Xz!Ezt-IwLZ^zRnCbmcMf+U$2PFj_zTZVfnu@I7>3KO&UG>`grwZ zE1&u}J6Jw+&GEcoJYOBVEEw-Pyl{$tQE=@#53dU+AG(gcI2iA~s%FnXmqzwnsI$2Wqlhq>;0d@~%rzP=S~EWT@dS1=yeJ@vg{c69as zgJAxbr`7)t!{Kk3*6c^Y#^H_IGPXYscK&r<$n5I(XTkj5oxXm59*l2}zX+Bi{#7|& z+~a>4jIW9uJJd+S>x_&GP};lcY^sI*YCTL@vZH9!MJfP zz8~3I)GwJ`{r)gmj^jN4C^Ei&?+(U|bM~{y=B$3n>^f(^3pQt-*E(nSg~N|?c7J62 zIA;$`c66OHe_$Cq&e=-g%$c*yxkF~x+U5oGJ^I&u48Le}+MKFGxv#lcI$2r@2vZL#qJ$mBC zIomdzIlD1&%o&+oYuhfE?{UtykBo0^I|bv$IeUC$b5_4(cJ;eUupGxZ+ch%2e)kB* zjdQkVWOG)(WOh9RR?Gf!2KfG>XTa*=@SWE+gK_>Q>AbELjCW1nnsY15Idj9j2kqRC zJBK%gTamed-uyNQhacx>qsaJiem0)$ z=o)X6VDrPD`608b?@fdG8EfAxvf4X_%&xUB2*$Pcg~7P7_AMjxOV?bs3RZjllG(NP z?Sj?r+-QC~gu{=uKRz;kto;d-9bMz?9IW=9Lu7XKy<0FpW9_>~R(r>g*|qjPf^n^V z?_k_m`#zERrE4zx2CKb($?RJD!NF>G{Wrg-gu{=u9~K!u*8cR#j;`?z4_1566EeH{ zUKGsFSo=|t)!s2=cCGyx!MN7`>|oqj`*R}mOV?bE308aklG(NPCBbSRpJiVd4nNlZ z;>h^1_Lod{bd7gHu-f^v=gI6^`$@t4jI}R~toDu}vuo|A1mjx!X~DR$_R}NtOV?b^ z2v&RjlG(NPR|Ttmqha084u>CWKQ}Udto^*nj;`^R1*@GuwUgPk_SXjUGuFO5vf4X_ z%&xUx6pU-_mjvU++AodFFI{uFELiRJOJ>*FuMAeZ?`E3cRpIbs?bk%ckF~#TvZHIf zYlGF!pW4an>ifS3^E1}|j>u~77&5!o{?1@rYkyBLZmj*ik@=--F4qUEy?)8;TKkQ` zYWKHo^ZRf({8;t#F;|Uq{B*=AK|{Guqr68DE><1)Jy5=J%2D zZ2XOSU$Ey@^CXtPW&aos-@hI259V9W^bZ8H!Pn1Uf<2#(ek>TiepZewe*LTxnGOC! z0Sk%2{4WpwefH_Y*yLlq=vGU^nMYWB-yy9L>^W|Sg z!;9H5`o?@@FrV^Z%intv$Jm9za_W3-8O$CpuN~6+$ct{PG@QKP&ezt#bJzP(;qaZW zZGv(9-kIL#Pr#6~V?Vi{6<35$q!uy4%ul#=+|6my9QWc9Qkx!}YR3 zu4DU*&4$6>N#8a6w_tjEsQc=36SqzJcTKr{KA0_?Gv>SL`SFn-{TI^k@`2Uxt@O1q z{*JV-q_Kftow0KKaxZhv)L?o<&5bb`DHlzagY3F;ib&uR1Y>&vP z`{9A$^5y;#4nNlypPdhLAfEW_#SvFM;_An_R1>*#t*@`mr@6Bpv-3RJ_FU1|cJJ6) zhXrZIjed5E>^i2i@95r}q@8Q8$up(%%I_hZZ{~HE^gi~+O&PaSWSsorp0Q5|o_ogb z91gFhj?v3gyj_F&lsjAgdZz9kOfM&S>=W!G54tC&;p74Ldf7AB_2PFNy$9_T4&P^; zy@Nfo;N@07`sTK8usHH%FNQphOz$HPHv6X?ktPq=HJQi2nFE8(b@MtX^4#nA$>H#Q z_B=S)IDGva8f-n!$$7=^QzEm&cidsY_!X(Cc85pCd*)K3yyYm@QO2v8?|l--ICZ$c zu=U=@Z&Z#7_PZ6=FFSMV{rAz~>D1}BE6)h_{?zlDXLK?fbNBr8KApQ~M}A(Kyx{tL zZm@CkG=`qtDC6ZVhy4=E7(V<~hHYch8MinbKl;wml3;l*OHOS1%zRw<*7L$(^~}~i z)qH$7d~IGd+3?dEUK|d8Waj0a$>}A*#^JlRCj{f`=cU17_4`k?ofr;Zo0BG+n}%3R z!{Hn2*^zPf49tgT$=3wWJ?G91hqs41M&Gk!S+H}?zGBOtm=~t^5tGgN zY0J~ZgjXb{>+-e13)0J54CC0a9c6s4<<+yVjpN6)%=Wg)-^IcH9%W%Gle;^!w%>U1k@nilrx%_V#@?XiuPaE^U zD8BJyYyPi`j@~{V^Is$O@)uYBWVUnj-!c2jZ*bJm{C5tAAM@WeGG6{i=Q@nLnXBS& zLHf#XjVy1u(eDxa@%m-AF#OfA^Nd^hn#g{;*6ZcsVB^NtdM}BN-uW@sdq85TSDxx6 zvu(Y1CboU9?KteUCj>t$-S{5)m67RPd;R`;S!Dc* z_-Oo#BI9wrSH5ntqjO$o8_ZlB?01sp)3XhkU435>%#ZK2JLdI~{g$$0$n1LGzA_ls zT&@bn@zGq~N}onozgGwQO=kU)*>(O_&X0}NX?>dCs^Rcs?Q2BFkF~Ek+0ixLTES}P zPwixO^}S9oKV$9dMpk>rklD5N^@DM(eS=`!So?;N`K4 znP0l*vQx0y>zB-~wLdXf?c@93y~5$g+V_o&A8X%lvZHIfCk3m$?_A03TKj>){EW39 z6j|*ZLuS|74-Uq)_NN5n#@e46nP0l*^0Z*J*Dsk}Yd<c zJK51S-t&Uh&YyKBv#akH1oJc2zBsblJBG}zwJ!WtakVF=67m1{8;-Lk?~{gFQ4q_8t=?twezQTGQ0XdE0~|L_P>j)_KqR5YwfQJ z#Snc&oX4l#;3|6~$q|NW*aQLzI%Oc~)+Ap8%=o;^eV72$1 zE16w=zaf~PvGzAcR(r>g*|qj7gK@3>s$kq$`&%RPOV?bk4pw{plG(NP>w?vO#jx)0 z4u>CWzdka4to?nH9bMzSKUnSjS$8tK*8ZVje#Y8wh^+RGA+u}kHwNQc`-g*ZW9=V_ z%r9MY`Dn1(>zB;V^V-|PdydfedF|EHvyNwrr{#YhI4rHYzaMmmPr5!=To)hot}V|& zM@)Lp6`oz_-0$`1-jeSp53AHJA;)@4i+9^S?ZCJZt`Yu#fek zyDbf89%1pl|NU&R?@szXnP=qx84ln0|0@{R_bUGuEEe9n*RS5XFN}}>2o~3xu@|$s zem+XyRr`E{E~2N>6|gwr02&+e)N~7;pGFX;i~ktG5(6QH>9zF`)=L=mw-6KB@#+j=X8SB$M@~+78(RYu$C)k+ok?W^1)`0H4X*lCxdGkxg zlixSV`quHa^i2i@92ED<@&rd zBT-;Fun%Z!n*7XUkvD)ZYcu z%Sj&h2m8o_?)PapdBDA1?hAIkjPHN{7!KcOoj(PCAZ?b<`1;W|w+Dj7kuQ5Of+c{&H#^qF~K_|~&kuzHrqrul6h4qux`O*Z`0W}9&MBQr1F|2{g{IDFT3 z+hBbCJSJGI@%`^(!{KYQ-DGpq5NrEz_{Mr%uyOdt+94QUKRX7CHNOAdDIC5wkDqMD z_rFgFhi|N%gN?&C)-J*L`q?#DtUlLTv)#hsYqR@gbNf)&6T{&fYmZ>#@Qt-+Fuw19 z_X;+z>Xiq*7~cQx9qj+r_8e$`klEXt2c-AuIk#_QoIL~c;aT!Y!E?{K{lnqyp^nk_ zEIBaPIXAnng838k(DXiHvNz4i6Tu*T&nEKbs>4n-hY)Z#^@*zHfbLxV~?tcRhITaAGiB z&-YV;y>B%yeP44~&U(CB@_ zZ_W1??EO0t^|K-zKXmnTTQGjEpN_dEK8(3Pzb`Q6vdHgD!`n||?2fr(isRp;=$J2s z!_OUKU;K0Ye@)X>6j->W0u5E*JS5#__=G+G5bwpjvK~2DI9+8n9kLr z=*D$CDzX~IqU$>1J7#h4xQ@q0HbyMEt|Pu<&IlgY@#T@tfjD$sM|{Ve8$7P#d6A6~ zhpy|0pZkv28UOjLgO4-bd&bYD;hd%LXQwm%o4IzKEzZ9Fmf&0A@Xo@%o4!2riXU6w zOXr_Z;X!Knb>%K7lEG}@0Rf5-7>_xH5`77_oj@+kFD{p zjE>$p+jxG*fS()h4_OoUjRixz`@-SJcyG>F{MZ`rEz!}tk2Kx^SwDQ^xo36%=;!V^ zdhgNgFMZDh@7KK(cMY=XZwkCWC*L`Z(f3}rH1=|5vvgj*mr8#__SC{~Kg30E|M)xo zdy(ngf9-YpyVCG-fyMdpBH7E6Z-bu|JHG+yIrh6?=Ox`s^9K&uxaFzO9^N)hPIu)R_s-96 zI;;h(`M0b(_kRvv7S3-y8Z)M|dEFGgb}NNrOIN!!g5^Hitr^bm?&!?v+UeD<@0`9Y zW2~L|I3fJHX`R#g!St>TYq!p%k2t$_^bbqJ^9whh4TFvKdzI$BQ8@fD$?4YVJbOfB zyuUAMw`pWG()AqLBADGH(`vUcSdDbe`BA~f|1dF}tB($DzI5B9;p7W<+;+kI`K_V3 zs*T=UEsT@xgZWS=d*j8}A$?=8d0ZM!47j!K9IOU^x3z{{BHKH3t#!9xcJ_xox_j^w z)9f?$^!tSO%)iH^>oc0acXAZW>=~X;e*2{NalQHXf%cA!lOHUGfA8!dIBj`s&4ufL zuP0Bs*JRDeheYNdZ?5V~Z?4#!mbGW2e)i&YZyXkEjm&lD;qb_Kapk-yy^ox)PR>V0 z#>p9$v)`LPJJ`RsCTBk6?C-|s1k;^9)o@HOt~ER_SdMr#)VIDhoE|+Jb+H#$ZVSTs z^t;owBg;#F{dDfF6a4z*?VL9zkGHcQ_0X-GhBFTKTOjkcesI#CJvYSPD_DMFvNfhL zHV78O`69+>IIM}FHz#_ton zZ`#8q{;}cYohh0fQ@`7XKPVi1=l*fQ#?mdHa@t|y=*sOB{M6Xt&YSEWAB=x`_{P{d zGM}j9v7p&IW>>pgN zb9F#CbLIX(U)uwN`J``t2L+oey4oEajJFo8_YuMD=xVoVu(hTe_mDj?%J$Z8^DjSI z-!p>k!FAJ&w{PqVV_ZwFtrumH+4UNHab*8?;qr`k4SLqau@k3fp`N^M`q_P-zSqBd z&&%R-S$weRy~@4i|8?(V>pst)zWz?kxRVEeeV%atH^wuFG5U^KnlVd6-U;C%s;lqaDV#Rz21tw@x6U! zzIWDcyX-ydLx&%J#GHAvk2>U-IkV=@K5E_}$No?KF-ISE=}$smEIU#|NmD_l;wiQrp=vw$Vc!`Cdp^ho=;nv z_IlcC@x>--_BhS|hJDA&)xo3Fc8Mq0HGQ~S#<9ooQFg}Q#&WL!G zKr#~sUMom0Q`6Rr7yBA^kylSYHEmSCAKSn8++X;$qqkPZh`(tYZ(r~|IkJ0@4Q^`W z_0t=VFZ(IM{I8Kw1L@hB13NN1Ic}TgD%m;O>z#aa$=eUh?<#qx$jjY$<@P1-8hP5o zL(BAgME-5>#%1zer{u(NrsDoTP5rLPSmzr*+cx>y#?fEbF#na$;$Zut^0_J4 zRr0OD&e5{JBiI_1e1Fqh_h*95*}9j0bc$geOI{_iGoAg~!2`{H<75Hj_jX^od5NCvB}PF9Q{)b8$(Zap67qdWM??Ja(yWR|K-m)PG7QfoXp-Hn2}9x z{xyD+$hB|8-7T_bz=oN5sHu_K>>n5lla&$$JLl+&AX(o?w3Qn?+{3PnvUs ze&43YS8jvB?vs+wi0r&9`JBk!2}-^oa(CSqM1FJA;}=Gr+L3=4d3@&LoE?{<)jY=s zle_b@)@rcv|DrQzIf!vy^tGa zIQoYhwif(&KFXIL&p$G|=h7R4~dSAJe-*jRo%`^t}J8=0R!r8fr0581N~SMwZi zVxz3>^8Nkn*@jaajh_v(0qglnZ}p=Tx91#$==_b2Y1E? z`#%P=v5!x0+$)W9UskMF!#O9qv0jUeFPpyvJ3Fjp#rtby{AtbisLTOpY>gWo&bY4c zG2!qhw{c^Gjm4L*@xgNBt8!l<99z2bzf!Q=yLn9rhc92N1RKlWsV(NJk=fBr$jYeK zYQb^U+c1ruefe0c*~+JUtQ}51bd}?zU_4(nc5*P@v*64Ye_G_St2#^%Cm*^RyFoDC zv(TKclV4=-3*HH8Ej9{=zpeS-I2bo0t@gvF!FXqUac>P~r+#0{*n#w0M#fvSvfHZJ zIq%DE>u~t4-8-9|+Lhfl;qdI@_KQSEsy0&H~1sjJqZdzC4Yq!T1{U$zYs$^6{x)e(?PGZt&?~=Vs+g zW>;~}4HoB|^cCm4V0`6!ey|$iEB0rD@#W{jV1B$Wl%I=&@ipe+U|i+@w2 zRxsb)_4{^Ye9i5iU&n+=8^K>!rc?g=x2CKRM5L%7dMqbFyl-1i8kzcu>G`#q86eMeg5Ju@7>a@;G}SbX_y?j0T7-Az|DKR7bIGy3J;*xo%&ufDHFravSNFD6{~*rCDpq4DPJ>^LkO zzV6kJ1e>G%L4SBK8+`fsXs~_P^)oLVzWf{&tTyH6=wLSZYvX$$<3AS6|4{HNx!+C< zc9{p=v1vH-fNTAZ54L`zGwE8tkB7ro{1bw4|K4JKB3LZEH7UP(Ya*YMg2h!c_F`79 zrv#gWT(4|CPmOF0o&3*C?~*^=U>Z*TaOFQgSpL2XRQ{)j!&m-8!MMu*jNpF!^ztvi z`pW;TU~$cfy_oVkC%sEv+vNWHRAijIV0lf8{_J4qU#+W}d?p-T%o?MwnCAxbDG#>% zElM0?&kvSU)$6mt?D6tiklrOPx(m{9@`9^g7Y2`9?~B6Wt6moe_QO_l4lVE5Qx z!}FXL<-k33Mj9lnOr{Ro)&7WT~o;+?9LFK4bHv;Puki8jcI5 zw})z9t=KrZT+wn{DI8lm_xPmr{J7*tKOqe-AJ`n$NnbX`ubTFzG&b;w87s$$!SXN% zW9ZpA=jhi6_RNrza}Kv=xZ;ajI~;zbFFvatY9OBY?8On+e8kmvYiUm8sax>&ScR3*qp!N4^-0Q>z^_)>V6CgUIsH*B;q0*qGWQGujwyK(|pE&Nx`! z{F3qHjhn2mdEA%W-OFt54}Mqr+9R6<(+?%L+9PjmoLp*;Y#EL%ojtNcdVXB;qyL*U zynJBu-Y$LF7{688+tb*sQ&ta_+{ zc;d4cM_ls}SKqCrIgzW@`ttQatDW@_vuqb6PrmeJ`@86^gY(R|uAfIDyN~JYJG!g# zd&{}DM!st}uY9+4zNzaK>0N5?y=Y+?PE5GY*vo=Pp0SsQ!<$o$(aTf3D}(uzJ6rzh zOuY(DlaoBY8SIh=-Ph8-ktPq=z2H6V>R@xM`{$a-Bk!MU!{O`w=el6y@a5w z^-cVyVBg7yVn^>;>wEIe!F1;CTlp=)i}DQf-1FW~W}}`zNbjn8-WK_eG;@M)Z#mr_ z?46en^E8eP+pdhSc|R3fx^mCb{} z8)R;?(rT?A3TLg2tM`vzhQrr;=EK3p;@!*E>Cq;eXT94$7A__1zuU8CQ)~BlF#S$x zmCuvG_}$WKot_Fly{!{{t<%%NbhX!?3znCA!M!M_HM8&Wa#BNMC#U~b=4(y&O77Nt zlhg_4DsI!p$-i>mJNLO-`R^2!^SHgiBYygX)RZ~G>?b{f9h zeo=aHw+}~VJ0oMf_g7x$<+q+({U)yb&ku+1=KtBq_-_6e zMwWl~8`MSN@ZJ2^O;=7tgm%{OZ=&Q$s0zNckPQ~V&B~t>;|IWD0a^6l4nHrox0X*Qm}E|R`Z=49ld?o zo$pDBWxnz>UozX8@1u#W&e{(qwfR-BvGzy# zdn6p5Kj->y(!2Pgdn^sd7hE-6khL_TLXaCO2E z>1&Vd)z(EFHpzb&+cd3odvuK1w0&#l=OO|J&(O#h=ayqK^v&-co^gP%=r zyffTyj(fu4>;3&F!RBKf=z(`O!Tb-! zm;Lfcu**E?evyVV54fJKzYHGvTlL{^_=^9lU|hZP|2kMKyfrDmdUYR2+((1ORWtTt zR<6GZHV3)Zv+}o*jiHnOGwEIOr~6$RPX2J^|3vV}_rNE^;Vb{Af^n7q)4}57h((S$o2kHIDFM>X>gy5-xHhP`rUb8^Qo`j zo&OSSO#SXWsEv^e-G|a}a)HgCUoxINr^$NtaKC&y_m+Le=Ir2)r>`|UIGElZs(p26 z~iJ2H(8T)#W#1Ot|8UJ2o7Cq%S_J9%>+-`0T|I*L=j)cWY@*h*B=+9Q7n#;Mh%8SAP&GC#6>^tDHZf{m#?a%LN24d~8D!x;z5n_n`Xd{&e7 zHIECDyL*|FklOr02&aKl=01@bZDp`%CG| zrXMTv#m(k|^tDGW3YLdC7{dp<+9Q_)+aq$S{qXs4#TWO*aQKnF_^f)UK|fwN_Tq?Z zKH}=TwKOMk)mmS^E^4*29%7d5sK|Wj%XU%p*1>sZT-VQ~k=@61_8pyfRA=J)dAD<3 zdB=6Wsq47tU25+=WlUt8m~fr3%LR`-W0wzyH>Vn-m#28+gZY#@TmI@yT``zmPV!hQ z*d-6T32AGl$pdyTcrRQf*c|KrSvB&=`)9Rq_5YysNUUn3IC(56b)1Z8?kBuOIG))MIgQ#hnt4EnWTQeRFVqo1m}X zyinf`yxG>biA{pNBeF5x`yqe&n)Bwt;yIt#c)w)h-LGtR3HB|(dvMj|AH&tR0Q#!S zuEBJ*xAqM7Ex_KZ-+WWD&+z6d=lT}#QtGFMd!-(J%WRgsaIWGuZ(K^=&$oUH=&L4v zFZuoCw}x2eQo1?e{eGfTo6?DQU}V3a=**+5^LxqfCpvj`b$&1T{X{1}I{U&mg0=E{ z$!|`-7Y~^@zUS}krljGA64P(MEz^4!uX-Ofe%Suy{I)DV$AsgDuKau~82?6oPK~br zJGJBIv~c{;m7n>+_&4%%PIUbEyR*vYym0)`m7nv2@gx1zn76lD`2A~)zbkxK8h&s@ zE$oTiWA{(d`5T!-b0CoaIkNg0M^}GCgRe1{B?o^qQ)4cVtX9U*8AFe+F*gSL8=4yP z-N@=<44pCb_!@I>u)nFPG51AQ17qlnp~u(SJ|3)=*1PKRL^!pit6Dx8j2~Ic^7CSJ z{P^3y^7B$Se(1{2AA|94%TF(uAG-20Djfce{7i_BAAf6DepU|Vhpzmr z5{!Q%KkG$je{}0JDI7m^{Ma{>gYkoJux~bxuDfrxh^(f@(bc}e*Lv<4+}$^SA6fm3 zp{sp^uQB@vclXWvBCC}#bhU5rHRiD3?!K8DSzU~wt9^qX`Ay36wy*XzjzT4_s+SKUioqxWi)pt((jQFrl74MYf zD;}=mof;0`jW>|7_-?Cs)1sqyrdB-PCGjKU{VX{<>$~&&c{qGG-t>&ccU#5VAUb+y zf5kf~&nkSy^IWX@>PM~#y>DLXudjR5x3V4LcQEx}Q{VY~3nL#Ny)pWFKD|5ka%VFx z&&^LnSKn<;4yIoeeg8LIWcn;ie=pE~Jne)ueqnJ=ZL+@N?AYS4SuSzrr>{6?1kFBX?Ss9?*;oN=T8Rv`$c*3QG5PV;pys~?Cjv@;;{b4)3fq3;oRqRAD%d_ zFB>hJPpq; zTywcP*w}^1v+}++9NynV*L&{b$oLr@yKh7`Z@Rj7zZ1;vhP3i`W3V~WRn9jB8}IM& zoJF?;SH5&Nr{Uxa*SOn*`SahY)T(UsYBdlicLeicp6rbm*}s+eLz;g-A!k11JTCo>=nIlR=k zn!}RFa>ScM`PSDQc8;D+e_r9mmD}qkn5(|4JQG=7`md%&_R6!tnTOANY~(s0JX^V|9@3HERC z_%H62aAf}`?}j#q~Mq4Zna>%e~VZ#-V_@$AaqTEY1JI(F+s z#&`X#8yWB4G?u?fk@4(KYWr+Tu(_7a)ZpT(Rs-SG%DT~)?X+M%=_|kd3;BMn=*sRb z!FX#?^Ie+7QB%6Ido`T3rt9t@d!j4bTTAltoUXS^#@mDTr1AERePN7y$-QNf+0{Kb zD)Q3sLs7X0y&vM(iF0Q-dr(ilvdQ|o|2=y=F9tJ=&1KETv*rJKp0b_aZ1v^OJA>!p kVDne+7^lU?81E%=($^U8GoHUS=J;mgedYD6*gMkx3)0nn?*IS* diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32_h.spv deleted file mode 100644 index 12a63e408587c5e8d6be35aff162b53e2574e499..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30676 zcmbWFpiUKMoD(XrW3l>0Q2crV9 z0Io(^X}dHPTZ|es8VhI;>%$gE9d-i=DLQtpZh5@^UO2fUwheAR^EDz znN?PrS!ZUwnHx5knQilCR-RcY$hg)t?lg@D&s)y_aHHkp*&ENytUa^l%rg#p=283Z z{q&=bJ>`^Bk9)~6r=RfBv-9n>-!t}m)M=+b?@`-7?lBg!{>&4GxNQe+or!zoz*#F_iS6Fjo4D;W?v-8d zp9EN+J;>Psk(cLe)ma>Rrt7BD#MIay6pmgjc>b*Bf6EI7`E}8cb1jVAxnQkxRUYR$ zJ2ssQo*#Q|<|dDGsS`PQ(4<#0WI3~L^`r5H=^j2SuQIdmpi^UXn@zTVl|#1&XD`m9 z0+{a|2G+WV%Htj$6PxY*!L0%w08u@9F$9(M*xpTo<=c+u;wO?#H7i_Lo(#0l^ zbDbA?oa_9^oeS1BO;<;i$GP4S8+x%}b8)LY&h`E%arDlLkNF=l`Ktr+=J~HX#|F1m-~ERb`u!u{HnU`f zd@%V}pI#v!8u{Ey-?qa4@W`v5`Py08o?o0UhmC(maM}w0Gb8+V)yK`s^0hGCKG~BO zrE8Vn7cAZuc|v%Gn&bbc%ikyB>v<;5lP3AHiKD-K!s2)S#lh-=jK3ybtGqPWbF=n0 z20Nq5U!U~O{mx)}bnex!oXtDO%Iin=3}?S_@chaC;aLUz*Wn%dX%tQ7J#5K;vkKcXTjaRU z^6bf8J8<-SOqdTn*=q@XWv?HV`FqVETfcgHPUJDSM@Mdriu;<#&f*30)|h#1uzIv- zyzWlh>+6E)D=!Sjk*)Rh!Q$c{8JX=l>7G&aZuIcaDHHqIn_zBOOht)Dv(edVQ*j~?V@^n?BNkv%7Ce?#Okzc)r^Q2kAjFC6S| zj{Kp>+a+)FGhUz1A&-xP9Apc#%@O_~jCp4Fn{{uRHP{Q;zU~el<)00*`*3!8^^41V68te;e8BsMpH+Jv$tJ*)-o#!8p&NnjBlK4gT?V$(m2-zTMjCTp=j9b@acFo7`!D5f+w?|}r_qJCsZaj;> zFmBA*!I9;xamnnOvm=A$%#v$$?Tf5 z*9Oa(*YM`-^}+ZtXNw}^$DExr+0iv;Z<@F>ZKitZ~Wg8uwkn);Q+u-I4K)`~G0un6nQ=mb1nsv+Fb98_AbvfX};q27EId zzIpxkVBELTo7Y=|@$U4pJh#@3|5sOCZs+y5IsAG!uPJoa$?luMWVOF;-i6@46*-}1 zwn=BtrhC6Nn0{$=UH5k)TlbCWUH5;4!*`9}4d#n)tRDvBab3@kg4xkE{_Vly`yAc) zKMsd){GSAig>U@3g7LV<|5-3Qy5{Pa!PX#Gf6RNmUj=*5XS{pL$>O}vyC!HVV2H!Y;308MwoO{FJ8|S`YxoI35WrM#g!u-z5>Y<|4;H?Fbke?YK# z?TNiPt?NMAPzIae)Y>o*2aK@hD+k@%VQ0wY~ ziL;iEOlx~bFk3oz{398~(X5aDo#}Y%f$ia+GS-IwyVKv7&IW!}e68`l!Pa39eCXME z&e4A$*lUJ0dCuW33@%^pgW>RVV~N>($btFH&)z)dwIA~u$6VSIxw$qr){#^0oQFBt zwnnxIo_nVr77pJU*)|v_S6|MkwMH(BY(2)-$Q8kSS|f{tHD^Hg$#fh)*t*3fQ5?=>ti!&a_@c7Uq!~56IQ35v5yX(`;2`|IJ`a8$7r4A`|Dsat(~oSeWq?7Om9ur zv1_nq9dwUR-zD8TV0XcLvK@l$vG-@k$aC+{6T{*A{^v=-{P2ylQ?NXqmHmmkb7XdS z`>`H#Tf->h?T^nH@#klKp4V(U=Tm~|y{B3>&9PTF@9P%Fw)5^CjxC+ekuC@O1VC!-h+(m1;Ds_*yCOPEmH_yAWUuSx7*6z&Tojl>R za_^Zq>o51-iDO#seG})q+wScrnNv>v{#5R#;e5XF8MgbqGaOsGo2NYfBG}nlbH6|Q zWjMS&sJoSX@tmga=zf!)Ezi~fY;E>`cd)%_?xf$T?g@vl%^xQl`>f5M!r|HYJ>$>8 z{Iqg^nK*vs?hVIREBD_M$L^rqX}?|F7fwtu_^?r<_N9*PWt8#m1>b#B^Zf18Z)$AS zuHVsC3HH03yzU&1%zoRLjd@=!JR3Qd&j&`)MkTl_~vB8V18P;c@t+#>g`9i;soF*X9$GO_sUB=CW}3=KS(tepC6TX8mp`Mmup!gg%k1+sSU$)4zdGDigUz**%~=1}g~K-|ON04o<(5s{ zn9u9O@g41Mn7A>YH--~a3_fheJ=jb0d2=v(`;t%lZ9YFAET8fvpJeuP^ZBpg)VuF! zTkl^Fhac-{Ien)9LiPWrdgorlrh*pTUZKHm-Y8l|<~ zzZVW)o9|CH@>iQ5gv0kNei+P8EBB*`<5%vsaD26Lw@)0q*87jci75sjHsc=brRVcs z!R+nJ`PgsI=Z;|KBd^Yf%zp0q{4^Y!*8A^+wdV7#$akj8pV3;_km>A2z5hH|KF50h zML2wIel^*Q_5SN{_~zs{!Thvxzn!=-pT7&oceJ~E;>LX56HZJq_^=uGU@y(*AA;H2 zmwei9^ZCbM`IImDB(tBJ&y`c>>fQJ2t@l-e@ngNO9vMH@`F_WY00WZwxwn_-tf%neM8r-yZtC#`?kZe&h7L=~|OM=Cid&zh*jKZs2}y zpBK#6bEBVkHX69j@xX9+bExf2gEh5Hw@Erqe7I-!_gODL?;9KcRpIcyS7`i&k@3D) z>Gv>;BCE%3(&@(ESlMN`0@AC^CIKN-%rnb@ESL?xaXu03chE7;W#RCRb49Q`G|nf3 z+2H#d(5HgMUzAwt<#WNBeb9Y69cLe~`TdRDXM!)wxNC+S{Wk}j*Phs$)4IN#QL`?;S$-ih&bnahS`hs$!M>yK zxys3x!r{$XA7khIO0byL!B)Kf4*IoVYihp!HJCl#y1tiDvo5-?r{k;(Zod94cm_HP|BilbQ{{nOG9OSc}_9uCb| z8~*#HADGSt?r&TN1Y3ta@S$htd29~{2m6i7ntTV2dwRI?^|ma?d~$01;Kxxu{Cl+F#T8~FPb=OX^or| zjxC)Ud1Xd%H0z^3IUR33uzkNGV{Q1qIQ{f=HgId?v|#J72R`)dS|cwFRwLHbdU#p5 z^5xD9ho2iu%;rN5%x8Y~<}t7RnAbSw(w@l8wXw0bo^t0r%vsw$SB$mYK6>ZidB$&y z^XSO#F`c@j>+hgfZ;)Wu|Eu+`;7fjIJ`a8$7r3N z!M6vCY3*#q>ofHy!SvQ-9lr_gI_U05|8=@`!0v+gigyOvWAD#hk>}o@pM}HsecI21 z`QaPqm%;Mb_iew5%nolq)?;pK7-hWuSiC790M{SJC>u>W_!>z?01$!z3vm7KfgPRZwe=_`k`C-{cM zu!ohxd4DX1eez?&c9ikm_YT@wdUpr%>77 zG{o5D*s>e0>f{yoz0{}$)N(K)AaJs%0@oalN^7YF0#o>SNJ$>_ux*RwdBICPEksbKuv zIL+ye(T(SQQ)Ia?KV8ooU!Q-YAI{+$k@=X5uIG)f&mF*Ir-4MmiRv8(sZ0>Hhfd=c1&kJ7VN%z#`YO-UY;BH@mcY%aCm=b z(Y(JmGJgD>;=;(DiQ@Hlii^VGJ;SYSaj<4>bRSCpWV*G%p5fMZaj@s4<~_g{QeT&Z z!`J4clTB1BY(5qa&&KzV9}niImHWiR@hf*(IKEoB%O{Rqzr(#EoS0(pVdJ&KzU;wX zMj7w*!rxu2lGj-N_UU^_w!K524t9s+)%WUT_S&C(s20ZWLYpjt=#n!Cr{;W2*+0|cjLsdYd&uZ zC#D#D*vOxK*@L}|GQRm-E&1VZpYqAJ`TSzAe9D)6lG)GA=Q{bVPHkV5v9-NkFn+A< z4I-;;XVKc;FdV+My+yF*Ea>i+J}=!qjP}8XOeZhSW}{%Q`&w)L0pakq*?6*%yV`6L z4&VH48q7~Cx7oz;EBByqe6@0$PaM0R&4a^`AD%e@b ztFs`ppL-Vb!?9^?KRQ@zKDUm%AYJ~9*20ENXD{;k&|vu-Ykiw=_}Xkc*^IUR@NoF% zWV>K~TDeC|+?dZthT}Wh{nf;c`FvD3F~#7+X553lG@p+NW^Z5eX}`_qV}s>WzT}h4 zer`T@4yU%ip0TyPYcPJS?cF1*ZLh_x?LETbTiXW*YtDl1Dd~Hr%e~P)*pTU(=e>fx zZ`E42dxyibV18P;=SuxG|r{gcDN?K5WK4*h}+y zY%qKKl27|>KA#sXpYo-)$?WIm^TpxRw!f8VZJ!nnKi2jck=6FnY@oIM(s20J_QGJz zSPds?Dpy;hXnY2lLa)oi%a%%DpBW zU#;A0Cyrff`*q>O6oU_&aS!&=vv_?ld;4-0_S>^KJJ?ystFs`ppL-Vn5ROf2`+{Jt z`8+4`qICH)S_>O8oxP~-HwMe+Sle$3hp)}Klg(J$ZwZHQPTm^KPb>Gfi5v6z_HcYh zyYnV)%;)*x#1w-Mn{f~J(tN%nn7w_;r~Njc?+lht`I1jE`{uLn!M~Nd_8#2l@YeQs z!{NvG;NOpoAM5?L$iB017OnT&!{J-+_XKOshwi`9?@o6faC3cUu_1?Yq1@^dH|BGVaD27etvPXHKGzB-rWkzK$e(@LgT0J0 zzWH1!@2UCQr+l()KGzGDPx+EhGW%__9{CirpG%E%|JH(j?eMc}$k@B!Zw~yOz``j; z|IWhS6fB!^eQof*nVZec6Sr*QoLleO^?cHoPO)wbHlNWRo!ga}%iR(GeXHjGBhT31 z$!`g!w;#Wc+#H!+e$@m0P3d@XVf*^>BpW-=l4%|`@5wx0%h-ATEttOZd^IwC=b`^f zI^I06?_vErDc=e{JEL`q(eD-hJv?1=@t?u|rp3R5oZ`*|?lxz_i0 z!S*xy-W|^Wk3(1A-v{HJb$$O5EY|3IZ)E$VtM7fmc)mT0RaVV=ipc(L1E0ZHi>x_s zx>eJ0&H=99TEW(Ge7;9f1GO=#f%%ECcCZ-Ek3D~Ltea6Y2b*=$apr(~UYiFyFYDeZ z=bx|CXC535KlgjuC*&XLV&^{8e}{1V>FU2@Fn+H8?&0`5|N8F{jz3-f_YB6*_1`}n ze|1v-1H$pAtN($)___Xvh2!7v5WgRMcsTxa^*VI4? zey;z?;rRP^th)YF!ttl8|Ea| zGEP0f?*8@}AD8amA9G*Tp1ZFm9zW?`lY36yF|s31WV0DIXysLu6TV7ww<6RxD@#q`x znqa!dTN=_yFcVu^WuVBsHJu!FpsgZH+F5KTw4+!2b-CYyIS@>=7z+k$*XM9dDPV0N{qax!^ zi;sUR@aS|LA7^vy#PR8w9v{wr>1uc4WS2+u3cHiT$r)YOa7XfH4eYvxpN7MC4R;3P z8vCwboYpn`BC@lhtKF|ByKxP_3+KG)x`z9LtsyC0u?GJzCBAD|$v;$?-qANX*HQPf@Nz^aU!(5S@Xm(L zIgPp(g?GMm&U(}x5?)==sj*S_^zdqv?)XXPE^U|he_H>(YU9Z6k@25X>+_@AH29bl zh1W4Y51xG7Kf2A*ar|I^^Xram9xT_}rknps8RgiVZ25GKErZSBwapycM%K(hw^cfh zAN>019xezzBX;7Kdq_C4e|Pz_(;ogN9NrrIjfC}W9~r+>@XqP$PW(jV9 zgx@n9ed}z;V7_#VrZqir;^@jfIe4Gg;a)e{?G%jPKYZudB{E)&&arDS{@}rG_sDp5 zjkiZI{?NhhDUtDGyr)LS`**B6@7|H|>|ULn%g?^S_F9|$gUdBn2ZWO=ule+~Jup~I z`mXPwV7a2J-P42d&Z7I>CYT*v?H(5Ftm(!YQWK-Bw(icqu510D4_1SlrSn%e>Vl7F ziQhLD=8)N`xf3&Ly$gp$R&Pf7=S=$6{gIJzY~a@Yvx8gr?qus;e71aA_eTehb$>!+ z%^b73$T)t-WNbc;4OaK!mpd*Td93@>!{NueKO-`JZr#5uTf01uMAfAxYog|g7IVBpA{J|M(21%(ukBld#iZ~0-WDuZbhSG#81F2)-{%LjqpRHu zf}J&;I&TfBiBZP4?!OqU29HeVuWr-@pVs}GQ}^r+%CpGd2Amt&UBG+);d`661mo5I zfq9M6dN&>wSxp-0|9aB*ZagM3jt$(q@z~(r4XZ$O!Fy&d`kv#Hf}JB>@4=HN zj;`EJ!R`jGwX$*w|g*tyc>H&#*gv#jEo=e##18W z+4XKbHP~Le|Gk3C^=|APu6KjJwoeNdlfLWQCs?lNYPWAN-dS|N`vtS3tKI&=&YI3$ zXbq`}QO5Uf92Beu)g*s)qb~ULZX6uUPF=q=qt?1VG_ra#(jPwQTlde3jAH}0?hgxY z-Ftu6x)+}ng7IVBA0HVnM(22bFn+B27evOh zYrGSJ@nhY;Ffx9OcVcAxSobfAjAz%nKPlK=yZ@7e%eC%L3D>%(ukES9V$yegFAkO~ zy4t-Y81F2)-_wHG(bewsU}sII&RaujVwCZ%`jW-65cjF_G zHFMCtDILcTZqD8u>~4r(?%Z(X@ou~&9DckTZ;gzfdpF(|u6Kh^3R>&pE$a57X-T-xYo)$g7M?scxPn17@gxEgYo0tcvoaRyT*HWFn+un?}>~bDd+q)|5L~WzxZ;O65~!RlW8a-R-I9_#*J!r{ld z|4d~3+`9j4xYj+N#=SCF-P8AsmION^y4LvTCXTM$Rl({W*E+a57(dqiHIeaRbdGC- z@nhXz7a7m4@sASu!1j`j&?YzGF z>ubWl2D=ORz9xJ<7_av4%c%8k{AXk}X{7(rr0?DML1Y{oxOd}+!Mz*4W9!`zpDmx> zjoX69yYb(VHFMD2o{r-OH)lT%b~nT?_mgns@oxNAIQ)1w?ud+^dpCX>u6Kh^f01KM!^{aIKYJ1mnlM@yp0~F*?Vug7M?s_;qAFyTFRnLIIVTRT4eQR zq+e^&x9-=7jAH}0?$-=%-TU6TbuT_!KCSz;gU7mmXk^VCbnB$!_`%J`y20vR{BrAs zBad~zemMMC_ZvjU&#n6n!?o`DH151$bx+?jx?iv}qHB%cf8yxMZ4|8Tajk;~1mnlL ze_&+17@cF|VEkD3n?%O5YrIW^@nhX@78yUrdr)NjSofPp#+s3!TV z8+E~_b-!&eyS^qoJlI{p_cdX=V7$LM^PAg}#PEI6D{_ClUeNbf9pf`_eNx6ef*u$m4i?J PuF2ma-J8?fA^m>=xKP$| diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32s_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n32s_h.spv deleted file mode 100644 index a025194b9e558f1077a0d96dc00cc87482de0ef7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32888 zcmbW=ci3)KbuI7@4$^xE5zYZrR8&Mk3|&DERf?#fh(r;QUPQ6ciy)w=5f!AwhzJUZ zC{4OzG%@zXF5orMXeacps7)Q%5s>47sV-0}m*2PcK~HD6p( zwTN4C;?^Fxr6z8@fioW8xdUe}a2pTYvJV1;{ z^Rs3-+b{CMoGq~cho0%Y>0DyE*zX^XUMzUt0?mKR8xHaV(Hl#?_$4+`RLzXjRs~;C%nC?jns&Q9jo#<)&;TYSg^6?q=`)) z$NGBYajb7d?pUyEoiue+c^vDXV?!@CY%Fe-$FY78g`BcqEu$D`US?))@R~v9GB<6F zL{Z1Ei@aL;xoL|n=$BY<)|@B&TG87pKE{8_*ar)IX1Ytksp_yKfdxhmfi?7mt1 zZw1?<%0Hg;_Wjqv)@a|WUp9-kkCoSq>>kd3gW!3S{gW~a_^-m%c7Cw?se0Wr*nN%P zUcuyhB3suBg3Ss4xY)Yq(yt%vo=Rr-!t~aNlOM8sDvti}3G*c%nOYTby~4-G4jJFHd-k`?GMT z#;4=_M`X`2^ZZM&d74-C3nm={`CKduV1qOM;_2DqjZdC4ahqmEY}bw~7h~hQ?qtWN zV?B}oF!tV&PfauSKIz%w`0tzE*mz@;_nSCtIXze&#AAO(3*^@n+aP((Qm=8VK zvx>g5=Tl|=o=N1+|H{a%d2#NItQKFh%2G4;1*>ms#B1&m)^>j|edW&v_UF zRwv_H+z;rk`yX`f;f=eFO-r-xb<@iU&N?2OKL59Xmvi!Z6IcKB8Dwnsb0=Nz8|J-n zxcRvmI5&MqaBjQWOi?#>~P|cJ z#YpNG#PMuz9M5jDINwaq2PY2Mvm3W*BB*80YG+q2HGX?$gVlxmDS59+-XM0(*R3nf z%H|7(!l$A6VcRZpIiV3nPh2hu8guUJ2Wj`Bg1 zyklnJte>6cexQ!G%^c+ZoQbb|US#)#%HB`i6B^@hg549w81IdYKQrTqc}XzNUe@pZ zk@?xz#{NKLeElvB#(6IA^Njvr*-N_a$G-(% zA3wS7e*7dHzOjB9%opFi{bev7cls3nSHbM)8vnn7#rJO5_>Y9c-w;20_M2dSczzp( z`)#oE)x40|HSQmS#r4c++&=~5JI6lxN@mI=mpzRL#7A--cT7mRP56@ta_>}#AA zgYorQDHzxJt{f~5zVlrr*!`;WC9`X-Jawgod*bZ24mJyiAM4mEOQ z;>JDRDx5uTU6a{0A6o~DJ?`H&k?~#I4#Bu_FLsPA+zh+ z{yJFfF=wxgjPKeG3&xE(drf3HYg{tB#yuj~9LJm;85!TW#|GoZoE;Zg&Kj4@t~q;C zu$+0uH)n4R#*aBWEi!)0+3AxVU2}HU#Em&SJDi+7G2_S?nO)cRwqUWxoShdL-?d#B zj2m-yQDix5Tr#`Hy*St$$DF+@@&bOkm%KMJZp_&wk>#v$$?SR$SR?23`sgo9>pfu2 zaQNnR?O>dD;^uXoV7xPZZ^rj*Up4O_>^zT~!!^QrrqEeGyS0PKYJd5>mEhKioKQ1Q zNMp~YYkzDo{XNnBKJQKI1$%$eKQj3rnRli;qYpLnmHf|F2i@U=?(37T^V=XYz31I; zW|o-w#-ulwKSZXVn}#HLVmOm)Wfz5~0K9tsRHV&q@2IE{k={pYnfk>wSVQNSUFJ{s zqBNZO!=3+2g3aIOO`ZP%;qaaRfx)=W|E0mk$D4oS>O22~f{klU?2Tz&uT1YUFYm(# zN5+{KY+f5ie@L+VZ1+`8{yH4qnDxw@fgfGo$CT+~i{} zbjzmU_`%jME*Vc=evAQz31=Fjc*43&LXD%O}=C)cmwsh`W z8?i}qnIHY?X?XL2tzmBZ+VEd1ZQV3B@LT!DV4Yy|um(Q#?A%Z2*9-P+GAH*F+~dNP zFSkKB{Onj_HXm|eJma%Bj&ZHWxcV`d)p1$Yzs|J)nDf8jc@q-r|z+zUz2$=I$)Bc}nmu z>02Yu3Z@@x*2ZK%9qyFAlQg`!HIBY>J0sXQ=F8p~=JB5NF7se>cG|ns%ma2Ny@Q?` zEZ3da+ak|C$LEE^_kHL5V1D?ti$t#tnf+ezLY3 zBI9dw<79KwFxDr+;XBq%!Tj(Y>yyFw#<@AzSp9s>Sf7fFug#|?o4(Jnxg{LFW8E6e z58ttF3&uCj?ZL+Cd#yNkM8?)4BYI;Pum+w=wv!VLQtBp5--D*ZjnBmf3!GiuaXZ zf8(;@|JBIi>3cT59&EgxjrAs*Zwxly3%)e($m8d>4~27|>sc`Whr{7-$oTUj|0tL* z-t&~rzfH1r_V4(ggv+)q_&c6Go8I$&9!$S#TIc+WVEj61?pwaI{bleM(mTKO&Zj#0 zRWMzjH;)9Hmwoa)G^fwx*@HJHIppj26q}}o)X_efyIA`rPdHb(7fzh{m)n2hlCnkI zizbeH*Y?UhljYR+$>m-h&Sx1uuj+aa3&)o3fhmti1lwD4?so-8hQnKfx_fJ|OWn~O zllGQ0H2|Bh^&cN>Z7%2Sw9My(aQNDsG}%~ZZC)P^&&D}^Loh#AxsxZ3U%5Ai!dYgu{1lt_kMnD)-Te<5%w5aC}|muA4Y^t^bdO z6H^R6Y{oTMOZVscVD{E!f2_Crb3?HGkyraeWZVbn!^?p~dtNFYs@+Z>dPj4=4 z$aL1C{%;PJ&kVUJSDy-pugxu!%~=1phQl`}w*~WamAif7#(drpj_+vqnTZ?od1p8= z#o)tcT!XbVpLYkdw=Vg#-sbb3VEL3U`6RQSozE|XbAR`JOzZuN;qYU8vW@;AfbYx6IYjr`T-TjB8C zi*E<>bCvtf#Etv&-Ee$IyYEfhxIfhjc_-c_^){cs4wg^(l20=G z+4=lqIQ8zkwATBd!r{ky|4U^2SnrF)N4?*ZMfLu^csP9PedS=6{h?bTZBCl~&|4cD zGF@}OWU%)yS8I8xaQNCRGug;rZI%s(?_Mky%+FPB`H36%XN7QlN4pg#Zrq=h!igyc zA2#C}tfl+2N-%rtvOm__{aH2G{>ZESA+w*oKdXgf(|Vs5+0}fm5qb4A`O}*V8!}yU zwN|ivX2?akT00!RHtS9{W4%8%9KN|*FPNXJ+~X#0%;);y_>OiPOx&2yx#7eVgAbc= z4c5|pZWzqoy5!S(o6n7c#Nv$x9kMfRI7@0j<*$7N6Wej@VSY4!kaes2!;w~edy`l)dE+T1$X$Wd)>3x{vM zZx80@DtE`k@hkV4aC}|m?wmMweP_8VoS0(pVPg-h%Nnd@l=0n@rSs0n-#YCH+wRGI z!TW{lJ3Ey$gT&E3BQ%U$2k z$?TiEAEws4S8IN=+I#no!RN(qd|!X=#ND3rU@r3`o15NT^@KT;X&u`>#HK$7rHBF=`M@xZz{S=XX!49?0u8&y|Z-hitO1;ckwLUg^}II=q?y^ z)^J>8XPNHN%-HVI;cNe=&`j{`MJ^EeK@Nxt9@AiFz`MPiP@16Yzu4BA7 z9Nrje`@r-rwM}#(y*%-scL9KPNxz#QR*OpTjH_PCY&$jc)De z8h4rK=6r+$m?~b65Dg2i;kN?%qk)xvUVGzQ3oqZ_=BK_0q4HhBqc` z4l7NvzH|8EG!C10r(K-ZaaIkcw+8$6o=M+v=vPa_8wYL<*9?{e{?646iLrL%>(lqM zOm(wPIDCKaux>D4d~>-$Fdo;>#pVXHqig&PgT?=Gn$M)f&wu?mgKzwegT=x({*!|7 zxW<2SFgrT&{hngeV85r3d+~iw`qXfE&kOR?gT;~u@@Bzo@Qt&1u-{XRah@3t-#A+Y zn?vJl8O#RX?~=C)7XRGDQuEsfyVL;PbJB4325z3V4wk2W?krE+gu{3IZG&+)W*zgQ z-!9l#c(v5HdbPx6hhXEX1NO%3T%Q|k4d&X<7M~ZH51sk%mfmIlbUUTt%pdOjpC4@g z{ruhhcL|5@{O1SbI{#gRjgL3~#?^QJy9XQBn%EoDy!J`&GB3Y<*fTQDykPU%F!~n+ z&&%QHzRJno;qb<+kG^B>8!V=IuobVrr+8toIW=GV2eZeU*CFX$=0*3SG@NA zXYcn*!r_~*1A=kluD0~Txc#pDk?>;b`(62OgZZ2pQ$D{Bc9{#^@6vGm7U1;alJWBK zhe@xO59j6EIa}(C&3A%Vj!*aS(Qx!?sCDH(2*Vu^UB4?|EF4=p_pMdZi{mms`o+`m z<^x;9is@^^|1pu5o^0TLSH4uRF|C1*80_3ntYO(;-&L7YzZYLFT={Y z2gWl#d*c|_dW@?db7@WF=33ua-<@)2Ka9z?HS)4>v(MB);qa}Imj~nIYODCVS|jU5 zHXnU!WW8WMt&#O7AA3OexHKF;*u2FhtVBS<;!gz4nI4Vn9YYA7|;0ZjbmKvF|K~hr8SY8Ykgz=Xv&@aSdiz*w)Yi% zZRg~j*gm+Q@f+hj8reCfQ+IUzJ;j}Q*LA=0H=X;NydIXmd7Tsd$~2t$!@b8I8a(?R z`|5CbYpRdlJl%s|6D+2=vlXxR)Yk^no0EAQ8{B!&9g&7J54h*$s9@)1{5{3d;qZNb zIwp9{G`zVrj=pm{F4#Ec%ib8~aYp*igUyL)r>B_*>`Z#@ofItBo!9Fl&pyX*2#4?c z&dI_2@Qw4PVEgHPL)Y zR-BK9!`J58$)WAzGHnXm><4leLNW7-&0&4EU(sU9`wfWdx{%^{kMPK1Jwta zz1sX#dY9Uq6a0xZoSK2f@Gkku;Mw=No5SJNP<`~hmwY-{Om)RpJY(LG-epWSx1`;k zW=#10jOkq77VPhA^EL)QHf%>3-?RLs*zyy{S!UbcQ`{BoZ(KI~?~W{51n<< z^_g&dIG@qnxzc~{>4}l?KKrWO8-m&S988V-UBUYz``noQ;dvQ7KmCfe}8(!DoZYqAB=3>#;0@7pvTu|&&0bTYp%~;k@*;ljt@P) zKCcMAZtyuIvb-3Fjt@P)K1T=NH2547nU8Vk_|W5fhR+DLCq9pAe$NbNPw2WQX9eSD z?@8mlBRX;JPW;AsXE<@_8t39*{QoV^2cxr3<9sd;XP@Z0PgexvXYW(zb8U3ujPtoJ zoH%ri^RZz3>^RNoozac^eOF|;Fg{)P8(*J)pda?(A0zWI7G3unU!Mnq$NhdNvRoO5 zuKSIz&o6?<{r+WSKE|Qze&an${yq6{d6xKG^`~h#_jUMlGYg+3KM!`^-D7(X*kkG3 z$HR^HihaZ3{a&ki-#;>b{J!&m$nJ^a_502P!{Ob-&F#owm$}g$ly*d#xxwz?o^!7V zcAs>4zI$&vBpkjrubOP!=WBClI6NEg4X+O7=PGyD#PKWlns9tw94u$9wCs;n~Qse4ZBU zYCeyTd}^Axz}6z4uM3vX46!IzCxye;=H$sH+rP->jp6Xk$(w@txyrqH;>LWwB^=+; z?yVCy=JS+rVv50sjr>`cHCW3i`cG(NM<vYD~7|@ zX64C7?rO72IDGeE)nI&MGtQk&BG5D|<*I+H(i?xH z*pTU(=N*E*x4K%lJBGv8=6REiT-9c$aQNnZ=U{%Wa?hW*aW8fW$9J@wKXKz;>>5r? zG5D|<*I+H(i`|3STbI4C-tNU7!S+I4?FE_r?7i4C9GlkmOM_j_=L;h5l_r0Bb74cK zYp(VQmd^~iC|CQ^4>m8HY{uH&KODZfdr>eySGgBY+?daogyTEf9WZfYJ`W5hrWkzK zjBBu#=JRF2?5#^at+)9+C|Ew_OFqf$XXo>^;nen@(zmvc491VOeRO2CeNPt9+CC;6 zzP0_9V3)n1J1*_mG`ZJX2OBb7_u_{SCqVT;)!l zxN$Gu7>@5~_oj&(_u|dr#1w-Mn{f@+(!F?VFnjB=7uMUoI3?I#$g8~|v!A^er-ozG z+P*N@)qI{F`Ls0o)0+z$GF@|ZX0Uu_$VIt2D;&Ny=S((ZZJ!$s-`u?|n4hcMc@sD0 z^ZampN4vLA+?dY`!igycA2#C}tfl$9D44x<$*1);pYI5kPx+EhGW+JU@8B1uuDyf% zJG{01o^bf_9sH8W__5wEjqEcEd(nEoEF8Y|etodZ{?J{K_VG0P0XNrw8|-roS8Mk} z;qbM&da{wP+I%D&zPZ09n4hcMM<;IFk88v69qq20xN$!|7EVkt_^`1b)@2RWGRpYw z&n;OGf9teAY`Z@<2K&2FUhNN={U({v_|AA!cs8x~`-5H0=gpDtOEVX^xw<7-J~PCk zT-_QDUz6As_p-5Jc!Rqn2d8}oU0IKHFZJrg(P^WJb`iou7C{8^VZSj#Bm zo6p;_9{$!TpKP1Y&jrh;e90%7{U(`^e2Uq>OZBsVvqisqcyrMAEcoq`-y5Ad#pv&~ z{5I*{jN|W}l=gyuH<-->6E|n#>|4*;65;Uogfrff!N${DqgcmhEN4gj?}uL|?S{mh z7jF4rdh78W<@20Ji)^S!3rtdf_Px_8Szfv0B zIIz!Q{o8751wWA9JjLke3Tua_Yc4hj_S-oBwxT_pn|@xJe|wScwX5X2y7c_c&HC<* zZTIrQ{4KiR-_}Wv>?5pg9NX^OlY%b^w_|KN=2OC%msqviG#p#Ha!(7!C-frUErP9Q z^xZO?|E`a&zFP(3mq_cl+Xjm@`feB5I_c`WLol9i_hP4D^YQN?`5XNC!7lqvw{sfK zKEU6%x6S_NBb%9* z2jgddPPGYEZNl-VtN*sa_}Tu?5654f)PI+7 z{ORgHKNvsTf1hyt`x)Zg;C;jKr>p;d!T8z!FAK;2j_I>Z{s)EQPgnn!2jge^9~O?k ze;c#&e@!_4boDY3jVHJ!fA{ynfQ1mUB*iV`O!LZ|%P&WAa&89}DBXHC*G-H{L11bd7g<@aAdw z#ydY)yoL3$Fy7n4H6DHAT@Xyyc<-P~!#Cdhg2h`{KMUi%KV0L{H{J(==^F2Y!RiX% zcvlCDx3HcT#`{RP#-ne%Yl7(-@48@hhHt!^gT-4|Ukl@XDqQ2yH{Pd%=^F30V0DLY zy!(R1TUc)k2~VgViP8 zy{xsTcP~>@yRE#?#{G%Cx~y*3=*)-C+(zB}=&YSi9!A|R(aAHNoR7NaM>l`c+22vO zb9Cy5pIRDqJ4L5f>D2A0dtP+T1)VcO*JsG-!JY%|=gsMvk@51{cagJ#@xG^bcFzrV zIlI^8?4A=D=j_7${`8_?|BkS;D~7%B-Qqif>H5xic`(k^cknAB<4=f>e}DMOG#np$ z^P!33(>=X9ob}Sx?z+j&dq(Z94<~1Iox_94n>n!S93BdX?;IWu#x?ejf^n|K{@2L% zimrA)o$SUr{5+ifrt2Jj7i7qIOJ{-Pt_(L$RvHuv1b9D}Xj;zk;YPVQk1lf&q zSR$NSrRyA43O0wl;w+lO%Hi;x!z#hJ#$GiT=jt5Ri0q8e)o$&{Zk)qo!#QVk&SC3F z-?Mg3^lV-gj=dVI?v(Jxp)=;FJ1M;N(OK)LJ2t!=(aG1SJ3PF-p|ek;?$Ge|m(E^~ zx;?|ID>^kc>UIyWHtDXObk5SlOE1h@f8+S&$j*`eqp9_IQGYe~suYFi7@u!WKF%NA z*V1tOVE^raGxCjKx!xqr_*S?r$apb2#_xmi`wVt}jEra3cz+7U`~RuY zcz=$JALIQcGX8+UZn69wfM<6~RxUqFgtOMhS~7CE=4z>Ma^;y%U)!beY4n}nGQo01 zSG(nc@%EzY{XsB0y4w9PoV}(SYe-Fuvf6qi|6;%SpB8^L__Z|t>PB7gaWC=x=4Z3X z?9|-G>0Lbwt3+0Bdiu2{ed~Vp$T&7|>weAP*1a>?x)+}fV z{NUzeyB@_fLzApI!Hxg=^jOY23|&)jfUp=$XOxh^{sMtcjy5 zw`H)p$F&Zg9gH9A{yCBHVswnZ3dWChzfEL3yT;o#7(dqi_L1>pyd5Ir$GU%RWIVgp z{m#MG+V$@eT&{ILKV0jczP7sti%H-4?G`LobhX z7-f9x{)}KXxOy6Yb)zo$wC=x@x@Wg{?nQnZ@K2GQ1-$orG_Sx7xT+ard#(ie+P8o~7`}nM2`$*Swu*JmDmD@7d*}%0{ zo*j%I&&F1f@nUq0=LF-&v+-Av@$4FJ>tOtNHnxe3ALDHs89$zl?IPpZ^=xb(Y^`1Y z4#DMmHg*ixvq4|m=LU;O-}yZcmmJX5Zl_?pz36&(4rWJJyXObnYdUA4HKZm+8Q-(9 zYp@zrll;|G-y4Lv1CXTM$LBZ-C*E)E4Fn+B2gCpa`=oqgE#*cM>NMt;_#{27F{8;y| zjEo=Sy(%((touVFpv{GTlSGyyE@%EzY zJu;XbUG0tvw%2s(yfvgIMj79_KQ>qms!9IpMqTh}-5(dsuFr(ygPjF@p9v=f0Lb=r$knhdipaaeb2^ek#TI`o{iIkdp7)C)UzQzTRuG-X9kaF*_pV_4csAZ089&B*Ph|XfHvT3uo?Xwz zdxNdD>%SzpT+hb)!u4#>*Y^FvV$yeh9|)E!y4qbDjJFqE?`6U4=xX=DV0%sHEVPEy z#3k#w)6;)+(zotE5*fz^ZrxuK+`9K! zLF-<8wtQOm*9MPu|GCI6W6)ifhT{h}A0G==_u`lPcsTM{_t%HRk9B`TWc=*9zcF0v zo=@X`B3RwicaLrgwnucW@lQ@1UAdcs)jh6t@Tp+@Sofcfj2EM0+!Bl*>;Bfrcy^6< zTQGjC``aVq$9Q){#*cOXnaFr{t@}HJt+ngFE4WajUR@?k7wh-$oSc3t|e@#ER}X=FUR#{2hR{CGBg78yUr`*~#i zcsBkcGM-(}#xH`cwd?<7aJin1Uxn-0ps($(gT)V@>e(Nf=|!JAA;Gb>kU@Opt)N2e-5wS^z@5Q`quqS zWE>l~b-!3}>)z+it$Xp=@@d`A2_EZy{m3q3&@GXM;|Dh%j|o=y;+I=89C@t!rNZII zx?egnesMXqve9_5nXG1`H7<|w?eSG$F&Yt494#nzGJKu881f1 zSUDI!*8M7x@$4FJ)nNQs_p3$5kMUNIj34WMjmUU*t@|~Dt+jJoE4W5uh@IyJeZQyscKClzJJ+73_uu2-_?TyZr=kCU z{Z5AMDzQ=T`hKVQ-C)0SIWzIutQeWi%IV2|k3`nj=lj8aS99j%vt(p8OQ(M{ySG)^ F{{}lfDP{lw diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n64_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_n64_h.spv deleted file mode 100644 index ca2e6f78b253b3a32a5190dcfd635b523b753013..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 33428 zcmbW=dED>SxIXZw-EJOsb1J0?IXZ+g)M->Ijk{DTM{*jJl2RxkY1rnuGoWKSr3^(l ziAs}l%m%Q(`t#z+^eZHUX@a&eJx7l(t zOD{FE(#$F|uU&m+wyi!hZ)T|=*A=Gg_S1D?Z16u=Yw`8Gb!TQ)oLPS6!2J(CVy`_8 zIN~2pI_ddGpK|1>$G!0ABThW>)KiXo@&DGp@U#=2d*t(uIPvHcPderFBThW&sH0EF z*vrn2xBK1)?)|72oO;xww%%qd6Ipd;X_wjmF33!l37(&R;*rNa5C7zOGc$Xn@0I?v z^wkoF-ZIWgw+4OpMEFpyF`3yTQDAMe#g`=b z7Q9L9xfz>0j-^iIWP?esX2^18-s(r=3)4MfR$h8$??I===+>WXAC*nF2752gqXU@l z5eC+}hRWj_9uu3c0k#JI=0V<6fHd+`BaivoD{{wzwT@MJ9BZH0bS&6dOQ(xX9>;oX z_@@P@E%84+!oBminU&@1ob=k07o}_b z$?r^;hmCTFa1S-c|4)~{569R2Oq}f|`Lc~4g2M4 zo;YhHyD#GC51BAu@?nw9muwE~P7AkjuEufC>^Sa;WPa!7$`)rFvU?z|aW0O0)*xRJ z*=vXMbycu?zVq^_V6yX3-%m$oEDu))v%wkrnq1l9jZMCG;>7!0uz2c5Jop>w+Ll?Z zShr3bzFq8ZMHZV)=klZAu?Fssyi;tdUtpkYnz^nP%oiu#>bZ)CH&^l+6KAeF2g{lG z?01PA_gS7j*=q@oe%A@}p(lH7p|9-qq%wc6A!PGae@8?fbA4pw)~vW^N47W5S$(;g zR|cz3Il$}gP1br&Fn#57gK@pDofj-Fe#_XieRaBfmbJWQ(&Ia~tAd@;%GXAAudaM^ zWHnUz3z5f~`)1^aOnUqSk(bZht!1wySuKofah~Z;_`m3mAL86*gOW7s{&ujO;H=|2 zxymQr9+1B~apq|rw@0=K?C+R#J@@o?h1(+!N6uVd{k#`#GpBveIv-%SvkNG_@ z^1BB8Ns;du{GT8BeS`kw$h!{uQzG9L`SBS=Ub3v&W?TOd#@Mqt$XeEmo%NBe^}gUy zzI%|Jk@M55_bhPx9C^bGLceoHao%@Je{#Co-9BT;_fsam@}8017b<(EbYHkRG1%`H z%*LLaH*p6{ob%kV4hrY~F^+X`Wc)dk-yy*``&_?6Bl8o#b2uzAzJ7-Xu&v?84iE_G~TmG`AX!H2N zrg3ni&2E#;MnfFjXtVcV(>S=%=J3hJ{j>AIjW*97Y#Im0=2g@Fy(L(^%42Kqf`L10 zvU^WBwMEzZcz>|^u=lNxOM>x@b!jkPd~5CUU_9>ZDgH--+0ixr$AZOQl-~Fs4~M^e zs-aH=^TYGoG`9a9?A$jmWOj}FsbF!vzBTTrgYli?)xqY7KQrgrYxOn3_{RB6usB{9 z8|S)Ue0{DD##yHrHw24=7sqD8b-rYF9p{!{vumwfzF_g5cy?_aToDdG*1;zt zNu1mphKUPQJRjeBEcc8z;; zusM!-z9llgala6Z8*}!>$a2=WWOmKjH-pu&`)_mh?Qr-pXWxsAA9Hs5WJlMW{b1t8 zoc%DIoNbhOxW|#%b!|Ti7JJOuk0ax|wx0###+>~;vYa(8nO)=lGT0o)oc$^?zH#pj z#*I1qZDcuXTr#`n?15l8^L)~rEftI(b2dL5e$3f2lO0`iw*176Ia?u|oVm}+8JS(z zwsNr8W6oBIjPKf355|o-TO+cZH7=Q5 zlCR66_g=pDfLp`io7ZmzO`$VScDDwT z)&9zPdcu7(azf2KES)`@uKl)P`sRAPFzzGMSSth@3vW*v*HzxlXQg1{ z${BlOcCM=gTZ6e?JjGl!G9NngUprUL{OJ~?j%64bzkLVgK&6b*2lGD zZWt`4d9W4l{TYYvLxRnz`Pw*`J>I;w%vCclx=qq?<^?xjn+DI_@6E#Do3G7-apGQ* ztM;d0XY4hR#dN)5+Pl{U^EoGaK5q!t%!Tgt={SC{^@~f!lixVWuJYl${6juOQDD5r{>VkI7lZ$7X!yf4?<@PB*yh3RbI zm&Ml{-x+Kk*1(6Jo%%2jKPTprncTw5a_4d&At`B<=K z59qE)$MJ*BTU;`p{P9V4?K+;Dxi1x&%?ZJu&$TsjWib6%BcGZ$b7_ryI+!h;8u>!5 z;&guWSEu962e$6dW=P`e9qv$c3!@8-Z--F1l-@`^|85XYQuBUqv+F(39D20*sX%+-eVsV4sT8MaWzll zJuX;Gb7w1F@2T4a)0>ld>>R9_2i+6WcS<)8*jeyQ_M~8I?D^R?^4#;YT{wK7fBr6* zAHH#R2$n~$^WyFpnH}DG%*WW~Fv@uAK*dv_hx+`Ma zarX?zmd&5K2O~%n9kY{i+=Cm+w&UcboNkgL{w?i+bw zx;4Q!PIKBn*fXyf*2#|z+fl}M-DhWA{KT+{F4;qZOVd`>W5{Mpl-PMGBJ+5W_EN#X3XJ$p9Y zyOV?Iy&vd&UJx1YnZNsVYGmJGElM2v?$c?(bglK%gU!oXa2D0h`Kf!nImuzidCP*u z^Y z8#$KGbt7xd=kk%)NjDeRTI6%3VEMHF&DF}`@U>Ym*{FxwtQHR6oU9(qPb;^^#7R%N zHN)}M%B?kV?3&ND!-**dA2#x5UDjYNql|Ap4@rLbTc>=oZ9dlvmQVSTPcr+t`Fvp2 z#r5tU(R%+&IJ`5^dY_lGh#%{HemM1Re_HR$gu}Pq*9_L|58ZO<%cfhKtF^Hq)5(?n zS|Qju*18ufhQrrp<;f<~Tw=3IIDB)yYA`>o+=7YYS8la%e6@0`PaM1M*Bar(6oU_& zaShhe{aGuRy>;0i>+Sxm9c+K()&7v#&)uJO!m(++Zyu~QpAU+>Zo2%rnhP5;owdm4 z`oZ!!*8c|K@U_`+vKi}tqj31<9TDhGj zj$gT*!|~P1?J{xfTJKK|C#D#D*o z4+zJ1v^#L(#(X|4oS0(pVKc75TAI&;gV|e`d|Gewc}TE)%9nhS+0V`Av%;x&-x;*t zj|_(&>;0(6__5xPiLBnQ%c6RJKQq`;&5zQ@8<<;&F2}BPfwRWS94)Qrn46HerB+Ij`jZ1 zaQNE1e6ktq{j6~K=HwN@{IqgsPu!T#SBB#|+MP3TV?NIfC#D#D*o*?Y*K|3Bo3AGYd;g`i z{!fDOjUXc$?H!f`6>T%y-xf||=7eAoZ9V18P;gC=gw-NE7bj&_Gk+?czkhZ9o_K5XR5x~#!kMj7AS9hhA3 zw@$fZ+uS`PSnm3~O=jQRwLZU=XK3}gQ6}2@{CYTi>+^@fnjF*Jn*NP+^K&&nHe|Zy z^IO3eCbrgEye%BQHs6_SlrOt}4GH@=Tt zcH;bA(wvu%Ebp%7>^^b-@{7-db^JMar9t*4NMC z76jA#t%2_^R-E)PpRHf|71HtY2KPR)MlfHmbA6A#_P}+Fb;IF};e4zgtT`Wa>!stw zhr6FI&Ham?_d1RL@^E;cg*E=Uk?}qs?B|KEimZkomQFYR*5Ngg>933Kyy>;#4UzHV z?-<@189)Ax;cb&0U48#~vKxP+aA7$7_#1_Po$Tn;;(K$oX4lr@*QdRbhvo94*X7fz z`|6-uandyh7bOOLe~+-zq?ZGEroT8HZ%o*^^jo+~g740izt1>*&v%YywLVT^NmIDF%LG*})Q=Zau9_6C6;>mbg*U}be~AaSqE(V z%cH+C__AE*Wzp85_CFa8-|?>s#(g9{n?(PqU}NF!N#nZ8JDaP6jVovDjoG+cc1 z8f;F@*MA1H$D7yfxoYM`_qBAKdBM%s*MsNo_cy}fo3C4gapE4H+SmGB^Y+tOaP4=^ zI|lPPCw6>xo_x%OZl`pdxxm&hE*VeWWs+Ux!+F^(&td9}&E}rra_t`O7EG^(T335a zoVi>y)zzNi*wW4Z<|tQjH1nf>YWkt+<^x;9!MWCk|32yar?Y|kUGsjy=3xzd=-Ig+ zTf+gtzH2ro-$mgL3|GF~LE-RoV~N>($bs>U&)zu3wI1WTj=8iZa&zt4SPz|YXFrU| zwl#8F<~sLGeLEb!HS(QcoLrq1U#&Ir?8xTh+8Q}3m``iun90W;&>fwQ;|H6!xMV!} z*hzNnIv$z1JIidg41Q^@t&!t{>Bky5apKISHS)Z0Z0Xd<%W@S*Ge7#1((&d4TlY(H ztquQE(oaig1Gh#_4K@#J;6u-@HS(fhHDXS!hZlz{U+#=>__?vfY(C_`c*bXM9OGJ# zab3q;S`)dsc5SS`opNVCj9J^>S6pkmL-h8+{fyrj=Sh*BV>)$5*WV+&D&G;hU->

SAhVIr`Pp~PnUc>;d1mQ!Yl3e~3~QJd?D<#>>*U9V?I`2B?p?Aj ze&Sd++kPMInb0#L8~&aT#dGbNR}40u`xBe@W`1lu`_<;5!M^{0RdmhE!@~9ZbNc4x z;lXsRtw#p?{$K6&`~O#`&hXZ1&K>8FB;oA~{`WXf57%+%JI=Dp+n>E0umk9=3>y`%S%boQ@0^YY%&dr3OE9(CS3dM`;QKcmijNAD%+tZmeJ z?|9dwv&K>9y`%S%{H%4oHN6TL)SPj4aWbqIE$ha$KO*n&a1o{~U~;8^<~Ex5w{F41b^Z+|`$zmERrNJ{X_Q zy^kJWpNkUD-|N-q;>diAMaPF8U!P9~`}@86Tou_q8HbJ!J-$9S2m5=z`rHzkk8$Yu z(Btd#z2MyjpW7qbN8`}(p~u(f=fVEoujAYunU8Vk_|W6)vsC_G+28-wXI^CcY#cg1 z^!WNL2!8h9v)bfi96CPq_}2X9!D`HV`_{`t!>KX4*4V>>@pEgeaUK($IR4hRaUL5^ z9JGhbI|-86P-9?evS<%4qf9sCm26BPWSlq=*IJOMr3+_D_@jSgVvb{479UpppeJ%|i&(jAY^Dzz`A9{R!t_>c~ z(`O>vTjS92p~u(fi^1c0`ch;*#-ZawkFU=UgU9o9XJmVC96CPq`1;%vJf5dtN9JQ3 zIzIGx&ut&b{%W4veD-*0I?nSE{MqUC>O;X^!#vOQb?vQrEyRznfBzB=?{|2uqjyHe zkKbdzJF;g0@%o+1d&1#8FPYoNf;Dra`?vHf(#;L_%#z#0%=?4A_G_NEKbP}zNjQ9M zJ~-J#wZ!H_;qYwyF7Ly^{IqhHO&q^+mxtr4mHWuVvFmq&9}Op_7<|}xR>Z>4quzqCmXq|%^KnG z&F`AQ{IqgwO&q^+Ylq{jm0M@x*mZB#4JW1;eAtX@u$Jz{dco|i%U)P-_hS8Edm*p( zg3NyIUThGKO>6rR!CLdVVdMv=%b%;cup!e~i+nyLSU$&E-#8q;Hk(d1W36u%4&R(? z9?VZG_t1$O^ZBrFd`G*7Pu!T#Ey9T@1|K%#8my)H{F`9*)+L|T+k8GUSU%-TKFPE8 znZS6jcw%_9y;a^{wzjtohaYSE?;@*h&laui?Ze?)+j|9T_JVH5^c~XW-qkwTkm;J| zorAp>(^|K?gu~Zn*U3h%YO`B7eDnU4V18P;-6xJ;xjn-1)ynNTaqL>#PYoxg7<|}_ zYp|B?#ooc}t;=3mZ}(!KV0$62_JYiQZf)-yj!kR(8NpifxqsyS(&f+9T-cE5tVL}f z5Gv_o3XYJ4u@|}4hiO`m3#Wcjrlw@9N*FIu!$S$tRip+abrGTA5Kg$_^=t*U@gt(8-v+fmwZ}p^ZBM=`IImDB(rZm`wafg)U{{u@w27x zg~N}};CDpEkM(|6WS`O5i`M&(!r@!*zYo^z58Z#I|1RBrz|Hl~gMDtNwRZ0ghp)}A zCL8&x%{}4p&Hb-~`Dx{TGjaUN-5ZXtR_?bG$FB8#UpO(v;KRm#SeG?e%P8Z!KP%)J zkiT`>AGY0}{|@#(SYGW9nSJ-C_nAM1XVZFLdCF1qd4F^(g)8(=5y6x`IImDB(vWl^BJF0F3cHF+keTmIeu3#eyr{HL{{5;Tifpqhi`3v zBv`W-bpMwAzI5wwwGK99y5{}jV4p*4t>gEH!`J50$wuyK^MP>q=J$ia{Iqf(nz(Uq zJ{*qkXm{DfjeB!>I5EZG!)9EAwRA5&8qD6h?1lAqFRlo-7xHQ^$n59t#mB<2X>DI0 ztTmsXi2U(%`ExZFHe|Zy>dIjG9BciP;qbNj)MPW(`lrL;o4c!n`Dx{@nYb~Z*M{Re z+I?o?#(Z8EPE0ZQuo>52EzRc*!R)O|KCQR;yfIilY_HT>SB|`SSeTx zd&-`_F;>Y{GX|TL({aXtyN4SF+e7o-J~iQM>WzZ&TfymF+kPu9wsS`6zkfLXboE~t zjGya&SUBT|S^vYs@u#c*GlKDR{f`dEU+vWYm~i~*>VIr7ey;!X!}0HT0C&XiVHZwey;yZ!twX-sCN7_!||u9|4W1MbN$Z?$Nz-E|NLFU2I7(dtlP2u<-nL6tD zZw|+wuKsTc#@AmxY#u*NJscbV%_8H}1MGgVO|Fkm_wVI8!)nj{KutVh(!C;QpS*2k zb%Jkwydz`sIX?QvduO=Dqi?(mgXtRY-N6gf@r`$Buy~8>W%2wz5U%m)8}Ea`bd7ge z@H5i!jdx|Rc#G?2alB84YdreKyDFHj@vaV5SNO*JY_NEX>uGVko5M98edFB{OxJjy z4_0UR#`}7(c#G?6alCJYYdreKyET}u@oo!NclgHpVX%0M>uqtoJHs^|edFB~OxJin z308;r#=9q2yv6mmINqD z=Zw(xHDs4yuLJJq&FQY;@bcQ{liecY{YJ^z-6L3Yc2CIJ-90kS*@gStoc)6LNq2U| zuor$ivVSmLpBs+|#%X;He^zAt3*zJ7wmvc)$H(3rHF12pr^kk~Ub@tjI7S+YPa!ZH_l&Ufd?cnIyY#xri8mn&K@W!Dt=BV2( zy!Fvp>!{l?yd2TV*Qk49czZ)96#9KlQ<(A2Fvvp>BiqKS2;E& zTRt6Q<6vWWZ8OFrB5TH=+aw*w4?cG~hnoeT7CZ6FJv1EI|2O2yX$@P4!<&QO-4#D_+ z!gq|FBICvA7&`~!7Y=q$j*Ms5c)JGU4;t*A5*a_n+dVSg|A(dH?im@+?&Vpz{OlcU zt+m-VxLk9!UpTq)nonQb{e#7%@B9`9%N1Si4h+WIi>~+K!R+X2w?(kMrWdlq@h)LhNe@0{+8@P4< z%;46&GugTqpDmx({gJ_A-5(!WGsdhgGLGM~b8SA33Rd^xmpdjLd93@>!r{lde<9!W zxpn{IaIJejjeAD0x~K0Ry(HKk(Y40UoH)92FAG-pxYohTgYjeCzalbTjE-?OoW3yr zoy$3q@$4G!++h4z_vc5(kMR~o#*cOX>d1I@t^3ynTWiOBLvXp){Tsuz?&)j$reHDY zJHIyv%N1Si-WrUz7hUgjg4xm4?zrHi(&^NBYe-FuGQM^HC{ z&u;(Ri~JVm4UwG%yyp*}<-9Q%ulA44YmL^k@s!AF(v|+HlfGwTkH|PSaL>k`!95#Z z%X&7%XUnH&W3S-xY#b9=GX~w>={SCHbGA>gvmt)DeZ!H*v$0<|{CGC@kBpytHWr5K z+2GT-2L$h$vFN*x2L{_mx}JllO&ndhgMyt6Tx;dvVElMC4vCBxqhmZh7{4(7oy(z- z@$4G!uweXnHV%)BALBhEGJZT8e;*mou4m(!!PeR_j|eW;v+=BOJsb43Ju+BK`p)m! z!E!}cyMGAA+l#LEs9<(MXqcei-5nXHiB@;(i?#y6y zk82&gG#EeD{mUZb#poC>55_Nyf9G;mWIVgZdqpsQtoySg;C-6t@~O1^qjXSyqNTz->ZV)E(4vYK?If6t`v z*?3oE92>Z23a6PtV4CgU7RRO=Qg&bni>Y@q?SQe+zat#4mSIIP!Ql zE)Itu&&K;BR@|K=Pa~_)Wj&`dp153tOnI2 ze|4iS`1EXC7tBsw|2S8zb$?T2_2x?dxk=x;zd16F4cxlFCAf9(vx3&W_-y&K?*Air ztot8D){H^-`E(pVxcT@(u(}t&+!w=<$GZPgIQ&@mUyh8QTlZfH*ShD^xL*xc_w?PP z{|vTAbgl8PO&ndhuLrApTqhowC7{4(7oy)f(8 z2D789-Ce=I5u$4#(xL*Z1{|=XG46pe0nzi6g-}db*4EOgYM7i zIDT+*c7L$5A%3|B!jZ?b@t1J;@oZ$GGx)h@W2x{x8+;mfUU+ANzWcazuzjTKIha3j zbmf)_b~bRWm1TqRcL{tcYbRG%N1Si)(pnmi>`OAV0Lu1TRYfZ z(>V*RAvH0|_@0di(Zxqi@>e(Nf=|!Jdco|}^$xjeeNEUfvU+o+-(=Fa?jI5v#|CcQ zZyemZ_qlWHUVOHETKAgVEn@P zcP?8+#z=;0PYf25 zzVmxhuw2pAZrfnIz36(k3uZ@GyT1#z*L3Q!$gcJA+ofv;pZ?a#Z%|D!;lIclvYFs zI;btRYCFj0xV$J!CDKWvETtSOmG<-g&GVUk_V;?Vf9&&eyFb_Gy6)?`?(4pv`};hP zpJ^*^vD(D6l_u7hSaag=)h8yebtYDxSSd)4((gL-`>wNG{@HrV`%^ZUn0Va8jEVjB z*?;aHb6!684T~1NdHxaej#_Zc{J9I~9d*Ql@pgIjyV z$r+b&t?;+%z->6--ko;Gp+NN1nXHLg8Je)6R4^oj3I zTRtayf#2^%?wnv(=Tvz-r(0uF-|%L!H(p-k@tl4cS^bvlr%n7Sa$~`+#;UxOI5lkD<3JQ5pvzG1PJ*J!ysj}%LXUN`-$w3R0N>67=K`wPE8^w!E4@t-z~w=dXl9N9g{ z1~)76Ch3jGua^uq9r&jVJeeOkJvYr&dFM%K@#U3wSuVe@@@|n=`PwI6PH!*&H2|YF7V81m{{Y{+I|4PJh;< ztUgQ9)PX&DS(>ZzIl=YE|3zu)xM@y|i__!&KTVxKo3YL}es&q;&kY>?zYLiF=5tA~ zeL=>5Bh6L$@?hs^?XL{BmX&WB^w#~C!RBn;t5*!`T6tzDl7M zAv+Us_4Dz_CSxw22<8W8{DtYQE8ckWMFZFI7e`i$?USLgUyrQ*K$<6J72>jQdbDO@hbDPZGns1d&Jw6Y% ziQIi6?wrVe6AxKuwTYJo+Y@5gUv%y>b=*CezVaTyII_9y8O$$!`^aqfNpo({?>p%6 z&24e8`=au)$j-;gr$_b-Q2Bk4$LoAvWM>sU{`|XJjtU)#)i(=ebHSc|1>R zs|H(-bmp)1z7gZh=;!3&qA@RxyjFN~ywTWTcJ^?8Gu_`;^=Q1 zu(jaF@1cD8@q0&RcW-)QaQu+{zSYnD12&c)zjgKFw~Wls!|9E|@k91ohU+|6F|pKQ z`|J?=+;169Z8U!J#67tOoHyhx2ibX2KQGCG?-PCH*^xb~RGt%gyOI3T$bJjlUk3&| zi=?t$<^wMb_w3xQNo|r(7tO@PAHukY%|{*^BnSJGEYAmn$MU@+*$oSg)DcD(HZ5r>-k?|)Fz8?$58QXDx4K{AG=@4^fWOj5jvoh+nX0SB2PNQdEAL|ab@~MyY*r(Ao z#|?w=e0A)Tg7JQjP8;GsC35Xrhgsp|L)WpJ1>=o1XTR-RMD}~{H@$1IWjOp5ga56A zaa*NzKWrO}cV?H{KA4^QU7WG*fgK{_ty%4!G1xhuYxm4>__5uy2RpT^-E+d>+3lA5 z-P!%}H1`BfdqM6poHM>QJLF!hjr`c)#x~CzZ0PF;H@11v$fka9Y`Pw=33h+UyKDKn z5!W?3B%Jl6>v|j>Y(3Pv>#-;pUtezyHWuHtJti2B>z+C`m>pgHza^OeWoh+)LOA>v zhBZ4e*f_j#e#hP#?3`*{$n5HONie^Q)7S6PV0?2tDOis96LSuE1~@qwUq7b=Pv)1t zeohU>cg#D2apuX#X~F#9`SITW&S2+e^Ch!uoHK&OIX!*joEeO7zV8WELwsYuHyB?( z?+@n3?|=QA6^!qg4+P_y?+1hV!8hNtgPr@$m&~qlJ{&C0Icbe^Za94R(?^1Fo;|u( zE(peZ{^<_#_k7m8_lzv> zE7O|yUg7Y~aqnPb@%6P|Fdo-@UJ=ZWuKxEA=HEL-{l6+4zW!ew%oo1?4+zHN>i@uC zc66=PykI$~)gLETo;WDj@3;Qp;jG^&KgQP0Z~8UiuN~>0G16T(=vwpnk?Ga8f6yNm%m!aShX>ntV?T?+;p^w0 zg4L#ejtFLh|9pHq=Z^~J-?{SpoUJDWyUc^`=ro*pz+Jy%gRP%U)%80r9KP|755`?G z#Cl7xSa@qvzj|vTpA&<{RWtTtHrKZWn}b|GIrw~gWMk;$e@c3n{OJ~_;p7iD|E0n5 zpE=6^q;UA=e{wLc`7a9=A20v<)i?iBgT*x`_F~HG^z<%yJtz0yX_0aAg5~ABepj&b zy6dVY?+%9-vt#s)c}6gw@?gu~rxM55_XNwS^?GkGd%V2POYf2w-TTsT@`78h_Xkg1 z@3X?;TdxlU+9?Cp?20oOt#%49||}1p8Dr-`0kN&f^ll~!i;rwk8BoM zKKkyFErN~d9@%ObV-4uGOv4!m%bQ;^p1k!S>pPFDle>GF%{9T#Pv1TA)L{B$$*p^2 z`+<{7_s9<6*wWb}yQb&IB|rM7r{U!ToA=J?Yh(N~(soK?19y-7U9ddN!5DgW-6PKq zwnyaD{qUS{<;y)U9Db@VK3fkp5Knye;)rWL;_An>G$(Rvt*@_ZhuT>WF>Bj%MPJ(= zL~k9OXU2{F+!EP+OlRNGeLC+B?$6id+1+`yW6nzFo4S5Hy-V$19DIHnPX2Ju*pCKJ zJ!3x>4sT8!qnD?6p9tnt?riz%nR-z$y`1Fn#bB2_=suNpNt!%h_kwrL&jg!e@1Kh! zPrZLW8xG&+pU(vwhp(T{2dm@aoD=+hAu>C>`N&6XIgDkz`5ALt#v3O;=QZ2L{7Nvr zXR0fP7+(#3MVjZU#{F7kwshX#e4qV#uy3}@Vn^>;!MDwC1k;)OtE0ax_`)3Pe)qiN zk=dx{mFZor=jD;FNHZt+sv)Ou1$!pr!#s^+!*(p=JMTMUYaBo3&GzcS-!;LW5!o2; z`H(+-=X_nTcuO*t%|U~WXTRFq9PAzbb!lDeTfDt{HOut)N^SLV+KPRp0^o!t=45Beyqwg~*p1VFDf6|a`zCj5zIEz^b2Y!0<(^g--^ZKZzTxoRQ?4E6{mRJJLX3V> z*gqUz%zd(teaoAhh95V1ivRjSFAv|}t%dLH2gcs_dU0P6j?C6KMBh4^*MjiA!N|Ay zFARqt=f5a2UjF-~7HYI0{^VZyO_Al@8Xg(@aShoycMgr6Gp_Psk$s=+eCGulH*THp zLDA7$!|{AiOf2)2r}>iEcD_GNZ1dKx9?kVXf_*!jGvclrINxAflN-YM?&#gMwYxD` z?dWRv?O?pxxp!^~c7N%6f6mQX*ME{>@c=Lce|J#F&wFm0&j&OMX zoO3@(@8XN@$7wje;MVQDte0`VQ`P_Z;qdlc{a+Xv@80isfKNwGxhKyGy4lgy@5Pbn zeFx|}_~#?z_Zr##Yh*mG=hCHv9bLzMb+CKYXzVw`;c*>%`Cvykc}}M{=PT29AM8G? zi#lwxYW8Ycb+bph?FL#yitIgI^yG z-{;vIg3ZS|(BBx$246ql34U;jpPRzr>*wZRb*P_Pg4y8v4Enub{+Gp<{c=aJ%RK15 zpN2CJxW6?&2%h@xc3U`nof2VgT=yIlls-GyR+#>!Q!eJdoi2qPlC;1S?v1T z@zco0(8>Rv^e*|+{VWY9f4KSoJb3DJ(Ou#2&Hop{xaNO%u=sfS*RQ_$|1wxyb7C*1 zynda&d3mqCH!@CMu)Msh-xusTvin0#?hl6-vt#s)`I}%q<-wM}zN`N>SWd0i?}FLm z<@H#4mwM4Xn1+)V+qW9AKG zEXloPpRrjQd|djj;rw9wWwGnNI%MGF()X|h;n>o- z$B$3Xk4t{^Z%V_<2R4Ue($~iLg=ue2V*~d+Y*Da0%)uCXcFs+6I3oC*^m1~h;f@Sf zzTDB_@Kb&9*?Opfc;d4cM_ls}S3j<$IgwjyeSMuY)XsW{$+mmskI8lFJ@sffeD}zo zf^lkfLB_heN0vsGkG^~4 z>AOeX8BD({xpj}6K5%mB9y!Csld*tk3dqhs%4<8CwzT7$C@Kb&9*?Opfc;d4cM_ls}S3j<$IgwjyeSN)a zsGao?v$j1~^tC-Vdh6gkGj8nX?8xq8I{S{!bEq@%4S9ZZUU`mnzNzc<=v`{>IVGl( zoc!UQu`35pJ!7YZ!|XF( zxJIx!_Wqd}dFuVMW;lGGryd_{9KL?m4pv9M^Zc$8nH}DI+OQYb3U>0e96YMUv2(A*t@ak;MV0O;reb&-@42Wrt99?BiOsKz1R1;gR{@@ z<|^mD8$Xcxso}n~X9@0hic=Xkx>j;vPV&~?4>9kWUBc)d3r z#)w1L^~O*AZsRi4y;gu^?x`n_p>;^4=v z-<#eT9lbNf_ojYJ!*3fO*1Pe(8z0WIalG$^!;j?tj^6p!cqgus_kdA6 zXKClFpIRe&-*D7J-@WDAkMA}144ZzN@@`+5}&)}V7FLyT6^E>dC==vSvt-KntVEV>c5}Cen=ohEq#ex0K z?VFt65$tcRE&ZRKOULhGD0_Jzp!twwoY64|`zdM|xBnBApmo!?i2&5^D-e?8cEe;?*-xGcE&(tRThCttYZ zz7@=${|=#6wb84UXYMP4`7lrR#*1-P`o>^$Wg1QlxO2TO*c`kgbq?1@wx8%a*BgS_ z*)#U7zmNWQn!U%Ke%J7xlfN_Q`i$ZkJV(jIP2uU}cT0MgHQOO|xj8aUey|w+cKY_T zThr`kwQz6n^}|88B|_IKL9PV=|ha^^$M)6@SZjqb={4i5}m=kUA8a>ScMed{}i7e&v; zyx5B?w?|DdSHBb76IovR&!k57%CCaAP2SFaW9}cum=E2(X*lCx->%b5+?Ri4nJ3Q@ z@%?`Q@)MJ-F^%z?U@@G7V*DYpOANXP(s0JX%ZBy)U9kV}hyQX9g(Lg_fBtQl!?ehF zIru#aoR}UCzjN>|X)_0Y)o}98l1`4P-x=X|3rFADuO4hH-8+Vy9yf4w@Y83O3i;Y!Y0qwc0eCTG<=)wS7u3pY+XdR$_XV+k^I`@%D{o>(4WT-^2RrGscqG7~{D_PWq0So-uxZ RJLcHI#`DTwvSQCo`)?r|6bJwS diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n32_h.spv deleted file mode 100644 index c99e99fbc550d81861a6f5af66a33e9bd9eaeaa6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17656 zcmbWfoMppJZn#xCZkvXaD?7!lxvkc5+o#_(v!0#; z$j=;hP0jX7Uyv7Fqp9^rL%^N3K|~t74VB=?LA=?aA0w zELbeV#3Hw`mPKx3Jr}uR!LIRX_EpK7_Sum-ybv3DzG1OOYBVCZvBpNBrtH^F6mf=< znK1BrL2{Xxwr;%G*RYGccKV5FEA;y@{d>>-gUVXcXxX0{#dX%TKCeA%H*wM$*V_prn6r^c&M}ADp|m<54Ul!bJQN1 z5bT^Jo5MTO%M)jgWalJ~e%}roOMY);`I6`cU! zpGzZ8O!3TRVK6^^F7l$t?gzZ_wpHZX zC*tfDx!q^GNA`^Tx3$Lh_6W8|*epp;=RQ-{>B00R?-`6Eo69?b#ldeAneEwaf*SV%!jUjr8X9-@$5)w}0a?nB(wWjdNw%gO!f@K(Lc~( zwdbe*JPgN==N*~d6P+E7AF}6L`FX0t=E_h1Sr^QYXBnBF=hGX5~7hTA4Pz&`fu zvbWVjGj6jiur+qRkT>sS=SS7&vpJ;Zq=qGbI1BjZ*q8i~$RB9r*^y^7@=1}s%ar}e zk$>3uJ0-dtNjO;up+54OG zpnSX->^x|Fyc8Kk>F>uD#xD~<~*ZLk64nLb-OyAM9r#UF}Xc2$Q-;p9VCV>b=Pd&ZfwXa44qy$5*K z*IH~54u4zce{wKx%e18(HzgSFEH7@`V0P+vMeK&sZxDY37rkDwK=r2p)WtU*5;VTru^X8 z)Ozd}?7oqAt>vtSt2H_-ob{xu^*AQjdZ=}+$Fafq@^xIWvG`is*}-^R?WvQ3+0m8% zQ-k^Uu37$13x~h9TeH)Hjl&x^DYkQhoj+97?9T_|%g=ei{G6M<{QP?`zQ%kZ7+3k8AIuNF^8I44^StsUv#Y)GcwS1Z ziF>#9!5_lm+kNm%WPH01p6%@DYK@n5T)W24g|o)BugUDH9xn#--LBtDk?}RRzXs#l zwRk16wJ5)2cI9_e?jJd}bsilKUw+31zklm&~qe_Ud3Y^SrN`ts9JQYqmjT zd|R{eogH1(Y@?2AYqoJXHFK6ZcgXB&ZWDv~ZfiCa8DDeTJQ&y3Y>UWhR({Fs%I{Xe za%^ihB{IJJZX1kiYqnivH7mblc2%<-gVoGCQPu39g7Iz5-V_<%*6hul9bMJzpF6It z*;~V@*=~uWW@L6Xw_StzZfiCzGQQ@vM=-9f+4RV2R({Fs%I}O|Ikq+1D>A4%rjqc7)S2aI4GQDT`bG>&kVx1f;7T%ha zU%fSv&#A%Usu_DRE7#M5%|WhntUo8 zUd$S!ub7_>=2ISQ`MWG}j6Ek; zFBr$~hHk1{e-3t!y&RrT{o-!zUI{j4IC^7N=*Gx}E@pksIN1F8C9@@u>SVooxL@we zy=9-Vxj%S=^tFa72GiR^wXeo?oLm-mxxFeJTRQjn`1Jg^Zzm zZOt?`@M#$<$2EfGVGhR7vvbbTzdG16Lr%^)+}h!aFK*p%_(5NMRz1`}Jn`9!Bd+;~ zt8Z&*PUNb!zI;8<)y{f|$+q^$`N?(gp88@qeC?47f^ll~=8SdK9+?zbKKj}tn*|#) z96MvS=*CzBy3NyY#=-LDmy9QG*~$8v$3w~8z0Bs&d*mO2?GZWEe%K*g z@x}dPIQ*b5KC2#TAfEW_#Szzh#MQU8G$(S^T3^0?)78#;h*`FEuIS75PtjWk=b3S> zpQj?bkLm0?y36ys-nn*R-m#olzWY1h)b%UrU25;W=;Ab-{NXxdzZ5)p#(p^*-kfTT zUY_Dz8qBBM+45Is>Y`wJImu&juuC3vm#2L#O&+j&!F$>j!RA=^&y|q}@1Lu};p_e9 z>R{vW<>%U9b@ZI)_qxdJ@a7{QvE|Upc=I!6S;iYDKj$^uig_cu=)F_j*2TCvcxKw- z*jC(cL}p9p+loI^`DU;`XK^pld)D^<%p#c1+}|7hEy0U&%zN&6?fAQ$e+IEd}pwD z!x_uwz{qU8`<2a)gMD9{l~!y0vvAgWIIZ44?hA*n_ssi)jm4kQ<@AeAHqUyu|7Ez8 zu>Wq)o=vUYZ-VLHl2-ZrHW>f*v|6V}g6DPXL|^OlyI{K7>yHP^%e~-Uw09O{-{a+^ zhQ|6c+v_u5Yr0o*x8_q)C!DLetvgQs#ck7ZOpAMM$NBbMbDNR#Ku!I5Sk3L7;qZRr zEbVIcuE^F=e)X+z-*9;O&d5FB+xx|7__mp&+%M_$=HSoUtfN0)n-%-Z(u+Ga9GUIp zjPX0V^1345Fy!j@T;+ddIDDJ`Rgv*+{?|m7KYx|~wc+q>{$EO5eA_DjFGokO-fjLj z#9sd5%Ad@3F#m6dm%r~NmH(1(_%{FBBIDcqZ;veh_FL^8;qYz#Ux{yg+baJ{qocPs z+Wfy4d-;nie=^&_{O<`bf8Q=D|9ivX+x&kT8QO>g{~GD|wt0d{3i0F|u`~JE76J-;aqb zce*rYUC_!|M^8XExPi1Wn}s#(bspOYa`?LZ0xR&jK|fp z^6Q-)U5&lDvwL?l_SSHCT#fy9XGhn6cSvu}x2LbYwQsjB>M%Kf7`|0n>Go-KTX(vu z!P5AkukT0Obb2*VXZr7^;l+eKZ~WQCoxx9}H{NsB@4~yn;p;cn_k+#HI?(?hm<_)C z{4m(}qt?%l!r{x$kAu~r{QM-C4ZeOm-y6*T{P?n8eje;H54xYG;miZB_n-TM2fruW z9}Zvf9|*=>mw61u=Yzpw;jKye)vG(3Uj&P*X6(hRTz?g84sxycnTH}9Lnr@7)4SwP z_vk!Fo(+~$)$5PJ?D6t?CA~|%=>C+3 zlNVg|S{6LG-p_@@SG}GO#_@Yf_P(pW4ZOd*7xeXQ-~++N49Cux1G+JCq1!(VCl}cK z`6c7Y2X?YvJ=`xB=H9Z;*enR1oxaxa;9z=tsP@$%9VeIiHgIS-wsg+rlhX6!k{|tH zX?Xd-=J4V4Wn=u2X~(3of$Q7A(ZTXC2V>~jIXBJWIQsN*a;D*q4_AC~Cx*ih`r@!Pl9)DxVW zcE`!3_Q4t3C3C zV0%PPwI9w8SA207gu@T|;6=l&NJg$KZ_%~kLm0?I`5~>#INR^(0S$k*7>HcD@N~9d+#YLM8?Sqs3TyDyE zIAhs(zhvXxuWa5H?0b#(;Ht~6;p%%0ebr@JFkS7fJ%W9&vG?lt-WAzrcypC=eXn^o z^;5&WQV+j@woP6*S8>~QTuQ#2dtJvdExW(#INxi`t9~=+t9E{m`F-X$iJZ*2bbgQd zeMYB-rIVN6V}75}nQyD}d(7`MI{CM{gCqNWMrR(a&hIh5&*;pj)%iX4zD{ReboQO^ zKkH_Ke$VzQ%kn_|Rs|_am!;ICRF)<7>?Q!Tye2#d#pIG2+k} zLyxcf>8W7rWlgFsPlr<%x~l&(!T9+DwJbk>iH@IB;=la79F8Bl^7GeV{QLnwHD;CW z8_=ooQ){wnIQ-z6)R+yrF{d?S#)rcXj;UHriLPD8ts|SGSah|H_!_fAaJ!B>Mm9z) zx>`qkjhP%4IIHr}-vi*H-SyDmC< zXJW0^x1a(y-t5p8eT52I3Mj~eZ`s9#bGlh zafZ`ZoU?-IE6&`=^c9Exj5NGBuy>JtlJm!d{rzcq@=^EnC&JU!d*G*neZ#H4WA6ER zb~xuH-JxrZ?aRi^&;0D+T91cvkNZ1zz6o0kSo7~)s`lpv`#aOV6<5qJgp=2j_$s^e z!?C3+yNiS6zI(=&-6i4txgDLFemT9l>#L@_WsJ2`k5Q@V!t_k{Aw(flST4el|P2P(mn`vgFK1Kj-w%q|qJM z&EeUOt2z8BvK;Z|P`>pwhw0I?F)#Mw%I(i4n5(|qJQ}&5SNgA}M)t~M!BdjAbKaOg zbYski?(sC7aj-v2NY{HZ|H(4<&kgZ+3YMRkY>lZHPX~+P%oF3q$SyJHo=L+Q2cOxk z-yefdh#mjMEel8X?|5(Q<}fBQUJjl;fxQ*O;dc(6nzm}kj}0g9ENOpC`F&ORw}zvy z+OHgJEZzJrr&T(RuDI2LcZ(fvZfCc8Fy6mst{7`Z#`94zauoOQdo^}zN5-=&f9nL} zXEt{0MaH-O){l($@1iU2_{ey6r+52oqhNC_n+d_iRjoD&r&gX1^kq9Sm{0o3Zzx!; z=*lkt&l^3wwW#?%pT%KES9ULiv(|L&9AiJVfm}~iMg>c#yf}{^fku2i)U?(ncdlVH+eZLwnN(g0BXs@QFu=hUuoco=R zT5Z*>)|gpsm6?altTXf9*O{69tUt5r%ql_phfe+8Q@>>W2l?AJda!@}rZY2Z&8#_d z#Ns25KIGuTkAC54r@i#}Gmbs$gqIzE^eM-lb;b#=_`mfpJNx95jy?70Q;t97v@^~* z`jpd-JO1RvUVS#+LC-ki89SbN)^R)Tw)@T!dHBp~y4nA(nLr-x0l7=F8Y$ zMQ$wEr?D!JvF?mbW5Ek!ZtnIv+y|W3_WH0c$Pk`YEhk<=sQ{}Oy{bEz!u$qjQ7kLo}`H+`H9@pyd$c+X2 zG*;y?)?de_v0$-QOXHh7#(G}lG1lKkZY%raAA6OYGOxbC+EG=d@4IvXJ*d*eC2%}l=rRt)W{F{ z^au8-{Pf6+e(;8s`a>iCWaiT=%8Mr}FY(=W6Y534U-b=goXG*cdkae{0g=YyXpAA}uLZre`d#@M@sJ2$rGrm89C|B@?w+SdvNqiCTt8n z*}aIqvinbE>958o5;@g%J)Y0{8IVPkM2@N;kU2j^mIocY->>qf7RJ%a5Y@$K8*)9qXL zY%;rj)3d=@JMw-LNB@)wn{R&HujR{+`!Si_5tAK`AF}&z(TxaUtcIOE9f zt;TJiRkkH`onPdGCfWJa`M58Oc5cR1es1P!zl^E;yvWB4^4~`8 zxhMYfBQGBGM@8OtkdKc1)xrNUk+&Q4$3{MH(7zz^*CM;-&JSJ~+mx0u51ym%3b#kD z=$Op%>7tpLxi^gK-2ABXz|^RAvUylIM}EX0JO2+$QMD&ObB?@tke$m%4>J3UljJ!5 zk~#9FgKYe}2if&@Lt4js?pQZPjq)=ldH+=DEIB;QdE+|RJ2hI1BPPD`k&&G_l|A1& zbL!&-!Oon~#|tClYxAOD=a4;BKQE4qUz%LRJs}upY{#7#Y~1Mkq{#Slr*S6-8;h^6 zmj=7f^7Z=3?(|@`boGB`u-wPI&Wen$ua^ZI%inpEua`$=N4IunSpKgF&XUY*lSa?J zK3+Z9%BMbF6D%LP=6HTEp0AEw8jN>czHy5Grr_Fj9xezcAG(gcFc|N?s%FnX7e)45 z;+l)EgLo%#E4>=vefdt|&dtKAiooqI#=t_+7C z?cP1vnY-HkLpVIU{c~!#Z+tGzUdL(s`T;sKwKiS|$n?olX`uf3*Hh(?X z)DMnL*JDMn>qg#P%P$UG*XZlvtS4R9kKEf|mMp89bx zJG%P+X)yoG((3>AaQKf+Yxc8X;`~F~j{rn}EAI~23^FT1ZW86$|&3Bc^{NS7Ks=@9j&6musd*#bH zRIG_>xBK9haQJZ_d?PY`+y~#B?C842-ygC5X|?uem{(iZ*8{) z6z_;Jn_MaGYFw$)@u*E!o} z;>J09OgM9PZQ__SGP~BcT`=F{oINfwzP0TXj2q`{=g8))e#z|Wch_J!j&rtKWPJVZ z8H^j}?1_=hS^bjP^$b`&%kB*D{YB4!HNxRLuMZ8z`J1Hkx>hjWHGNagtt{uvHS->{ zb3g7JUK?&j=88_9?5+n!Pc3~uJOMTEdEc1_+Jf&Z~R+=#lqL`*MdC@ z*Dsk}Yxs6BuK9j17{^ER{eCdNboKj#V9(U`OJ>(KTsIT1Mswc$HVB6w=V#-{_;G$V zne6BqZ_{A&!=L#fv#amTg83P1-#oI~JBG}zwQm`WYwZhyabxWZBlAnwTowhZy?)8; zTKjgvYIkllza7Hi$J%#}j2~-%{A5Shc)J9vz2^{_U48E!%+FZ+9+B1FF=TeFea~QA zYu_svH`e~-$o$eZm%W44UcY2^t^L4YwY&bC-@)PVW9^4T#*ei>W3r=byhDT4-t&aa zuD%xs^E1}IB(mB&hRm+D9}$de?MDXV#@hcTGQV`q<=Me%uU|5|)_zQ|+Q(nC$2p?|%fVoj$dQ|^U~ileUEWmzUR6z*Q)QqF35i_OsnqL zL3h!l>od!L#s|G?>-RJBW-gxeo=^T1nf|9~crjr)+@7ASZ=BWgeWD!L925Ja(i-O% z;pjcrh;!_uZyfqxrs4U8yUxE3w$8?Pe*P=6wZe6-e-jyBo4bOo&1iFXWPEMz2{zB8 z&F>@Q+4vjv-eAwG=1DAn%lF*C_gRh^z1baRm{X7s3Uq7p)e(~#P z)yQn{*Tnb2_?{Qc|FYmeWS>4PvXAwmdq^72Ji_Ap4rz^G&v82@htAoW;qZx4rTx-T&%;x&=U^U3~1C!5nBO5~}|4q{S$e(V#G@Sh5=D$I( z{Jn2${u_qFH~)=-am|0@VDa(tuU~!h-!xcUHL({{UR$R3k(ckY9vK-YFIZlSqTf8& z-$`9pbFxJ^yqF!MZ_N3@e9D6@e;-U7V;2O=sq?ikm_1%zJEZrK7u}*XoV?)9*H*!E z*L&-5_|Dg(gK_-+D80{p!LH-wk@?hLH?7^x!Nx3&-k5(5_K^$SiZq;Yu=@EW1$*BEot9KV*|fBW99huV0oy)7LTxtf=EA3oh9 zw}+RHzI){7!Nzot{4&_b8qobB4QCuIZ+^*m@*R_2-#Ru+?yhAvn+LCwG2J6~hNB<% z$ZscZyY$^7cLlShvqx5q4}O{-{oQGJ`M~OaAbo9&|9#pY)7Ze>BliZ&Lk-5zv+EwY zFW4TDQ}@ID!R5>SB^-XPFFrdT=0H60*^48tdc@U_bEzhB=UQK1TTF9jJ!a>5vhBH| zukBv3wGLaR88`abJ+kYV&c36&I^X|0*It`vO6Qf|Lpa~e>#pg2?2T(PZl}mN`NQ_9 zGxqVpbI;ga!r|4_F?xB5w_7lua%an5&(uAF>E$GkCkOk;gYF4wIC;RmUY;22dht7s z-h-YL4&P^;y@EZn;N@07`sTKGusHH%FNQpxnchbpZ1zh#EKMG;Ych|4GfxRN*UjsI z$aAmbr-sA#+4I0)Bomf8OE3?;EZ+T;&3h(U zo&6sEz2UNKGux!GXVWw91HtsyWL}%|)sgYOpLcHg&Dw_|-6ax#a;`n~Cv*+cfxlH|_UGcr#&pK^yzoczljHgTD7$NXQP zxcIR(|2ITOZ@$O;e;9lDiz|OJ+qwDQ9$x<5Z8ra(g~N~e|2#5&%>S2>x z{Fwi;_{NW|`M)tbdV6Ec|E}1}UtIZ<+0M=XzVPz*p0WAg9}Yj}|L4f~G5-f5%YXTh zzm1O{^M6x(&t4ENim5gK3!XoN@$!uHit%+@)Ydhs)Us-GLkr{X0Q2(wI=l6Tg-X}$NKI?nFzc=&ke7ZP! zyeD|?LHDl6)|Kw6LFanEJhI&B-Z4k_w#d!_y0^~JT^!jpMt9L1-J2rYzjSY$qkCOs zdxLK2ptC+FMYdja|10^cq4$XWgXz87^>^TtCwdgb|0=zYdoSI=X*j;%uFnZsN8?=M^?y=0yuDcer$@%SANBrqcI0fc+4~pW_#XM? zk?CD~{r>v=$oTQQ@i#`skKc{IX|khpUS}K3ToCMclIGL14VhhiUlPoZ@3lMT(#U>G z*)e2xy>DL@jO$!n9*pCo_w9E^=9jL1uL$;=%=#s>>-?>f9~-OF`ZT|J;qYVaYedG6 zwXZqZ(KX&fgVoNT+R5zdd+lI;#@g43toDu}vuo|^2IE@$2En+o_6;NROV?aB3RZjl zlG(NPErQkV9B6)z3WpzSUlbWX*1pwbN7r~;2dkYwwUgP^_hW+j8EfA*vf4X_%&xU> z7mRD|I|Sp#+INi1FI{ulDOm0GOJ>*FpAf9}@%`_U!r{l-_l}GoYu{(Gqieic$J)<~j2~-1YqFzjyt9MV&Y#-J?CSd!!TgN1 zpA%W_9Ybc<+Fu!rYwfQN#*MYVCNjTt&E?!+wbw71U29(!tak56o8JZD@MG;4MaGY{ zUp(2-HQpt`YVSK&GQ0YIYcM}!?Qe^$_KqR5Ywec><68UW!ML&ZcSh!yuDM(htoHgP zvuo||4OaW5!@9pe9Dc0*>d5%9_76>VbdC4nV72pS-O21)`$vQM8Ed~Lvf4X_%&xV6 zJQ&y7KM{-@Yrj4+zjV#zlfi1QUotz-Ywr&4IYQs(wf{Cf>v*?fah&T%- zedEwyordQZ?mB-Y*g6~E`T1yMYlZ7vUlSQ$n~w)uo6+Xl$oSfPBG^2SHrGeSv+@4- zQ^DT!)IlkbPai`g;y z#{6M0pYmYKU*G@!C|FLNuUmuJB+@<1a~jYZ@E4 z@AlpjEDtppL(k6Hu7x<9MhdB^WeD>mqs~&On<6NqV z+_~1**H@>xvmRoy?H>7Aa-Dlk{X87Ld*m0vICFJ##`<)Ryf3nR^xY#L2sWmBAOe%DVTm)a_b)X*u=@Dd*tKc*wWb} z%hU7YBR~3U)9~_v)qP|7+8BR*+6`%J;O>!61htGs7U+#0^ z@N<3f+4(RB;)%~*9C6hnu6~?LHIX~l`uh6LGy(gGnPV%@f*he08zfZ%-1Mc;5Z?NlSeE<7Lc4>Xq`BU&m((rPtAANJX zA5N1mdokqk@XVKwJlH&twoW*Cz^+O6?N!2=>*lp;554N6t z)>@p;M>#zZ(P}o$;R6JfoA@n7hr=`*iL$j{L|pdBOGhh+yO7X$(EPQO3(#4u484 zWBBm?mu+Ly8RuD-AARS@Gq60DCMPz1W?m4!^(+cj&$8GwzpcXIYqRxa!%uA<9S*-Z z^Wy#QHo?Z>yS9%B#@Elb!D5Z?e;*qTUz_bFo9l*Hj|+!ytnGu1!#CCr!T9>wF<7ke z{qIiU@U_`_vKin1K0X}2v33bI4&PY22IK2zw_vgQTx-pC4~MVK9+S;2LtRe@hi|Ms zgN?&C))RyAegFHUVDqY8dC-gD{qJ7E{$Fj+f%XTPy}h}AdY_(idq>9EGqX58JWK8y zZtgj^UpTxy)G_*=B~J-<&e>ON`4jV?^gd#;IUwz6X=1`F64P~gV6g8KJBY}k%6 zzSr_iu{Dk#*D~9_|9yI}zem{^e@JBh^u0C?4HmE0#=Db0o5Kd1mjrv?dUkYu-+Ds0 zzHg;>J$UbMVlZ9L_fvztZ#6G{UvqKJdc0cYY^?XK&&`@ywpOSer+B~Oor0XyT%GqR-mlP^!|LSqj>z7x(5ZLSd7t9_ z3Z48%-9?eTU!haSsPjI>`xQF%jJlWxj7s^boH|$7(drf$6OU3#@v_R7Z`JK*2J`w*n$DjW{{iO9*pOshepC%TaYn&e6F+Ykw|G#?2+{%CS zV$m5xkMEeff)5?W+#T5*h(l)#J-%c711tW2_QqKyvN7V&8AFfn{BIm={;f&pWs`8` zlCJZ=X)u27{MXN-==eD%{_AI}aQx8K&(^{Cxqdq4@za=NiyZ_qj^tYwq&M8wIHYX;|9qAkA&S3h+`DJAK z#-aa38eSaOvxtAs36|L#;*UKi~Hw2;hdLrC*}_rvT@5& zpFO;7nw)OSHSV3C>)l$wnt$7>bN}byrQvqbPuKec!Sb>{YnS0OZ0TyZMmV|qeMs%r z4Ci-ubmnxe^lI04PEX4iYiB-Q5`LYu&gpu=^kQ4PwI_YV*|np8SQ?&RxcO`tZ0wlN z#^LbKNlrIS^*u5&-rpC^d$Y)Dr0Y5Ks9<*U(`vUMSdDbedFx=~Z%+*8>NdeX_8Q%z z({S>IJ8rvR{`}U^T-8Qzt`^3}1Uf|)0Vr<324)BDKJzYnxmWSsn9 zG5q^y2c+$nwk);_(z_1$dg`QmZPtwZw8;G9%~gHr%@v!|v-WJ%&t9DFjYER1k-6?X z92yxfuACRA_mT6J$@!U)adL*`?DytJ2K)EdXO@qn)4bdB> z8n%gymxFtc{2mh-zjyFHX=_jXW5dZiQ#3oKejgY9sp055_uB^>OSf#wX@`lUE4Ned z(_@D_f3n*-7=LK^#@Hn?o{z@ZH5k8Su-iQ{o?ZRz5sZJoxeI$o?(EWf|`p^sI|x zCr-~oJ$aq`MuYdQR)8cb!e6Z=g%Dv_Pb?;>BKF^=N{!YxeQwD#1o^by+#xsa9 Y`i?m{V@@8%+&bBKHu-Z_Y|pg+4O2}@?EnA( diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n64_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_cm_sg16_n64_h.spv deleted file mode 100644 index dae885548b47e50952c2ee1943f99c33c3c347e5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16372 zcmbW;d9CKZXfK^Y>VZE-3E(Zwnl27{n3?e9~kwK^k1w^Io^ZV}E>FHg|{?T`NJkNgiK6~%8&w0=L zelIsuCT}urFlEwU^}(8hpTA%*yk-n04<-faQTlCLzx#}F`N8$Z`x7@D3|1RVAM7#j zUGsO|>Fx91f6ze(FI=?X(EScyIRC%}hc4Rhi2toW{ICP|U+|&%2QEDDphZW{Kk%S^ z79Nn;s}AGsxXT{9Y<0+?`)sxKYhNRgwFXmk!~b79QC10_ns(rV{XT?$C`mq+_Pew{ zr2RQw$ZD$F0`5%{y-O#%>JWz>RNizhK3(kRg`?*S9vN!>N1oluqtW-V7DujFu&ZK~+{ZdG zHWdpViM?TBllxfqikfWH^!5!|&E#zlxr~MBwiwD&22XT4`;4yQ+H>TW3~i?l?w&F} zCwqb4dm`7IU{}qlQ~BHIkm4Ij|$MljAmNu9BUjgTe7vl)T-z{HBt3h`h>W=Wkc?+{m-< z8(XR0HS(_pm#vg{CqMDMmGV0ypL)S*E62Y(@~TIkuu?uWa;}y3M?`pP>T8Fx`i!QT zm-ytdG*`)A3RWkw8ZJvy$C)`XoT<3~PgB2(GS>OV&o)iIxN-ECG|YeHvpm?osC+ID zc9nc(uyeHRuL-tBCEwok*8S07bGGiKpPXV?$CB5I>`Z6BVem+^-y&JSe;sb)VCSek zG&9&aNj8VKrI$a>9Ldf}9Q~dR8%y3RvV6&MV0T!!?bD0n46QiMNV0KD(zC^hLv|+O z%Fnrxf0W{x%Xz{4;EX>%y>-PKPhQrz8h>GAwb(Kl8vDb@>fblMu4v=>IoufT_}IE1 zR|Lz;I>n8cbjEZoxRRxx6h{9Q>A%+3u0HP1C=l z>G758Xt4XFmpB2ah$W$GU=M<^k8y7 zPiw6PTXZ_}*JgG;Pmg}5yllux%u6G$on8%Y4OV-+JzdXG_l{>IncYv*v%y(+vS%WW z{=SB-1wWpT^5w_#kIe4z^v2-$A$#7HpC=kNmLJc)^5fY?=I0OTjluCl_H4t|Jg1x3 zBx}3sa6fyt;nYUsXVYu|d%*ca-mJ;am-2H_7JOdxC0`ubyGzNJM4r>h-;VtD$nLKj zf=7a-GAHwa?+^F7+`U<*A)hXq+$Uk&)8-?O7bgd~k>&Yxa4$dB$@cQRw9pMZ+0e<0geOuJ=7T9RBz=Zc4DR`0_O^*fW%`%6)n` zwshrxwP3mTd95A}U%u81HkQ8=TFkW~v!h!hE2Ca(2glW5(=>Ya@c9G=~q zbH6*g-@nC$7`9v_zJoz{|cq|RikM97d1Uok?UoyLjb6T)Cp5qng^k97D`{`gc#8>Rk z1mnxkXM_1UBYpWfGZf8&m^E?luG|OV@clkG zJn|upt9|g1W=B_Rd{pE5H9k6=HLiV4W>@t%Hkj{z{f>)_uepr|@=In{Yx9-V%i8!oRcrIraQLe0*}=H8(yFds3&x+5 zc2)M1^L&Rq*x5NJtA;y*@llHGzTOp#$5lRi2D76p|L+dw-|w>WzgIYX`F~F^U-uzhH}c{@7}2#2qG^+UntXn)Wj6wC%+em)#*-}QbLg~OMh zLxa_({2Ug{27hsUk7WEug83f{em2jnV}o7hL3czN&OG2+zoUY!-{efX*6-+W_=KOQWuIk6X0UY|-&(s1&E zt6rZCp19s;hQn9AJ{OGRcVjnIu0I94$NnokpZeu(?Iy*>n9=BsnbO9{g>G^h&N$fo z`6c7YQ=6<;5BJOUxwq^yHa7&nBz>*nv|xIBsP@%rjg!lHEw|Ofv88j5ZwCX2{7ohg&~f@x^Tz4nNTs zpH&Yv5Knye;)rWL;_CZaniIKdtuJ3Uw%S<_G1=B0`C@XNcu##P9KQC*mxFO?^_Glv z)gF0yWclc8k8B!jOzn})+Zby=w^0pTxyTJCLCKjdu03c{J7*t|5s^v`M~DAZThk?e(SW?r?G)+kGw8e9_C;S zJ-gZ?Zw$6a|+#g0}OXoMQZ`_v$`|dp!J9^Jr--oXVrZe}wqW@9w zvOL2)_xyGyvr*3Qy&S|wy4+o#v)``B>>5*W%+Ut)8 z%gepsUX;`I+4p!ksiCniP5&>MuQlC0xm)u&sT0ms+$$R=|H}F8xzE+ge|M;y-w_V) zx68Gy7XJ|0`iNWKX5JkRFOPZI+rEk3n1=70+~ssr)630&*RVeRJH~?8FH0}(cHziu zH_sUF{gu~V;pFN!apnJ>aQHs|_eRF|`M)o+{QGZE^TXl${BMqo?_1^nZ{g|HvCsbl z;p8u_{K;%5=6^uu~z%uUmqR>s!tD*68T%%YMGcC6@Wh(|pNnYrYR8wmNIqb#uKj*f->z zJMN~&`4(R_xh0(M&wh{B+TR+ic64QTTQFYjoCkLVJ2Uik=Io!fa?jkHIeakqu1@!X z$o2%?!cJ$eE{JU2bbC+Gy*IKwO!uA%x;-P?A9U~Pbk=bD$kvSRh$-W_)cJF7@}l=G z!u!pg>E-Hj7SsPE4R4Nc)#mG zYU~f1-Co_;E5qS&HTJ4zM>l*fr8nmvr>{M-ds`QEn3Mm|^~$u;?bhjD)pS*Z>*Iso zKDTCDHN6_BGyM%|crjsTp6``61wWSFcxSlZ95;u<*Zcb|!RBKf=x+^XgD*d~2m3bB z`?(_=zWm%7tPbVpu3$F!dgs0;nE$c(vR{4?>@p9!pQho=1FmQ5&w?lZR=qDAzT)2> zjH`G4p9hPDw`#`X`YQ|p7%JrAQ<{;O4R{kooF?8~OG`&mybiYo+$sex# ze-k|MJ@BD$_{#s`U|i+@NU->L`IldP<^S7Yam|UnnDY94`pV09gvTS}P7cV8ctqt)$7^d ziR=A`aQLd%bHPI{evfT_>v!kg&8NP8cm7kbG4;E1pEgD=bnj2Y$ptole#v<9!Y1q0 z!~Jqf?k)R_&8fjhr>`~KH<;cYs(rP8s_xoHlE1^eA8CubV&@NmT!cSJb+L|=SXJ=8!v@!5+b zuK9?o?`vsJiKZ^+9UrNj8m(xWvr|A$dbtN(bpas z3pS?q$VqLCHK1FXhBFS9H@{>&`QuI2*F4Th?(StapA9}MeeID`g6YSSTkVlgHBK(I zM@|dJmd+kIJ3T)x`O%-AhL;a)-d{;yHp5tvKi6!|NMC#8%wTz#gE4%tt3C3CV0%PP zwI9A1uK41<91cIx7oSxRH5kSV$6g$9%|~2)UrTc$SFQEs>&#X=>mg>@PKwNzzHFC8 zZylUx#`S)_7TJAFXW!9zM|CFNmUlbnm3Lg{o4QVm-lg{5Q>H}5i3!&kyGroHGj`Q* zcyp>TdU=XBJ(y3qv*oYO)YXFN{f?e{UTO)1#Gk;u z4qxw6GlGr7m!EZm)v?~&){D#zZ$9!7TMoU9H$S;tknzUJ&$}wyin&oR{XTiWx+-T8 z`;Eh0ntCh`uDCP9v8Aiuye|u`Zxi(On|D?)ow@rqF%rBY&oIwD-zLax)N`}+uBzuI zkvB~*t_KwKLc<+b&>1)of3Kq}# z#K!w28}EK)vs18d0p5eFE^iH2-va2XE^~wFYH#fp>|21nSHJmg&pyMOtDNgwz*DK8 z8t$HY_${+#^1``_dsX97@?pO9TR>km@q5YdC%-ksGMCaV4Da_7o!XR6yuBm){X}OT zz0U6?zn|#j)$9CT^81NSesuPQZv-3U_mbb7elPAfWBS0~&CN{1k0qwxfUimKUA*dj z==5=Wf7exh4iConnx!Pl76xl<&>2IIuQAJmmvv*l8`&6f=!~Jq*P2`xYz?hb)#dtd>Oxm*azikF;+mA7 zJEP<0lK3w_cZK7JuKe5`jQ>l1eifZ{>ht+^IDY7AomK?nNB_clKM@^2{zkC!`F%Kk z=*rKN!T5=Os-{yizF+UDk<~(cx>|31jhPYLulEbv7_sPTz40|>)8Kx+H;b%R;?UK4 z<7>>;!Tox_wv7>ouGSkr@f(oyd*AFOm-E{zSXy@pF~IRyz%X-zANH4j}Plz@!lFvJY2<_8xG&cyE8JrZx!#Z@bu2G zisw5Zeqy}4le4q6pWi*<@O`|yGZx>sig!FfcU+zg_=@K_RQ1(QToZcVnABfi z_m*!{+sE%{>cOVItNHdsJ}P=+^z}S=bL{2LW?G(8AC0cQlN=vRufD_IP?71gEW^D( ze{|Y0Y5cxwq-hO~dmG*IX_PHnz{_;&6C>gIw>G%Om6c zU1a6`-N@!mSNHDsgV|l0R(6*Kn z+tTbk_Vn9^_g;Q`)73kX-v~KM26u#~liywGUDnLs?cNy~CqGyWfB%1f+P!IGu~iHA z246pKx{u{PAU_b9f4o|iFTGl^`TMLr8}nx`PVJ2q!PZEvs}2uE#)~WGN7K9HymfMZ zBr;CUu$=u{gWsq5_Xl$3L(bFEKbb~%NSnh`jjK649a)Zeb12{Xn!}FKvl-4Syts0E z-UM^icZ^3P%S-=_)W}|WEO<`xcJ>?dL>ps1bdRUujDvk!PdE5o{x8csJWIsiCRl!A zvNfh+JQ*y8b5M*wMRtin_f#6rICx20zo&!!TQ>fSdo~=|ziGR)&0*EZcsY0;1rDZ# z!~3^*+oi41_!oqecb0TGru?oReur@MRr@u9jip=Ka$2)-bj7V5JU4c@(PpB{bp;jA@XzlZFJ zUbeTM&cBLEH`poT?Lm9ec>Bh_Fvh*)-m=K->K>dF`ML08QMm`bAL7`Fb8R?#P)|O; z$@;qgJ$pPaMl+1f*PD%J%m4K}WxJ%=>dT*Z2G7IM=C9r{7RSaI?= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -vec4 gelu4(vec4 v) -{ - return vec4(0.5 * v.x * (1.0 + ferf(v.x * 0.7071067811865475)), - 0.5 * v.y * (1.0 + ferf(v.y * 0.7071067811865475)), - 0.5 * v.z * (1.0 + ferf(v.z * 0.7071067811865475)), - 0.5 * v.w * (1.0 + ferf(v.w * 0.7071067811865475))); -} - -void main() -{ - uint ng = p.oc / uint(4 * TN); // oc groups - uint xg = (p.ow + uint(TM) - 1u) / uint(TM); // pixel groups per row - uint gid = gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * gl_WorkGroupSize.x; - if (gid >= ng * xg * p.oh) return; - uint g = gid % ng; - uint t = gid / ng; - uint xb = t % xg, oy = t / xg; - uint ox0 = xb * uint(TM); - uint c4 = p.ic >> 2, o4 = p.oc >> 2; - uint n40 = g * uint(TN); - bool reflect = (p.flags & 2u) != 0u; - - vec4 acc[TM * TN]; - [[unroll]] for (int i = 0; i < TM * TN; i++) acc[i] = vec4(0.0); - - // input column per window slot (clamped / mirrored once; zero-pad slots masked) - int sx0 = int(ox0 * SW) - int(p.pl); - uint col[NX]; - bool cok[NX]; - [[unroll]] for (int i = 0; i < NX; i++) - { - int sx = sx0 + i; - cok[i] = reflect || uint(sx) < p.iw; - if (reflect && uint(sx) >= p.iw) sx = mirror(sx, int(p.iw)); - col[i] = uint(clamp(sx, 0, int(p.iw) - 1)); - } - - // no `continue`: Adreno 750 mis-executed a `continue` in this ky loop (one accumulator - // of the 4x2 tile was lost) and in depthwise's tap loop; zero-pad rows are masked loads - for (uint ky = 0u; ky < 3u; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - bool rok = reflect || uint(sy) < p.ih; - sy = reflect ? mirror(sy, int(p.ih)) : clamp(sy, 0, int(p.ih) - 1); - uint rowBase = uint(sy) * p.iw; - uint wk = ky * 3u * p.ic; // k of (ky, kx = 0, ci = 0) - for (uint ci = 0u; ci < c4; ci++) - { - vec4 xs[NX]; - [[unroll]] for (int i = 0; i < NX; i++) - xs[i] = cok[i] && rok ? vec4(x[(rowBase + col[i]) * c4 + ci]) : vec4(0.0); - [[unroll]] for (uint kx = 0u; kx < 3u; kx++) - { - uint wrow = (wk + kx * p.ic + ci * 4u) * o4 + n40; - [[unroll]] for (int j = 0; j < TN; j++) - { - vec4 w0 = vec4(w[wrow + uint(j)]); - vec4 w1 = vec4(w[wrow + o4 + uint(j)]); - vec4 w2 = vec4(w[wrow + 2u * o4 + uint(j)]); - vec4 w3 = vec4(w[wrow + 3u * o4 + uint(j)]); - [[unroll]] for (int m = 0; m < TM; m++) - { - vec4 a = xs[m * int(SW) + int(kx)]; - acc[m * TN + j] += a.x * w0 + a.y * w1 + a.z * w2 + a.w * w3; - } - } - } - } - } - - bool hasBias = (p.flags & 1u) != 0u; - bool relu = (p.flags & 4u) != 0u; - bool hasAdd = (p.flags & 8u) != 0u; - bool gelu = (p.flags & 16u) != 0u; - [[unroll]] for (int j = 0; j < TN; j++) - { - uint n4 = n40 + uint(j); - uint oc0 = n4 * 4u; - vec4 bias = hasBias ? vec4(b[oc0], b[oc0 + 1u], b[oc0 + 2u], b[oc0 + 3u]) : vec4(0.0); - [[unroll]] for (int m = 0; m < TM; m++) - { - uint ox = ox0 + uint(m); - if (ox < p.ow) - { - uint oi = (oy * p.ow + ox) * o4 + n4; - vec4 v = acc[m * TN + j] + bias; - if (hasAdd) v += vec4(r[oi]); - if (relu) v = max(v, vec4(0.0)); - if (gelu) v = gelu4(v); - o[oi] = f16vec4(v); - } - } - } -} \ No newline at end of file diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x1_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s1_4x1_h.spv deleted file mode 100644 index 38307d1e29a4bfe39b790e6ddd26c17356625cd2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23000 zcmb80XSkKc)rJq}P^__GMPrQ}Y^X_$y5Xo4Q4tgsOKh=3K_e=nD2fF&v4B0ANE8GF zX@UxNV~I5yqo!DnVv9i~Arf1{ci-o{51DKK``*jB=2_2Lvu4ej_0FEXIW{$J+Nq(b zUBikED>t0BTtn-#T0>()JCOZ>t2LnO*edtxH?Uu~VP_uSt^3Z~YxtdprdE4xS83?f z(1E@^ZOAbv4^=Uj_8@Il#>?ZXp+?i*){F6X&^OY0Gg|DkX zxs?jeF>x^F1U^rw@$%zs<`zF&iNd>QNb-=ahny~3Kh3y!L3+v+ZLR$>d%B` z9V=H{clute+B`beIF!w)vau>pCPBt;Z02+da$8Q@w_@m4Tm7r^aZVrYr^3aTc;hxz zt8e8^3i&PMa;~?LGnd#WIgDJ+)q+jt5^s#X!(4}QIoC-zb}skOI8R2-Twa=EU%k@=53Y2P6B$y|}kxu#*0xx~)ZMANtOCOY6zegwIk z>rrI>Yn@B%vo6j3id@e1I5z6_C3dd19JR@tAp59)4!yOa{Tj@Ih<#IQY~nSsF_w*J z?PzPEvHq0TMz;pNvi`TJY}W&8+Zk_N3VuVdPwcE;=TO#1jph@%XRCDKqmi52Dah^N+*{zZ%RGjg+rxR(J|E@~sT#0uIxZKxi$exYv`@QtqX8atmYn69CWzQu! zd z-7ERu#PYm@dA5lB@tvAPn?Unj+Z`X~b23f-GHjIZt~lSfjJpTyxM{SE zn+}%0uo^cLEay2AU$emS@#TAWKh1a17~|`Kig!M3=Fzk|?|^Dwgf zB~?t1fE_0v?lG{uxlinmgN@-aTI`+zn+w;FJUjz751s{y>vv%JjC&UBSo!J1;k)}B z%{kP$S1-`}e9a!Hd!8oeeu=-b&;B7_QLX0{uzrjyIex9+V)th_d1H^=TLqU`Ti~?K z{#YaL(Olz(w5)MK#XnKS@jkMgpPAu423t#V$=By#eR(#<*CMd|GnKD@faS)}rdIj- zCs^LS9Z8Nl(s!iRTj=9^Ib?lzqL1(8!SWfi0$6S|e)X{;SU>XScx8HXZvO>umNtud z@m#z6(HnkM+N>(B&S3SP*Gq_Nl}b<7x|iy|LzCC9*c#LCI$-B_{A9*j7aJnGR%>Eo zu#Yt%zY$GNKVsM9`?)DtU&+H}VDn(!lZWrY$WdDk7k_O9Dlwg4N8F>CLf z&an_~OR#5;vF|~w%Gw+EHuOHm?f9*c<&0aLxVwVOxVs_O$Gsh#eBPVw!H$zR?)bG& z+&h4s*R^QxoW`{iy^nEevm;H;xWvg{cW`|iJHyGRj&=dd>GwU5zmob#Kw+j799)^s6kd{7xm?r#9DX->3u+$HwuW&_1PUBffyK*7-uPakvJ@sMpRj zNB!quYt@+YUM_+QAMT%U^7X#-nS7W7=W~AToyU1ykMr7>xpYm+$+dlatzYHN{cz6M zre^G8yD55W**bKbeo{|0xCyMUd>?zj$)}!rg5|Q-X0V*~vJy8n^RyQ-|21@_IiLHg zUEZg?!P+?Qcy#-Ky$|FaRo2G!dWM+~`9b2dSUZERQr`x#D^(}Osb^jthKLecaN95S~a;EhMs~?BnI_`(8-p4vt zzb{SRwTrVy2Z7zAf%G{u4+hKUvvCmE&rJE;6AlH-J8$;#Fl71oJ{+uX@7VEu1Xw;} zjs(kP|BnLeM?Rko$ACT8eD{t6`?xOoV`*}(OPrec30Pl= z>{9k#pY}Op+gTgf=ZtNt#u$sbMw*=C#I8-h%JRzXE7?Bpc4Pat9BR`9UXMO!Y$ve# z@{C=f;*2F{?22&Ost2UGHjZC~wmMB4an9J)z{cSk9HU;l zoUxt3oNp2jVhL z<94hu<^383R-YIzL^j6qtiK3OKIh12u;b*zT?)?G74C94`33khwkyHr#NOCEe?4U5sDp26xKtkZesdIqbzz0i67uE<(Ff7Shvcke3toEu}o z>fc9ijsF5!y^l4n{^vA#*C94<-Y0Jdd!KYKJeNFU#v{v5!bbTHu$*U)@5V%WAAPHv zK$}O?w^-lf(N72aIT+urV+OK(&heSx(KPwwe>Pa&^~7!tvb^i;!Kd%r)6@+^cCQ|&IQP=njnjBbSw4Ha7IQepwJc@4w#nC*9oq7hcftI81t*_;eGQgN zzB+h`;4xq7zNLLbGY)&>&_-EZ^7kLG`O949@4s;J$;(o(`IArnytK%_kA3Xg83V1% zUn6qzr#|^>0;@~@+JoiFIE^>?>Il}x*v*$~O1@Tsx4!aTny*#iNN9XZyeevt4qE*gUwgwGGA-L$tMqMfz6kE^0hWtzRcG;$ns^r)5nUBGf>oW`4cZ2;EB*v*$~O1?G+o3DxRc^|iclTW_31j{8~JAr-7m%6QKThYvq zy>V!xtS|?&v9ZEZdW`69ALmOpv$=6|E^Od>G z*I+pLE3Y^O^Mlatm!kIQ5D7L$JEU{1I3#bN(Hy zz5e6tbL99^A76{W>f-AkV7bKl1$Yd8%2=1c>0e#^e+`a*_3{4=SY7;o3zmz2%AobT zVCladoc`6te+O{QnX?TFyNxS5r;XjF zaOOkZMb+5Nz>Zg!IW`A72i>x>p^T|4fA-i66u{*P}E7yA#oaVWY?=M zV@H4;uP$TH1v|D}?|I0sS6#-A1Up_`xKUv5#OmVj8t?`*pRD&N56vuyd5_y#d+vs>|3L!H!p#u{VJoTdwzJWY?=MV{ZXFUR}7`zS8y&vMbj+1J3oT%N#Soj#rnlv%t<#u6H)F>s6Pr_kta-E@S6_ z9b2w_Q-6mn|8ZgWEV6ugroN1v zb3}d4kypU#a*q4~ESEW71#7SWoJp@E$Cvu}dIPL3zWxN3i?6?c^;MpqZz0E*`uKVq ztS-LZ0n5eLyI_5}S&8d?mxx6=v!LIv^g8Kq&{Abd#PfNhss>|3f z!NxY67Q3&&+Nz7)H(=*2-}P^iwN;m~OTnIR>N0i}zULaB?{&D<;EZo9En`=Q(^g&V zI)jbx`oeBaIBnI%Zf&sfmG9;{aN4TN99_Vpuv3?@+klPF_b^;nIO8kdxo&XUs*BzB zVB;&_uN~mDRTsORz{Xd;OWonLRhRg71$%y|%e%P`*!!MO{Plt}W_7XaUD=gqM;|!X zp)Pau1v_3{#_kJtj`Hm22j|+Ta=hP9Sw$4}rqPt@fdIuR^io+l?E%a`ZL$;k5Md2%{(-Zk}k*M0_8mv`+9uw3RG z2G+hjPtHb;FZJ;?9IP(B&H>BC*STPQmFLO%$nm8jC~$#e7>9EUIZIo`5wN6tgX7(y$m+K@|}AHSzC3ndlhVa<@@zVWNp(scS6#*~0y|z^xW(XsG$> z-%geHtH#)8jQ<|yZ^#+rzticrAN@0Y$MAb^4IBTq!oQ=*-&XX1llOPs*zH}}`F%5X z`@qSUcKcR#ed%M@4=i8W9a!1N55$uw%y*W6ws;SoIk@ z9IP&5M}QsczfES|bCEMvea4;#R@c9w?Kl3BVAo+h`c-cYm>ct8K1x|WzbP!C9vpA3 z{7pgI{9bR%-3%w+0vEeqBFmSbbGISO zyT^GSCL;Uq4aRC5laYOl!&;g|lQRx+?xNGd(`fFwe$C$`YJCP+T|PTzg5@&bY_N06 zo45G3Pu{*J7HwR&_Rec;bLoAI&G%srP0rZFD^+`TKbTM&w$v71=H9ttYoEEkL9dPT zJ5E2&HJ{$cxdwpe(d3*?i#_Gf8((1Q~Es%#Is|&X^SlPb=egHdvb2|`Oz6PGc zyW~4F09n2#xS6(6#UBLcn%dJG6TgGNec;raV|^R~Zl+gvdBq)Caq7Yi1|Lw_UtHN8 z4kteVK64z2EU%BuaTHj7P+@m8vb=WjcMMp5aA9{GvV7_9cw~A1&N}m+fGn@wC|2&; zP6oTy*qj0mmt36+XRge#`q&Nu>r;K=8wzg5PF?Iy2g|z`S?{*sX6)3(t}EERR+oC% z4s1O*f9$qLwjR`l+X<{(*26i-@-=Kz4+9h>xYUDV;`e;8^`JhnUjS}~Qd|3}yBFmTlevT|(*27iE z^4g^yt_8c+tbZIhT)jdLjGemJ?E-eM z)ukT354Ii}v5nmikgW%G;eG^GF6-fTWceDlsfY2%^7Zv_2VCmGG4VSAY(1z?>=VJw zaO$%6cU7FaaFfB-gIw}61uS3I!&GE>ePoVlVEM8hrX$O17k@Lr@?|~DLY6Q6%|@0l z>tPPEymqOF`@ya?>z@Y>mwK2FmwHei+Xun=RG;`B0ymQbb+LN{Ebm@qy?cP0u~Qej zJ;Cm^y3|7tu=T)y%j|k0SP$wlc5kq9Sr1Pm%h#|?Jv@UfUtbT;!lfP@6Ti=ctq1jq z{RMC{oVx7&ixsCX-0#8GgIx0SGFZN>hd&_8>mzf#3YIVH;WcD=?c(osuzXn$e?pco z{rwqPzO0A8BFk%+dUzY`TC@HZaJbaNyKt!o^|5^qtWWicZvnWO9H@)k2Vi;kBJ1r1 zZpKbs?0SQH($wXC*$3Q}=GkGrXyaWacMbd8RlY@M?RbB2oc9p#S>C&z1gp>e>?!2O zX}O>IcfxYo=|6W%``jaU<#N$N%-+v)PyHdB{B$_)MNfi#ycen4jV9;4NSyoU9^m?W z>Yi}&`ior+EMMMJdmzh~_tbrm{hOI3tkF37Atw&+slAZpj6<9}^akgis^8pG`+(Ku zp4t~I=l=O`N&AAGQ{H^Vw|(|wzskmSYwx_q)}KDHc~9LRSMQru z*xF~V1HsxjzvJ}dT#wQFhq2q4tA`%oH0jKoVsvFf<1TS@*W-qmM_npA0x}_ zBXb-LmM_npW02*wi@#&R^5wa69I|}r?|5YS^4$3evb=UVcTNDi)~x?TaJb~^B)FVA z>SKE{SfA<>-%r8jN?q(u0n57=S?{S}?bO9?2-v+=mwFf~ra6D?PD8dH)P?(*9IdQ} zGmzzL*rpzaA7i`wPyXJ!QoO57sI6< z)W`M`us+o%zDvR8N?q(O6Vu#_tal7pyH;KG9q)2D_gY=*;R-RW5!?8?64`oC7w#8e z<+2{GLYA*#n|in!S-!p=u7OKEI3|9t1zQj56Z=@Ov8&78kE=L!;jRN)4|2)R^#~ZJ=_L%ty%xCz~NF4 zzlKXasE_UKV124jeB;69N?q)J1D1C$vfewu+Nq1(onZG`UFu?R^x59-2A z0xOsGFd13ChHdI$3bK5CJ=_hKdT>nqP6b;J>J$4ku(7Ml-rrMk>cUM2TMu%{&kV49 zSr0Rj<@J#{W`X6)dYFwYuU-7z3zjeIVGgo<>2EHwd|40oASH?}tWWic??JG+QWv|2!1C@z*84D6J9V*p1ngd`%RTi`@DQ43hxMY3_oBz> zy=&O#uJSn8+UZYo+yGkcsZW5_=YIBE<;J#qY7emYR5|VR|0KQs?Q>6k5!^z|V`#aj zz6>Wn9nO2vVE+Ep$9s{wS7>tj66gN;M{xZ;^))zo{l)HeuzYz>eFIs(yr;g2?B9Lo zp86hg;_#mO7P6dih?9r6!MUgEH}}+c!0K{OZ2`-9vlD z^@)8Yu=SuWd%tqUsSEcVu=OC9{Hy|&FY94dWO;pLj@7{OWj(BpEU#VstpS!V>!CBU zeCcmZWcjil)cB*tKT;>wv?h9@d3RJ*bcEdSHF3Pkdd#=1N`c)(6YG7g_HH zVC~e!?z>?3T3za4L$LMGh;8gPg0mjfh1&$IT-L*;$nrI8QxBUV%h%V#_ux_wj)~vR z!PbNN#J&aC*wtn4x2!mI;kE)>4|2)R)?oRv9=1W2*GJ~q7A#-ZLsw*Z?c%Q+SiY=> z?U3b5f7>I=m-Vm%vb=VwhaJJLHS6CA94__H9WM2tKDIl9^{GDb?E*Ge>SDJmSbj5F z*86?1cIsmH1F(CoF7@z3u=UV{ZR~b~vmVri+a0W2*25mi@-=Kz4|^iZ*VjW0F7@D; z`0W9<9@HoHo?v5Fm%VSUICbIn0$UGq$b<*tKT;`-8)!9u9y@J*bauf3QB)C%yy0=1N`c27u+= zi>!AbSUYvGI|%GvtIIw0U~m)7v%`AP#(U8qdhZ(cxvLxkwsyR~IL>=W?x}|&tIz%H zFmUdv-p{IrqTs}g%A2qFw$FZysBB!f_RedLM$#uX@2Te^%Nd(E_tf*i)}8lMedV4S zTl>s)K3E&)cbtBlYazW)eowssSzVvr3vV3{&j765$F6YiTuxriwuLFn6xpO^S&K>o! zy#cIG^@;CBu(?tfyPLrBm(a4_o59+ti`^|?_gY=*;g?|R!TDo%E3);VF5Iuc%4I$L z8d<)EZR+86Wcm7f7!Q|va7_IE25dd3PwaPqja^;#{?3Y17j6RBdXP(gCW7V5dbkT& zULTob5?H>hhsntD+Qr`#uzXn$cO%P}{-z?!m-R3WSzf!;!#!Zvn)Ocyhf6)ofJ;57 zkL^sbKGi3_SzvReE_SoQ^6o{}doNf!b+MZRcCXc?9_E6rhem8;cOSC#pf214VCAwN z<{``1uuVP8N0zUzhX>(O4~~i7hrrf@`o#V)*x1!&?;oi+b>SWbTMu%{&tqWuvK}5s zme)t-cmgb6*28a+<+Y2yC&BV%Jv@ahU;29*S-z}?XOQK!OFjG!>{_$_XTjl856{7+ z9@NM7d9Xg!C%zZJ=1N`cUIfd#7g_I1VC~e!?)PB#T3zblWw7zXHrXF5Hmange*WpqRj)~tlz}AEM#QrC+v8&78|GDDSh5HNGdXP(g{tA{a z>)}mgd3|J#x4`mcJ-m%9uU-7T1C}rAp#@pK^!F~Zd|40gASOyMSfA<>-$!6`r7m_KgXP_etoLtV?bOBY6R>-&F89>GgKwdEc33akcrRK= z?_I+_ca=}U*3L+p<1V1(p86SBeeP$UgL6+E0`{INr=9*6(d*w{`T52++5SIdA*h^wGF&Yhu z9fQ5M_miE%sS(;%gG8ubeUBXCJvK3THNQ(-h7=ax)apJmqF7 z+~f^6N8x%k+&qPIJp1-8+>{NsP~oO(xWx)Lb;I>7oVgm$1DB#IhvFlhP@yS?X$LgRNTf2_|c(fnEu46rl%|G2^iGAjzxnHsC zSdZhQZY;55Ox6=ONo=Nt0*yrAS!KD5>VNX8fq)B%7A=TElmTPbzT5p<<<7pq%ZT-J| zkHG%A=Sto2)rZnthxnfc&V1E})2xBs#Jh&K=ZXA$oJQ9+So{CC8Sgs!dF~iz^~S!w z;mpT99YM?ZH#C}KCEuIDo*D5U3HEG^J{s)4TL*XZwr}Fy19tB6j<4-GC1-y3HP}AU zJ-5^o|1of9W1H_YG{>|5r@URST3?@IdtN5rpTXwe7fXP>tH z9&3E$?5FM7XTQGeq;>c__G7&LE^RAQ%}{txe5~CcV61UtxUcik+*@t;ajnMwo?KqA zGtbXp&w>}R?eDuEQT(QEq+L(*TU&z|*2PUU`BU-HzNO*(-qPD5T(r*-4{)DFA z9bh@nk;J+aEI*oNEWfk+X?}a=m{<=qe0;_{Lg8L|0P)dnI`^MU^)BU z+N|N%V0rg@DC;&k?Pu%?4KYjQ)MmnfsdLs;$3yFU8x(yzSo?Th6@2$$J@a&3jqw*1VU4lh3!bJlKBn z=AF2{llKZ>$8|3HJEnQ9#M{Tb^jVQ6XI|p0-^$?DJXV2|PaUlamNV`Te5*b=d;i^h zyT0#g*8456edfZ?K4Y6c=A!l=nw+_aottsB<+a~#Y~QKPxq41qLw)83f6F_y`7T)9 z+8y2G@j=6x%MA_pA)LNy*6??{jpJi}>L01m%t!1TzUCbt`+q|Fl%|jPIQm-W6Ts%- z9PFd6pXY@7=U{8qobp|M0T({pS8(#Jv5cAZa19*K@%48c$8|oA>$|R{bJEUQ`%bKR znzeI395cSD8Q<|;5Zzj~4((^0)Kh=B>zQBu9=qV=Q%`Gyk(9UFZ;m;iBw+4HvmNRrSu=;RxznhJ))qVVK)HkHbyVl~=#Fk)d;%8dU=&j)7 z^Ye3Sus^rub643GEbq9fo$avY6MF}+vAx$P_Kslr^w|k4mwMY7Y#jOgEZPn1IqJ8* z2iV7X$?r~+b6(=q;GSS(C6~Rx=3-4Hm%ZWSvyS_K?I&-YC9ZGl%%4U3g3ZM>)Za19 z>j2(9=A}Lu`v{tOiIc~H;MP12f|Jjh91NB-?!2t8PtNq;*n4C8&Y9i=AN%A??`Zm% zi&{HP&VFL&W?XG~?VgS8J7-CI&xvcOPX~A|-Z|5Ifz|7oK2^h+OV0GE;q+CrHs|JT z93S&jpN1xHK4RxR2k-dUe|p+XG=0Q5(`N*mhjXxxx_&v+X90WOnNwp zr)G`@%Vj>tfaUCyb9*Q@|8$?*ejDzce&(2QeJ97s$jdy<+rH+M@9PAxdU8Aw+Z^l9 zmXqM*bB>%0wx4{sQ^7gA!kqyp|32}|?JTfs;@jLj?>*x^^UZe_-scw0Gu|`5drtfT zM%6syJ@eJxYSBF7J@eHbYtcO8J@eIuwP>F4p80Bfw`iX6p80BnS~Smi&wRCIi{^Ph zHEZj6ueLhh`nkMwP7MdE`|m$~Q|Dl-`}j?%pG}i@-Nf$Gr|8#!y)V1>o`argf5w)- z5g+aAz;d2-ev3Eo_A$2F2-+B$vBk!|27NTxpYw_BJZ{I9&(DoJz}A6$_WLfdyz`0Q zJ=pTjvmdvPd%^e7M$(M0ZvHp%zMrOcD7LlnK*L!(=5C(mTW$H&(rk=jALr78rf=43 zuPG+3SH3;hXKy(9tk*tZxvbY=U?0~@Z9g@ddH6ODeYDlGe*1%6zl`Pj4Th7?x*Py@ z{p7QL2ZH76`W=KVU)S$o?5v-9*6$FoTGsDSuw2d4e6wDM>q|3t*ULF&y^aH0UpWU{ zujAq5vtB2F<+5I9fqh&rwUcNk(p(?k=An#D_wDFP8dYuZE zuj_RhwtQW$)3LK&>RGQdz-n2qGr@8-PxH-soekE<++8o{l=Zq8?0StL_k53+z{zL5 zE(ObFy{-lOxL#_P(=MaAKEBOEA8oa)*A-ybD`UA{SHj6>z5WPxz2viASApg0di@Do zzOL8R*jX?2tk*SQwXD~l!E!ZE^UZo)2iC{jT`%X9^%@Cwz4B*Y*4J%t@>#D@V7aW< z17IK5OYL^rXqxNe+dTBqR?B+b0d~DImg{vVoP5^nF0kt*pY^&MEMM2_9&Gu#UiV^W zz0|W__kq>2UiX9LYM$ns^%?`#$J|{n=alvO3)uAy@!wujk<8vtG}GT`&2p*9&0zx?V40%h&aK2|Mehp7r`G zSS{=IH?UmI(|ogDe+TPh?yi?}N-lrHcFfV_o}6EX^G>1`zgNKW{&{2k{(&t&0WLqY zKExhJ^JkWNa{dUcmYhEZ%O%zXu>QtRtS_(=OFgl^1gj<1S75p1`W@Im!>qY}4`+O} z#Qzzb`09!O3s^1je+A1Weh=;^{<~+5-wtPdwZxwsocQXA-wUjk_)~!8HsxNy^{H(> zus@@H&V}=5d~Z0{Of7zW8b9bsehb059%?5xeHRAXUoCwX1v>_>NqrZ`aIR|Uy9C(& zYU$e-Y+tU)llm?V=RDNXcNwt#)xs?g_8zE~c;l$Y!)ZR5@4MK}S1o?;HGXxz@54D? zwT$rr*#2ti`ytpd7Z^X&=eeAUu-GO+#C(sy#O zed~OC!8u>G^qm51f3!$8GT(XNoUdB^=574yeCLC6zG@j`ez5)3 z(ziF*G3tC5fOEcT>Dvcvf3@^o5NzK%--Y0uuUh&p47R^oxW&N!yOLVYvK7Fd1^zu+ z&Vm);au%rPELaJwma|}Guv}vO4(!=iW32|4Sn7$jI#?~Seh-#QtWL1!T#eNaF0s@T zt3OyRvAV!=i8TQ1nO0-12bWmti8T3b;H{5*@o9gb~&_1rlETVJ*K9R)VO zdd3XF)>kcl$AHbRo+m@G^;JuL$AkTMdbRXD4{Uy(9pNs(HorDn`VI%{s}{eDz~us=FV!Z>FORRUn#;SLl53mzUJ+VFnt0mS)V7ctmH(=*ozoqZ6&A)z2-(%~mmcBoN z&8>b*KVj>u7QbJ>j$6N_U$OO7OD*`zH{M6o(su?p^Q+&|OmOB`zonVs^i@mWS;6L4 zzopsW^i_-B9ANXCoR(bXgwt0o+&p0K4{GV#7i@m@TUr{<{OY%~44l4d>AM`*{Cbyh zmxt3=Eq*J4&98n-E5Yfjmi$%)`}1BcKT8LI{r5AU#9I%}oYmqtu<_%wo8-4Xobyo2 z7#o1?ua>@pz>dMzP3pTLoO4r4-;KcbS4-cG!S<~`>o^ELx z9RQbD>WOtASS_&*0?Q@VVPL=U8tZVl#8OYJBfx5jbtG6WwJ;3qyzBRWDz^F8@BcJx zebv(U46wP?@Bd6}ebwT3HrR3N_kRwyzG~Ue^TE3_FSYc&Mvhj$|7)?$uMOYyy$)Mn zwe%eUHoyA)-+-;JTKsMTn_vC@Z^qVFEn|!X`}1BceIEjwU;X|c!8X78{XdGWuUh&( z4mQ8~{Xc=NuUh<`1e;&|{-46uS1tKH3%-*4)YA8Lu=x!r+`qx*H;|UTZ-VtzOW*&1 z&98pLW3lyBi{Cq7^Q+(MIBb2@GRFJh`^Zl%|L*=R*gqS1pO*Um4$hzdYTJ~`8L$^ZM4Iy<=f~0%Vo@-VEv7se{bjomsskFH3e8Lv8Dve zCDznnW7VJk)4?T{dSXovR!giIz;cN-6WCbw=l`s5iKU)cvw_tTYj&_)_HH4t^RD05 zBG~3%zpq8H^;JvX#lhxQzpo{*^;L^sU$EoW?`tV+ebrJQ{vA{s^HNLSPO$ma@2fwy z`PJ{M3tL~c^j!yRe)aoW7h7Mo_^k&vzxsU*#MW0W`T6%}-T~CocL%Wf)$eO3Z1bz% z*Us4bs-^F)VDqcr*KXMQs>N>)u=&;R>kruaswKa@!K;#=T7JGB3HHw$eG=~|ICEBu z-;l3clbzV+w-32@F=EqzY} z+g~l*FtB$3wR{`rgZ(z@&&CVj@@=T++ZYa3%eQeMST3{l;so5!i{Po>(`4)e`GQuv}{4Zm{#N z-~YYX=3l@6`>^#@OWy~;=2pM|G1&U5#qS}ov# z-~Z>>`l_Yxmtga&-~U(G`l`k68?gD+@BdqDebtiR4`A;AYUw)#f3|CWM;2}>IP)7q zOW&#C^i@mWX~E`Kzv1cN^i_-B3}ExC-|LKU`l@9Ne^0_YfSP|d;_nRjcR0RN+y1VB zfB)g%LF<#>z3_Jlv^Q@0_)Z^xAI;z4NS{HC&qlnTX`A@_{QdCpcd7km^1Iai;pF{2 zhxo1A`1$uQ@f!dqU;Q>{{BACOgW%+=-=>Y9|Gh@yZ3dRtFZZC`uwTJ8p1P>Nzv)dk8Ik)zkOTMoZtr zvF+)+3$@6p)ltDe5cfYml>oA^66L&46&e2lAZ4Y)S0 zgX>Xk`TQN*chrOZU8^3{fxh|umGi*k$jARq!?nHu+sCz5JD(=!7~<^FC18IK(SDiV zWpMJwi{BO4^7UT&M{M~C_~m;TfxQ>sg1MTVJ+QAlQR!-?x3T@ifhz-3KO> zw#D=%mTT`=@%5dten!{F@$F|E#~Q=i$FclY9-zrNmN<8ZMd)h`pHukGt@}l>58$m9 zZV9mV5%Blp=ik|HiY?y>zL{^yzMEmo_Xl^;rfK*s;G7f3Qg@%k-3opnoVsgljIF_4 zywy%`xNRCvE!=kCjT`@y8^7)0CEq+UZ-D|bf!!ls&!SUm_EVlKa7H$QwcCCk_u;n}PO+5_3mT#?xW8hK` z_DS4h!PbL%az75-1*ew1KfdAA!kq}V9^|qfst}`PO>4 z7B2N*pTxZ$Y(1za_YvSOIJNBk4GpIj?k2GHAeZ&I87yDx;Z|&UV`PkxVEI}Pqp;=m zOT5uw`C1QmV9VEdcVf%edbk@~Ucc1CePHLB`9A;-mwFfjmwHf-?}K1tswcmPz+J3? zTKpaX%exnu@0#E){M6#N7TCR3OFi@hTMzs*$*(_#^`Mr%>wvXuJv@yq--&PP;TdfC z)_QmjF7;ra#C-v5J*X%57r|X{YT5gj8cr?T-@w*`T-N7huzanDSFq)ckuhEc%h!7N zC$_wPiT4^la_*nMTeSh$G38ya#P*&2*s$?&-ugSPxoyJRC->Bi zu;t86oO|lVVC&9%sN%P*`eXo z!tDt5+>y(7xD!~uo;y2Z%NrwO>;jgr=gzL!^7|U#- z9u5%G96x>sVp|Vt;SQFg)p|GtTfP(D)Wf0J@~!o77+mVXK8brc*m_V;?ni*lT`hZm zWW%Y2I|^()$Yp(ofaPmF9E~k+jEr#%SiaW7P;7bq67N{Be65G$u;pvKTVAo15ey56Q?nUN%8d$$>t@(Gn)8X7} zwba8IVp==CiFYQp^`I8+Y_N8%hjXyyJMm3DoQo~rS`X*Jr5@~)xaWhd2leEB0odHt zviHLqPA%MpVCz9H>vIuUzShIV*z(567?*(MYdu_wEw5kVT?Uq~^>8`1e2sSnwtTIJ zE3xJEOFjG%>|8VdtH9w>4}XG7J*da`YOpcYlixL9*GesZe+J9D7n$$1VExqMcOBTh zR!coxFQ#?i8^00Q)`MEO8^PMO9&W;x@5DFta5J`iYdzcomwK>I;@%3j9@LZjNU*u9 zW$$lmIJI!2z}ACY)@L+WzShI-*z(567YdzeFEw5kV-36Af^>8<~e2sSxwtTIJ zd$Hy9OFi5NcCMNK{ort^hX>$N59;w912(35@_P{MTB*hFA+WrAk@-Fh)=w>dkAU54 zwcJx51rMfqc33a^crSX4w|5QSxvM-5wstn5*>5vi?x|0J)pI}li+20Odul(h_f$Fk zjQ=EWIY!!&U>n{a!-w~?~L^k zSRco?pK%;(tFj;ddFsdTa*id=xicPY44>5FC)l1lYT+h;wd=X_DYkqkzBzY3!Z%OR(RFTF#iS8cr?T*I>^bxqOG;faUAC^DVZ#F*3$?VEKCP ze2*=!U*i1$mapf|kJ$1x-cQ)__1yUxTVB7MJHLRPYv%teI9$%1|H0+lQIGF$U}LH$ zKgzOut<>Vz!@Z}u7nyH+^RCs3-Snz}ACW_I}!iQwuj8*m{u5`b-a&uk|nkw!ASi#*AS3 zS`RZ}%j=hTGlS)8Jt<>T-4_My4$b9Do>!%jK`M~bATIykbu=UW6Z~S`0Sr2OA`hc}-JuHYV--&PP zVIge!)_PbNF7;ra#9ahzJ*X%5MZxB-mc3uB;ncz{4z?cTvOY_IZ)vc6t%qf>EE zC%=`!u9aH+RsqW|M9X|v1?#64zu$q~YqiwFYGCW31K;?q4re{6gS1HJ)Ps6_Hvt<{ zJ^5`4cCFOnw;5R8y~unw2kWO6zb(M-wOa0}TY@`io*mYUKHiJA;_Y3-ckU`%gRLF! zFZS~ul6&el*y_2TZ41sl)%%(ER5|^Oza4Mm`_4UbcknoJ9!|?Wbx%0?QE=Xiz5x4p z2UFXNCTA>h?w|XD$Kq$d#NQ82-gxobA1pt<_zlLEulLkLu>Ifi$UXHaY@ggy55<-< z4{_GvFmUdv#?3wTaIjkLsYih2+&}NBM}i$w-t|gs-`S5LjgRxz-*H`|p}c)^Pdyr2 z&fLVgryc{g?!2cOEBDm+`p#I#g7tBH`x(cvCh+!gEbpnuVaqv|IOopsU}N~C9#6pb z+))d65?H&QJ11kycjB9KXBf78>$!6ZT+SW)B<`tT&mHxABd3A=M$~e~oZfJ1;m!bi z?#SgkJQFNm&z-Zd<&BXs&IZfZbLSjvdHoXaT(EpSch19>ukp^umapf|1=#ZX<=hz# zcCMNKh2U^GcP@gTM$3FJ1M8<2zsteywOZ=o3b6Iy z`0=|E+j>w7cNJK>*2AB$J&i42>){z}dHqrk z&w`z6=KmZxT+O z)WZE6Y(28%RTkK z;LB;A9oCCJ-is#i_O9VOca=}U*3M9x{f?vMp86SBJ@>QE!MUdn276DH)6e){@HW2h z+#`PkdpGp&{d14}Nq^dCIPXPM`agQggn6VTxjXr4z_Ro-QbSWS3P}ug4NP@GO&Gz6EZnZ z4wszO)3+B`Eqgcx*m;>HA$D!< zCGQ2mx&Is2wej~)`heAP|6dR+m+=+?JEpvAl-RzrMvF8)&Rc)SHMhliCpUlZcTsFP za}($OzZjTQy6^wS%KbmSzBASmV0|3le#UXE%Xw!k@Bd3;%Q==f=XqbSF?{l!FNN)S zt`=??uy#Gqm&KOv#5d>pa@g|y;XFTbMlBELoO;silejB@JBAn za-OdRcCML!CpcWzsvn$dow#1g#FY&el%h!6?8e6``+Xh>{*2A{g z^7^G7wgWrY%zt}uxYWZAaH$9N`0fZcrh4+*3G7;_#cyY@ynB)P?gG|NEq=R#-D|bf z!){>fp&j4&?T&3dsD=9jSi9E4p4jr8_@*BA!j^BXhrQua5B5pieZba(dUD?vZ0>5= z`~4bDE!_TK>p?E-GZ-vi>)`-wd1GXZ1HtmO9uC5m*Dvu72FuraI0Re1#yb>SzShHG z*z)?N9u5aP*UbM2aJbaNk#MO8_4pnIHl}*=8v=H%)Z%wESl+$Je2)R^rxw4VVE0-r z^>8fMdg#D6e#c>34{G5~0BhHJI1yXE6W`RsN!aqO^>8v=>cKvVI}B_+s3-SRz~-)& zy+5_#)WV$xwjSiNKBt4_YdxHSEpLpBaVA*4*27uY^7=70U}LH$zl*@Gm0J8R2FrKv1@|iNwU;!0YVo@i z>|U$o?syq^U7BZy^`ejWqRV-E*YKUY$`xR1XK|YSmZatWe;3a?Z2vc?r{ZlMV|e@I{&^3!oOy^dzk9*Ce;POU&-=h?xqsddma`7M?>+!_OnKKK zv3+NqA8dS_xBiZ6ZjbQxF*om@4`Iuhn>hE+hr!m9fBtN&+&|;%J7YZx*2nSfXB@{` zy6lJd&&S~997~+@~KNOT3rC z^7TCVJGOj{_X@UrJx^Z6me()m$v?o(HS_-`I9$$?*WhxVsK@tTU}LH$zt_R8m0JAX z0L!}%GPJB}j)~Cv z)PsEz_dT%npq|{{2b;TE_Wpy0Qw#SY*m{u5`g{bIul4XTw!ASi#(1!Nt%pyr<@HOv z|AOUfJxsusukk*`map~j8MeHBsfW+O&NcJ@0vs;&@FiU8K|Q`-fsLu2{JsXeR%-G4 z1}yJhWWL{m^;3)AcVPEgE%op{*m`KkH-0~0TMugCegbRPdiWVzz7yZn!!OwKt@ZFL zTGlTK1m*WVf4B3)chAdXUTdw8P2Qdg#EGH%7+j36`(*Fd4SI zeu+0ZSiaUnFKqc5ZwhSrS`SlV%j=hVmF3<=u>Smy_j5O#TDW<@)`MKuXI`*;t%v!r<&BXs<_F8ydgzTU zuV3OV0G6-y&<9(-#^e9>x=p^;!$R2d`lTKg20Pcxe-UuF)Wf21sR#A=E(SKHdh%Nw z>{_YCZwauxdy)Ar3D!?7etp62wOa03_)pGyb z5G?1uc>ml8?3nVdLt^{RI&aeWIB)$O*W5Pe?UVcGrr2`kCeHnHGqCmK{nJ>vf5z8$ z#@Yg`kK^0VIF9uf-ad}y{c}rfImZ&`JlP6t44>55*4UmWYT>p8YuEE+J8bz*d~=>` zk1gMNp6mdZ^Ta-hyCc~1L_M{?6WH2U%h|DW!>NVa1?+hum+xy=uzWpFcEgr8M#k73 zEML!)J+S5VOT0gTg2Jx}(=me()m$v$A`n)&Yw4wv&}Ke(JH>hawl zY)tj!HyG?%sm1RAu)KSb`5p+?Pc422f!%Ah)WgAG>%sBkcL=uipcd{huy(D7!?EQ% z@l8D(fi2%!4@bhK9_*92M}e&e_2fPTZ0>5=`=c99E!;6+>p?E-GZZXe>)}{zd1GXZ zfvm#_0W!Q{LaC)9@N5}2iC6ja6Yzt zC%&nN3$W!|>tQ%t>cKvVdm-3*P*3g`fz4eldw+4msfD`)Y(2&ZJJ-zr8gRJO!=K?&59;x~ z7HmxQdH-O!1wba9nVC$g+-}v2xZ9S-ky9KOW>)}>x z`A&RO4Jv;=LdQgw= z!(d~oC%;F)u9aH+9tF$07n$#4VExqM_c++SR?GeK3GhKQ&kpNFAMZtf;q6_+ckU`r zf~}p+Y4+Qamiy;ZVD;S3o(AXsxi;AQr<{Jqe}=d5eS6>8VG5Gv@xHSse?R^zyqrGb z+&^CfkHv2kE%(oV!O8DVzxch5EkC~ay@4%X@1J9_{r~RE{qucnpWHv+#+EY=apw09 zIQLKE=KeVjtd{%dyI?u@#rx-bV8@hq9TMAj*7<|R$9e1TxUSL1ynS;2{198t+{C$m zegw9jynh-i_s{tH&RFBY`Z&J*jN@3_l>PAj`3bz7V~KN~{1v{4ywtOePIZwX8mTx^zzJ$wpVxPqQ3ha5Jp4$H!Z0)P%?D(eP)WUrW_B@fx_w^lE zzMd!FW6K*OWBdS?ujk2+*z)=%-cMlpdY=4@EnnmPf-PUqlV7pr^~-tkKd^Jn{C@+7 z%X!kqKIJ@7k8clnW2z^=cCc%u7QYU#ynB)P_H6vr;x`%Cy;e&-Ob)gl96x@&;H(F= za8rV{YduVbE#HZ6>S1bZ`PO=v1}^nrpTwOOY(1za_vygSQ7wBveZ#4Rn*nS+$Yp(I z1k2ZYmMsh<4i1-n*i@tY4U?_OlS^MmzMi(hZBd##pwSO9E2wBsAUK5*89 zTDXP4+O-}Q#+L8IH}$XxwtQ2m2&$7ub4GPws1j&0Q^fzfQxcg|U$o{<$@HGMZy=KWJnKjZJf+xWhH HSe5pFrPP}+ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x1_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_dot_s2_2x1_h.spv deleted file mode 100644 index 89234486e02212900031bb57db3b81e3547e36d6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16092 zcmb7~dAOHVwa0%PrX0hI6G%v?Wrb)8rgD{|M?-{r5y*)&#SsAT$-m)Y~j+SL=7Po+uKA&^m?{PopzxzJyXMfjs4SVgi*SpWlfwpyf zbgZ>)Q`@GtUG0z#wWeo>T3xLT$a&}vHFO=CARnX+t=DQ*=zgR zTE|*@=5~xRBhMYH;vvSPj86E=6RK9v(B3fs|9hG17)RqR_SvI(YT>k%Ge+$6k^4-+ znT_1G1*eZ(r-Cz2xg876t;_9Na2+eITfw=Wc0CGitBTvR;I^)~y$f!eitANy=4w0( zn!RmTaebHvuxsn6uX!k2Q)P2io=t(w-`uR}9OUMjwrj%BE&EdE1xm;^CHtLNfcCF?bHOoDbebhgP-d@qZGpitC-?k|?@lM#7OAkgH z#?EN$KjmG}b!JvJ{#Pp7u3&9x`m;S>3YCTZ%b$PLYM|HuQ8 z+pm~?SmYtdJ>Qt#q92C5zP6%8K3@65hg#$j$k)xivBm#e$Q>qL(IPipQngxtYj6R& z9t_6`cs`W!m%X1VTQ z?_Ondn9J5p}v3$=SX87)zV`43@_}DyB*%&J}E5P!xc?|43TE8{sh<@Rx@JvGqAOA52?d5VC&$!pS+$0%g66Iu)gvO$-{T|d4_AK^Q>NA_SwW4 zs9VF3YdSCHjhvh3sVsj@wVzkO#xbwd__czI-Rp4j<{rDX1(#gc!D*ZGu}9uwxX0ZY z+2i_(f3nJ>5n0Zkk>UOTwwL5muMfb+@-9rQ55e-!RI&a7mNSR>RlPm}%X_xTwLQIV zuJ+|*>ehj|1Dt%;`wZCiyic>yXg0+mX4cIvH)_pr>>+ZaPw@sUmcp0sS zXE6Z26Jv3eYiF?f2hjZ=d40ChGc}#B`t2F=#ueLl#@z+%`ufko*Iwza%xs_Z0Q=Y{ z^1CzSj3ahWzOy~S#!4Ob09yyoB6ZjkPX56v@4dkK$-D2wb#~w8@&&NDn6vh-=^F3B zeG%+^WbTKNtFrdy{bgn!^Va`M$a3Z_PTsx1W#0QBx6ZpaoP6G;eZl(4n|Ib~|qV5fboVkeIn{k!pm492w&gsoA?V4+-O;_;WnA4lT1FN@p7gc#QR-C!a zs<^k|v{h#hZ(uf#kNK&8haqo1V)yV7b8Pf~m+>A$8}Sr;?ejl@&BHzDqh34TZT0Vi z?NxKid-(xe_;7!LlW!f%n5l;~a6Q-8-gR8p{kX1kSxfh%oLW04)^1hpJP+55ZFs{!)v;N9D-tYSLqgMLmZW;nsKLfq}I0#w2kNv3r7>2xi5U19|!Pa^hbMC%x zfaUYEZ~|D~b#o>sBFiWC2(YpJeopLfg5~3LGFUF>cna7!^10(r1A9OD z4xJA6abNNy8FKDRoU=U>Y^>yR7T8=otK@PvoP6qd4p={V&o6PEJwNaGQDAeihT6NP zd5vcFF)#J!BadU4mpFNh0k_U$ES!95asgP*xZS9)PwvW3ID2C{=dQ%8rcdt5y2{5~ z)U{#A=_ht?##NSAZd=LDc}LseZVk1m19xH0UD*MwzPu~9syK7WUAZ-!w(9K7uFS^q zF+cU&Fyzfg?7nwlj*b4?Gj?ETBhFpf32YwjK_B(n<*w`u_8u{(^w^GY;lu3=C*L}j zF;fp~;CimFz3aHH`*B_8vX<^iIkk39thQC{JP+55t$k!HdnY|J9z2shl)ZZioP2s_ z0$48lnFyBCC->nbWd1bWhn}lvr=2-wUFYQ3j=ap%y!ADwykAqm>XYN;$mUqy*;C=< zbB{~|>n9)XDsb+uaMRT@))UX%t^r#UXLIu&_3rd8HQ)Bk*A_bOPVZ86@3#L`=RLYL zIeU+)JD7J!zr6o9fYmpmw^x6Ftlr07RsVg4y!nXTzu$Lvg8ja;rrs~!6?Y-a&#t~3 z?g7hrhuGsYnSG3{?motHhOx!Qz8n1lus@d)+kGrVmd|~>2yA`jQ^&<%dG`~$CCKvb z^KgDjE(Jftn8z@_dh?&fyo{mlB4lg+aK(8h=5C(mTgvh|qn%hoANR5ae{EB*HR!BY zdfIxu04JY%{TwWpdi@IQW4+Y9#CVZm9?s^Wjk3Db?`5#{%Uag&6*&3S_C*^Uy|FUF!9Fu=UDX*6R;&@~OkSVCyBHdi@bB--vze z{)8-F*6Tgw)JuKp^=GiU)a!k)T$!i&rd}U{wJ~?=<(^WnjbQ6_KYZTDO>pw5*JiL> z>eYd}!pC~4` zUTu+6FZHQcJFvRct36n*%+q{Rua01C%-wpqr_`$xy!Cn-KK0rGPCoVeH?Uml^*OMQ z^-{MZqcg+$IGcwy%IZ?Doxs*BYgwF-yP%ZqUS>{1uIm%;M>z9)9CBFp>x zsocSh$Zs-shf|-N-v+Bo&hLQb66=p(?Tw#U?;|Ic`o#JGtS+%W1j{AYzk&U|Qkm-p zIOD5J{EgtmSD*Ns!0HlzGgvP1w}AaSN*SNlXo{~c@!P>AzWT&(4_24>9l&x!d5Qep zbT_a+b9}CY^Ji9fIBTXZc0DRP=oY)4aMnZJw90o6u>R`eyBF9sm|A?lh~Qq;#dmM8 z{_5iUWw5?{PPO>RTtmBVExs__du||<$k{g=YG}2_aLzT>caH{`;DhA zy*>c!`{j3GxPfr;TLY&ck5`s^EExvY5{So`vwI}tgt z)F;+SV0DQ#0xXwUCxeYuzF(&zCzkreIt{EYu||UB66*}Gv9?g7yh~>zCzkreItQ#S zu||RAa(3guevkO1w=RaW7V2U*zOrM<7Q0K}+^4#n=LE3+>f$>Q>>6cnO@eb@>f(DT zSbuf#oeb8u?5)e-+=sgOP66w$F5EP*?~}U3y8+yh;gkK|i0ppV#cl>zyK=ubA-i97 zS>tA~{_5g;3s~QBzqca0Uv=@l4XnSq`2G;AZ@J&wk=?Jl`2Gm2zq)XDfPJ6TUkmbwwYXx$8PknmtQLwu7-eX|7tho}beffT^ zMoujCiS-0nU1B{6mP@Rkf{j(aU(X;Xmiokc7OXC@o&(Dz)*7&}wiNHzi^z$kKCxZ` zt4pkx!E))XwP5$%yWrjgn}2yHuS3>WU3}jHo7(}5obh_Fw(4T{HrRC!WW??ru(s;L zy(?y@OTYaC?DvmP;%!7WXLYgL1lF#6A2uVq4|Q4NpJ4sf#rI>dzUBMy39@@r7vC*l z{nf>nzq&U0mhVFwIQOeAzI9;z)rIr-1$V%y^V`qAo zuROT&agL9_m-TlU@$vV*#vR1`Os)Cfi~D2azu)<9wfXnEdN_H1FA=*VD?7h0V|Nss zd}((~WjByHc7wq3rQNZWo&O&oc0%i;a{C@z}`UYelYpw2jhMa4Nb4Is-{e8TC+23t&^6TJYcRR9tc~AcsS>7|w z`!Ex^2k(Npn#XKpAM>!6W-;WwQ+s@ zjN@95F#EXHQ1EhwoNI}57wv_wF?_zudv4!<0eLL5x^Q0tEBk-TmSgAdaE?WmuLsYi zuKEr|mOldAz}U9phr_w2b_{(I_c-uCIQ7=p7{`MfnALr+;!dbIb>U6|52@_0tn5a> z$q$3i8Yd&m8zXC+0+v6%usao5Uc1CQ4J<#Rusa=DzKnMUvb_IyFYBI#EU(?=?A*PL z0=w7PoCgkrAJm20AFN#V!zIY__1LB#CLqhV?uSWm=?8rhcQV+1P@mi{12@2_%h^w< zICbHsg6#*n)Mpx4zU+r9k>!n%HLe27m;G=xvb=VQHytcr_QN&E@@2g5Bg>cla4oXD zcIk)f!R|HtzY!cR{V)SA{h&U!H-U|*KKb1YZlDJ0Vs|T8-m}Pl`+ys;Qy06wV9#1z z`r&I}`=Jio*d2syKd1}W53F4F!=1?T_1LB#?n0Jt-4FM`r62T3-21@xgZkt?6Wjo& zE@yv##i|V3~<=}AXhezPj59(vP0&Gn6$?s8c12s?=yOm&h&m#Lh6x@KFy4W2C z_N>*VAL_yO1AkiV4o9#b)W!EmuyWZCKSOTfu}wcb4JY5aAD)FvKj@RV&x7p;^~rq= zxB*UG&i;jpQy1<ByH`n&{|FZ7DeVZUhOw@SV>ob#=+8J)f3_ltghhxnc4ch^&3_4z*gDe`JY zzR&!BXL8yZKi`(l`Hno0kBfEW?Du)TQxArdUkK-S(Nka_zl+oz!jSX3NSyD_L&2@T zQxAiaH(u=egXPQb)Ouw3@;mh?WdF@~1A8=&LCDF&@6@j&%bABbbr=B7cdBvooq9A_ zUA|KXg5^9ve|LNg*fr&?S7JNoJO)=b?pu4;HMb$m$<6Q7|3H>AH*vmG{}XKA`JHO4 ze5c0NIcpsY*2eYqGmdMmV)n^*>QH1k*AnO683s0nPx^5#Htz%r`4eUAU9M-aB%64^IKhm-o)M zkmZe$HBJS~m-o(T$nx4H-bk=~dGDN#EMLYu16jVjcg{qX*Dm+YSzz~?{htjEms*_z zmwQKjY)65OsXqCg3$|A3Vs{=`-m}Pl&j)L#E_S2Ap0&F4!x*sr;QFx}i)=rr3wI${ zx$K8;Bg@xgn|}BXvV7})xCk!&piknC1KSVkll#SBb61zMA763m!d(KkALLS>31IoM zA0{Hp8zXB>0?U{Ea4E99c8NC`EMNA+WytbnyeY`?Wj|bwEU#VqVJg_YX8+T`;nELR zz@;D5$M#CFG1Vu(tH9PuUF^OKmiH{O->bpesf*oouxG6<{qQ}o{ZOZE^>@K*knIO` z;eG&CF8kqHWchk*(+~fJEZ@2xu7gWI=##kDgY5_P$^8bfxvR_B-&k?#!p#8N4|1u` zO!2j+9lp?VEM8ieuykz#=9L^zU+q|A=Y!2%UC#c2ic=SE0oZ)mJbo>S|_ZQD1r zZq=}1!{-{FTCbu0bZlsBXa({;s$&CMyShDi_`$<_pLO;(diU$Ui;f#Nw66Q>yGcX) zhPJ%h&`v+@)Nu-nXv=6F=r2#Gh8j(O+hF?N%)5~`gx+GGZJS@~IDO@e5kLFLnQ}O@ zk=v-?>?7Bq;LKC5Q^B>XxGn|PzT&zUoa5QIN5QRMaoZK#1{JqM!EIP^y$a4;jpu@9 z9-phYe!K@WYuC}f=ArGHYMZO}Y!=A;&CNBPgWbHQZR!}h?rVHCAIJ33e=b~NiMMH% z>an$Z7WPW)a;!gMXDqQ#)-ZNC)+&55mUtWd?dCkR%dt)-uw%K0=6MQs#uEEvtk~sP zr{R;a#ExY*$I|X;0G|39VwdxpgUx^J32P|e$yl+=vF75FvBZwmnr3Y6o(AC2z7M+` z>wawhtB)o2=}L3IVwYpBz(?I!V#jJ;qh`A&wvYN_=+=tGBH9)Ozctt=e%7yJXd9zO^NBsMZcSVsyQ$gUH}(P8 zZSS46PwWxcJztr;PX8MAs}1+Avqx*cy=a|127Ah!sq6Y5jot42N$c$TA=S{ZuWN7y zS`V6!<7uB+xBf5RbFtrR)4v{HeFDvOi2r18=Bqx9W)1Wp-gMrcC-Pfy8r@uB?f-8x z-YxX=+%e9cl|7^4%*Q>wm6q{mR+?ia-#K8cH)6aQdvxprf)J@fti%mn-M#&w(p);8}g z(LAf=wgP(=Yn#Juyq&9@>!L*v-bvoQTU z>$L5+yz-H=pZ1E1>qT{2hmT=fQ+^I@yR*YkcsD=R?nf~0yD{A7?lkvW+kM@;vOgr3 z$L-9s7VKyL32giO9zYa7L$hczXnwZ#CWdt~n2 z)$cB_oO4O6`C$3UG-LTWyNBjy&m0qLNyW!!Y2{Iu2tpLlr_R0MLusN)t#qVLTYvCNS4v&Ie2S5AC>(^lU^m`0!U-^0D;b--6nq#QB zS5NZx`Gh@Cdx9oc-!I-Dvv2OFw)|z)e4Ygx$Goz}FBDw-UV@W1_xPOqkTT60@{|4Bc%lL0$8{hLW@!tl^KU&3q2P|j5 zxz!rJ3zm1k6TcnxZ~mUG6Ijc3yxYUcXV2CLJDz81#@hfapFSIcKeH}L*P5o=2y9P0;}JS_5pcysB~WSJy+j^CU0D^wP@TfV8^%r zZ2DS1TWj;ScD4cgSUd7PXmZ98J10NK+k%aib=VHd5N=r{lG2r=np5KI@%p9XWS3@ z{QBfPeyRFA`(9A3_aDIa@lI-=l~o^eQG10ZXD(vrW?XG~?LStw@6=}3HqC3OPdD)U zyi=Qh2CG}UcUF0j-)V2s^bvoT zzSj90VDoSe_EFc*&$jx%z}Bic<#YKKT=;PBz{$6aWz4LHYv6c}ufO9suJdtR-{o35 zC+)1Y@5I`oT08f{G2@$>@g3iu=+?4zXg}kmo@#J2m|yvM8~`VudKw6p%Uqkla@Na6 z+~8cN{jmA3VP~4-xv%==^Rz!$AIF`JHVEv!A)irgeVngnnCsy?=aFZdzK*F+&O*;F z?V(j4-_F~+?G5YkKiK*_2|lpUrWD!(EwoDtZE*|j;zFBSXr6WNk)J;o9B(hy*zt0v zjRdPtL${8HW2^gE$Lhmq^3GkHJ^DJ>Jvx|o&dfu=^7$Dw8tl(j`P?TC1Is&Z_VRFS z`NTc~Y;5n|iG3tkK7Eb?%VqzM1{+5{KO2q%d#?G}I|1zDyyTCk$vH1^YT`t&v69P4 zU~_Sglgr6)@>$1I!1j}O-xJrj`|i(%Q^Drq8tU(u=5;!6AM;Wli+v8wyu`_49Jpm3 zXTZs4P0j?%8Miy@>ytC~Gxpw?zH`Rn)?lBUv5i$9b5UzWle3@LxfxemUb}T=`_5;( z72RD!eHy`CdFPC62Uai7*!3&UTyn;40H?2-wb_lgaeT~AeM6eO`G}qO=Dg!$|BY!K zY5ItB#&!UkhjXxxx_&uhHwAnCm{V%36I}Rko59JqjAhKMhil+?j<3JtIIizV5rY`$OTeN~}(=6VLJ?a@N>%=HXb`?UT4_Vdj33|3oJXr8|t zFjvoCwXgEoyNY+tjj3Su)#%pvmDuV&*0}l=G0Wp)dB#k~mY;=> z_RU~9&mKP;xAOKew%QEZQkt>F#-5Ho5A4su#C9HcV$0_ozY9ExCZF}c8!Yd9;#FXgXOYbTY`ODFSSmzO=;%g+dTBqR?GTz2D^S4%k|p~PCo0hIoS1+ z&-!%%%df^ieqFKU%k}Gqo%K`C`fUML%ldT(%awVWZ`Nxous-JQdO4@8*B8~`^0{=q zc7T)5dhH07%X;+#`?y|eJJG&GbA5c9hd$bBS+8DT*DGVWUOU6dXB~QjT`&2pS0Au^ zxn8?q%a`lb7dz{vp7q)ltd{lK4J=pYX}(#n{$PE~-Su)#S+9M-uGg(d`8@6mC!h7I zf#tGZ!@xeSm)bzu0GjLL+dTBqu4~ojvkA`i%2=+~esJQUmweW15Lmukuff>z z<$4Xl&U&e5y$%4YWxa-i<;pzGH|sSVtdF_7Ud}1&bqLt?dITx!H5yJn>vbqtF6(tX z*vIu!JDheH&GqqZ9{OmjrM||1U9U&z>sYSW5peQZuOq>(mweXiD6o9FUPoigm+N&5 zcGgQh>vb$xE$ek0Sgy>|e6wCBfb}tV*ULF2m#48Eb1hPGeg@8;d1~={7A*gI;rAT2 zyx%$H4E`(jDw=1odUAdZtd^Yr2A0d1Z-DhTeqz0aomlFL^)^^7vEBj8CD%1zzsD+b z{Q%DRYKi{|IPujJe=S%o@jnI2B|c?P|F=yUzZIPE)e^rgIPujJza3aD@!NytM)5hU ze?JcPXO7PlIDcks4d;J`8jO-wuN&&)Gn_2ZU?r%TE^HO>=?Y(_1yu(xvHh_ zj$r$%rSDE)`*ICi*LP<)=b@Iqy}|ZZ3)dIyy+$qZ&Y~VY1AQ`&v$378TKvwb{L1;B z3+H^*GRApe`>Unzc(7wI?7A_|$9BGI>3ad#{%YxaA=tj8)d9%aEYa!Sl5Ep66-p!Tw?tM z?B}42bv;~SsVCM?!D@+h16VGxZUkHNWvuCNiKU)cH-Xg>>t?WAYT-^X&Dsw)AKU!P zI=>rRU$yux1e;q~(~Ge6Rg2#}V8<=%cL}z>YT3_aVDIy4>HCD3X03*M3T%Go(9-v5 zu)b>X`wiIq#?#{WELdN)_&oTDTX%{#!&Xd+{3B^W1;GgnJ!MzC7dq zjxGOs;r9=0`SOf^2Rr+yo_%~5td@Oz4=k54-v{e&{M7x2*omc{SRaAa66<5ITw<*S z8>>9eKf_Ke^~CxQSS_&{__vf?Vl`qL>wnH}xWrOVtajkGw8Ux;mdjbODR?Vm@nr8h z!MPS{@#|dqu`TQTHiL7XYT3`t!S+{6-!5RsD9^I4aL!9DeY=6}ua>@BfbCnJW!>SN zhg$k>3AVplxE^57Z?(kh1NJ`Vllkrf=X}-T*SGR3=esMM^Hs|jyMgVmmcISKj#19H zKb-SbOW)nW_E$^aFN5t{&UX(u=c|^!Ujf@+E!t~M{+(%A>odUms>SbHU~}t2i{H1w`l`k6 zY_Q{&^?MGszG~?^9=wb-QcK@U!RFVe;4TN7Ute1KUIEruEq$*7n_pRjQ?T__i{CY1 z^DFCZsyae<`7!D`vt1z@?1xe%;>d7j^comlFLwFIn|SWCfjiM0%Dtnxg+A3L$s6KgqGEwNUB zbu(8VX{9)|GQctW$z-o#0C|E9g_dM8npT)j}`yICVmp|KI#MW0WelLN|t^C>k zGPb^I@p}dAxEC@;YGEa|zG~rCfqT%@a=yI<_TP6tiT5^~IjhC*oyxB~f8T|39%>om zJ+S@N()WF^W0dFb8aU^smcAc=?XQ-;AA;>$p1&W#IbXH({TOV2wQ!$;_oAs~Z`WI| zd2h?JX??itt$Ox$1F%~5c0;gSVr>jwNbEA!CUA+Ro>(2gYKhemESFfF!PaOQYcsgS zQctYS!D@-s1uU0X-NDvq8EZ?p#8OYJt-xxDwKZ5Swa^RfyvzFSjcxvA{r17uS1o<} zg3Yb0-(9iwRf}Igu;Z5X+aFtBwd|+geUE2eYUw)!Y<|{ixS`nQSJve)Y<<<@cOcmO z${IWfTVJ*KjRc!tS#P7T^;JuLes6yrd8qkair<<0osaMAjo-EVeTm<}>XY9E_+7g8 zfmI*h>ErK5{hdPk_$FqZ&T`{f&E`zrPs(C-3+C@!P-h^ZT0k4T6&| z{f1V4{!Ll@hQY~~ej_VC|HdzVqrmd|<>%0G*#2FS@%+yJBy1nQGc(T8qZ;->$UuJsaD;lZl&g&%sV#_4GX# ztd_px!S?lc)#-aacKWKP?*(ABkqyni^SKc0Jj}ehg3<2tw=r7fTDfa;V-~iLd00y`X>#Tv&cEsI0MDhl=f-vY{5{M(uv*TlJHc`pZ$8*D z@FV{U`M3utoYCf=ypv&CRiY1n~gEDgtsukVcY3A#Rx zZ$INW)>7U+j^$@%2~Ezi#QE9x1^ODp=Mp~W*8TR_<9Mru+Yzko--|E7&)s`p9UL*Ng9Q+JJxF&f;&TkX<{JGA1|!W|ACQTczb z@*4vu|26oGaU{09F*3$cVENI7-!a(o`X%16VEHkH-|^V;WxNxx<^6m7jC&%synYuk zbLVyn*ty2%G;p}A)i>c>E7w>(zGK10R8M}VgPZVEi{BYwdG{jo-3i=;pIZESf!%Ah z)I)Eu_2BsN>w|4QsD;}VtXp4`6!Zh}+G z-ha2^)WS^!TMu$spNqlrWj%ZkTizHM7)sd>QX@Z27Vt zuEdttFZFOW*tur@Q^Da<57XdM59;x~7HmxQdAcu zxCu@zdw*-isfC*bwjSiNKC{8{Wj)-6EpLpBF&8Xf*25jx^7X=hKUr~V;hqLt4{}+bXTb7hJv@soZ;XubTd;gt56@%E>z8;hfaS}2coAE^ zjQ4wN`LZ5f#+KJF^{^7`Tr>Yafy1R9R>7qn)Z_bSurbw>-(SE@tbtnmR)gi;i_CWr zxCuYC_zecT*J`<64gvR~d3IPY`gm8#UBh?oDr?cK9q%vp^B&?o%X`;DVD;S3euce) zmiw81H!i22@pHHIoqOc&TrO6Tv-k7dQ@;WyKM&4((L-P#??q}~rOA0O66gN8H@M|J zbsso+tzpZT_tgEd{Tr$^%+WlCVJ8pose`cP%tM@Y7!1xm)wsE*4gss> zo_YXS&i(Uuu|vU*DernEw(so6@XE(|>+iVcHiCC@^PYMjww$?%b5A`8Y~6WJHI{~B z#n*Sn8VT0N@$F|E$6C(YC->A**m8~~&bjk7urYj6j|XFW?x=-31gu@2JEO7XYxw5e zITTxdAY2nIpQXd#oD;8lpTs>J!gEJGpOGN%PIilj!!W{|r+>y)Y@F=i+dF~vI zEpLpBaST|#Ja>-8me()wjswe==g#ri@@2deu;t5h=Ns7a`sLg?5$s$u|C7MsvQ{U< zxmMm&)#G~#*qG|c?^LjBr53-_!1C@z=KD>seroX>3wE#7QV*wttp~@C-#BdRK`q>w zVC}LVzJ)Dc!#DNtZEX3L^>7wk>cKvVdp6j5P*3jXfX!Vkdw*`lsf9ZaY(2Z27VtF2a`AFZD1H>|8Vdi^1Vi z50l_h59;y#9@v=b$?yAM*GesZmw@Hni_G^2VExqMHyP|+tEC<;1zQh|`d0rIybRlV zPz!ejSi7u;E3xHk_@*AN!j^AY4^!Y$5B5pitHIWTdUC%8Z0>5=`>7SD7H%5YdXUTd zTnm;j>)|?Vd1GXZAA;q}diW8xync!IW3YT#4?n?{FXLU0Enn8dPqF3oOFi5GcCMNK z&%ohQ4?l-XJ*da`7hq$mC%+rPu9aH+ri10(i_G^XuzqUsyBX|WtEC=p0b38P@r~aM zZ0kWS+)S``Sr4p?xa&jp*iTK4|-ic<@B2iSU$ z%lga%%a`?VC$_vXGR9qC`LZ77W6SH8cz1*4%X(OVEnmi4h%H~%!y;^X{ZbE$!Ok`F zzXu#H^{@mk^`IW#rC?*KC%=2au9aH+mVxEni_G_zVExqMcOTfjR?9v0e(+eDXNUEo zk9U>jyuEAq&Rt~%*xDIEv)?FM?x_!e)pI|4P`k1Do>~KYPnFZp_z&?mzVF;qp9Fh1 z^q!h~3e z1K4?(k8#z@b#OhhhbzJQyEYb`Yv#J1+16EUURS@T{u{iU>nP5C`~%$b{{IG?{3`m# z?@es^^8Wu%Yw`Vd<#N7l0G2P$^9`}( zjgc`n0?U`@`RB0Z^-H{s!SdyKz6rK`8LtDje0iRC#Fp1D=lSQs&NcJj6dW#V)d|kE z^8T+L-_BrTswcnAz^;{A{5A*6x1nXeUBLRO#jh*ay;e&-bOT!tjvv1*;QaZn7H&(h zc3BTwVawO>O+9RlE#I;pdcdU~?31|LfUO7hcKvVyBFAcP*3iAgUwwnd%sV`sfF7YY(2U}O_4p108&f^`4F|hcYVkV|Ebm@qz6XKzQ;XjSuzRhRdKd||9$MoYzfo}3gIc(Q z!P;d#d>vc9hHvWO5N!FD^)MPP^Mdo`dSU_Vn6R8x&M#DR?q$H4DgD^=KH_*Gw=U$`WgRB-p2RseWz{pcX0Xs zc>=tgKH}UfF9Nr`e@=vx-<@&dcQLkndHpZ3Mao+knuDMO+?PG4!1_48{fy&St&08d{&_9DoMVY|o?HhuhEMA3huEGc zYTlmM`mJIktQmZw0n|Sq~3j%j=hV zco6JdGyjLc;ZhI3f=fN9$M<2dG1ZgbBVgA`Eq;%J<=ucKvV`z+XcP*3i^1)IBC_Wrqw zQw#Sz*m{u5`n&*^FYDoV*z(567%zh5%X;`dw!D6c_YzpXtcRDe<;!?~z?LuT;T3Fo z{ZbDr!Ok`F|06hD>fujtsR#A=t^ylVJ^B3^>{_YC?=N6^_agIs6|A3H{8oeAYqiwF zU%}QxYkcGP8n*SI7VdShc3BU9$Cj_*n|k;MwtUNacmpo=V4uW&6Kp-GC-;AX&0Q^f z|F4Qu3-=b-dXUTdybYEw>){=2d1GXZcfs;yJ-ml4uV3Q550)?MVGXu?8SevZ`LZ59 z#Fp1D_3#ncxn}+!gTtjBK7mU;sK<9L*qG|c?^CdAr53+`gXP_e%=a^}eroai57@m{ z%l(tDXB*~Wdv>g*>Em4`cMadUt2BbGovAeYUCTT7&(_%Lxu3NGuV`$(e@?>B`=^|K W#?Rf-xAx=sdbjcYZ&qUu+J6CiNfNjK diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen.spv deleted file mode 100644 index 3e630a8d46f9669413b2d2e80d06e6dd0e879439..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36836 zcmbWAXPj5%)wPEv7#l`WvCyos5RHn83XYgy7dtA}XzUeHiAGVtE`kPo?;WhMBrzIO zj7luQ-g}R+cc1GT&YJAZhxfz#|2^k9Yp=D}UDmm0=16qt(s}XjEnPab%+WGe%eQm3 zw0-7n>DJ03_W=s~ zb4}*ziC=zRE(fyTYdEYub8s!(R(EMRmV0Ne^%+gyc={R%gG!!1#`A2r-Eg>&C}moJ=al{4V1d9H?A zg}a%Vsoq7J`ZeyZ*S!trnY_+@R)$^MFl9^jq2|Xq{Mi2{cXh`W_vbR>)C0hNtj8hL zxsIjITw*_&D|MY~JUy99+<%s(ZEWj0*ZK@x=h}cebBX?kUAP?EdVmA)vEC4&&b1LWEp2m&{q*MY z`KMmIjn&_!WfOYT9ZT$7v!4B|y3Tbdx@X1y`IyCd`ZES}8}5bEwe;ia#I*p5=goS# zM)T`JZGIfR3h$VH(l-$7_|}dwn9EPDnpn{*lzPjXiFNd)y1$kIT=3)cbOmGlzcE`*C-V@@qD= z-<#xIkG+0VbAR?9$X!i7`XS(TN<9W#{YOyuDEg6LKk*+8_WM#F{{OMu9sB?1a(&0) z_4|EJ#!1Z0o-Ld7D2dQ1m<(aA1@|<3KF5y!De_dI}<@EV|G`U^}?px|VfPE0r zAD|ps^cm_rXxIHuplg$J-H&p2Px9ts{aC|gyqVOVtE}hgrd|y8HL$t#0l&`O+Hv3D z?waKs*ZR$dQ~#jBdaXa^?mDb9@2B9FPXC>^FSuLLm!=-q)IQ(g)Wp!f&P}QPIhA#9 zP3;*+Rhz!zpvNlaU1m#&A6VoJE_O98S}miY~FH? zcQ<$UDerjJ_cUDg{Qz|juG9~M`{B_)1ME3-OzVG9JDzp=zk|y@T~8mLy*=*JpF!ED zKVz&NXG8ApNzQdy`!hvOpFdCH^XG+i+x$(BoW6f?_vb_M(bLz%!;y4-)=N@<(sk9L zj_c2kjO)(^YscM~yJN^XuC+fKI2cPaJb)CZON3F^tEo=JU4sh^^rUg~G4|5WPds2?r$3)C-{ z`bFxGO8qi*Po8P=Q;#V1yVSds`hDtyO8p`AxKe*aJ-O5$ zQ(sc*PpPMu`g7_(mHJESM@#)R^~hsWRYC5+F~GvfE)i@1Dt zt+4(~JBVw4E`QdoO^;s>2Xo0!pvQW2!};?rzQe%!JXi4@4wjE^ELhH;0r4FL)^{XV ze7^z9$9D`^&YzF^CUQTXYdqHkuJ}%9`1G9A^qg3FP6o?o>{Gz{y^)5rhq4N)xDfSE&sPByR*P@Q@QSMxbqs$pPxhc8uyu8!j<)00d_r?b7ehO zg5@*rRbV-v+x3jwi~H4F@~$U+*MptkpNU^_7H;J3nehG@C&A}Cfdiwb8`l61&~)z8 zn=x-|boc)`=VMx%#{c8xyu za5K5*IXwgRIr;NG&*@pPeD?Jm*m>o{y$E*RNnBSqIllz9SM3YV#H-x>n6cU`Tpw|{ zcX7_;yI{w1{Osv{u)OoO@}2tw?jLf!$<>=n-Sy7J{oh<_J~QWZeCMgPywA}78RVkJ zH4Nsm*Ev4s{t1`wM&@T;o^3fjj(0cM@x5oQo&&S?=Ttp6KIim&4fYv%mXqQ4V0kk% zuP&Vce9TL&Gqvvs&L=*;IRo9n`cC6=K7IDsTWk58+e?^RpX2y-VsG-F2dwUR$2D{G zgmVtH%-gH!^ZQP`i@?dt#k*+Jr9_X=RW=hB)xmyUIcL7E)_{}Go(6+mhrF3&9Pei0_ro>8o(1<~zjL^bwYmFo z9rmonCFeTCS;tVY>oE7MV;!)3*3k-fpYpCF<9N?H)&)C{YqQ@uT*rFc{kRT$hH=Tc z4sq78KG<~(=bm+J0G7{L+7N6W^2vW=u)JsNDCU@rF*X6qXY9?uj=d@Oj6DJ@AJ67s zInStLYyoy0`HZs-*m1Vzo^iGX%V&)v!Dc0&HEs`<_uOZ`9l-J#dq=QikK&%OcLK}D zvolyOYup9wIPzxwOYVNm+MkoVQJ=|W)?&x;XT%<0_m{bU1$OTFxia^jaPl{ihv#fB zus(U`+K0Oz=Th67OU}8(S@ZtjcrNZT%kvF>zsDR)ZNK+hn|a2A^|%)2nb>%oM{NR^ zoIbJV(Q&Qit&eYN@19Se=XG6Y?b)8|!5479xta4MuzGL$rZwED4d+^}YPi$j>{au7 z%!S+?$B*k%pTs5adc^MGJnreyKZWbJT=s~M#_Jl-1iKFRphw+4&zAaG;DOv-Q_jHI zaN)z93n$-kEXPb9X5f6zZ@=?6ulsRc?=?&JWSy+NXRI5V?0g>1nckcS@9DiA-LvO; z)aN)km!si4(>a%ig2%I8_ndP%22MWb@-VPId9yx(yC0v2+TmPs?n#_;c_cWV2Uvrj zoXe}I?f0H@c@0=k&gFHD$9dGQ<&x7UHY>-qmbbpXsl6wUo5{dE+j9%}KJGb}H;Liu zxxBgITuaX7EpTnU+&!1~b9Wp+u220|E_v4@HmiHNr$_%CTz7HVBhICf%7@P{m$dO?#Fq(*DT$Wb+Y!Jv2Ja$^LaRD zdUGDUr}r*&pMz&rpX20QPK5K^=6n2!)N-Dm^qdTKjC{{Oh1x#(_$GnneUD1tWNLZe zr}7><1w5Y1=dSMA_r3HCu$uezd%b!1{XJ`#3a;PR&!QfWuYO-Y8%{ny+s^^(lW+T8 z#odqRM(td#^SGQ#oVhLl*RRzJshz8StzHBtpSdmu>yvj68P~hd!JjjifSuQU+3%dL z>oV?sT$eqUa>=y&zCVLqgMWr%PyeQ8aqchkybpl)dVt#J?4NheZ6;U_+DsyYVo}YmXGgsuz%j_*zvtV?R;wSy$P0&?|ra;?pS>v!0A(q??bSB zp21h(^XPH^;l6J;|J*Wb=**v6n5|mI>H;UfIk|X-x`XBXOoB6)9@J)|mcBWfKIrzo zIpNGg?byaU7g)bqygk8f^A_*CaOR;F?~lRy)#B|1*2}lo_IVeeHWRgY7X<59bN@5Y z7Xn{|&+m)be{VSX_!b7s>G$lZ$>kZn!#OkWF^rL{--VO^4&J#uUmsALwOZ!-5GxoLQ@7?Vo`4tHt{%*g4ojyZ19{vsR1ubFhB3c)tMa ztyzC5hgU7$ufY1%+`n0W4L0li?C>0Y11BHfw_rK_X00Zdth@7vZD#G)ZnEwHCttIk zgW9aslJ%Tm`Rrvbu$-S{JvX&ktEF$ArVqNkuP2;Yt7Yx;g7vG#`(vb7&uJ>CVu`qf%Qx3?R^8`%vvpL|0!6%TD&WQ zoug*G617>Y#rrd`ezka42J5X^uR?9sYVocL*00vuWIYIM*7@^H&-H3>^6{+>meX(6 zYI4bX7}%`+-_(-zdT{bJ>*3U9t(L6U2g_$K8-V5fBvOrX|E=NV zqEe1 z-Iw0%e>9wYe20SN^qaMsT(Uj}Y}Wp7p~-q2oP5oCJhfS?Wv&Te`RrvPSk6zfKAzgF z)zWuD(+Az&cOslwt7Yvcf%U7!dotKLYSyPvo3&cJr-JpX#d{i9Z_RoVwOOmhdpcOZ zT5FT_WUyIJ;L84|z{$sV23StNS*yt<>kGkV?f=f4tS^F-uUTJ8ZPsd;>oTx>_HsE` z&QG$wg4(Rr(syOk2i@Lx6`WbCW$jmk^{d5u4cIwq*4I*-wOYK_f%U7!dp%fh&H4sv zvsR1uMzDUh)+Xzlz-E0SSN1;*PCmZrU^)F}ttOYO?*<=&zrL5>1Lu3WTE_i7SU$dc z!T#_5)%PbjeQNRD2bPcT0ed?5{vwA@Exx~k<>Px0?7v4)=lchoKDGEB0?WrY6Z}0{ z`c7+ZzP~;R=ebdf?sQNOUkBHJE_j1l&R)-mnp`sc80@=# z&F~XAGgQmCpMvGH#?QcVev;wm)MlubzAwP`)eOI+Hbb@S#Z4nLv4m? z@qP=|uNLojV7)cN@2SmDE#4o%`qi@6PW+*Bec$g)EoZM8s>vn8dB8o;YKA@G%up@k z&I^{$8h;Fys~OHmZH8*;o4@I+8TNuRL$&N>0kD3xcozgaN6l~{YBN-ew>Mb7TD%K` z_0|j*p*BOccozlhSIb`efES`KJ$}@-TXmZJLdGJzjHNzF)%up@k4g||*jXwp; z)eKjpHbb@ato%3%F!@vZ`Pj+)`B)Mls_?;x;#wRl$p>#Z5CPHl#2 z@vZ^Zua>>8DMw#=)}ofP*9_I8lxT24{w9S?36_ezkZv2Rld2a0_ZPREu{@uzt08w*u>}8E#E&hHCL{1J>>M@21F6kWE#8B``qkn+7_7HucnGx_s>M4RtY0mA9b-*jdJdzOv)2sO zOW%o2U(N6&I5SksI!^}cSBv))uyfQ5 zPo*|PwRleh>sO0+5?F7|@N{Z3REu{qSif5KdIq?@C!a~(#&I`8HMwMX3HTzon&G8z zW~i2NF9XYGjhBPvYKB)(o1t3zu59{hhF8Ivp<4EGHCVq|yw`x8qh@$5wHd0#dmUK6 zTD;eT_0|k;pf*Fbcy9#jSIb_ff&b_Ims-wVGi-bR1t-HlfR|>2^}X_saK2ZnW!!tg z^6}jd_TOWzzCXk1Q;Y8buzY-f1N-m0R^Q*@^r^-7AXq-Whr#}PvDNnooIbVqW`O16 zdm8NfuYb||D zp4tr6;{5@vUoBq#r?ob3&9D=k8LGwG8LVF|d+iFY@4wxsHMwN?W3cbPHN*Mf z%up@k&JUK)8he4|YK99?o1t3z7Hs-zh6}-&p<4FR8?0X~-i5)=Q8QeG+6>j=T@sO0+HL!Ek3|FT%L$!F<0P9zacQ9CQ&2UX> zGgOOrEwFyI>~#pZzW)xTmb2Fk)#Q@l#$exn>p9#6&J5Ku?xtY*tZ_52T+MIQIckPmQ=6e$yxV~FtHrx5SZ~d6B()i;#k(C?zgqUX z1Gv8bj-r;c*9_I8lwY2xo?B z*~>v-{c797?Bz7DezkZfft{mf zcsjKis>M4QtY0nODPX-d!!xMOP%Yjw!TQy**Qs)3lAg1uHMwMXIoS8#dJeCE zGefnEdnH&tYrG09S2Mht+6>jwcTLk*GrShg4ArvE>%jWe;=La195urmsLfC<-W$RC z)#AMgthZ)3joJ*=;++oGua>>u0F--$QWv)Z%*>EFa%L!Txt$)%PfzKDGEB1Ix$v z9N72Yne??b@4wH(c}CRYdjTw;XYe9e&QIQdU!pb#we-CVwy&O%SE${$TAtObVEtyl;c`t7Wh6g6sS5d(?9FdPdablHr%? zUH>=3ui(s3E#rO-md_f$0n614zoj-qwe)=lwy$RRJ+&FCWiLN~^{d6}f6%J8X4nbN z4AtW84A!p}Zx^uMnqgNsGgOPW8(6-+y~h%h_v&YI4c27ufgTdJY$WGefnE zyC7ITYg`B{S2OHQZH8*;Te#_~87=~6hHBZ%qG0`M@h%2-j+$W~YBN-ew=Y<~TD<+h zdTWOLsm)L=-T`3!YT4@&;QIc%B(%3RpgCToo)= zGaN*1hHB|st?8>7t`290YT3&gVEtGgOOrEwFyIc-IE&tr-rXHbb>| zhl2I1Wv{K^`u@8vwVb_Xs3w;THv{|rTQeL1XNGDScXP0O*0_b3t7f<*wHdZ)d~fyN zvD>QYs~K($XNGFo%Qj&BYMEnOF;~rSB()i;#k(C?zgoQ8i@9osJ5ZaUTD+sc`qi@6 zoxt_|cV}ujd(BWyE*b6z_WieJxIdg3s%6{*!17t+fnd3s;X%}9sFuEio4%UiA#i4> zmc5Jy>sO2SP_T2<498HLp<29$f%U7!dpKBc&F~0nGgOOrELgu<_If0^zW*LYEoZM8 zs>vn8lf)e|JQ>am)iUlWVEL@^RIpsl@HA>OR7>BarmtprI-D7*WiOM#`qko{0(Op? z;ThCss21;;VEtp8p{ z&J5Ku?loZftnpf~T+Q%0YBN+z-}Oyj&F}^|GgQl7ZUpOBi}xn5bJPr{QJbM!ywkz@ z)#AMwthZ)(3$+=l#d|AQzgqTsJGj38-a##AuNkVzCBr|1KP8CzUiko=@0Ds9_b*`i z`2G&|_XPxC?C&Y4z9-=Hsm1p%uzY+k zf_?w>-^Xih-hW?$^Ngs)_cB;M&)^lXoS(e^zDjKlYUz6oY+pSiuT#5kwLGgg!1~qV zeG{y=o{_hx-Md=6Z-e!##rqCeZ#^ULQoDb(c;5r-SIb^M0N3~552@ws^^B;=CBtvP zzW>$?zlAeHwT$~6SUzj~9xPWg{DImG)zarLBeAb$*a^-I)v}k)VEtfaEoZM8s>vn8h1B@ks~Pr&GefnE zyD(TjYg_~@S2J9c+6>jww^-9xGwcIrhHBYMU$B0)c>95!qh{Ej+6>j=9RSv^7VqL< zy*0xnsLfC<-X+2M)w0*6!S(%j8EQFu+s+|*$tA;8!M^|23-1OB9*Mu`ewd`druzt08*9JRB&2R{{8LGuQ6s%t@-gUrwYlf}VW~dhL zx?ufk+3R}X`u_WKYB_t&P)#lwZlT^G!!6;=P%Yzb1(wemw+74A47Z^+L$&m6+w|28 zN5Yw*TK2LXSif4l+k>5>X1D{j8LGuQ3ano(-W|buYlb^fo1t2~JA?JBWv{z}>-+C+ z)N=Nkp_*JWJP_>rZ#{sQNOj|SKG-(#rd>@`C*xny{% zxI>1g!I`02#+?L~&l*n$%he1gQ=6e$`ld8}HN!LD%up?RITNg3E#BXPoug(rmD&u| z;ynwjUoGCV!Fp?k=TMuWTD<3i^{ZvC=Y#9}?*-Iy_L`xZTr#{C?E7y$hu6WGp<2eh z9xR_V-T;=X8Qw^3hHB}%sp+d3PJ=T;wd`d&Sif4lH-nv{W_Sy=8LGv5D_Fl;ytje% z)(mf_Hbb>|?*Qvp%U)b3j?&+2Wkezkbt0qd=2 zsO0+ajsO0+7+7!3a6M`>REziL zVEt;@>u_*=|6QM2&R#QAlS_tMgMI(48Eyk-hH4piTd;iAI1(&ZGu)2a4As)NebZMn z+yTxE)v}jSVEtj=-3_c?EqmP^T;G59 zpq8`O4Atb4;UQ`rG8_$OhH4r2P_TT~I0h_NGdzsi4As(ic+*!iJOa)P)v}kdVEt|?*Qvpi}!b6y*0x- zsm)L=-n+p1)w0)n!1ew2_tbLsnxUFpGJFv1?`y5^mH&YAy;3dXJ_MGJZwA=k>so#P zgwv-M-=kpp_?`g!`(LZ?UvT==;+qMUkMC))zbCf(o`KV+7T>dA`S@N3`~K^Hx7ymg z|Goj|8BvSxO|X2P!CPQCKY8zXo7x=I()SM7zIsO9rFP$Hc~^^AN(?f%u`{SR2bTK4)0xW4~>N-bxvXGBdd8Frdu*86YGurr() zs%6|RVEL@ED_E{(*p1o@)zUXx(^oU>4rhjH*-H}@-T&J5Ku?%H7atZ|4MSIuxJwHd0V zZ=I&EX4nd6hHBZ%x?ufk@eTt!N6m0OYBN-e_vc{!YVrO8thZ)3oZ1Z4;$0uCUoCsx z5M1AXH=>ra*9_IQn-Sif4lzXa>88SYMPhHCNd0oJdUz3vIF@4tId%h_v& zYI4bN4A}SIn&Dw^W~i2N4+qOyyt2Uvwe;QA^wkV+hciR9?Bx!yezkai2X>B{;hofGs21;CVEtsO2SW3b+OMn0i-|7!7m3f8Zdy?zd^@4sJA%h~H0QIkuCUCGWr|E(EzgEK?5 zj5`}xK5Og_ma7@|pf*Fb^v&M%)eL_GXNGFo%N$_+YVpnqc8;3iT-0W$7Vq3({c7>f z1J+wJ>`85gYVpns)~}Ym&IhjVzw=Ye*=vSsa>=ki*!SO>;Q%-@RLi)FgXOcvCBSku z!zHQBP%V8+HGMV1rQys_Eqhr8tY0nOpMagCX1FZ18LGv*99X|vyvu|2)(lsmHbb>| z2ZHshWv?rO>-+CY)N=N=okQ}HONK+hzW>$?hr*emTE<-mET1*Dg5_$4>r$JcTKa}H zeKo`N;LK1hd-*w7zgoP%06Ry`a5%LYs>QoLSif4l8-VrJ3^$}UL$!D}0_#`HUN-^P z_uoyaHMwNC1K9W9n&BuoGgQmCJA&o2#+|@&HN%~$%}_0UyEJ_@!(HLbP%V4e z4Xj@+-d}>9qh`1}wHd0#y9ZdmTD-pk>#Z5?No|H|@$LoIua>>;1FrAC`%=r8lwY4QGaG*~>9t{c7=!13O2} z@K|ayREzgGuzt08$Ak6O3@1>Vp<28X!TQy**Au|?{r5y_IeX1eO)eQu6?e$+EI2b% z%eZHQ<+H|fz;ZRibE(ZxEq&)TeKo`L;mlAid$|CtUoGAX!Ol@LyolNi)#AMvtY0nO zOTc<-hL=*Cp<29`f%U6puUCNU`|p+1a`u{`np`rx1?>B8J%_i#nW0+7y$vj%HQo-E zs~O%wZH8*;`(4vlGrSYd4AruiyTJO@;=LQ}95usxsLfC<-rs}utHt{Vu-=;CAF0hy zE#7;<`qi@6`@r@6_kLV`i?Gr zgW%-to51_-Ufc&$``+xl+1Hxj(T(Qs5FbkI$KN6D+J|t-nTNQQu_mIog2!{u?+A7L zb=A4@yUd4y9beweGLHBBF7x%k&SQ@DJBRBS&fSmeu;&+Cn{l}gaefc{Cg6>@{C&sH z<@5KyKiL$l=Dd!jPk*iD^ZUOi@@(}uw!dH8Ue~cDcR$Hv3u-yX6DP;5!OriqPe$8N zcg$#8IQe8W60BF=wPkGY$!I&U8M$8jo!7OE;_k<_*|P)Ju3WB7oVD!){_omm{XI}S z!zDNMxE6<&)d-bl z9dp|oPCmKq1J*0=+A_BHF>5WK+)f}jJ#Ui+QbwTMT4wz{vL}H3I`&MOuf18*$!Pyw)2#1dr@(d0 z>r^=T4#%&e2`?b35)g^L?@x+*GjoaDHz1_l``V zR`+MAIjT?Ql6PEjzF(gMK94@X|L5;uI1kS8vaj>0+xocY?;*GtEbs3WPTwWeW2x2h z^XiJGZ))kg63)M?OD*5Wt_GWvbavtDgeTtFwfiF1mtL$zoV~vSb`4|b&H7%Y&id4|zSqENS>GFA=k#kh z>wA-0-rmgd4%j(n{cM}{eehlC%%Pq+-UF*;jt{`j!T+_-@gar0y_w@bVCR@w_VKYb zmpQ9vj!(d9nd39CbMP&zeU8tmPiKRzS1@44h0OPsMfv9ONMs54ylrk=fZ0jp(i-N5dxKBL*-z{2J5eIQHiIfcNyChVHYuhfAO1crJS5@nc5I!!1NDXGY>Vn`f~I*k`p2mt$Sb z-MP~1J#$S)x5xSQIgWGn;qJ$|2B9rRE$3X~WZEC>7=9);--`yo_2p6vw^UOv$^B0H z{Q36?Sl;)PYx$n#_puq&@@s+D=9;tNABA&IPNgT~J`NrRr=IUMPk{BRo!G4DUk#@g z?n&?l^vR8H`kn&IZw#L~o}rd^jLh*YSbjw5d!Aa}zKr(*SbnS0_Y$>y9q(mo`H`jX zRcd+rj$!BK^9I)2pd_4~yLV3r?9GyIvT=hJ3F$XUDGG13|`Fb8^qn5AZ zb*GlE=V5kgdHZr6=A?G7+5gi+8=IuIFK0eDd`?41<&JcpiQZm-C<};|_-( z#@y;z`}$zlu9j!NLBpwq+X(D=kV`%rgXQaa*wmWKF*3(yVEK9;Hm8=iFXL?imapew zD{A>V-q!YT)$_0|wY+^f58H#?YxX}194_Z!N4T5^_4Mupc1-oGZ)dPssikjMu)NPA z`yEc^_Nk?BeK?=BTHc*E0Q+;q?;V~Odwf^v!`*ic?|E05!dRZ2-f;RB`KiPCoCZBf)-r7gF1nOV0I)GuHOtypLyHqu^YZ z`DR@^!ew20T$h}^?mc6B&)7SIJHA8j1}C3&?hf|jI@Ny3CFeTD8S7WzyhCT5d%?L* zzfWhKd&6a&dR(WRy{ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_gen_h.spv deleted file mode 100644 index 12559433df3f5afb043ccd911ed8a5f0b2c21c25..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 37080 zcmbWAXPj5%)wKsfu%W16FNnQk1iRvhiUk!*>|N{)W9*7l#TpwnRIp$#7&RhxqtV1i zqcQg0yJEv0-|K?2Ci~^X^Wpjbay)CVwbxzNxtTc-9oo;Zc&C>3?OJAS>D)4F=a#9T zIa_9EX$SJY^_(qe9jDg)`fuBRr6d11aHU?od)hdAOZ%z)_Ri7Lsbxm)9k>qN>%hYl z4&^$Bs|$X4hH7yj`@Q5-eMaIQ{n zw!-O=>ryz^E!VYh9UE@G!gXr6g$n0RcoTZXezK3$j_1ez54o#5ws?k?YSJ=x`zYsvHm?moojn)TBgn= z_A@`1&p-9zQ(67(TXv*J-Lb^ZH7%oQb)9QCx@X1yxtYaz7Gn(Pskj?X*V3J<9oIZ4 zo;T~|8_i!AYF%-3E4*X+N#9Cf$G3KjwYdDGUT11GZyV&s6jiz<%N%4EF2NBK&hGcgOy} zT&`~zUcU}Gj{C75-f+(6GatbxKB|W!&Y=!1Z0u-Ld6Y0I7}T^32q01x~L$9<($LXugh}kUo=>+^|#zzlX+(iQ@}0l{=0_7z%v$oS?VE8?X%vL znn>E#Jb>EoS=qx5)Sjo*yHoph#eMiaA^Y%azP0Nb!QC~>xesf<#@A;vk$Q<{T+iNp z)I->e+20Q~dpXB@fV=yYcRcF{8!r2Pf_f&d)X#vswuxHLOtzV*cJnQtg-&ZoO-#4rscL(l{A?LW(e&3K=fv*65&GY+&XU4Bt zUdiJXntGP{Rq9Pj{Tg*WSFclhp5lLl+RRhGNj;#{Z&B}A>bI!}mHHj(b4vXl^|(@h zKs}+QAVjDfMU6ZdB@4lDZ4$BJ2H%x_ha=q3&7g@2J-=_4m~N+KK;1 z>OD*SGxeZS|4Kci)W1=mQ|fj+Q2S;F2FfkM)ra=l8<+jt1-V zT*Y?`SU$evz;b?ni0?$Oz7x3OI|(cw-^pM(zjx{z$^8_r;anrQ;ybnB({ozWGph8Q z4wlc@XMpv|$9E>!v3>W+*k^&|<2xIyS3Y|=7kmy^-OG8@@{c#!T?m#N&GksbUD|Md zZ(WaH<35vdTv^XGVApdsSJpEgET3_&18SjOLd!Boq)5~C=(@R`=POpIFv#(dd&MP184Y2c`&2?Rq z^P6CM)xPIUyu;m(8LPd`^);7!7w24l3U(~V&z?R9%R6r?U%9{F{w3FkT=R3OyWV%X zPv%lPlDaeZuNuy0=o*~Qd2222GqwM9a?<0P*5b0)IemZmj_X-2^E5xtx||-zn*?@z z?^(O&!tAGTujk0;o}OR8J~PjDGHq$s(jsrB<~Jj?AM;b|z%>h(^NCMs&cw`MeP?kw zpFVr+t+jm4@ofyL&vE>9V{e|voM3gw8`jJ*7o2mbW!`z4KEDpdyD*%*T)c}keQG(M zJ;44t^8L>>F3#OGd(Rptum<~F_v2hkaV4we#OS^+`fF{;MrX*Wd8seKCGWW6%)K&L z?k|^Jf|(e z@}991nWH0PYz3Cj*js}gdjR*0y$x7Co^8Q$o>9lx4(vGc8D~eZ+o3-kb;T>3tO4=iqtN=Quf+N5Offb1sht4`;vbIp^{iIQg8* zW5N35&H8xmetaHk$8pKICvnc@N#J;%V-0?CF7KkY-+RvG-C#XAm-jRt=TV!;C8tkp zR*q{eZ#}80y(f={$iO|@^Dy`s?m3tDgVpP~e5m1EOU~uPa8rA^doG{l?l^v2pZX(Q z@~%g0R!?(JkN(HFp5U@aoOAg&*mbxEJ?i%5T>b;>GjdHigHOVR5BC(D{PbfvX7Vrt z=W~AhoyU3IkMnx3S-L0dWbHj;J$?-S@g2hXZL$H}=o70z>;ukoi- z%Xxm%b0*j^@-_b~YWw8lI|nT9dsO<)qn7u5D(|)DgGX}t+|@n%?}9G?tL43RG+564 z`?cSE{Cb}?T@3cuT7ErU0v^sd_3QklaPs+HKL)H%-ZOGJcR!vZwad8VoJ*X!#)8c+ zbBzN#SN*knC7gWbx(cjM-aTYo?>-mbZ>|PAulus!IbGLy?tWaCJ=bu_xh`?maXonY zb=&|apXYKT*geZ1gqtF-UXI3 zi|oa_S#)NvuX4Wp{fPP4>pttV-?#L>La*n;Gp^5Z?9IEE_l)~uGj1O^eU6ho-UIgC zcp~>?aW7cj->W>r_a@&d9;TM}dFOrkuMPJ+Tz>!b7~F$g_N)8*Cpp*P`j!Iw483Q* z$HA`QSjMqu@up`uq1W3s9ShTD%K_^{cu6f1xh~z5<_LH?#kR;pF36 z1T3fDv!^DPXY>Q-%)F0cjAZ>IoVsM=SvR)c&)_!fx{+EH1k8fG9oPM)blS|gUz-H~&;$+<$PQGTn2DMqMWv(^B z^4ZH;U^zd@dTna6R!iSHO&@ey-@0&St(LW~2iC6^Zy&I8)U4O1HfyzbTfzF(;@tqO zw`Sd!+N{;$-4Lu_t+mN|Bd}TL&p$oa8^g)R*AFbG->lW-lJz!Vv-ba!OV-=M$=9s6 zr#5S~WW57eK6}{_EaxX#??i3ZYU$g#>4R?T+Xc?7)w1?o!TQzW-3{yU$g!rwOOlWuA{*6*~`&jIX}tz7;3XtOW(0gA9P#aad2j>mbD)b)~^=t31H`_ zS)WL4)@t#d1lF$>?_jXrn)S)lW~~&vOlS}ooy!1~o%o23!Hq-dIGgst7Wd+!SdP5pTTl|lJyK^6@sQNOzX8|Z7ko=CXRl{OO)eR>;}2kc*RL7Q0B4438Mi%HK5Og% zmh+PgXQVbmwe)pt`f7%q;LK1hdzlHWUoGC5!BgigbIbx~hHCN73f8X{?`&YbHN(!- zW~dhL>|p(B*=rZ@%=D#aPHH)O%}`A)87=~z7p`WwD4ZFpW!&yy`K)m( z)zY_k(^oTG0?rK8vd$&J`qkoH3hW&994<|5hHCLH1JQnrSif5Kx|tk(>DioG z&R#QAlS_s>g13dM8SVsUhH4piXRv(MxC>aWX1FW08LFjkx2CUVxI3H~s%4#ffc2}z zyC>K=YKD7Jo1t2~dxQ0>#k&t!Z_RKZwHd0#yDwP3TK2j>xW0EEKrLsl8LG)8!=u1Q zz|{(sz8*S2H{T&J5MEmlMJI)#5z~>>M@2!PI7` z7VpVm{c71M63d_j0h_ zn&B1HW~dhLSg?Mz?Da};eNVoMdMd}=4Atb4;cZ~weQSmj;LK1h*>ma7@w zL2ZU=>ASP(s~O$}XNGFo%U{6y)#AMy>>M@2iPUDO7VkY^{c7<}0_&|A-b-zUYVqC& z)~}YmJ^=pz-hZj(>@~xw@4w(=_&?xf*8tt7qhMYWJ;{XY~bGzgoOsg7wxjGMU=FtHt{j zSif4lUxW45Gx80!`&WziTd;n$>~#vbzW;ttEoZN1L`^OkcIY(i{kLW~Bb*tkW!#Qn z`K+-MSgvL`6SWztrElh@uVy$4oEfTRFSCO6tHnDT*g0y3ovF=GE#BF|`qko{1FW}Z z*oE2*)#9BKtY0mAof}-=f4fr4*=vSsa>;NpuccHN&Oh%up?RSq7|ME#76p&QUX5j@k^>;$0rBUoGAhzsO0+Q?TBe;bzoks21<$VEt;@>lWbp{<|f$ zoV{kKCYKC%0sH=2&*83cW~i2NcLU33jk|;8YKD7Io1t3z_H6oUhI_%8p<4E`H(0+~ zy!(Kiqh>ge+6>j=-50E1E#CdWdTWOJQ=6e$ya#~wt7WeTf$RJ4!PIj0nxUFpGCUUS z`)|$gI5;y@%ecpb<+H{Uz;ZRi6RFKmEqy07eKo_uaAv5My_^izuNLnRuyfQ5hfsQNOPgN(A^o*jGv)2sO+Vs^7$H1APTGn|PSif4lmxG<7W_Sg)8LGuQ7OY<_-f>{P zHNz{Z%}_1gtHAozve#?mN`~X9HMwN?XRz2G{rBN2ulOHA6MIWcW|8|2sO2SYp~vWM!unT?`rXW3)Zg| z?{{Fm^^8oRcK>Seeh=2Kmc9N6uJ6A;QOnuu8BvoA%O<&EhGn^T!WiPXX^{d4@2iQ4khFz%5P%Yj$!TQzWoeQkDW;i#s z8LGwG6|7$^dz}Yd-+$+&mb2Fk)#Q@l5@6qdYlch0nW0+7T?#CpH7*U7s~IjsZH8*; zTej(|87>EBhHBZ%@?iaH@vZ=Nj+)_$)Mls_?@D0(YVocN)>|{|No|H|@vZ{aua>>8 z3a;BWg2LOW(##U(K)|oEfTR zFPnh%tHrx1*g0y3n^BvgTD+Ts^{d6(AFQ`#xCONts>Qn{Sif5KIsjbXf48QVv)2sO z6D*%K?gf^s8SYJOhHB~Cr|GL14ums9wd`eIuzt08_X9ge z&2WEeGgOQB0I+_wcn<{Ytr;FfZH8*`9t_s6mc1SduJ6BxQOntDhH7%j@C2~$zcs@X z;mlAi|_?mD&u|;vEInua>=@u1+TDIfGixUNcmaONJMNegCZ)UIJ%^Y8m%Zuzc1y z1}s-Iyo}lm)zWu)(^oUR0?rK8vd*z!{c7=!13O2}@JebkREzg2uzt08uLkR_8D2we zhHCMS2kTeMUate!_uuQOHMwMX7ufgTn&DsI%up@k-VK(|8YhC~YKHevo1t3z zCN+IE!+YV(P%V4853FA;-uuDMQ8Rpi+6>j=eGsf)E#8N~dTWLcQ=6e$ypMqOt7Wf` zg6sS5W7Kl?nxUFpGW-{K3PIHO%IDyGuT;yp{|3v)_afMzLr{G$!Rb?r?`5!je6NB1 zc?H$?I-EYW_}&1^$M-hapKDNk@4)F(i|<{qe0-mSegEyj)!MxOegWqhQH$?Ouza4u zWU!o{y#IbhZ4PSb`xsQNO=LgsK-vy}U>@`C*xn#Js8eGkA88|ak%ec#e<+H}+z;ZRi<*Ch3EqyCAeKo@s z;mlAidszvrUoGC1!Ol@L>`85gYVocD)~^=t@4$L%hO1JWp<2AX!1~p)*VVxF{daY0 zIeX1eO)eR32=@KAX1EcY8LDO6jluF+V?VH5&2STHGgM38rcGbXa5FeFRLfpA2kTdh zw?Ei9YKB`-o1t2~TY~kg#k&<)Z_RK3wHd0#yERz9TK2jvxW4~xM=fWs8LG)8!@bm} z%W!WvGgQmC`+()M#(`kDn&H0GW~i3F{hGd-;r?)DsFuAP0M@S-?}1?Fs2LtaZH8*` z9t_s67VjZoy*0x_sm)L=-owE9)w0(?;QIc11ht&KW~e5Y3Wa%eX_p z@>%0huw2b>7_}LyrEhrCS2G*|XNGFo%Sf<(wRlefJ4em%RBAI+i+2=QzgoPff%Vo5 zPp39RwRq0}>sQNO&r&Co^qfsCXRjHm$tA-vVBddhhL^#ap<2eh94wzTUICV?8IGkk zL$&mcYx-)2SHhX0TGn|LSif4lSA(6SW_S&?8LGuQ9;{z2-fO{nYlhcRo1t2~*Ms$| zWv@4a>-+Cb)N=Nkp_*JWoCx;)w`O<`oEfTR+(}^htnps3T+Q%4YBN+z-~CNr&F}#@ zGgQl79t7)Gi}xY0bJPqUrZz*hcpm}lSBv+rV7)cNN2$$FE#Ak#`qi@6$HDde_X%n_ zd(BWyE*U-#_UB{O_sSRGe6Li?xG#d`<9h|{&(W&BSK;)j#rGOmKE5}>{=BW~dkapV zT6}MV<>Pw~?9b(@zW3qusm1pJSU$eV;2C%b{x=e>&HL|HaGnvh_`U|q=NWthmh+SM z-*2hSK`njXf$ghjWD2$WR?D;c9;{z2-XFkv>lyix+P$mA`x98dTD(7l_0}`;3$^=K zi}zQsezolNH*kIb^@j+`+3OimlS_u3!T$TdHN)BA%up@k&HDObJPsyp*BOcc;^M{SBrN(u-=;C{M2Ts7ViRJ{c73k zLg4!TyD+t!y=JH;mkgH!`~F)qTprF0)iUl1VEL?ZMX+4Wa3yLpR7>B=O<&EhC!86o zWiP9M^{d7EJFs)q3|FN#L$!E&f%U7!+Z(L6X1E%)8LGv*I#|D2_PQpxzW=U8EoZM8 zs>vn8eqi5!YlfS^nW0+7-4raJHEsr$s~K)iZH8*;>)-U%47Y$YL$&N>OR#>mc((#O zN6l~mwHd0#yERz9TD;qU_0|lxr8Yygc(()VSIb_v2iN!C9jN8(HA6MIWH=D)`)|!~ zUpO;V%eec2<+H~9!E!aj1E|eVEqwsO2SOt9XX;aSvX zs21pALVlAd#^HMwMX1=#oBdJf0JnW0+79S4@r8m|P))eNtqHbb@a zUETE646lJRL$$1PJXpV4yw`%Aqh@#=wHd0#dp%gcTD&)a_0|k;q&7pfcy9vhSIb^+ z0oV86TdC#jHA6MIWO%RobQ#_UXNGDS_kOT^*7yKeu4eckwHd0V@1drzX816i8LDM3 zkAU^7#rs#VbJPqUr8Yygcpn4nSBv*=V7)cN$EnRwE#4=<`qi@6e}L=z?~~MW_L`xZ zTrzwK?9X?t@0BmZ`Ch4(abE$;$M-tep95QcZ@}qOi|-haP^^Ngs)_Z?V1&tM8z&QIRmzNa<^ zweMv` z`qi@68Q|*sZ+mJvdp#p+a>=j@*!SO>;hb=0sFrc(0?TKObA#plB*U)MW~i3FZcSg! za2_}_RLfrG1?yLfcRsLl)C}jRHbb>|7Xa&5i+4e=-kRY;)Mls_@4{gHYT4_e;QIdC zom$RbGgOmHhAV=7|E(FW1ZRe78Fyu{eAd_#ELSsJh1v|&()YWjuV%O^oEfTRFTKF} z)#B|9c8;3iYSd<^7Vqj{{c7>90oGeHT$9=i)#6f7hj!v)2sO1lZH8*`?f}-Wmc8x-uJ6A)Q_I&l(Q`%he1IrZz*h^c~Xl)eH}XGefoPP%V9@HGMV1)8WieEqgfwtY0nOGr`VLGdzph4AtU28?0X~-amo$)(p>~Hbb>| z&jssO%U;h{CzJGCKrLsl8LG)8!z;nQ|JDqzf-^(4jC(a$K5M)NELSrePi=;3>ASY+ zs~KJgXNGE7=k;LyYVqCxc8;3ijnrnS7Vk}9{c7>v4AxsSyoK5f)#AMstY0mAodB-y zzqeD%*=vSsa>?)kuQ+I_WjqNCDPiw|9%hW8BvSx2e5pe!H-}$KY4fiiP{|0 z()TmizIsM}p?2SDc~-xI^{d7EAF$qfMt-Aq?`rY-mw@P3i?97>}5MWTx#+D9_$=7!|kcf zP%Yja!1~qV-4U#}X1Eiz8LGv*Gg!Y`_PQ&$zW?qvn8gTTK3)(j7ZGefnE zdk9!QYdjPzS2H|}+6>jwcX-oRGaLkGhHBZ%5n%ml@%{nq95urusm)L=-amr%tHpa1 zSZ~enXlgT5i}x6?ezolNIBg7vG#dmdPC&G39`GgOQB z0ttJxtihi)Mlubz8jjp zn&FLbW~i2R-UQaK7Vphq=cpOpLT!d>@!ks7uNLoZV7)cN3Djn&7Vqs~{c73k9pL)@ zdndJ=y=JH;mkb{U`~F)qd<4!6)iUm1!SY$-qhPt3;bYWhsFuFJHGMV1$KlLSEqi$a ztY0nOzk{8lX7~?kGgOQBNw9vkc>f2iw`TYhwHd0#`!ra;TK4)3xW4~BOD$)w8LG)m z|Fe_**~v=hP##0}~mA|`bc{uqKO5e&& zpZ~r}`g(%pYhSOX&!2gmzTR;1wQtR)Z*b{b3r^m?{Or8-sC{pC-t4On*uOhT&7UdW zm%AT-rnqb0fJ@Fi#I3wLk3`=HJe+%eM(6?Hjp5|;v(5W~9beweGLHBBZ1YXP&SQ@D zJBRDooVy>_Vb5k<+i|%Lu|G@OzpHN>uC2KIdB@J>^XIn*E!#Gl^E#G3{k4|Q&;P!F zXRF7t{rTedx{e*W`$-- zBiC!c^SZXZxchN!_Uy^EAD3$rXKnj{|GTznxebI%ZtBTxU$B~MaeT*e%v#GQx6$OL z$N9|7Ue|UIcR$JPKx#S16K8FQfL&Yu+g;Drq14mob{L#|ayuNXSKhT{Z12f!5ZK&Y zul>&J+K%Mz$F2d-neT$?y+I|}^YwN1IKE>!X07Fu+ePH2$N9|7 z-T^Js{!ZF6!Tvh-Oq#E~S<|1;{=24WU&GFUn?A2|;pCIod0@TruIU2quF<<|;y=Z1 zxtMwU`r|r>wP#-LxAS$f8{DN}^-cM{;onPgA+@^SOU+SzG?%>Niu3jQa`2_}`Sm}4 zC&O4c$IHIPQQIen^bC zl=qor9Pc^9e*rtUdD`zBS=ZfQb=P$dcR#L6ej=Bg>k?E0t17OE> z9ro*Yj)%Bs4tpNtl5-BR<9Q~m<&(vu;OVn?3{F003b3^@A}jC z6j9KweEa$H`_xA#KKW5k){5+SO`w{!S`U~{e zz^`!mYsI-8|6TCwV72@mAa8)>@*emW*g5^U2EV_*O)YQlHjMAOj$zI6>2+=IQ~Pmk z{#t&It6lqPYZE7n55cbKD0;KrkEpX=^{n?}uv*sp30N-c{S53H{578SeoifKZ|0l~ zcFt+v`=({|6?NuR&zxU_)iUQdV7bis9oRYfU)!8hDCF(UoIiq{^ZByRpQtmZdglBY ztd=={0n261|A3v7INRp@jauH`d|iHt_iUew8Mr2UJo7#q&xp?>>uV48Jo+5+HMk?3 z{JHQx)0x12e5Puhxa1s5oUvvByVv?mXNAjN)w9>xz-rlRXRuuMItSRj)@RxUPTt;} z!QmbFcLAt9i}t$Dtj)V$D?F=X(5Lpm>8b0U8_osLjhIX9nVA zumE`aGqxaHGEh$j3xU;=!NOpqh@;V0qtzZs2R6Ul(7emR|?FF4t@g|0bM!aw607Y^nC!9?+>3jKBAU)jLh*dSbjk1`;=PVzKr)7Sbn?G_XV|l z9q&tO`5jB&SJd+Moy^Y7=UcFQ&Dy4b!zHWl;mm45F7@>O0Cr6EtnWv#S*fM(XRy4_ zBKz$Hwofg6y}>?fwVa35!JY@_Pv07Fo(HvXYd3X05AE^E*YnT;PJa6H(6O0=+f+}+ zoe82ZoO;$iGuXAO<=M~DaBAUZ1A89il22!_d_50yP|G_;=I8>JujgSdYI*xI-rQjM zdLFt_%h&Pdp_Z@bVLobk`*I!@1iRPle_?RAoQFl=avs#vyC~Q()w90tV6#$7Uk|Xn z&m#L>kLP5cTKf9H`K;A)9$LYk2fl=~^=$y>c~FaYqo%IsVL5#A^*k&OCqMmpSP?E? zAM|A0mErp`w|dsz6YSd6^6Xb>IJI!Af;|s%$)^`szMhBGsO23abF2=QujgS+YI*xI z-dbS!dLGuHmapTjOD$i|Lmz5+`*I#O0K3=he?xG%oQI9zavs#vyD``?)w8~SV6#$7 z-=<)BpGEe&37Olc*478-vsTM_*c|M6;MUgHpThH?7VlO~UC+bT_~h$(*al91`tz_I zT+V}@jJrL2U*=ZN+IIlEcC|eF9UD$9+|FRngIx021uS3B!*0~_j*&Tb2g}#ghcc?3n6V-(g_0 zQcK?;u)NPA``w1j?NdwNws1acwY*zz2lo4jUpqGCvd4FoylZ&RyUIAm^6V@Dr*9dq zoU2{H>VACxdzbsJaPs;6&~D%zx$^rr{~TS;zPxYl0iOQ-eQ!AVyt@tr`|;gKZ67W< z-;Kl>Yd>(_-?PpG;Ks2|^UgXCgp@%CpZoiIa-ABX8XWhquvu?+5-E#K1?#$)gcg*9t`*A(~ z-u5_ZIoBi3dQJqp9)Hi5^$dn{J>1&Xb25c|)-wd0^*Dy>k+aYBWG?U4`7;df&h4L# Y4MV$_dAfq_$)BHHNYAf4#WlG8A6XA9H~;_u diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k11s1.spv deleted file mode 100644 index 95df567158146d5e0e25bb8c6826dea26df8f115..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36516 zcmbWAb-0#g`n5MjR42-cR9R<5Eusg<1 z>>T5Fo_jxQdkx3&{qw!YJoBt;t!v$}-h03M0c6vq^TN}$bm`PGL(5DppU&9Q_L`-o zb4w@C3@y{P-3JcZV$jkDA2MX=75gn~!^|yR+WPG2(K21jG~By#?YHw@`z!3nbrM%k z{PMGKIgtHc17Yo%fora|x=YJ&?wz^TXEd>|h1+i(oL)J{OrIXP=?mxTgGy>Qbu+#H3QuHoh>ob%~jpm5z9Zjr)uZ@9$^=f3s!Eu3qWGvKUwriNRQyP27( z-npClRqn3Wy$$3&d5wE}hF#k*WlQ#<=F2&J+5ZN2b;lMj$YsW<7XtgTJ^-Q4bs%-- z68p+rsq0(^)04Tx3$|O@#S+9mr=USbbmbSUXzUJie{ZGAc z8>_!d%bN74JC@kF+TZ*3y3Vy5x@X1y*_g$77Gw?A9UVzJ2AL@O$%bCLh)cbOGkMgTEwVzFL zuE$VE-Pha$-!M@@j3ifkpKK_3=cgOz!xm@1}ynYTj zj{C75*>LKk8tnMlub(S*y{A+6;BtMQk+ZoxXXf{3?!L#?^YH)Ya+hYrV9c<9)zn#+cr+)&tf2kh=`^LJD zhk>qD&V4+>-95^ill7#A%Xm*vdyX>SQ%yZDte?&1G#}W{UTepFox6LGb6o2;xa8E| zYp`DHkGZ=J>&!a^+|udSd3%BVUKV{`>S0ao{#&X2K9qG1qV}`Fb^5uUb^5t&?YswZ zcTIAx)7sBzxy&<;dXZ9JOFfLmn&)`1dCEE7b==*DyyID4-*6f4uhi3XrM?5a03Q7l zz@96|w7!qp@vO72cj0*ZJN#3q{aVO)J$SG6xNpD5WZ!-dv3A_mxVuL=*J0hb-rG_-%V}i@!vC5eJ{12#nJDh z-l^30Qy)<32dK|1^+VKSO8p4+&82>n`mRzxPCdEQld0b*^^??}mHKJwo}7WK?^)^v zO8q?bvZa2JdaY8wOg*^NuTt++>es3Lddc|zp&nN1x2Vr7^*hvKO8p-7&87Z;`mR!c zL_N9GpHRP1>M7KpQLn_i$Wh~Tw7)nGE#JZT-g*}OOg^8>_p%k%?_mdU?Z@T!wl(P4 zlly^O@}uanKDgoh{ubY1V11sm_znll$9E)H&hH8F9Rt?q_rCa!1m|MGO&Ecy&NoOj@L78FYaTw=j^@>b}Yxwp56t^J8vtWtnYDupX&`S&$GJgor(JgTxz~&&guBhQ)_wOL;EL^ ziyqhDbCtc$@iF&LxF&I#pLuz<<@7k7@3-T7&ssePW<7;_JvYA3>G=xmdo+%FGW-rK zZ-(ae6L(+crS>CNXFj8xPdvIg1D)XFb3T3c*jsD)oZGXQTc6_`&1G-$?*>+P9V41K zy2Ck#TFz8&uxH^jUn>sSda zpLP5W>^|jPN5=7As=Q_m6u|L?odRC61cUnAaQOjqo zYlB^@&&^ru0I+;K>wx7vYmTul*m303vk}<+WZaFxjys5Z#@z%gAJ3*>xr{p)>^So7 za|`ai+~*kXn^TYEa-U+y@iTTSuxrZPTZ5f@cCO6*dpP;OG@pIjfc42c*LK`}IhWeD zTyoAO&YE`s$8%NE+=eOT^oY(z0ulJgzd$LZ}-ZR#vO?JK?&Y9kv2k+^<8r}E7^Qg~paxQm;^GxSl z4grs3zwSBbayK~noXg$8`sB@ePwu{aKh*Z%l5X)I-p1W=e7QdLaa{7QM{HI%b5D={Yq+lCvPYbAIUejf+=Cu<`*JRC0Q(-frkueW z;lhWz2~K|Mu^cmbn1S;-zx~eRyza+&z1J+=lXbH8p0UO?+4+7rXL@rUyr=g%bl(Th zsy@fbxjYokbDQUhBdFy(Kj}FN>=^l8ax}Gl^6?D?%a10j^bMny_jx4Gfy2QEbNRlj zd-i><8v#~xzkb%6hoA3R!|~vCx$^Zh8axtT{ail*PCnmPPXz0e_e`A3-IwP^?IbQa z=MrbGQ^ECX^)zbds$Z+8!^vl^Gr;=f-9yIp?)%`g#F=2{bzk;7r|UYKyD!&e&skh@ zu1lOfp9`LP9p}Nx=Y2RI?4ISbp9{h6C-2KeVEg3byA&*6-@nVL<-2p`{Tl<$`=_4w z?+UP*d(YavyLaE;J2^i-ubYRx?yJ7nzUS$^gI>>rXIP)(*t;rsGx46@e>LN-0H@D! zvZt%SK6j4ep1ofKmLJDj?%{in&kQ$G%lrQ3S@focYyb6D`>!u!H3{; z=yCtyK5aOEUoLC^63%SZGS*jM`OV11GxQx;&e!R1=JGwY*{G%ON3ebTucPlL3bRlf z-gtio>sO1n6TknU*WXvl9Gxl5LoMDeVEtC%==KrNY*dI$xnfIuKT#ZMs3z=nd^11eD?AN zSk6~+_z$&NtEKPFrVqNK?=3j9R?FJo2J2Ug_Z_fvu!RoqyVPc_7Vmpt{c7>P57t|= z{(#!7)#CjStY6LjoApOvv(EPp&(X(l^6`BFmeX(6YI4c?JFr>%wVSNJhm)^a|441t zYRUR1uzdFNGg!`7vi^nItku%T^4t2LJNi1onYCKh-WjZ4E#59*=crkCg)?ilc&7pD zSBrOAu-=;WbZ~9wntQy{gY~PmHd%KAo3)><*?)I9`S@l4%js`B*WBfjb#Jg)KfxHu zx(}Ru&3Z0svsTMobA#oxmwCW)zLNF4)Ml-gzWJIy=#IYm;mlesYhM7YUoGAR!Ol^$ zUWnSP)#6b^6?D> z%jq|3HMwNHIoPcI^QdIK1)O}%dTVO4R!i2u2g_$K+koYKCF^ae%~~ye+ckaA9esa* zGi$Z1eS5HewRm>`J4em>kJM(Z7VnN={c7><1lC)#-kI90)#BX+tY59Q$$D3?S@)th z`yT=)AKz|ZIsIm>CYP)a0GqXc)|adggp;pXA3|-`YMJX$uzdD%7+B6%vOb*Jtku$Y zMAHY|(RU=AS*vC3M}hUL#d|c^IcnC&P@A<{yvKs|tHnDMthZ);9JN`i#XAhFU#+#t zdN|mu{j-ehe*~O-d?Uef`psHRE?J)rHf#UfG+CbkCttHZo7$|^GS@j^`RwJ-U^!pO z`dn(WR!iS`O&@ee-}!K6t(LW40M@S-?}cFJs99e`ZPse>UJTZ+7Vjlsy*2Agsm)p~ z-pj!H)modZF9(~of8Lz^kAahq?+UP-ezR7SOV-ze4`Ph^Tz&(b&*f?v_eQXMd^dyr zbL{H71x}w@e7Az-| zKLP7ki+2iGZ_V&iYBN-e_cO45we0l^aDCqYl3LDQGgOmHhW;kC&+s+FPH<+ZmT^0S z<+H{vU^!pOuq(A0s-VEtvn8Rl)t>YKE)9nW0+7T^%f+HLd}cs~N6IZH8*;>)-U% z4A+7)L$$246|7$^-nGHbQ8OGsZH8*`t^?Mu7Vo-Xy*0!2sLfC<-hp8KYT4@s;C1Os z&xX`;_L`xZTr&JUcni3i;Wlt)sFrcJ1j=9Rk*`mc8z7O<#KUpq8`O4Atb4 z;UVAy;cAA5!kM93#yt!ypEVv1ma7>aL2ZU=={vINs~H{zXNGE7=h0yOYVjTec8;3i zvD9X$7Vl87ezkay1M96B4x=_hwRnev^{ZvCBh~3k&nW6Pj=LGE$tA14p=^G{4-dtW_T{O8LFl4yr!>acs`sNs%4!Qfc2}zdm-34YK9k4o1t2~7lZYy z#d`@@Z_V&hYBN-e_cE}4wd{2a_%spaf7!?x#Na5DT0cwRPGpDS;I^SM$jbvL_5BM@pIUskgXQD96YPKIS$z}W^r^)+5iB3y z!(g9({e8yP=K1##IM0Y$e3QWPc@G{1%lXQ4$79sypq9SJ!S>ZN@&vW}R?B-e8LVF| z-Y3C&>lt~9+P$mA`!ra;TD;GI_0}`;EVcVri}yLOezolN1#o@-eUVzuUeAb{TrzwY z?DKEU@I5#)RLi*UgXOcv55RIY!w;#=P%V8Qf$ggqeoSqKYT3&tVEtHN!8c%}_1gFTwiNve&P{_4)T3YB_t&P)#lwcA2jI`L|}+70wLR zGVU~B`K)nTuw2b>I%+diOW*WOU(K)^oEfTRFWtfV)#9B2>>M@28L7=sE#8^H`qko{ z8LYQv*n`>()#B|5)~}Ym&I+#2zq3)x+1qvw`Mi@$h6{lEz|{;Fgfm06jJpt6K5OJ} zDz?bg3>Tp`L$&n%rs=C0E(&LcYT3(TVEtj=T?(vUEqh%CT%UiJrIxeT4Atb4;Tm9{e`|(o!kM93#_bQ5&l=YP%he29sm)L= zeQP&;HNydLW~i3EtOM4s7Vo-X=cpO3M{R~`@eTy*SBrOju-=;C2GnM#7Vm~&{c73k zAaH&D-I!X=UNcmaONQHleg3T({sGPm)iUn(VEL?Z2e4et@Q>7HsFuDRo4%UiPH<+Z zmc8r@)~^=tE@0=V8SYALhHCK+0qa+bcQ>%!n&Ix$W~dhL9$@`y+3Q~5`uw{$wVb_X zs3w;T4+s1FTQfWY&J5Ku?vY^mtnnzYT+Q%kYBN+z-!V;J&G1+_GgQl7hJy90#d{ps zIckQ(sLfC<-r->VYVnQ$>#Z4%q&7pfct?r3)Uwyna%7U86R73vHA6MIWcX*W&%ZUp zbK%TTE#saCmd_f`2g}tAFQ7IUK1*uPg)egA;drxxEo!SeC_8|>dds=hnm^r^*nCs;ndyTJZErRuvIPM=zQ_kiW& zdlc;R?-TU3HqXD0!Ffj1;(Ht{pZDMiu$-?v|4ybh2etG)3AV4Ek*BELw_4t-r@{Kw z;(Z3Jx1Nz_solF;yw8F4tHt{~SZ_TeFHpOGwRm3y>sQNOUk2Cb-&d&R?DdSO$tA-N z)Vuy~h9AP2p<2fM2rQp9ehikY8Gb@-hHB}X0=BPa_$jp+s%0;qf%U7!`#D%|&F~9q zGgOQBOR#>mc)tSctr>nzZH8*`egoF8mc4!luFt>UQ_IhJC5cP%Yl2!1~qVT^g*nX1ENs z8LGv*ELgu<_PRW{KL4&jEoZM8s>vn8wZK0A)(l(W%up@kt__yY8V7*oYKH4jo1t3z z)@}M~hU>wZp<4Db5UgJ<-u1!GQ8V0t+6>j=-4Lu_E#8g5dTWM*sLfC<-i^Wf)w0)3 z!S(rfFtwb$W~e5Y40izg{97~pBb*tkW!xRX@>%0fV7Z#%&eUe8mcCt@zMA2#aAv5M zy$k{CSBrNyuyfQ5cc(T(wRrac>sO0+Pq5ya;a=2cs21%QRn{JS5uoV{kK zCYKD40{i@1Gdvp34AnC3F<|+u@mR22&2T8S8LFl4xTddWI1J7V)v}l2VEtnW0+Nc^O#0TD+Hooug(rhT06(;=KZ_ zUoGA%!Fp?kW2wzhE#7fp{c73k)!_R4dkwXmy=JH;mkj?3o0v>iE#SV;=2nhAK$%T|K5G|-3O;nEx!A~ z^6@|zX9u4i}zcw-kRZe)Mls_@AqK+YT4_5!S(s~M`}5H%}`A)8FuT|{`^}r><(v! zY8iJ1uzc1yBUr9xI1{xQs-f0(Op?;jGkVs21;RVEtp9#R&J5Ku?k-^YtZ`ScT+MI@wHd0VZ?~qeX1F_?8LDM3 zdw}(;#k(iiIckP`QJbM!ynBQ7tHrwySZ~d6UurW{i+4Y;ezolNPvH9edjPeZy=JH; zmkf^u`}|uo913TKY8m%9uzc1y3@leO98PV9YUvx%^wkVU!kM93_A*M$r55k;VCSeA zj;1z4wRlee>sO2SM6lkP;YrkHs21#6EwlAhD3HMwMX5!mP7n&HK8 zW~i2NF9FMEjhBMuYKE6lo1t3zE^qp3hGXE&P%Z1c0<2#x-Ydb*Q8OG%ZH8*`jsxpg zi}xz9-kRan)Mls_?=@ikYT4^}aDD#0j#|!MGgOmHhW`Nj^PB2(eUoGBm!Fp?k-%*>P zTD;$b^{d7E16Xg(@W0e%s21;!VEt;@>(Ai&{QC>FoV{kKCYKCn?B4$TTQi&q&J5Ku z?#y8Mtg#1Ju4dSi+6>jwH%rr3Gn^I94Arui*}(eM;+-Ar95usU)Mls_Z*Q=EwRro0 z_0|mMpf*Fbc;^J`SIb`K2G{4`d8p;=Z99j2R>>v9CBQ!a)(n3OXNGDScS*2(*4P&; zS2J9S+6>jww{+82Gh7DF4AruiWx@K@;$05x95ut`sm)L=-W9<5)#6UoCrG8C;)#SD}`(*9_I5U0we)S& z^wkUp!I`02_OdZpzgoPTfSsdexGA+6s>M4PtY0nO&A@tVhMQBHp<29Kfc2|ouUmoZ z^Y7Nwa`u{`np`s6Rc)#ahrpSkTE^WCET1**4wkDK?m=yaYU$gv>8ly;1!sn8*~{Kw z{c7><19py@;l9*ns21;jVEtmGyD^^8LGv509e0T_IePwKK~v}EoZM8s>vn8 zVPKzs>p2__XNGDScLZ2IYa9uds~L`>Hbb@a9pCiT3`fJ6p<4EG0$9IVyeER4qh@## zwHd0#doozRTD+%#_0|ker8YygcuxcCSIb_{P$!f0oJlQbuNkVzCBsYAr^@g$I5Sks zxR-)MlubzAKx)n&DVDGgQku$AR^$#d{UlIckPiQ=6e$yw`yBtHpaQ zSZ~d6Jhd6B#d{rCzgqTs1Gql_-bgKHuNkVzCBxgn{`}GUT={P}pDWce?j2zH_$Gq= zIi=Ni7o0w|`0fVF$9EstpJ!Tq_rvK^i|+xje0&ds{kf;r_XwOmwfH81<>Px6?DMaG zZ?&~~{(TP4GolvX^I-YB2QPr-eC7G~MQU?UOW#Xi`|25anc97;<-K|ZtY0nOSHXJg z8F`J`y{pChI#|D2yl;T@)-&=SYWJ@e@0(!#YT4`C;QIXg4z--Uo)I;vX}3{`qkq70j#%X_+M%>REzgV zuzt08e*){R8U9RdhHCNt0@kmVy>^=6f1iIlQ_IA-O<&D$c{npv%U)Ih>sO0+MX+#Z4{L2ZU=@tz6Rua>=@txhKCIfq)#UNcmaONL{>KL6GXuYfZ{ zwTycuSUzhU3zn-Hj-xh1we(%p^wkWnhBHI8tn(VMezkb71v^L0a6Gjcs>OR9Sif4l z*Ms%e3~!(|L$!Es1nXDJUT*@|=ii&DHMwMXC)l5#U!N-{z)d0?wTwFvEFa%J zV1Lej_1z0M37=Yg_krc(dl2l;B*@W0e%sFuC_2-dF_?@wU8HN&5& z%}_1gU%>j+;%(tf=&c!cf-^(4csqmjt7WfU!S(llr=ga!*9_I8ly`gEK?5>}4geezkai2X>B{;mXuzs21-kVEtBqk!_DE$P%V4e0<2#x-Yvn-Q8V0%+6>j=-5RW4E#BXQ_0|lxp*BOcc((=XSIb`i z0Itu!+f&QgYldoa$#5^Q&%ZUpz2VGIE#vM3md_gZ1#Z4{O>Ks1@ty=@t4=2AIgeV-UNcmaONL{?KL6GX z$HAGQTE@K!ET1)A4VJ4JUPEn$YU#VS>8lxzhciR9tn)gsezkb72Rld2@CIr#REzgU zuzt08{{q%qGrWn~4AtVj8LVF|d%YE0pMP(omb2Fk)#To2`QM+N?9WynPayvMW$($@ ze^1DNyX6i-u;=Qg=UVRhSee*PZ{`V^Bn-?r!`xb2a z{F%q;TL@0R_AT1<`E#7pw-}tfefimWOH%vX?7Z1mU+}?jYW@uIWx4zEXNbG@Ww_+b zL)^-<^C^$aZzjL^bmAU(J9rpZ= zYb`Fu~p# zJO)t9Ii5H$Tr`UE9XoeYrM! z261i9<=Vtq+os@O*Vg{;C=G^7ZtBTxGq9R#aeT*e%v#GQx5LRzkMo(Ey{>I*?!J=S zR@8EiC(hco0lT*R-Cc9rmU`;kwu6&TZhrvlm3M6!+k0}`9&B!|*M8@9Z98)J<=X7| zBi9fv*Cx)|b_V~tw)Wh1flF@c$!%A#nrm@<$8yYC%O|%Z$W4#)nVY??ZBOpLlG`5C za*ij?+V%#!wkPP#*W^CbQ|GoXoP2WI53E<-wPkGY$!&kIxw&5Zo!7M;$laG~v*!S= z!?;|VIBPo?{Oj7F>5WK+>RtSJc3f3#{nuc+Ajow`oe-*psDCY6=$8`?r!kFA| z;d8PV+-R`+yL{j9-yIo2t?u_yb5tMBCGWW6d|saf&d=H3m*Tb6o*;o$|gX8OM9h@ReZaHc$JV zBkLLqR(D-jarfoAG7U9uLC=->#$$HbKJl^bJ%k|mz;Bm z9nUjiEuSoI22Y*EEpYNVBe#OhT|V#iU%~Qbl)is}G$(Tspak6oUc09u`O$qPp@m4Ozq3H__g~4*9Tm#MV!4q1$GVX->=(0qn@VD z`qZ<&XTWM%-*aH+^lLcld!Aa}-puh5*g4w2x3$mlGIi!q&m6CS)iTFxVCUd}9do=+ zA#ZQycoXa#?cWF5=Xi@cbEs#Ix4~+e<6W?G@R8Lq$9vTB_U3c-cAk^X**&@jdpxsd zZLa2<^?U^OJeg}gL#I&7d-i;fKI87o_ekwiE;+{%XRI&4Q@=-F!ewvj+1poOwe0O1 zuzRcT(YMs{_T~)j)|E9;dzS2VUs;QHzqUPV$D-TgUiH*<{s8BmeV?=EpQz=t=U>3S z+_T!xTypMNoIQ8q&6xVkbcV~G)wAa=V72Ud8nEkRxQ_R8S~z)ov*$hOkx#FC&RV=@ z&%@}m#~k#y&icMjF1^R0drsQE#=wqaZ$1aSXWXIa6RB_D(&spyi|(w+ml@3q*Ns}v zjKniG@5PK@->b#B9P0w^&Xr#8nQJ(@JW&u5mOV-u<6R|l`bHDkly4d9eU7d!w? zJ)brAf%U4L*sSUPhEogoAb11%^=iyyy`Kiyt z`*1lAdNS^Z-~r67p0$4jcI|3;?>}xhwQy6wo(H+)^C?)qo`=t=Q5Z zPkkPGz~wyX$+)w?4`6QftbJCnYgfyAKU>48h3f_OJjf-V-eCE99_FBycZ|$2Cs@9o zhqWvz6#j2tL434wc*sltq%4)$R(dO!1DDx^rx10jLfkX zSiYWzwW;Op%XkC8^7TBdOD$i=TaQ}4o`?0Rb~;pskPzO{*ZDKHckBEffp_Qf_nwBJ9mTk_gY6lH=6XhO|A}{H HWv>4N-}E3)lX; z?R|j4{#>VV&5d84p;{csey>5W_H^f3xV7%wasv06xHe=ov7g1;Zta|2Imb+&9=UFX zb9HiVD}8$8<}RG;mYc6|vozd-h3ndIJqzc2dV3Xa)`nZ6aI-bs(uH&1dY3DlYn3zL zta*-xTba9=nW^3$P5l~o*J}oYcqXqmctd){ezK3$&f&-YH@K@iwzxM}r&9L;`%ymx zq0Tj&y6t&&X%YL$T&e3^Bk0Lo;@)kR*0HVYTx&D1bNS3%=Q`AxOYA3erLJ@JrzdlX zolBehu?y%;nhDeH?&~^=}aBT!W}- zX`M^#XCW@1f9l0sS^b?`Hljz}vBb{RmQh<>=QY-@oAt7d=GTSVd^qMWykq)F-wI&Iw|0y*x%{N=*IFI_KUC2Oq@MYn8`ep^ z5p|zePHERSp?<67p7wgM^^Duw>mk%*CY;^w-9*2Q@ars$*dSC8x=Fp3J zKkn{PeyyhVdy|~&vDa^E?$7=MxvR-XKLp&r)FZ&ve*|^6q8|nJ6aTSbzb`Gy|387d zWB-3H*LNabzYjT%`>`I?aO$HQ?D*NQ-&g8-$57A3<@!7$=Wuz>%ZEC-kUIqJ17XkZy+Ims2 z--oT8V;Xn&Bj+5}Z*a+}zu#cJ)}L^9P3E69%mBA^`e_Xdfc?1^eKG3eo7yY}Qu{M0 zYu=37?-Q=m@9|lu-@~n)_fYPxNzQdz`#rlpi}BP;G~*8Aelzv)EY@sq0h_IyL zeaJhW^~8qDy6>Xy#+CX(a4$Ui9|C*E9Mk#{YR9wAzNW)$C1n{aKe))^%KeZe(14E?7J6hTI)P&T*~%xgfV3C*SXB{v7a(_&v!h`+uCKzNLPG zdc#sbNnOv;Q`Grd^8Pfn&miMH)1iKrdT`O7qu#aDf2TgE)X!6oEA>CAuPpV8)RRj6 zGWCO{ewF&^Qol|;t<=+~XO#L)>bW@+S^wMAy-NKqb>C9IPu;)N)2RoS`XlOHOZ^G; zL8YESeSE1ur5;!6&#A90^%vBWO8ph}gQfn4`sq@CM?J07-&4<^UX`cFQR8&9eK-y+ zU%~inJu`m4zL3jjIRMt5ZwGPh&*jg(wdwKe=U^`R(ezjkZ#aMU#djE3pXV&T!@=_L zjRed2^CG^Z!TS8U7~e5q`S^|l%lWfX-)Qb9agE~geI&k<8$M&7PVLXPI`$dV@((wA z84H&4=iWmNcW%S^vu_=~2F=KyRhj!Tuygw}DRW;Ame06XfaT2bCdQqg`;}bst|xuh zf}Q^wuCIAE*K_w7dH)Yj#=J-K0QLCX7T|$S;6AAt^Oi<;?_cn|#&cKq^9{B74P5e$ zD|VhqP3=ANT+TeM#hy>Orf_ARJHhJyoOPbt8l79~JnECVc4@w2D*!1B&JfbXpDbN_(r z4KB~Ky6b(F`*bd~;nZ_*|FGeFhOWW+oVV8UK2!UjBqu$t$#*b&o%0j!|Kgg;WuE5e zS(nq}cs_r}_nx(TF3f%g_j-@&NTdouk7EN`af_ak>d=BM@pR~No#IiL9C z=1lO%6zx9e(`S#pwU*C0zLL51InGgB_U3tX2dle|6Pr0^hjVReIb#cg{aWyy%6S&z z?!4YJ&sB_LpL5^Iy(d@p+Y7AjJD6WDJsO?X);+5)%q8!*;>^7ySnoOXCex+hMpCQg z%rDdQ`MoQB%fcN;t!B#%2KE;-jB&N@~DyAE^DI#vhEXB}&R-KV_k$T;4!jy1u~ z7517derh+>-u2V>U(w8x&c@|p5K7w zJZp|I2<$lW>Dd(QelqUnV8`8zd&V6MmXBu(uw2I766`qg?sF)2KkoBN?n9_g=5n86 z$MJjaHelD3xwi#7_k3KLdpkJ!TiJupczduudFR@ZyC3IL+ks2Yxx`uXF5q~s=-lS{ z7QbJkCs5n(eM&RWD6k&a;yfoc9_LXT%_XN#>{)PJYkBLFo7%f)!)I_K&(uBJa})Sd z?vtB2PXnv_eRyKSo!M}%<@$y@3(j6OzrHWy?l^v2pZXXsdDkO$50`LHkN&f{&f~I2 zd@NqqcrMs=xCcGz_W6BG{Q|H*|6Ehfz=d$(!(9v~-*GI*Ode+7e9mvb^Ej{jabE8= zOZQ}*ti5Nfo15%>9?qHGoCojey&c`>;Ca;NI60TUhx1J5T(3hbIG|Uan9xb;CTMR8vNv3UQcbm_ngb|U_CjPH#Q#UQJcUer%!BF zj%zJ%eN$6=Pac!Wz&+b@8~9J$b1rWc!_{*+x#3((&gE@zt-ahmmk)7w96zp4eF~Sn z>k*sP1KiW2{|>IZx$F_=T;2tC9qvJox_vp9_kw*!t|@2mKDhAV?uV1_IF@534>NE+ z=eOT^oY(z0ulJgzd$LZ}-ZR#eCOe;pbEY@v!Fzh|M)x^*R`oef&gJ27p4+@n97!$b z`AN^wV8_VMl4GdtlaKFsu>8qnmA(_H<$WK?d*LYX2ri$yx@X__!qH&0yceDXmUI7p z?>8U6-)Bvyf(LQs>*+M`D8{MZ=TC=|&(GO2!20ApBWH2<<2h10lS|II#F=X>xPI-9 zqjs+PwR<+4eC9d_tWVxOWL)n)7k}=Y3wB=jWxsQ}uJgJ3ab5PD$0g^w#M$#j;EwCK z7*0OVBNKGKZ43Yr) zJz3lYmcN#@JjBl?-zg?j%lo|Zt~#aR+Wz^h?VmqxhnvV{zq^Pc(c z1iJ?R48fj-nw}oqr}6v`fcLtK+UNcV_eZ(>=bW)z=6xRb#~ME4K23dW(-Yq_jZZDU zXTkFEy#RiY%dz8ok=psx;(G}!AKx^ve-2lDZ@}qOi|S77-q$i*}C9azrK7&vqJp4x2G(nrj#eYU2r6P#J7ozQq^0_#_ccV@72 zaBKI@LSY_i@pc94SBrNxuwK6XwtKt7nTcAwvxD`kx&MdI=K%ZXHh$mC{^x{~k8dup zoPN)qnp~dI2b?qW9>EyNdODo^r|{0@`TCgJtkp8tCt&&P;s0uvz=PI9V?RCttJvCAC?rWv*qw z^4ZI>U^zd@dO2#dR!iUVO&@f7-wJSMt(LX_3ano(-o9Yxs9FD-+N{;$T@kEbE#8&D zdTZ7zQ=7F~ysLoqs|{$fUKMQC&vIq|tHH^~w>nr(zgervCF_A;v-W>;O4jSc$=9q0 zQJb||=GqV}pS^4Zmh+RWe@kuFYU$g!>4R?X+XT+6)w1?Y!TQzW-3;s;HS5i(%~~zq z!C?Jr@ooXuTeIGh+N{;$-3qK`psHRE?MseHf#U) zt7N@9oP5pt_ta*smaO*#%V#fpf#v)p>tWPpt(LyMn?C6FzJ1`#S}kke7pz|`-u=MN zQM2Bk+N{;$Jpim3^%>M=trqW@VEtsO2SCa~U`_07~~trqVsVEt+XnyhaHo3;P@ zboM_HPCmX#U^)F}ttOYO?*Sje81=pUUO3;&)iUmVVEOnS0Q3{sYc)qZZ#k z!SZ=7FM#FzsO2SRj}TAZeF8y?`rYB4%V+0 z?=-O9dT!pJcK>Sez6sW^mc70WuK!%{4z--Uo)I;|e+27S%U(P2htlYlc0j%}_1go?!iI z+3TX<1?fxAV$^c>nxUFpGF%S46kN@4c{npv%eX6m<+H|Lf#qt3eW}e*Eq%Xk`f7$N z!kM93*0~Z`zgoO2gPo&hxC*rys>Qo1Sif4ltAX{_3|FT%L$!F<0P9!FUe}VNFFpOJ zHMwNC5qN#Ln&EHZ%up@kZVZ;s8aDyU)eJYKHbb@aZPxVF3^#`}L$$1PFj&7@ zyjy^sqh`1zwHd0#yA@c!TD(KRdTWM5sm)L=-mSs<)w0)Z!9(av&vw*u_L`xZTr%7f zygOXYa4$GBRLi)-!17t+-e9?!;Xc%6sFuEco4%UiesE@}mUZqA)~^=t0bu8-86HS& zhHCL11lF$>@4;ZbHN!)w%}_1g;b8r0+3N^v`qFb4wVb_Xs3w;TM~UHThNI!kP%Y!0 z1eVVlPX^1?3{RmpL$&ms+Vs^7PlGcQ;`s8LG)8!z;m;!_^G0f-^(4jC(a$K5M)NELStU zmf8%}(sy0cS2MgG&J5MEmm9$P)#4owc8;3i1Zp!>i}yybezkaS0_&|A-b`(VYVqCz z)~}YmP6Yqg`!BVey=K_@{tHfq{{vo(4c7O{`{8`ARLi&zfaT+R2<*SdS$%(o)29~S zU%>M5Jq-5W_pH80;Pk1*_b6CCzQ@7-d!g0$1e`v#_?`sI$M*u*_h0{*mS^=kSif4l)4+P`8F_=+y{pCh zCRo2(yl;W^)-&=pwfk3#_Z_f)we0mhaDD%MpIXjd&xo2_GW-wN_urb~7jR~%mT|uX z%V&*Wf#qt3UsIc*TKc{L+gCIEmf8%}vX}3``qkq79;~-!_ye^Ws>S;wSif4lEnVBZ zHN#GDW~dhLOkn+L*=rYYegB=ATFzcGRFg}FKLh*zTQi&&&J5Ku?tEbRtZ{y@T+Q(3 z)Mlubz6F}Tn&Epf*Fbczc5NtHrwrSZ~d6QED?(i+3@w zezokiH@Lq4_Mw)u*9_I%14V7Z#%{?ulumc9d;zMA2IaAv5My&MGAuNLpYVCSeA9ztz~ zYVi&S>sO2SP_W*b;RtFoREzg8uzt1d^$2i%{~bv!XRjHm$tA;+#T_y{1;n8LG)8!w13s{i6C_`6oEvE7daYLty#%{tEW*Aywbs;Pk1*_b^yK zzQ@4+eWmJ~3a3vkzQ^Ub;(H40-)pMAr|rR~7T+^q`S@N2`~LeBeFK{J-&f!~BWm%z z3YO0^cnvJ)C-1+nQ=5ZY`lf;Ht7qg5YWJ;{XZ0pnzgoO+f%Voi@;0@5SBv)@uzt08 z-v#TfXXHI<_pcW3`(XWQ+3R$0egFNCTFze2h?-n7{7Svce`okLoEfTR+;710S>v~0 zxtig3)MlubzVE^I)eL{2Hbb@S;h+o zYVpnt)~}Ymb_Lh>-)_`$_L`xZTr!*=?E7y$hd+liL$!>%09Zb2To5c*GhB$;4As)N zaMM>a>;Y$nYS~Lquzt087Xdp*&2Uj_GgOOrF|dBMczc2M)(m@7o1t2~eZczFvezZR z_5F8AYB_t&P)#lwt_b%1w`RB!oEfTR+?B!dS>q~Txtigs)MlubzSWw(n&IkjW~i3E ztO3@q7Vny1=cpO3MQw&^@%97jSBrOTu-=;CI@D&U7Vo-X{c73k0C0W(U5{GMUNcma zONN_+egCZ)4u&&BwT!z3SUzjqQp{B|+=|)^TQ$D7`tR5cY5Ho0L*dL&EqmD-tY0m2 zY$N8X8E#8$hHCL{2iC6^@AhJ@n&A%AW~dhLj$r+2+3U{W`u@8MwVb_Xs3w;T_XGR> zTQl4r&J5Ku?g3!=tnomwT+Q$xYBN+z-@#2^&F~O7GgQl7hJ*F1#d|2&IckO@sLfC< z-owE9)#5!IthZ)(1hpBe#XAzLUoCq*3S8fRkEWKh*9_I#Z4{ zPi=;3@m>Jdua>=D1g`JD7gNjGYldoa$?$rx@4xjN-T-HYY8iJtSUzi<0G6v6-bihR zYU#VF>8lyu3}=RF*~=|p{c7>v3U-c~;Y4aPREu{KSif4llfimxhPP3hp<29C!1~p) z*W1DM{r3)PIeX1eO)eS!8SLNf9LqJJc|ZIMobQ!t8TYSX`S>0I`}gXr?@>5?YVkb= zmXGfVuz&x)`ksW-rxxE+VEOo-1^f5(tM55DeQNRj9V{Q;Yhd4h{rB;*zt`bBBWm$Y z1Iyf0@hnI>sQNOX9L&w-|p0M_L`xZTrylxjjz3$;X-g`sFraT2FquSJ-~7`!=BV;sFuD( zn!cLhqHt!Ymc1+n)~^|_iKy8L<@vaBfua>>853cXO8&J#HYldoa z$#6^c4jFC*XNGDScL-QMYa9xes~K)hZH8*;+otKO8Ey+_hHBZ%c3}N#@oo=xj+)^P z)Mls_?~Y*oYVqy_)>|{&nc581;@t(TUoCsx4P4)Ucc+%K*9_I)_5~mu4Z@(wHd0V@7AWT zW;hYf4AruiNnrhI@lFOiN6qjyYBN-ecM4d)TD*S*>#Z5yPHl#2@!kQ}ua>>u1+MSE zcT>yRYldoa$?$Jrf8SGmuY4HJ_e!;l`v_P*zNuh;FI4qC4yR8oz9+!)@jVUp_eWLV zGjRIU;(Hb>AK&v}f6r9){R2*)T73Tm%g6Ty*!N%mj>LfG{r*ij&xl%lZ-M3W4BiII z`N{k5JJjZ&mcDnv_SG};9<}>c%d>hPtY0nO55Ris8JSM)-qqs$5UgJ<-jBd~>lyi& z+Wo7=`w3XTTJ|~vT;G5HO)Y1yXGBdd8U6tF{kLZLBb*tkWn6#Ro_yBWiMsU{@RbZ_ zf-^(4^mT6fYKC3l%up?RnHj8KE#6tc&QUY$N^OQ}@pc32SBrO6u-=;CY}96`7H@a3 zezoj%4sd<{os(M5UNcmaONKqbzW>$?d%~HaTE<-jET1(l3YPPebGR6_8LFkPSJPKB z>&2R~7GgOOrNw9vkc$Wg}tr`A;+6>j=T^g)kEqh%CT;G3} zrIxeT4Atb4;p$-De`|(oz?q?1#$6LEpEa%pma7@|qc%ge^sU|W)eP5xGefoPWnHj- zwRro3oug(rfZ7bz;$07{UoGB&V7)cN^{LHJE#3{l`qi@6LE!rSyCJoly=JH;mkfu3 zegCZ)ZVhLKY8iJMuzc3IEm*E*xE-|_s-~#-tegFMEwVb_Xs3w;T4^iuo;cz%JRLi)B zg5|Tu5n#ER;bGKfsFuFNo4%Ui5pZUxmc5Jw>sO2SNU(F%43DBVL$!F12J2Ug_ZYC= zn&Gk3W~dhLabW#w+3N}5`u=+&wVb_Xs3w;T$BH{-I1bJX)iUncVEL@^9I#x?@LXy$ zR7>A^O<&FMd^j^y%U&)3>sO2SLa=kx3@@TKL$!D>2J2Ug_Y$z)n&BU)%}_1gOTqfp zve(PO_5Jq>YB_t&P)#lw-mKms!&~6YP%Y!$3YO0rCxYc_hLfnxP%V9vo4%UiZE$9& zmc2{?>sO2Sk6`Dh8QxB9hHCNN0oJb;@10=1HN(58%}_1gyTSU^ve$dT_5JrgYB_t& zP)#lwJ_`2tmDcyl$KZUgRLi(i!SeAv3HJAzR^L-_`qbik8Y~~*b6|h}Y4!abPM=zQ z&x7UTdjah4Nv*yY;qzPG`x|NaBs0nPjGJ8+&6wfNoz%jX%q2bS}b_m20e z%|R`FAAs$vXJk6H`&P@d`Vg#NE#8m7dg~eanA*Lo#rp|ZzgoQi0_&}3WCpeSSBv-G zVEt;@>u2El{`)z#oV}hAHMwNise9Y|ufG>K8O{V}hH4qNGgv-r>;jgn8O}^?hHB}X zrRl2~c7-!Twd|!ESif4lvw~aaEzfB-I5Sj>w>wzBTD-G^_0|mMpf*Fbc;^J`SIb`K z2G{rBd8p;=HA6MIWVk4}_1}Le87>BAhH4qN7g#=P>|mj>%si}#mcy*0ySsLfC<-etl1)w0*+!S(%j1!_5a z%}`A)8LkCx{r4YAhW+5oP%Y!G4VKRu*HPoD8LmrhhHB~S-}KcC2f&%3TK2LYSif4l z1HsNwGhCnA4AtV@0IXjv-rs=r)(i(xo1t2~8-n$#Wv{;l*Z1Fzspaf7Lp8Z%xGlK# z-+w3>ZU<+EY8iKXuzc3I16ZzRxFfY0s-19#}qWJRdAq zGrWM>4As(iVbfPLya>(=)v}k1!TQzWy#(wWHN!tpo1t2~mxA@H#d{f8Z_V&>YBN-e z_X@Cnwe0mOaDD&1np)0YGgOmHh7-Z9|NcWchm+vUP%Yz52FquSw}ItqhEu4`P%V9b zZ2D@3x5JsCTJ~}WSif4lcY>XxW_TC18LGv5H(0+~y!U|h)(r2ZHbb>|?*r>s%U0L#bsGT7hqUwyB@O~t1c->YEx_}&Bi{_F1&8PL4{z7OXaQH$>b zuza4ubg-PCy#IbkZ4PSb`v`1bJtH4eyKl8Tt53lC)#CjZSZ_TeGpOCWTD<=T>sO2S zQ?TB8Mn0o<|7!7m4%V-hy?z0%@4sJC%h~H0QIkuCUC7Qq|E(F$3}=RF8Fv=2eAd_% zELSt^Ms0>_>6^9bs~OG)XNGFoOLwq-wRmR-J4elM4r()0i+4`2ezkb#0_&|A&P{EG zYVpnk)~}Ym&I_*Zzw=Sc*=vSsa>=ka*!SO>VIMd%RLi)FgXOcvCBSku!zHQBP%V8+ zHGMV1U%;86TK2LuSif4lzXUr+&2SlNGgOOrS+IV!c$Wj~tr;#)ZH8*`t^n4rmc8}` z*Z1FFQ_IGkzcs^k;mlAiZRLi(Kf#tKtoxyT7!(FJ&P%VACHhne2-QdhnEqmD=tY0nO-+`T@ zX1E8n8LGwmd$4}Bc=rVBtr_k`ZH8*`4g>2~%U<^Z*Z1Fjspaf7Lp8Z%cm&w@-sQNOuLall-|ML5>@`C*xny`7*!SOh4yVAGp<2fMBUnCbyd5l8GrWV^4As(i zXVX_RybI0@)v}kn!TQzWy$9?ZHN$(U%}_1g`@s6u;{6}6-kRb4)Mls_?*m}{YT4_b z!1ew2A!<2$%}`BlTFZa`?qq+r@&p3$_b+=-#{Tz*Mt9-w!V!W!*Ec=mx#xF1_n_x; zYJU$${@X`A;pF{qJ*2N!)91ghlD^(x`P#Q+)93FzPTx{+^0jZ7rqAEwoW5n@AoP2(_ z`RZWDmp8MF<2}FId=0Skn4|s9;X3+p_v1S3S&M4}F4rOU_r&|(q8rH7pUdBO>|8#7 z|NhDPjpn?LrB8pY<@5W$PvY6?acqCTxV^4pBkq2Z$A;8$jweoz8-txczk|h$HlgmA z(WY?n$!IgMUU}D+vArjw&B12mdhK^!*R~~hKd#N5Ex5Mfa&6+QZ3y_MwYB9o6s|v) zxv3|&t-)%p#qk}>F>5WK+)gGpJJ4?tYTne$;Y~C(haq1iQAU=*`#ULDU^{ zI~Yzrxg7%5EAQGew)f;V9Bgi`*M8@9ZHIC9m1&hdAU#K`{ev^XMokG^K--hF3Bm> z>i#S>NA;7r@7H6%`JMOqZ!(O7bG+>9Y-;=D^WPyjA1v?h6i(j-)FY|Y@^k2t zrf*E?`vaW+9WAw-tINRVl;2V8_xj7>*z5(pq=4rokWL@LI>aOcX?tWaC`~)sJ z*Co!Jw}L%CCvrJwdc0@OiD1Wd9ro*Yj>+6Jhdq!`8?~p!SZI5zWc!Pu0MVEgXP_qb3VkO{1e#UCF>dUYt-LI{Sch|3G`V11uW;+ zoBMm1yB{-L3j8-NIrk&Zf79g&@Ki3pR-D`M{cpNF30BL01LP^NT;2np0XwH3*Wl0h zXQ}1w-IA|D*ENha%cs}1{e#+%Yx8UQd9F{nT$?yqya0AhZ9l);{@a8vQfIyDS?^0= zwXFALuw2&rD%dsnHJszpUtg-iLd;K1)mc9M}mdjqvNvb&&@S}Ac^2(;pIMuCzgKux&qTM!z3Zv# z?tkT&_wVauWXK$X(0$`ur(p-*p5qIZGulLL~4&5H-*XKCSwJ>)- z&b2JsLez53B~GS`fE~lnNzM1RMdAE!W2=Sh)6~7W-$kE4U!MTW`yMob?}2__e3Dwe zA9!u9*&F_8IQQgKdNS^_;DK=J`Cj!LSg+ct&6@t+aBAWH0p5r{xlv8uKf&^wz-NvZ zspTCbbG!tWA6)ugp_aEV<$V^}?~-8q)Y7*U*k`Sl^RP77^WgmH`z4&` zK`q>JO+P^B|Xe zegMnY^WY(rcZ|%@$%DyN&qHT(;Ic2{b)lB8=V2CV`8r-#YWaE|W~G+5FXv%)YWJG` z&q*CF=V30moCo#v&JA`<^{j6muvw|4Z(gvx&m#N%71%zt^!0`FS*ztdtO)iz@GYdh zZzZ@MTx#*I+SK(t^uQ-y&qGf*`HtsdQMjB3JsGzb{6OYb&)R!~UAtPIeV>L?3%3N= z^B|XemITY!^Y9C5dB?~cOM~U>d02*8-oA{tELgsthvli|>v$_r%h&VJms;MwoQIXb z?lt>g1spEtVO6-C2le!>26jyKtZ#L&S*fLOO|ZPrBKuu~%vJm-C>W-krgYsh;)i0yZnP^z8kYvE z9PxX{nq2nyuCg$9-!;7FUFB@X^6dEjqHiItoU2X1>VACxdzJg9aPs-*x6QyCapj-i z{Bv|U`|`dy7~JvweJeQmyt@ts`|;gKZ3vf~>lJ6LZNPbd&pNk*bDid$b#4zQpLOm4 z&N>~#b;{XiHkr$N=GqC|@tu2DIQgu5cd#GVt+pGNoa+{6tUbVa=gzwKgmc|~AJ4k? zf|Jj>hk>(h$8g`v@@}0!!|?9h{@K`Yw9}YpKCnIc^RrXv`JSh^I@kXKW!mU_ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1.spv deleted file mode 100644 index 8d3b6209903d3103d712123089f3534e8d8242ff..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36616 zcmbWAb-0#g`n5M z2T<3RDO>OT`fuKU$%76awB+(DEN#Q|EuC8X?CI7rRm&9IJ9F)`(;oXO?89{;S9kpK zGjKVO{a*cG?di%jTWj5^Wd!$*TkKELSb9HiVCw+S4x);uM z%gt1{DI0E_`^v+YbX&P>U!gXo5MGEJ>_4Y2DYn3zLta-YITb{d_nW^5{ zoB9pzuGhWw<2`wkds~K`TQOxz_Mzs>IegjwHg|Q$7SGGoq15w%eW@RSQ0F?3I&+DA zWv<2~ryBKDQJQrEd2rYCcWovRaGE{dej|D>|AZ{eOq1U+7;ciVt)^2ah`b@1G*LW#OYe*;p)KE3&rzhy?CSfb)hyB zj-G{gOke3+672ZajLNwUcIeeGj(6;Dekyxjno@Z&-dEscD+CKe_HNn zuLoLxd3$@k1@)=pPjB~cMLp%=Bi3+_gTeD~`I?z}FYa>YFc0XeKN9RK{-Iz$mr@T0`&lv<>U-Ku?=?o?BCCsy8hFsyK%Wb&&-)zo;&kAi@Wc+^&I^F+1%yav*Vw`-Oo?? z-XOK}xI82E>dk4j=ln*u=l^wO9T(B#=Vo#n2cEms{{S0&`tPK4{pp_oUbEDXfqkRh z$G?HDRnC1p&fPuAo0Ihu4VUpIQhTm4-*ZhpC#;{<<}?@B&thxGeT%z$kaJw?x4Goh zKWwmG>(98m4(rVO1-PZdAM^GC`@JptT-3vw+WoIh-KEq6sJl|TPCw_fPCv)3o%bN_ zu1U^yTKl;zmwB$BUZB+DsfV#x^SlOZo^p(=?39yxsvarg5-`RM8H;o!)*KI>l8-*jGKO~>_nL&o*9-`a6k=k6G-TvPkmFV~x0 z`nlz2y=P!Zb0x34X!3a?_1)C#m--&+dQK)#=j+4!z0_tL|9z9y_fz{>9Q^_69ZUTn z_5P)Phc^;WD)r;k_m=ue>WQU(n)Pgf~ zm-;2@wMzX8^}tfUM!jRH-=N;V)NfG_EA>0nV@mxl_1IFsPkmFVKcv35)E`q%EcK_< zZMyBR5%jJ972X;U1`*ZEX<@dJL=^4!Z0517a z^jII%aDIP_?+~y)&suzkg5~2o94zPefcTCA>+^eGd`E-j<2wc{=l4T>$8tZ8YZTY4 zT=9)==iAnKJoRuc`OJ3`*zaR?zLTltA8GbC1}t|P*ZmFmw}$h3+!}mcnw#H~vYv~< zuE*~^SW{w|ozV@TThV zz3jo8d=2-TnlW!^boc)~@8Njv>b`!aR==7{-f_jwb8}OB&pf9wk882#E3VtPGS5H3 z>V7|Uo?9E8TkAaPw{Xcjk2rbW1=f2f_w4)LhMT~B7<(Da{Xs6z+r3=j9tFF9$H;h3 zfgNv2?iufyhMUMe@6+?(QC!b)<$ZbqET4T%0zb$lAMRza^B&K2Rg?28V0+bm;7q*E z-Ip1wy~gzkmwOlIJiiBaEXU8DJ^;%*Z=aSaTfe`51b&;#Gp_D>{a)}fmzwXHb2`5B z)LP#6(EbUX+dQse6)t<7<1_A`b3MUje&*%bmeb>SzTb}TJ!|zGn6=+q>bdcKPS1B> z-y_d*GW-cFZ-(ae8+TvkrS>Zqzva+6pZK`u40MFkH=4`&^tmQ`Yb~F1dpdLLa~!`; z>`ne%!0L`SqM4&BoO7sU-szh@&riHF!pY0U+oS1I%Xyp`?AMLYT&{5z?ylK;)_4YM zu+Meh#(j3KWHmQf-Dfnvwq|Q|T3ho{pOs7AamAT?0kGaP>COAGAl%{9YWezDwCVG6 zE`5u^`FX46d$a_1UwMyuQ_DH0IA2#wfk)BjocX$18csfYS_bSox%+Y*_AJLG=Q_k$$BJOrVeVPSpTP22$4X%LDepQmj`yr%Ww7(OHv65! zb*#$Wm+P=+6)ri~Ax@5Kf!(WT{;<>MI$mdjk5f*nWRy>G$YmwUgI`{vZ6xZJzg zaR$S032t2{Yd)6xFZOYti7Rt&1t)(4J-+W-gZ0Tf*S6e!IhWctTyoAO&YHIe$1}zU z9A5v$@7Mf+)b@M7wVCH&upZaqJcl$M=TRHNC8tm9`EXoodFw-)+Pmk)_vM1lZF{!o zLhvcvZ*Jy10<7-m_>B#Bbi=upD;sVooV{v(&7aEMaeTQx^<%i?U60s3oWwmn`bTgb z%Vm#vC|=h%3hX-EgC2GJ{A^Pn4fgw|Yswio9xi;i6XE11AImY5hZ#7Z^V{z{&g*`h z*L%&u?Wx)a}c;ybkPpEDdR+PX84VtVuYRr{4=11RwI_h}$+v!1arfo9Q9F_ABrfL?XRcGg^=tK1 zYUiq7tEa)qXRa||ee&)h<9hdf@cYc^VCQvT_B*HRI+MFE*JaNcTym~UoIRfno_rnW zz_syhzh*g?+C9r>Kj(wpPu`ac!1l@K{ks?}U*Es6)bjQHyOcWbpL*WE%fM>xJ!|*w z-hF@X?%ekLZyxr#ulip5o~QRNdOZ)GVSSEc@2cF*#Cv-G)r`A5oIc0No~{J@EP4g^ z?EPx6yuUWMpYJ_BGu%Ke@B5qQ)0-Nu?bmN@zh1co?m8~})&2E~oNI7>vx8lq_sn-2 z*fse34))B_^vuTnE#B{a;l2EQMc?O#xIfJ0uZzZTneSQLA8GiE`y}szq=rsU!o`VlPW>r^;%`H9+W)YA70*gpQ(-uEkoS*Q(fyuX3e10xw|6SnZ z=MPZPse( zd#CAxZtwdKoLQ@7?eBv1tHt{s*g4ojyZ3!+vsR1u1F(Lzcs~T|tyzC0hgU7$kHPxY z+`n0W0ygV>@9-Rb3MU`mXJ9$~X00ZdtbYWXwO_l*`X@N~;c(7n*1u4jwOX?N6)c~< z{05fum8^fKHfy!?F=T5WbbDV1II~vE+B<^vtHs+1>>M@g&TwX}7Vi{b{c7<}3D#S) zo(it@Tyu|iYOsDa_uqQ1!Dj7eYxds-ZW^xmx`O5Ox1MY6a>=?E*sT35PS!KS$q$Ef zF0-DE+N{+w*X&^V>}3wHoUdd(C$(9trEjjL54yc?ZaA}6%i8Aw>sO0+Ua)i2tmmUP zYqfai2kTdhcLA{8n)QOzW~~oX5E+CtkvRO2drPs{d=z01)Ft#&&jOUgOiW1 zA6QPmS*yt<>&?Js?Voui>&@ZhhcljYne~>`X04X2{{oiJUbX_u`AXJXQ=7F~`nGBM zpxgVlg)?iltbIGMezkc23U-c~_4d?etrqVNVEt$9lMS}lENH+|6UedoZLwOZDGE?B=>yytR{p{NQB1JK^-H z#djB2KE8*+Uyzm0wC3iyc?8aLqZZ$zVEMc+kAdZU<=p(6+8or<_c+)-hHLM8g4%tn z<$ZV(tY0nOr@(saxp|t}y{pAL5v*S=-e$!QB+Wo7=`y5!mTK4(^xc<6e61AMY zo)I;|zXa=7%U-_**XR9jsO9W6Lp8Z%=xRy{tY0nOu3)`2!)d9_P%Yl+!1~p)*KXix=u1y`YB_sb&mo#zGMp1UGhEGZE;ut( z%eZrc<+H|lz;ZRid8y4%Eq(JfeKo`R;mlAi>s$b=UoGAR!Ol@LT!`8X)#6|*8%HSi+5eH-kRZh)Mls_ zZ$Gepwd{2R@VfM+XG3Z^d(BWyE*bs>yg6LWa4R@7RLi(qgXOcvZNPFh!)>X}P%VAi zHGMV1zrvZJTGqKeSif4lJAj>|X1F7@8LGv*6Ij1mygP&S)(m%{Hbb>|2Z8mgWv{zg z)0dvzspaf7Lp8Z%crf??xSHV*I5SksxQBq{v&KWgay7%lsLfCTwXE}8uzt08&jUM0 z&G39`GgOQB0j={RXUGEqnbAT%UixrzgoOq!Ol@L zoR-=Q)#9BFtY0nO>A`wyhTW*mP%Yl>VEt;@>x|&~{M&| z7X#~8i+6Fb-kM==YBN-ecL}h5wd{2%aDDz=np)0YGgOmHhO2{p{;e6V0cVD48Fx*v zeAc)YSgvN+huRF)(zkZgS2OGjXNGFo%Q|5FYVocMc8;3idemm97H>bWezkbl2kWgF zZa{5@YVmFe)~}Ym_6OJJ-vQKe_L`xZTr%7S?DKEUa9cPtRLi*Af#tKtzk=mzhTBt{ zp<4QOX!>e~JHnZvTK2LNSif4lJA<90X1EKr8LGuQ2&`W%-d(|ZYlgc~o1t2~yMy(s zWv_dH>+|oP)N=Nkp_*JWJQVEnZ_V&9I5SksxQBz~v&JL9ay7#vsm)L=eMdEYHN&If z%up?R84A{~7Vj}&=cpMDqc%gec!z`atHnD4thZ)3lG+T_;vFUCQp;YClOvP#jHZ^e z*9_IiY+rKDGG%36_uVPOyK+srv4M)29~S-C+6n?gjgI zp{nmbIDKmI-4B+J?+LKazZ2=}(>$j?3FjG6i|;A0eBOhn!E(Ox{5z4_9MsbH4A{PU zMxLd1-)ecUo&)Pwi}!i3-g-t}pmy(S@lFEkSBv*Wu-L@j5p8LG)8 z!zrh3d;YEGa4I-6RLi(igXOcvX~1$d!!FcjsFuF2O<&D$S~xRQ%U-4f>sO0+da!fU z47*XAp<2A%!TQzWodK-3W;i3Y8LGwG1FT;yd+iCX&%eE>pMRI5mb2Fk)#Q@lT40}lYleN`%up@kt__yY z8vBCfYKH4jo1t3z)@}M~hU>wZp<4FR53FA;-u1!GQ8V0t+6>j=-4Lu_E#8g5dTWOL zsm)L=-T`3!YT4^1;QIVKkXp`OGgOmHhJOY7{97~J9?lHaGVTsw`K)nAuw2b>Cu%cP zOW)2-U(IkAI5SksUIu~ntHrx3*g0y3yHT5=TD-f1^{d4@7_7HuxCgZvs>Qn}Sif5K zx;MB!|L#LAXRjHm$tA-hz&`)h43C5}L$!>16j(lMJQ^%lGaO27hHB|Mrs=C04udm8 zwd`d$Sif4lBf!p4GaN~6hHCMS5_74=dn{OQ&G0yCGgOOrG+4h{_IiRknWX1LYB_t& zP)#lwo&)yzw`O=QoEfTR-1ETlS>yR&xtiex)Mlubz6+ban&CxoW~i2RUJTZ+7Vjls z=cpNur8YygcrOL(SBv*Du-=;C<_5$P%V4; z2CQE#-fzKrYlh!Zo1t2~--Gq5#rp$TZ_V&WYBN-e_b0G^we0o3;QIXg3$>iRW~e5Y z45yif0CtX= z;f&N~s1|Pzuzt08X9DZ38TO<$L$!E&f%U6pud{&b^Y5(Ga`v{KLq4nIlHo#NpMPtH z3&WYATE<-jET1(l3YM!GE=FyJYUx|N>8ly`hBHI8>}3hCezka)1UpB~a4BjtREu|M zuzt08mjUao87@n0hHCLH2iC8ay{-VR&%Y~D%h_v&YI4c2uli&et^;R=Y8iK3uzc3I z9$2ns*pJ!_)zY_q(^oUx0L~26vX>3P`qkpy2<#j+!~WD}s21-4uzt08HwNpi8E!&t zhHCK+1nXDJUN-~R=ikk#HMwNCBiQHPdJcDjGefnEyE9lmYup7aS2G+$ZH8*; z+qLPd8SVyWhHBZ%?qL0D@eT$%N6l~#YBN-ecTcc>wRrae>#Z5?O>Ks1@$LiGua>>; z2d>Y*`%}x=Yldoa$?#~f&%ZUpp>SrXmT`{(%V&+lz;ZRi;nZfRmc9{9U(IkNoEfTR zFQdd3)~}Ymo~%wL={bd3&R#QA zlS_u@gMI$38D0QqhH4r2La=<+coA5xW_U5R8LFl4lBTa_I2O(f)w0e@!TQzWy$tLe zHN(rP%}_1gabW#w@m>MeTQj_p+6>j=y$Y;fEqfgguFt>MP|MkChH7%j@E>4*MpS*S z{3o2xm1-IHUtsz8?gIO>q^j?3IDKmI-2;}7?>?|UbE^98htsDP-veOz_#OuPv#F}@ z5jcHn@jVKbkMCKq&%gd1i9XHq{c~`h5w-Z92g~O@cmXWuE6=}^sLeqweJ_ITt7qgT zYWJ;{_v&S^ezkaC0qd=2=)>D!F91DA?!Un&Dz_ zW~i2N7YEB{jlIEgHNz#S%}_0UOE!Hq!=>QNP%V2|8mwO}-ethfQ8QeY+6>j=T@I{Y zE#Bq9dTWL&P@ADzyeoqBt7We%f$Q_{%G7f9nxUFpGF%Vr^KZ?tADkJgW!&|_@>$~s zV7Z#%hSX-LmcEUezM5fwI5SksUIu{mtHrx9*g0y3n^2pfTD$|n`qkpy6s)&qxEZw> zs>QoGSif5K`e$%`{@s#V&R#QAlS_uXs7;pPAUHEr%ecFO<+H}!z;ZRi-Kot`Eq#NV zzMA14aAv5Mz3d6ruNLoKVCSeA?oDlmYVqy^)~^=tzF@sI!~LkuP%Ym5!TQy**8{=z z`S&1dIeX1eO)eP@1N;12&*5-5GgQmCBf#=m<4CYv&2SX88LFl4*ru;$cpRJ=s%0;u z!TQzWJs#{FHNz9A%}_1g6T$k`;ynqhw`O=UwHd0#dkR>;TK0OHI+>(r47HrSW~e5Y z3@=ijEW?Z8%up@kUILcS8pnd=YKE6mo1t3zE^GQ~hL^*cp<32C4y<1--YdY)Q8T=f z+6>j=y$Y;fE#9ladTWN`sm)L=-fO`6)w0*?!1ej}dTKd)%}`A)8QuZ*XPnmO$~)nF zu2jpocY)>On*eV8S*Y~&X`U!Rb?r?_scfe2;j0Gx7$tdsmD1O|X8oc;5o+t!LzIYWJ@e?>k`qYT4_%;QIXg9<`jk zo)I;f#qt3-&32RTKawf+gCIEk=hK^vX`I0`qkq7 z8LYQv_+M%>REzf)uzt08e+BEU8U99XhHCNt4%V-hy>{sOzt6uNspaf7L!W=;l3_Qn zfB&s!*d5Lc)iUl3VEL?ZMzCDXum`mns-f40ev1;Vjf< zs21<6VEt+W~dhL%3%F!+3Tv{`uw{ZwVb_Xs3w;THvs$mTQl4c&J5Ku?nYqwtg$~> zu4XuZ+6>jww{g=~Gu#Bu4Aruifnfb=@ooxsj+)_S)Mls_@8)3rYVmFX)>||DGqoA2 z#k(a~zgqUX6}UeCZcQy`uNkVzCBxmoKL6GXcZV}WwTwF$ET1**0hX&7?n!NiYU$gn z>8ly;4QGaG*~>m){c7><3wDm0;eOO+s21=3VEtSU6hGpXh5HA6MI zWH=V=^KZ@YQaCeI%ea?;<+H}i!E!ajanxq0mcA>RzMA2caAv5MbzTM5uNLpsVCSeA zj;A(5wRo=q>sO2STCm=l;dRtzs21<_VEt;@>y6<0{Cg9%oV{kKCYKEF2KzJm>+|nD za6VV6W!wp1`S|V!`?LJ3?*TY{YVkb?mXGfdus`#^`W}VTrxxF1VEOo-0Q+wPRNs?u z`qbik3M?PrOJJXW{aGS?n&|e+TPVi?@X{p|@t(0nQB7;_V35ua>=b2G`&Joq}4}6K4ezkaK13O2}aCT}l zREu{Guzt08=LGAm8O}v*hHCN74c4!gz0M1+&%g6g%h}s{4*9H-ONL8vn8#Z4{L2ZU=@tz6Rua>=@rA{X4Ih$I}UNcma zONN(&eg3T(j)OBpwTycOSUzjK5-e9Uyo%Zk)zWu$(^oSb4`+sIS?4ui{c7=E3wDm0 z;dRtzs21<_VEtBHGrW=74AtVj39Mf&d%Xo*pMP(qmb2Fk)#To4`QM+N?9W!d zhCuxJ%ifc*|2~ob7R=p*V9!-e&v@?nS&SQ@DJBRC7 ziMub?Vb7nq*5Yy<;`|)=HNgHGcK*C$=kopcXFRV7R&!p*(x<=H^7;AShw$F&acuv6 zA$wiNI^2CFkG|A$jweoz>w%p=KZC`L`cY4w(fV-m$!G(xUU}D+vArjw4Z&vQdhK^! z*EWE=FV|*If3D5AT$?y++XVc_+S>kIrh#zDO+C473RZJ1j_+8GS!?;^b||^&aXxdi z*R^fQ-B)t^Gqs%KiLn}fYn@!<2#mP)>=Nf9Zqg~oX_0s9oW+L zZ?7H(_UqU)X}1st@OqcU*BkuTKEyXWr-EU^ofR z@v^UzsqK@`ze6wvEbq@0PT%R&hf}NN`_NfUU;VeU&W7{f&Qi;{Iu|?&&hdQT{H#9@ zPX2~wpXY;}L;h-d-1`M!dC$&8+^Hl{ES}b8Ve_%xh@5}PI=#x zjN?6L_%g6_o2UKGk#$`TR(D-jaQEf9G7U9uK_!*>#$$H zb6m$gbJ%k&mz;Bm9nUjiEuSoI0#BaB&2Vje+rK$+3$?k+=e@oSEN@0ROaB1NyZ-e3 z3oP%xoYS8rdl%TBCF>dUy*>+kH=O)%daNga<@|bcfA@3uWrp*C@8gnlKjM6^eiZyL zmtQN+ozH;Bz-oE!dmQW>zMRwV=TA_}+q)TGb*^J;)+nD|*D{gXmuvBB_i3(=xLk`k zdw&+}8ivrD^*u+O^{Hol&x6&nzDZ!`^lLcldy!h+-pug|*g4w1x3zr^zDk`r)HBCx zV71Kg2G}|HU;7+yQpnqzIo<&~N89&-wmJSoojKGq$Gc#)%<(?hIefm(K0ct9w>O`w zckrBS&hF7Q*yEWsYjZW(=;F5DJamM-@Jo$U{ z4P5r7p1pkwR?FVL2fMfW9{oTqZ*R`fuANyEwP(p*_m#DH_iNj;b~L&@?p04+=g)BN z+4nhn{*_ujd;T5l%RQ_8#wF*T#o2QQ-i*o5Oh>rvSv`C11XjzQrvST7hHHO6r-YNY zH+vpTk9>OFbJpTLdmcugJ?5atb=LQNV(C2w-E-3VH3oJZd-FNqJ>w2VpFn*bmp;ew zTy$YgzRYM&xM`^6%t$ zE$3X~WI7|*F?=1{d=~Y9>&~SXZsw-$#r>c3^`!qEu)NPHv?#eT7L5L@F85zgPx50F}N>tt7q+>fL*&<-uq7*PA%N$V9$eG^7#TRU(dr= z)bfszIlczV*Yof#wY+^9?>n%3Jr6%n%h&OKq?WJe;b&@j`*I$B1-sYm|95b>oCm*X zavs#v+X3D&)w8~iV6#$7UuUqq??v{z0N6gY^eqVId##r9urS#3z(+`X-y(3H2eo(? zZ|Zs;ro|^;&%<zv%W>aW~G+C#liBv7uoNUWNx2Y`j&$8y;jS4SO)BQ;MU%^EQRMm zE#4KHx}JyS@X6Qnusodn~|$Hw@)p7E5rF-tL52w6|mn&{Os_& z*yFQGH|{=bc+a!S2*>B@0;g|!uAHkrV0B;l^~&0CYjEY)Q~tiToV|ITTn9Y)^LRfv z`8=C$0QTjxklOlOa;{IDu{Hwdc|7YH0Oz{QH|yFMF6+|cy5#J2?-|>B#vTZs{26+4 zIQgve&tPA!Q*8?_IoB!9SbqWM89M9S8qRh4Ih}QG1DAE`ah-DZy3UO4-Dk$Xa`)x@ zLm6m0YB}c@XZ{_)&Oedf%)b+y^K)yTe`ku!ugCf2>~;Q(?cF-RN8sJL{Jp0^Xh$&a UOkjJ4p}C$B+<)a=S&8fa05c82-T(jq diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s1_h.spv deleted file mode 100644 index ec1fc640dbec3e95b107a1c4f7c6853c432df63b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36860 zcmbWAcf6nT-NsJ_2{B8o1WC-=HDVGRcmX| zsx4x#qNvUDexGw+$EWA@dj5HSujhX5>vLV7>pQOdd(OFOoOJ2DShtogomzUd%--_- zY%Q&yxm!B7bOL$bV(u2SnOo}(H`!{F6-FMm=L)N?wvvr=v~+3hw|CB#ZY?u$@5*(+ z?)x05Z~)h7Tyx=X&C+m-7jD*uTe@)WTkmp(bFFd) zoHfthaI11RGc(oOtEpe(?t0B&L!Qa>1`nlI>?iw}-deTIA+&!6cXh`W_vOm-vF^v^ zM|~tho$D~_&S=gh_LI3%yB|N!bvQklOWe0jY8~6U&NYyMoy%vYcOB}?CH9lKQrEc# z(UZBv&ZW(LSod=PKGu&=*L^)o&HuITOYA3erLJ>5PEY0%J69Jj$F}b00DP=}iBRX- zkeZg(xx{`J|AXbwbga5!_Yk|_RqsC&eNAMpj&ZIoUWx0 zS0}FdP&{we%Ql){7i#n3=vjEj^pn07z>aV27;AC)Nj;#oI{v=Y1Ka9#QxBn@=}*_L zlX@fSelMTgu5Uv9M$4bt>tWWP+}>Ufryf7)>~{Zl)H9DhVIB840^Eno&-~Q;ahEfP zKGgejcaQRGH?`lJx3cP5k{|Gkt z^xsYC`qO_Oc->Mz3id&}kH>(nRnC1p&fPuAo0Ihu4VUqzQG2d3-?L5a*Ve0GpYg(A zzi(SF0-ny@+Bsh5?tZ3oIfwNOE;;r08m!m)WA3iW{IiBn!7ZJBTEhZhf6hhkLw$Tx zo5lLnvzB^u>K@dt)9?9Nr{B}9op%&>*CgjUt^FP@mw6^pFJ9^!sgGx|W;+FJwsMYl z6L=|=R>qn^_&)RCYQ8 z^m+L6#M*Jz}pHW{?>My9LmikNT`%C>b^|VrdOFg~R-%)=;-Jhq(QR8&9eK-y+U%~in zJu`lPzL3jjIT+TTZwGT7z~#@qf%N#Z;SetQvGiDvY&d`R#dkPZpJy$;Bf#?UjRwp4 z^C7;Y!TS8U7~e5q`S^|l%N@(5ZyfiNxW;lV$Q9qo?R?uhPoW;eC7=0D2m3Rw&UXg2 z{KL)uCV=Jq+4o?>o!4;w{9A{wOLOz*Rn~JE*!B2xDeJi$ET3_&0Lz)}^^Dt-`;}Z* zaJioJT@7~rtGK@Ad0orh=jQzjo|eyGEDu$W&vJeq@?`E)n=z*}y8HhR&u|iVbwA%w zt6#?@@3>;;xw)ymXP(QL$F+EnM=>BTk<8fc4(Z zJ^Q}D;qK#pJbM|z{UNTAT=#Q@dj#zM9V6pC1$Mj@xM#el8*Uo+Jg2{d$8tT(mFM&i zuzdFQ9QYwF`EW0Qo%a;3Ynz;31lz0jC1>Il?taWz?PacyxZJxq=lN~0V>y2I^e$N5 zc?a{|)}Qz9bIsuLjH|m|e-?ber8bg!cJ3cGoX^lTIG^*@THa@B|5IJtJg#XiE_2W-tzvFw)+CKxE{iodPIr6!u=S#59%(IXe9rL|%&pIHTEA|!Tk}%y#U<~!;>^7ySns*?=6NgyH=0^4UoXoxeSY6d-*RxrQLFikR^;v{&u9f| zIp-AT>+BccvGh4-zRp&Llh2-30lN-)Gs!sK&BX7ItAafX?#F)Ta2@@*`*9uiti~nh zI>cGW8erF9?pep0VEL?LEwKBPcO4nWd)Bcw*m+!={m$V!26Fe~I_w$1CFeTC$#Fff zd-bdwP4CQjhJfX>)(ya})%WtO^_O7zcs2yfdDa|bDA;l2GuNhI_v1N?Z!@rb<{Ace zuFbh;t}Vdw@oWi}%UoN59Y@~1Z^PY>d%u$V*3>6+xp%SSjDX(`>^d{|_F(6pmn(Dc z04G0%efivX1nZM`uARC2aW1u;xa6EmoHg$Tj^}b;c=&va->>;G)b@M7wVCHcupZaq zJmVUV^QevGlG7*ld^oPPy!A;Qtu1D-1F6N#d{bzHX$7PTBSiG+BT(IkK4|>$?^Lv~6 z`CxyZx~80g3*f?sy9iFc<5-TFJj}rPoZo)uabEZ1yxwb;?#Vh?d(T)mG}-w)oHM;S z58l&zJGy7j^Qg~paxV9R^GxSl?hPKxe%*7<=EZ& z-U)Uc?m>^beL0tRgMCJ>DQEB=xbWfbg_G|%mSZLlGjKlVx8Hf3*ZnxJ_nM`9vQF0C zGuEw5c0Ld1OmEJE_w?R{?sM?0>T{f&%Ol}Dw|U1nidxR|lb&P1j*;*A$5PuTAKw_T zyze#XJCR!6_nW+Djs+ji<#Si}?E9WM4y=~<%#*-!?%(hI=HvJKtm#znP_BGEodzDu zIQ9Gd>2UJ-*?R_9pS)*eJa<2yBegTRKxh8)q$#&zhA+W5AArn!XLJ+yOV0%YJqLc}C7PxW0wKuFresyA$jh{Pz_0EY$S$=02V0 ze;~ZqUDQ7JN4Y=7L(qlaFssu$+F+o|;^q(R-XT^B%<* z$@+ab`7v*~`aZIX}tp6Kb|i-R$$AcIvsO#roJ}8e zd*57eX04XB&kfeE7Vpo%&QY_ThuW;w;++?)UoGCAV7)c#`KZlWE#CRT`qkXOSuX%K zYrnT<{|my&$F~qzPQO{J$tCM0z-I0D;$*!foctI#=Q8V`Q=7F~=2`|UpS>&#mh+RW zm!mdowe&6D^g*}xtpI1%YFYb=VEtjw&tCQh%lS#xBdE<JXc_J4m&*2lxik6~=*GV2qm%~~yUjRnhRFXO;+evyq_laPngq*SXC4N@}xK%Ur()%V#ft0L%GF)>l!RwOabF zZu+3x`>ugAYqhNXTCje#c&`IHN6mT?wOOmhI~lBBE#B+FdTZ7Pw@?EeN|eSd+|rxxE|!SeAv3if}iufE6N^r^-7I9NWue}F$DE8l6&&2#e{ zoaaU@zJG${^IV<>%lXNL+yiV=@)#9B2)~}Ymz6q}XT<{jPoV}hAHMwN?Z?NzBHN(%~%up@k zeh!w;8ovO``ALTVp*BOc^nD4ouV(lawHc~qFJFW8tHt{bSZ~enTWT{@i}yRQezkbN z2kWgF{y=SpYVrOE)~}YmcH&=3*Z2L-)N=Nkp_*JWoDS>w;Z zay7$wsLfC!57VrFE=cpMjKy8L<@h%9~uNLn@V7)cNUeso& z7H@B`ezoj%5%2=^rDsuUIeX1eO)eQO2VN4cX1F|@8LDO66~OXY+W~i3F zUo?F+!j=?GM(k7Vqj{y*0x%sLfC<-ZjDc)w0*M z<>*V#0BSjV%}`A)8Eyn#AFgKjD>yS$%eWhZ<+H|3z;ZRiO{vXLEq$9ceKo_);mlAi z>l_BwuNLnXVCSeAZb@y1YVmFb)~^=taIoH*;nvhvrJb^rdHeYB_t& zP)#lw?gichu4cG5oEfTR+!0{;tZ^T(T+MJ_YBN+z-+oPB&2WD>GgQku4*=^|i}ygV zbJPqEqBcXdcn=2aSBv)$u-=;Cq10xm7Vk)~ezoj%lr??nIhTD%j$`qi@6bHMdI`CRH&j=LGE$tA-pz?Z?*46lSUL$!?id$4@g z_y@3D&G0H}GgM38)lFZ`@ESNXRLfqj1?yLf_d2k1)C?z4o1t2~lfnAc;=LZMw`O<) zwHd0#dm~uCTK0Mq_`lwNspaf7!`Am-a5DTKa33~U-z)Eh^Sx3nU$DSpIUrRf#u_S9_;(C z|DG}Hd;!igq88taVEH_Qm%ws<^6vIBwK=Gz?-j6p^^Ckq?Y`CWtX>1_SBrN#SZ_Te zuT#5swRmTM^{d7E23T)BBX3f>f3|dxQ0>#k(+AZ_RKK zYBN-ecTuo@wd}PoxW51Pqn5MR4Atb4;fmlTJ7l;LoEfTR++Tp@v&NOday7$MsLfC< zeXBNoHN(~5%up?R=?~Vg7Vqj{=cpO3L2ZU=@vaHhuNLoGV7)cNwW-ZeE#3iO{c73k zI^g>LyDqhyy=JH;mkc)n`~F+c;ihnAsFraz1IuTPn}g+QhQp}MP%V91G<`M0E#b^i zEqmDttY0nO;b7;e8E#E&hHCL{1Jyg-xtiet)Mlubz5|=Sn&ClkW~i3E91Pa47VjZo z=cpMTN^OQ}@s0%RSBv*Bu-=;CC~7lQi}!G_ezolNNN|1s9ZfA~uNkVzCBu`%9Wp!_ z&J5Ku?kQmTtnpN^T+Q$_YBN+z-|0QI1T~b3V14y=JH;mkj>^_Wie>!>i!TP%YzL4VKRu zuK~-|46mg&L$&l>*Ywp4C&8JaTGlxktY0nO>%q=ZGrWP?4AtVj5v*S=-YHwVb_Xs3w;T?+5$eFxB_U2jG0KRLi&zg5~4;3)uhGsrvp3 zr%x@uhr#mkJqGr_iK@QG;qOqn8k|10_?`yK$M+K0_upys4Q}4k zUxxFHsKxgRSU%6-Rj{0&y#KyNZ4PSbn+~?Go{`t7-M3nv)eNwHwRqnE>#b+xO=|b9 z7VleN{c7>P4c1%F$UD^TUoGBu!TQy**Z0Bo{r3ZEIeR@LYI4c&EA_7bo#EGTW~i2N zzX8i1!sn8*-LM*ezkZP20KU1a1m-VREu{}uzt08`+)V<4Es`>p<2BC z!1~p)*Tup0{dWm!IeX1eO)eR(4EFuEX1EHR8LDO6Rl)LE<7!~JnqhxxGgM38>P=tG za1A&!RLfr01nXCecP+4U)C||AHbb>|2Y~gf#XAtJw`RBwwHd0#yDnJ2TJ|~^T;G4! zqn5MR4Atb4;pSl9e`|)r;LK1h<8A?#&lQn#Sif5Kx(m3z|L#gHXRjHm z$tA=6!M^|23=e=aL$!>1AXq+YJP0gTGd!5u4As(iNYhs{JQU6h)v}k7VEt-}KcC&w?{Uwd`dASif4lXM>%iW_S*@8LGv5E?B=> zyyt=S)(j_7o1t2~=Y#dDWv>^4>-+CT)N=Nkp_*JWyaw$1Z#{?C!kM93#=QQI&&Zq9?prO->MgK-wRqnK>#b+x9cuTk z7Vo=Y{c7>P2i9B9$otgpUoGAb!1~p)*N?#U{r6wga`t*g)Z~)kw_x9YYlh#!nW0+7 z{T?izHU0pWs~P@CZH8*;^Ox+{S2OGcXNGFoOJ}ftwRpRL_0|l#!kM93yfcCItHnDr zSZ~d+8=M)c#oHaMUoCr`6Q{3&NS9TE<-nET1*@0?XA5 zdsCaCTKX1l`f7%Yz?q?1_Od8gzgoO~z|K)K>`QHiYVr01>sO0+F|gj6;o{V0s21-M zVEt;@>r&wQ{<}1_oV{kKCYKCX1N;74Gwcs%hH4pib+CNaxCU6RX1FG`8LFjkt){PL zxHg;_s%0+&!1~qV9SC-gn&CRsW~dhLx?ufk@eTs(tr-rcHbb>|*8}TU%U;(9*Z1EI zsO9W6Lp8Z%xTSiB47Y+aL$!=M94wzTZVi^J8E!*uhHB~Cw&|-GZU<+EYT3*7VEtj=-4(1~EqmP^T;G59pq8`O4Atb4;Xz>E zf9p9s7|smUGVUQ@`K<9!uw2b>B()i;rSGt&uVy$3&J5MEm&3vO)#5z@>>M@2BdN_$ zE#A>!{c7?47Ob~scoel6s>ORWSif5KdMvoU{~kvzXRjHm$tA-x#2qp`6V43PGVXY= zeAajtSgvL`f!Yk!(sy>#S2H{Z&J5MEmvh1T)#5!5>>M@2iPUDO7Vr6B{c7=E0M=VG zypY-q)#AMftY0mAy#!p}e}6|UXRjHm$tA-{VBdf1Ih+h)v}kVVEtj=y#uUY zEqlEaT;G51qL#DQ4Atb4;a|c2PO17{`7oUCm1-IH5wLuGkAwZ)Q`PqboIbVqo|NN? zZyMO&QB{3U+k;OnzGuMl@%;nr@3N}C=iv0I#rIFJe0;BiegE~pkr>>(-_L;ajHt!; z23S7N;7zccpS=IRMQsjh>3bV&Up*u5P`hunJgaxX`qkon53IMIk@u|{2 zmD&u|;_U&}ua>>e4zBONb5P6KYldoa$*>pL_urafZ#Xkl%eV`J<+H{`z;b?a4i}|1 zL$&nvY5Ho0ec{YdEqmz))~^=tVqoW}87@w3hHCLH0oJb;?~-7>HN&N-%}_1grNR2u zve#w6_5F8QYB_t&P)#lwt^xM_w`RB|oEfTR+_k{+S>xJZxtiesYBN+z-@vA?X1ETV z8LDM3>w@*G#XAV>95uti)Mls_?|NYUYVi&M>#Z5CPi=;3@ooUtua>=T2(ItHL#gHL zHA6MIWVkih_urb~HgIOBmT|WQ%V&+-f#qt3+f$pNTKaZq`f7$d!kM93_OcULzgoOI zgPo&hxC^xzs>Qo2Sif4lyMguA40oqCL$!GK0P9!FUiSpo_ut=8%h_v&YI4c&P_+&j zj)XHqwTychSUzhU1(vHB9!_nBYUw+o>8lwY31^0C*~@6Kezkai3wDm0;Zf9Ps218lw|gfm06?B#s0ezkZn06Ry`@Iq=cREzf_uzt08F9z$a8D2td zhHCNt4y<1-d%X-?-+wQsmb2Fk)#Q@l4eA{-yb;a})iUlBuzc2d6IiZhIF;HA)zWu! z(^oUR1!`rFNP%Yj&!1~qV{Uca!&G1fYGgOQBF0g*J?DcMN zegD0OTFzcGRFg}FkAnT3sP(<_F*x5V)iUnmVEOo-0=NF|RQd)t@0EXp)29~SG_ZVp z&w~9Ot98D=!|79t?;l|K_?`#*yI8C51vq_b@x2I^kMB*e@4x=k$kGA*Unqg-+GgQmCUBL2LV^^?T z&2T1aGgM38%uQd-up68is%07r4Iv&P^?6uNkVzCBsF)zW>$?7lkuJwT#;bET1*@1~(o? zeg9p7TFzcGRFg}FYlD6Ntr-r0GefnEI}j|NHLjz^RWn?d+6>jwH>l~W84iXsL$&N> zJ+OYYc!z+Uqh`21wHd0#y8&3gTD-pm>#Z4XNNt8{@eT#+SIb_11+MSE8&k{KYldoa z$#6Tc@4q#}?cvN&E#vM0md_e@1k2S7ccL~!we;=W^wkV^fipw3>}6N5ezkaa13O2} zaCd4mREu{Huzt08e+|}KGu)Hf4AtWO4OqWg_PRH?zW#Z4%p*BOccuxT9SIb_j={T*1pTD+Hn_0|k8qc%ge zcrOR*SIb_n1lRZ9-&4!kYldoa$?zty@4xjNPK7f=wTyeS99PzO3s|mZcq_FTs-^F? zrmtprJDeG+WiNMt^{d7EN3e6$4DX~iL$!GC0_#_c_fKHGHN(59%}_1gd%*hDve$dT z_5JrgYB_t&P)#lwJ^}W3`q%f$C*gdrRLi(ef#u_S8tm`>ufAvC^r^-7ELc9i=fM6Q z0oC_UIDKmIJr9mzE{BV@x23X{qw(g2RHBO@4|UT)Z%*&ET3oa zK3L9A-hV%!HV3uzeF(O%o{^8J-M3nv)xW^{)#Cjathb(#PpI9yTD+fv^{d7EZ?N8a zMn0o<|7!7m4%V-hz5WMW-+#ZPmb2G0q9&INyOLe&pZ_fx&IC3?wTwG6SUzj)29~QC zcBeK&we-!>^wkV!g)>97?4<`-zgoPrft{mfI6Ji&s>M48Sif4lbAt8O4CkUYL$!G4 z2J2VLUgrVV_uqM`HMwNi7u@>if4L8C&S5{W8LDO6#lZ4ej=T^6igE#Bq8dTWNuQ=6e$yeok9t7We% zf$RJ4FR110HA6MIWVjBv_0RuuPloG)%}_1l4g$+(jf25*HN*9&%}_0ULz=#t;rei9 zsFuBK0M@S-?=Qj5Q8V0-+6>j=9SYX37Vk!2y*0yMQJbM!yc>h{t7Wg7g6sS5X4G=_ znxUFpGTaf|`saVSC&Qh|{&o7xQ3;vE6jua>>;3$E|K`%%l;Yldoa$?!;U z>!1JSo(xBW%}_1l{uV5sH68_)s~H|mZH8*;JErNY86FE~hHBZ%abW#w@g5I$j+)^Z zYBN-e_XMzhwRlei>#Z4%r8Yygc*lYDt7WezgX{b6Db#ZInxUFpGMp%eo5npEo)0!d zwTycKSUzjK5G+?SyolNi)zWuy(^oUR1kMcAvX|e1^{d5uDcCt`hL=&Bp<29`gY~P$ zdj(i;&G1TUGgOQB_h9{M+3Qu{`u=-0wVb_Xs3w;TZvnUd`CsnI@K&%Hs%6~U!17t+ z?O?f@;T_awsFuDzHhne2JK@YwEql2OtY0nOKY^X2W_UNX8LGv54_Loiy#E8%TQj_u z+6>j=y$`HkEqi?cT;G2mq?WVS4AtbOxBU0-PWE>zPbLt5|FZXF?B8cJwku;0f<4zZ zJ(IZScRlx_=W=R)4@dqkrQUGz{#_60>(li4=T*|z7c5`bO{hC&v?-i?GTIERSKhT{Z12fvbFdk?Ui+QbwQb4Wk887M3$AUsT$?y+8xHG4;uqFmNDx0uoK|qlh=u0z4ESU9Cz2~-8J!_Vz->iJnqYNj^u~cDco=7`(#hJ z)4}Q=@N>hzm*iw>b$^zcqxwl)@{TLc_v`WC{LcIQoeUG;954Gio7z74{5=E{!Seo2 z;q;wPJ(^lAKZh=E`s&}UbqSn*x0YJY)urIEaE|A5^LzbeaPm`{eO?ZB4*Bcoaqm}v zngBw<#+Tt*VSy-DIWE}4~!`FhH+dS=ej;!lC zu)6D-%-xUclApvS=eoq1a|+n=a{`xhrpJ5cyb0{MuET!)&T%vM%wf+|E;;8AJDz94 zT0U9a4(^!69dK=Y+t2hrQk%Pcp7mW|c{9pcx(6)p`qOtWSl)d(r@u?~0kFSI)-&d_ zo(O&rPJRqM)(?T@{CabL4|DfphD(C~$|dK1#Q8g2o&-P6<=2XHJHCIX%Tr*r{2d^F z1Iy(-@M*Af`f&~Ze1C>o-rg$fS+9E5`yyB^>wO6>vhoNs{D zGUuCMxy<=C*g5&X_Br37kheE;z7KZJwx1hqbACXbIn^`ghhVkL`4LzybAAkVPT$Y7 z&rhi3?alY)M|scoxtM`#vd1&;v++6jJhHye!JbE-L%s)pNiBZ@m(TQT?tXlxYF}~5 zIhHtMeG7K4^_hN0oxQ4Ouit~!vezHLa@nh!bg%W9c5>rf_T~&8)|DPO&!WBVGi&qi z_X^MI8R+)7cRh98UGbZN=OP(&gOg7Nvw;1Wfm(MiIWrI^gC1ZrsAp_8xMZN73}y$b zC4)J@a>-yWuls;0Su;(`yD!Sb8HXO0)BQ1 zZa!~;-D}qN4me!0dKb>DdU2_z_dT#K#^<9S#FF6Ti{#_afuzWoa%TUYPm+_Vb%h&U;JhglsZv|@k zdLCAymbWkGVHL1@&Hh&dhs$~B50~?xp5E2Lj;WsYtpPSGwe+n8miJj?ziX1YeQND} za6W6boQDBm&jYvizJU~;2eo(yHFZ4?>*15H=V1t(e8=;!0bI_5o{YO8{1E0=&)SEA zUAtPI{YDL^7H(s(=Rq#{Yyy_A=V3EzF2~3mn}g-+dDwzl-oA{tC0M?mhvC%nb-b)amB zb((k9xdWVh*101%>vRm)DQBPAWG?TSYiDrBckbQbp>an^GH*!4`K zH|seF&h>C>U(dl5@>$Oz;H<|nT#uZ6t|xPOx6Z%A@b29HyRngIr!vpHV0-fK&rYW2 Kd!FJNT>lH&8Sj+< diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2.spv deleted file mode 100644 index e47861f0eb0221ac7bb0b27870dc2bffb0b05182..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36656 zcmbWAb-0#g`n5Mez*ffjW*kVPVHuSlBss zca8Bo&%K|uy@uoX{`uZxo_W@_*0t_f@4ess0J7=SalvU?I(2C2+A>4SXVbT|zGiCa z*wO*iwPotod;fu(4_x}7gNH1=;!4ZfFk?%n);@c>wM^4875C0u`|PyGz6$$roy650 zzx+&G4rISqe^`6Ea?R0NcWN2Qy(8BKj3)NAVB4*o(<|qg>C+=OZQ)#Y6mIHhqQTc~hd8g8+|xo^F_3+Gzp3^;3^q2X5KZf0hx zch07MoxAIGZ~b{s-r(MrVdqv%*^+&z`Em|l_P@nl-Lb{wHkHi68p+rsq0*8(382u&ZW(LSTEoJysYo1 zu5&#=&Hs20I=6^@Wv?y#O2u53pfBT>(vqJTx(F%(mI#e*X&%r|EU*j zW%YMz=|hjYV~L%s?Y(cS>s-5{dsgi4!7R=*KVv|*;+{BN%Y0lNxO$;@-mI5wG`}v? zX2H?3@Q&##eM^HK-`X)&&f$F%DMssGb* zdwV_D`itAz>n*5Hn{Yb@~bzspG|VE z$6i0w+@JmXaaWU%ejs@5QXc}Y{==!KE&5SlU-1tE`?-{Q1lZ4#d0?=h@tS&#mX;|Igtr=bjz^T<(5; z%J&ATozLYNsaJ1Kt34Mqx;_7|E9bYV4tTv~4zICG|q3oIbREmil4paixBg`o>Z}Mtx7IpP-&p>ZhpREcGPnFG~Fkb$8A{*7qFse5IaD zy=+`I|cNkbcz9YbLeh-N6Xs|xN_r-S%SU$dE!E$~-)OQ^B(bo(o|N@m z0(L!q@5y>D1^wI$wfD?(I`g;|d%onll`Hf7 z6Rhs{Q|Gy*(YdwGqkc1&yz_{Y=bd1^cW}?X?`gP++=sK5q1+$f^1R){748wR`*)0t z_axZymgb)Eo^H5F-19y?2OiD!ELYyA=fU#X*JSVmT=L;w0z2;sTvs%`vV-vzAhcq5xRy23e!TIQXx>GS-=J2RZTT)aJ+KDC_3S;2nY_{`-RXXEagy=RSQ zvIhHH_pRLLxfcTKJ&WGF9}B}BL9LdrkHwol zKj+f71e~9@YQ9HHarc$?s5iBobBgnIwG4PPea@M$t7YNjv!~_2u0!5TGLCmM>4|sw zR;_Js_B)5`SdqIg*I~~JTym~MoOP@Wb{*!Pb^HM=pLMJPcAxUDBjb3_I#vZck888v zIb6r;+w^%l|Kj(v;XrEpz2DNzb1+zsYjK`K8;|p-9l|B2PwY8zTx)sj z!=_Mv3od~dwTSbh^iI)W?9A=I)wu22OwrAMPYL`6C_rp2UoAcm3y;q@o_B@aJ94F^;2%Kj+ zUlY56N3&n|oO8JwoP5sZ?qGfLX1xb@U%nq|L%HPKlQ`#cZ*V-fu?Al`m*-O3?>*=8 ze6XIJ%L^Ni^Qc|GC8tkpR*q{eZ+%fydruyhlYx7-=L+!6+;c831*_L{d3nRRmYmBg z;97gRw>}SYcN|}?PklVsEnKcgY*sgNPmlhqxvu51N1Ssx0qi>5gC2GJaxSk2`yRQb zoWZ}sg%9^PIQc2Za?Ipm2F~aF_B)UBx*zBDUbA#h*2&s?#v0#b=lkKD>CJiYp5ANG zeIGol`Wz?c@=!R>Z9e0VpqBIeq~|EGW8{0z(bV?I$9F7P-shk6ji8qIIVjJcBf$rA z`M#@r_I;il1y*yve%70ZpYK`2@!<8j^7S(YJQ`p9Tt5L$KHrm11nZM;{jB2d%X6c4 z64%LG&Lz%Vr-JL(>S@%@RlinGhm+4-W5N35-9yIp?)%{PnKQu7>%Q!FPSOFBgLClh6Bi30S_qf8(g->-%>Z zb>2VqynmO2)!cj5?%lon{@&HO?biwBVXynD@3rrFdhevy^WYiQ=Q#GR&fQGBr}tmY zxGTcxbDZqyDzMM2S8~tZuK~;ZYl!>!-s3aF4b<|!zj=(pkgmbu;l z%V#fdg5`WAhqtKBS}lEVH+|6UegAksAds>S;eSihS4H|vkVW}WXHo}*9T5T>t(HE9Z0&}w{=2|U#}!{!u$=zZbIn~Y zS@!~)wV%bwdR9335pd3B)^kvswOZzy6D*&-%mtS7m8|EcHfy!?&C~QjxA)BpXVz+2 z`+Q*iYVpnwc8;3$0@P-$7Vm;!{c7s6@DS}j?x3YO1aRs+lVO4h4Wo3&c{ z)@b^m+xympGi$Z1y$@KwTD)t4oug*GHnmx+#oHIGUoGBszDAMvsTO6w*%`}i}%l9=crk4Pi@v}@$LZDuNLo)V7)c# zov6)PE#95M`qkXOS?>Zi>t5jOe+Zm>e7l0>^qaMsT(aIDY}Wp{W3oN~PJRSqJC|7> zOl{U`nd=a+eD-oESk70nK8)I|)zWu((+Az&cLbbSt7Yv+g7vG#dlc9?YSu?no3&cJ z$AI;##XAhFw`P4TwOOmhI~=TE&HbD82(VfEXDHeKNI3cUMuFw@o3)x;vOW!L*8Ul8 zvOXP7egxw>msy`lZPsd;>nyN*_VO37oUdekHnmx+rSF`k54yeYTsX5<%i7Na>sO2S ze6VxWtS_K8YqfYU1nXCe_ad;~n)SuhW~~H9em&gdaB3O%uVDH3ZUp;h>eY7>oIbVqZU)Q8cN^G0d#}E~!|79t z?;l|K_-+S3Foo|9IDKmI-3gYD?;-H#WaTrhxp{6LhV$I0#rFtUKJUw;U^!noH~*$K z2etG)2DXpk+WQ`-cHe4wAD#f~SBv*au-sO2SX|UdUZl0lb|7!6* z3)Ziey*>}Fzb=?eEoZN1L`^Ok{s-){e$DV5I5SksxbK4Hv&Q$pa=wz``_yKrmc9?b z_SFnOq&7pf?Bye{ezkZ%2J5XEenM@AYVm#w)~^=tXJEZG!_TSBP%Yjs!1~p)*RR0! zdH-u_IeX1eO)eSwo76tT*9<$rnW0+7?Fg378asjId?mxq)MlubzNwnNn&H%NW~i3E zOas=h7VorR=cpM@M{R~`@pb|0SBtkRSZ~d6dTKLNi+2XFezoki8+bbU($k$<&feB@ zh$fc|=LXLTS2LUk&J5Ku?z~|6tZ_cDT+MKPYBN+z-vUiv&2T|DGgQku7Xs^7i+5qL zbJPqMp*BOccozlhSBrNsu-=;C;?!oS7Vi>Z{c723Z#nwXvlO+Qy=JH;mkd_}uLM^! zTpi8~)iUlHVEL?ZO|V?eun)Bvs-DirH&R#QAlS_sNgAahK86E;>hH4r2P_TT~coj=9Rb#_ zmc5Qrr!PIDsarYjW~e5Y49@_c4p%cg6V43PGVWPm`K<9TV7Z#%+0S;SSif4lUxM}448Ni_L$!Fn2J2VL zUcUv`=il$BHMwNiX_~g@-pcT7H?OubJPr{r#3^icxM3XSBrN>u-=+sH)=Cfi?=&izgqS>Gq^tg_Mn!t zxAh$Ic_)_)=L63QS2LU+&J5Ku?gC)>tdYMp*&j=?GM(k7VidNy*0xD)Mls_?}lLgYT4^RaDDzAL@j5p8LG)8!)?Gm|JDq*g)>97 zjJq9JK5P6lSgvNcJ+&FCrEiC(uV%O-oEfTRFFS$ttHrxB*g0y3yHJ~rxEr+@s>QoISif5Kx(B#E|L#dGXRjHm$tA11Xw<6JQ6Hd zGdzmg4As(ibkkQeJO<7T)v}jiVEtbLGu&K3A$`+*`o%@%@lc>!>Eqzae?W>Fe6`)eNVH zGefoPWd^W*wRmR)J4em18?_m##oHaMUoGC5z%up?RSpuwI zE#4);&QUY$O>Ks1@h%0{uNLppV7)cNWvIj=-5so7E#9GE zy*0x%0CV7Z#% zFlsYYOW(0gU(IkhoEfTRFC)PE)#4oqc8;3iC~7lQi+8k`OD*2xz?*qu+P6W!}H+GP%Y!050=jwF96Hc3@@ZML$&l>)b!O1 zFNQNiwXE|Juzt08F9kbC&2SvG8LGv58CbtsyqAOZ)(o$pHbb>|$Ak5&Wv^F(>+|o` z)N=Nkp_*JWycO)<86C^jw|PFi4bJCEwT%0BuzY;~0{eINtM7I=eQNRD0hW*NZm@s1 zzxpP^=~IjE9~9jBXGAT&r@``h51s+b z`O5R}v()CGmcHk}_SG};Jhl5)%X>8$tY0nO7r=V!8F`V~y{pCh5?H@lyf1_G)-&=7 zwfk3#_f@cdwe0nEaDD!LgIdmB&xo2_GW-PW^KZ@YQ#dnJ%ebF`<+H}m!E!ajFR0B> zEqz~t?W-AnMQw&^*~`~p{c7=k1J+wJ{Fd4b)#CjQtY0nO@4HMwLs-E?iwzcs@yaAv5Mal3-$v&QMcay7#lsLfC#oG(4UoCr`4P2jpXQ!64 zxAh$IStXYY7XkbHTQgh~&J5Ku?qXp1tZ{L$T+MI^YBN+z-;zyV&9FC|8LDM3OM&&P z#k(}vIckQ>P@ADzyvu_1tHrwZ}f*9_Ivn8W57QD)(nTinW0+7Jr*pVH4X>M)eJ{a zo1t3zMmBvl!%=W%sFuBq7IUe^dmPv~YKF&Co1t2~W5D{=;ynSZw`O=EwHd0#dlFc` zTK0O1I+>*BRBAbU%}`A)8D0SP`L||xA)FbiW!#Iv@>%1>V7Z#%CDdl9mcC1yzMA1U zI5SksIxhq3SBv*@uyfQ5ub?(VwRp#a^{d5uC0K9G@G5FEREzg&uzt1dbpp6P|6WTi zXRjHm$tA;ofc;rg^||t&a6VV6W!!&(<>R{(?9ZI4zPsS`sl|6USU$dc!TxNj>bnn4 zpIUtPgXQCU2<*?Ws=kNe^r^-72v|P8XTU!H`gbJyHqZCZ!g)s2;(HD(pZDN-u$-?v z|4ybh2etIQ0Jg85kr%1mw_4t-m%#ef;(Zycx1Nz#sNK6-ysv`wtHt{oSZ_TeuT#5! zwRqnE>sQNO-vZa?-?ypd?DdSO$tA`rZlYU!J)>8lye3}=RF z*-HRz2tY0nOS;2a1hO<$dp<2ANgY~OruXBRy^Y2{La`v{K zLq4nIlHuZDpMPtHOTd|-TE<-xET1*@2Fuk9m!dX9we&6B^wkWPfipw3>}6T7ezka) z13O2}aCvGoREu{7uzt08R|M;=8LmWahHCMy4A!rfy{-bT&%diu%h_v&YI4bNeX!5J zHN*aJW~i2NHvr3LjRU}PHNy?5%}_0U8#R43!+~&SsFuA90_#_ccVn<~)C@PFHbb>| z2ZQyi#k(n3Z_RKsYBN-ecXP0Qwe0ne;QIW#CAFNrW~e5Y40lnRBEunYW~i2NcLmF5 zjk|&6YKFU0o1t3zhBkdQ!#&{4P%V4e6Rck?-o3!iQ8V0|+6>j=-3P2+E#7^>dTWOJ zQJbM!y!(Uot7WeTg6s3|LDX{gnxUFpG8_)}`L~|K5pZUxmT^ad<+H|7V7Z#%XlgT5 zOW$!#U(N7%I5SksUdDj+tHpZ)*g0y3CsLcCTD&KL^{d5uGFWfT@Dyq@REzgiuzt1d z^>lSINzYhnIeX1eO)eQ;tUg7Cm%y2!TE@K;ET1)w1IyJ6FQYa?we(%y^wkWnfHOn2 ztaChAzgoOkf}Nvgconr7s>ORXSif4l*MRlb3@1>Vp<2Axg7vFquh)a?^Y34& zHMwMXJJ_FvTAwTLfb+RhE#uw^mXB{Dxb(uUDE#5c4`qkon6RfwMk+-PbzgoO+ zgY~OrukV2C^Y6RVa`t*g)Z~)kS74uiYldILnW0+7{RS+bHGT`0s~LVrZH8*;`yOmx z&F}|mGgQl7egx}Ri}xq6-kRZmsm)L=-k-tx)#CjHthZ+PE43M_#rqprzgqU%q3i!X z|8}I7v)2rL{*_CH-N63+x0+#hI5SksxHEy}v&Nahay7#q)MlubzFC^Snqf~kGgQl7 zdV%$;#XBq5IckQpQJbM!yt9M#tHnD9SZ~d6PHHn$i+3)tezoj%9&mmBotIk9-qv%- zXO&zs><#w$w`RB$oEfTR+@-rNbxtig!)MlubzU7*}n&I+rW~i3EtN_-p7VnB+ z=cpO3L~Vv@@vaQkuNLngzHGgOOrOR#>m>~$+}eg55=TFzcGRFg}FyMcZFtr_kPXNGDScPLms zYup1YS2Ns`+6>jww^!3wGu#`_4AruieZczF;@ubQ95uuJsLfC<-u=P))#5z>thZ)( zAhj8)#d{D~zgqTs2)I7~9!f1|uNkVzCBso*pMPtHqv6a@E#n>smd_fG2g}tA$55N0 zTKZ0C`f7$J!kM93_Hq(fzgoN}gPo&hcnY-{s>ORMSif4lr-Aj>3{R&vL$!Fvg7vFq zuV<>0NqWwrmb2Fk)#Q@lIIz#ZHN(r`%up@kUJjPe8m|D$)eOf|o1t3zu59{hhF8Iv zp<32?HCVq|yw`x8qh>gP+6>j=y%wxrE#B+EdTWN)Q=6e$ynhAjSIb`i2CmP)H&V;l zYldoa$?z_)Kg+*9|K1JfbER6wod}kX?>?|U^S}D;htsDP-veOz_#OuPZv#}{BXIiE z;(HVPx1?DMZbOQdh}oce>|ThGWl)b3v` z-gm+J)w0+3!S(s~18O;YJtJyz$?#jS&%ZUp@8HZ(E#rOsO2S7qH%%;jh$Ys21;UVEt92G(0MT%Fns)#6;NY*yrDx z;UG9ORLi&o4%UiW^iVxmc48a)~^=t7GUS78UB&l4AtV@ z60Bb>-amo$)(p3zHbb>|w+8E1%U-ty*XQ5usO9W6Lp8Z%xChwh-sQNO4+q!h-y^8y>@`C*xny`e*yrDx;TSkGRLi(0faSBs6Txyd!;`4Z zP%V8YH+?n3Q{c={Eqgf?tY0nO)4v-6gu_PN=4 zv#;LZgW%Ns8RE-w_vOzJckRn?$(e_^FVD`$p)U^}%{@OObSQWQIQjf6^A*94FK=cU z$9sO3`AT5tF-QBI!*#5}-IwdI=MP+KbGZ(2ehz#eu>aPbKkwMNeE94hXe*X8NytjHB+kc11Ue~cMcVEe)AGMt0iId~{VCT=zU@@cq)Kg}(0i1j?8UWTS z@7gl9_hhso*o<7S{m$#!266Z0+Uyy~wHcRd6K8FkfPY_G+rQs57%sV~C$~+(YOclc z9m_FmEuY*DBR4(HXKwbowk^5)N^XComUBFD*0vScwdL>bn%maYQ|7h}oP2WI7OYp^ zwPkGY$!$BZxw&5Zo!7PPz}=T?vuAs*AzZFaoVD!){(Wt2x$O*>+|-lXE?_m+;`ol` zn6;KqZikbb9_KSRdtKX5?!J=S?$mOQC(hdT1iQ9L^yX`FFX}0C+Z#?kx$OhiEAQGe zw)f<=FWB5%ul>&J+796E%eC3FKi8pLu1%b^9R&V;ZEd+7442&0liML+HP_dgQ9r|ufs;>O!@zpw zUDI&xuF<<|;;&-29LYR>{UTTi&Be>)p zSDer56T$hJ_xU#%PKI;5?CTV2`{eWQ5R3)O`!j{pcLwzl)N1)YbavBM|BbD4;QTkX z)N-!Q1CNGtJl{7z>(7UizoFUZ1z_ipzlI+7ej!-ivvVQ53HE2ndd7UO&j#NGCqIH7>xp1Fzuw&6ecXMS z;R4`$x#ZlBINz%u0YAj$*NSuJGvHCMTAuqJ13QN==k)ve)4t#%BR=0 zOrrMXTKw95it9rz*CNi|p8>muL+H)=o~6$E)U&?lz-n3FWUzDkHJtUmKrL@?=6D(G z9Btp*+CB$gq0SuYnd4QkTIP5i>>T{BeU3LMrvt+OP%38ep zwe49u2HhU_s;92=Cph=)`-#>b^d5`uIcfbG13Qkr z`5f?`afhK#q`sa@pW}Egy09i+W;8e4bkuTYB%Z!`FQy0kUM4e**=(>MIRaPG;e^km%o!TsRW^ELVaSg+a%&6*x;IJIyOg9p$jH>&A- z1S~%gK6Ct=THY}-$75jm!KLpBYI*xI-jiVYElS@cYWX_e)70`?mA+@G3tdOnCe;ID`2xyOW$i?dEbldcMh<9YU!I3?0c=2^DsBq z^WgmHn+MMGpcZcarmpAVT{!uA9^RvtpYl9>0GIQiC*ytu?#JBfS^LLe*RGcL{*#7N z3-=k=^B|XeJ_pO!^YA6LyklgJufX#4JbXhfZ(qjy7A#-S!}rwkb-W*_3tklxi87%L6k^L?Nwofg63&Z(dtK~c_ z3idqk5z^kb7@X%pE#4)Yx}JyW@yXZoFawv@=; zTHd~#hlRlIHTz!#94_Z!QMjB3_4Fr>0u z^RNN6ynQ(j8-d+x_CE+5F6UumxSR*|^lk!nO!cg9Fxafr(zh8{-uEK=U4_i;Q%m2f zaK6`Sd3Ih6?Dr8rJ3KG;_^i^6yU!Zl^Q6?)&=c+GQ-B*6SvJPAyuKaq+ z-`AG2H_wynf~R~Q?++)RXVU>-Up@<|ZNMex`otM)BXFL_v#vpKuFHJ0u8rZcE_fwrTTbAEB=-vR9WljzO-JHa_WxAysWrpWwyoL|mf=g-*Q it@C>X-krlw-Y7v7asxc(38fW@u= diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/conv_k33s2_h.spv deleted file mode 100644 index 5740460b33f1c174ec6e83992315651f177db4da..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36900 zcmbWAcf6nT-NsJ_2{B8o1WC-=HDVGRcmX| zsx4x#qNvUDexGw+$EWA@dj5HSujhX5>vLV7>pQOdd(OFOoOJ2DShtogomzUd%--_- zY%Q&yxm!B7bOL$bV(u2SnOo}(H`!{F6-FMm=L)N?wvvr=v~+3hw|CB#ZY?u$@5*(+ z?)x05Z~)h7Tyx=X&C+m-7jD*uTe@)WTkmp(bFFd) zoHfthaI11RGc(oOtEpe(?t0B&L!Qa>1`nlI>?iw}-deTIA+&!6cXh`W_vOm-vF^v^ zM|~tho$D~_&S=gh_LI3%yB|N!bvQklOWe0jY8~6U&NYyMoy%vYcOB}?CH9lKQrEc# z(UZBv&ZW(LSod=PKGu&=*L^)o&HuITOYA3erLJ>5PEY0%J69Jj$F}b00DP=}iBRX- zkeZg(xx{`J|AXbwbga5!_Yk|_RqsC&eNAMpj&ZIoUWx0 zS0}FdP&{we%Ql){7i#n3=vjEj^pn07z>aV27;AC)Nj;#oI{v=Y1Ka9#QxBn@=}*_L zlX@fSelMTgu5Uv9M$4bt>tWWP+}>Ufryf7)>~{Zl)H9DhVIB840^Eno&-~Q;ahEfP zKGgejcaQRGH?`lJx3cP5k{|Gkt z^xsYC`qO_Oc->Mz3id&}kH>(nRnC1p&fPuAo0Ihu4VUqzQG2d3-?L5a*Ve0GpYg(A zzi(SF0-ny@+Bsh5?tZ3oIfwNOE;;r08m!m)WA3iW{IiBn!7ZJBTEhZhf6hhkLw$Tx zo5lLnvzB^u>K@dt)9?9Nr{B}9op%&>*CgjUt^FP@mw6^pFJ9^!sgGx|W;+FJwsMYl z6L=|=R>qn^_&)RCYQ8 z^m+L6#M*Jz}pHW{?>My9LmikNT`%C>b^|VrdOFg~R-%)=;-Jhq(QR8&9eK-y+U%~in zJu`lPzL3jjIT+TTZwGT7z~#@qf%N#Z;SetQvGiDvY&d`R#dkPZpJy$;Bf#?UjRwp4 z^C7;Y!TS8U7~e5q`S^|l%N@(5ZyfiNxW;lV$Q9qo?R?uhPoW;eC7=0D2m3Rw&UXg2 z{KL)uCV=Jq+4o?>o!4;w{9A{wOLOz*Rn~JE*!B2xDeJi$ET3_&0Lz)}^^Dt-`;}Z* zaJioJT@7~rtGK@Ad0orh=jQzjo|eyGEDu$W&vJeq@?`E)n=z*}y8HhR&u|iVbwA%w zt6#?@@3>;;xw)ymXP(QL$F+EnM=>BTk<8fc4(Z zJ^Q}D;qK#pJbM|z{UNTAT=#Q@dj#zM9V6pC1$Mj@xM#el8*Uo+Jg2{d$8tT(mFM&i zuzdFQ9QYwF`EW0Qo%a;3Ynz;31lz0jC1>Il?taWz?PacyxZJxq=lN~0V>y2I^e$N5 zc?a{|)}Qz9bIsuLjH|m|e-?ber8bg!cJ3cGoX^lTIG^*@THa@B|5IJtJg#XiE_2W-tzvFw)+CKxE{iodPIr6!u=S#59%(IXe9rL|%&pIHTEA|!Tk}%y#U<~!;>^7ySns*?=6NgyH=0^4UoXoxeSY6d-*RxrQLFikR^;v{&u9f| zIp-AT>+BccvGh4-zRp&Llh2-30lN-)Gs!sK&BX7ItAafX?#F)Ta2@@*`*9uiti~nh zI>cGW8erF9?pep0VEL?LEwKBPcO4nWd)Bcw*m+!={m$V!26Fe~I_w$1CFeTCIVXd_ z?)5-=lkIxc@}8fg@z0Dl1T3HRZvb}vzN=^bzXZ$2vmsc{^XC{t!Hy%J`8Ni;$2$L} z)bgI!jI|k9KJyO)JOAd~GyfJ~`FOSj%Vqwpz>XttM%!@rV@6kU-I;f2-f3ToM&9)aUQj?Typxvo+HP#mbX5ssl9uyd{&dYw(Z%T>%qU{eseSD zsbF=#|KHSbXEdB^xu)UHgtJ%8?+ur7cN{;iPklU>yz3FWhl{zVNB`Me=W*F1J{GTQ zJQwUb+=Cu<`#f9f=Y##Z>zZ-~E`SRk?jkt(j$=7y@-PGEbAJ1s$9dh4^LnpYx+m*o z?LA}N&}8THaL)ARJa|vytO@ z{kZ$_d8qBnCFh>RIhO~5<9UcR_{q7vhT4AbIhWUg_2gVmZamJTHi=74pV+J%*IM5C z`lj}tJZ>ff_iWEC;0L(pTuu?g)pL1s!?~86%Uj@Dd%1fqALQ;heq5jWtz7c1M{HL2 zaZiu_JGk!RvPYbAc_-L)xCcGz_T^mO4fYwirkufh;KGNy7f!z8SdN)I%)t4a-+t$D zUiag?-fNcb$vRnk&seuM+4(%2Grc(v-qU**y3fJ0s?Tw9E{}xs+~#}yQPgstpY$99 zc8vVYIhNW!`S`|w<$eE2--*=nz6a%fbS(IAE}y%)XW#eHabUH)kDdgUbN_ztHy^*> zXHBPqhjQiX=``?I#;M=uPluDw&*(G2`s6($c;%h%_56?LAcdY&VXi+47#el_?1XY|>@7vl5#X7)b^oP2z9g5~sk z_SEF^jNap%nfEBhNY?Me$&Z0^F3;CT)Ml-gx&8%~&t5(T%lS!$pHQ2%TKYb1`k>qU z{tai=YFYbdVEtsNFCX8jG= ztn;(PbM!5oe0<-5<@B4inq0E(#=lfEYrl4rb$2-VF>uah);*}rS}j@62A0oWW(Ujp zN!D{vo3&c{=4|?)+xzB%Gi$Z1eQvORwRnF9c8;3$Jk(~b7Vo@Z{c7>{1naF?&qr<6 zYVpnw*01LN&3XZ_S^K>;`(F@FKE8#(a{A3$O)gn40XA#D7boi_;pE4_IhR@goZ76_ zGS@O-`Rrv`u$-S{y&Sb!tEF%GrVqNkZv{BBR?FH~1nXCecO|fM)U1C&ZPse>t_;?% z7Vj!xy*2Aqsm)p~-qpbR)!e^X_XnHxvs~H#>TvS$tpS$PZ`NvZ$$ALbto`4=lJ)v< z@?#j+xy*V)YO_|$TtmU~*~>;?IX}tzSJY;$mcEUfKIrzoP2kL0EoN-Id=%wuY0BZyT_jezR7S zOV+!C&D#GhEm`jYCqIVqoXf0#Lv7Y-$$BraeD<<8Sk6zf9zku^YU$gj>4R?X+ZWEP z)w1^e!1~qV-5=~6HR}VY%~~zq1Ht;$;ynngw`P4XwOOmhdk9#+n)^5FL&0Xf066;} z2`3-lVPHA^X00Zdtd9emwf{S1vOXS8ehgzfmsy`kZPsd;Yb;nkdl?6o^OLMkqBd)_ z^qt)FLAUpv0%z80S^KGA{c7=^26m2`_36}RtrqVYVEt^#<*sSm7%Kp!WlaKElu$+FgR+CHCmx5crUze;egOeY_xXxwPS5lj`TITvaSU!9C z16a;avc8Jitku$Yb<+pk-ggb0S*vC3*Mjw{#d{stIcnCEsLfg}-pOG7YVlqV)?2f_ zf!eIq;=K{9U(NlS^%SsK`@fZE|2M(O$2S!$r{ApAiY|vKDGG%3YL%WQLz7;e)T;Dr%x@u$HDUP z{R8|NS@}+DZl0Ux;5;{K@%#gVJRciOH7Vm3d{c7<}2kWip=5=cKuNLnNuzt1d^-Xa7=YqGWte@CRx$REzgVuzt1dwG;nRy1wssrk1nU4Atb4;hf;^Xf?yR z;LK1h6^Fds~Pr$GefoPWj?TewRqoDq#I;@vaJXj+)_W)Mls_Z-20UwRl$t>#Z5C zL2ZU=@vaHhua>>8Ek|E^22jh{Yldoa$#5g^`fxSFU%{E7TE^WNET1)Q0+y>8Zc1&2 zYU$go>8lxT4rhjHS?4gYezkbF06Ry`a7$`4REu{juzt08hlBOj47a8>L$!Ff0qa-G zUbh1er!PI*Q_Iq|yTD%8>oug)W5VaYq#d|PVzgoP9fc4f452ZFkwRlH@^{ZvCqpaym&*9W^ z_L`xZTrxaS3|BK83ulIE8Fw65K5IM)ELSr;nc581(sxSJS2H{n&J5ME&eOp9)#5!J z>>M@2GpNl_E#5Q1`qknc57t{VJd4^4)#9B1)~}Ymo&&D$$>&nHa@@^OO)eQ;0lo~b zW_Tr>8LDO6--G3|#y^1NYKB))o1t3zu5S8jhS$KEp<4EGEm*%=yw`!9qh>gX+6>j= zoeb8m7Vq_7y*0xdsLfC<-W$RC)w0){!2k9BOD$)w8MeOvf|KF@fcvn)`d)c2obQ!t z8TUT0e0&dr^Jhv2x3s-i{uxf6T6_Px8?4LcY^F0EmPc6Pj!SeAv0rtM25w-YU1k2|cyabl>lXth5sm(zxeXoG+t7qg@YWJ;{ zXZ0FbzgoQ0!FuZ%d7aw5tHnD5tY0nOH^6%98F`c1{j0_M7FfSp_WBOEzW=^UEoZN1 zL`^OkegXFVw`TYsI5SksxL<||Df!Yk!;{6eSbiL$&N>L9l+ccozaYN6oMowHd0# z+Z(K3E#8H}dTWM@P@ADzyo-YMt7Wf!!S(&OAGMslW~e5Y3|9m%*&)M~;LK1h>07nws~N5aXNGFoOMkF_wRl$tJ4elM4Qew~i+4@1ezkbl0_&|A zu1#%*YVi&L>sQNO*8$h}-*u_w>@`C*xn#Hr*!SOh4mX7}L$!>%8CX7R+#D=dGaN>3 zhHB~CqUoy{ZV6|GYT3(HVEt1dZH8*`ZV%S4 zmc8x>uJ6A)QOntDhH7%ja38Smzcs^s;mlAij!%he1Ipf*Fb^c~pr)eH}U zGefoP@`C*xn%eUu) z5G)_xU%>u1QPuZXIDKmIJq(tQ?=i6d?Ns$W4yR8oz9+!)@%;_#e`8gB)8O=}#rHH= zKE9X0zW+|6Z*cRT{xY0rL@mBo!18$puY%?LsO2SZLr>YM&6-z|7!8R3)Ziey}l2w@4p{V%h~H0QIkuC zU#WNf?+m|&GefnE`wdt=Yy1{0S2O&M+6>jw_dVFYn&A)BW~i3E{0P>s7O(%&rQVuh zCpa@yi?=gazgoOqz&9I3JuD zs%6~y!SY$-0${nC;eym=sFuEkn!cK0FE}$)%U*he^{d6ZFxWY2hKo>}p<29)g7vG# z+Xt+-X4se74AtW82iC8ay)F)}@4rh>%h_v&YI4bNWw7tRHN#cl%up@kt_qgV8dn3$ z)eQSno1t3zR&V-hhHJo?p<4E`CRo2(yla7-qh`1^wHd0#I{>U-E#84(y*0yisLfC< z-gUwH)w0*Y;QIc%9<`jkW~e5Y3^xb+{#!E~24{w98Fve?eAc+7n5$;E6}1_*YJ6|? z&)5xb`f7$-!Y7|)#BYAtY0nO9mHHU!yT#3P%Yk_!1~p) z*ImH%{dZStIeX1eO)eSk5BB}HW_SRc8LDO61HtlH<3V7#n&H9JW~i3FLz=#t;h}J5 zsFuBq1nXCe_b{+?)C@;ao1t2~hlBO2#d`!;Z_V&XYBN-ecQjbPTK0MrxW4}$O)Y1y z8LG)8!&Ai_GCU2=4AnC3>0tS+@eHtB&G1ZWGgM38_@=LBcov))s%0+|!1~qVJsa#C zHN$hL%}_1gbHV!6;yn+nw`Mqz+6>j=Js+%JEqlEXT;G2$qL#DQ4Atb4;Wc32f9pBC z7S0USGVXO?`K)mgSgvL`nc581(szB+S2Mf;&J5MEmm9(Q)#9B3c8;3iP1I(n7VlKB zezkaS2J5XE-a>7LYVqC*)~}Ym-VUzszjsi}*=vSsa>?+|VE>!a30#Al_rr(ae6Li? zxPJl5$M*=>|Hi-i9);7V7T;rF`S_j$`?~tH!QdHYyKl8TtGB@V z)#7~{thb(#cc|UFTDm?XNGDS_j|B>*7yThu4ecnwHd0V&tJx4U(K)+oEfTRFP*{q)#B{})>|{| z3TK9D@y-O+uNLpjV7)cNZg6I(7H@a3ezoj%R&ag)?LjSPuNkVzCBp^O_}Z%(E(m9a zY8iJSuzc3o3oKVN>`iTkYUx|J>8lwo0%wM5*~_9}{c7>{0Xs*{urIY4s>Rz6tY0nO z#lU)NhKp01p<294fc2|ouSw@*G#XAVBw`Mq)+6>j= zT@S2ZEqh%bT;G2;pq8`O4Atb4;g;$hGTaKz4AnC3aIk#VxHVX=X1EQt8LFjk+orE( zxE-7ss%0}m;Zf9Ps21#^Ya{(Bs?oV{kKCYKD) z5O>J%OgJ-C%edpg@>%0qV7Z#%1Zp!>OW)Z|U(N6wI5SksUd{#USBv*NuyfQ5CsLcC zTD<3j^{d5u0a$O%@Iq=cREzf_uzt1d^%8J>|NR}coV{kKCYKB+fqnn2=WsHd8LDO6 z>%sC_;|*ZBn&FMqW~i3FDNSF^@FqAjRLfqbg7vG#do$QMYKFH^o1t2~w}SPn#d{lA zZ_V&_YBN-e_YSarwe0myaDD&1i(1ZJGgOmHhJOY7yQk`V<->5kSE^;)N5Jy&Jr4GF zR8`*-aQf8Zds2=ozG+~8msRyWZ4W-R_?`jF$M+Agzw@g4o`ch;7T-U?^6|Y6_WjrY zMq+UDem?`wGolvX8({f7gEzr)e)9hN7PUF3rSEO9ef5mIL+!rR@~qwk>sO2SJ+R(- zM&74(?`rXW0M@S-?}uQ$^^AN(?f%u`{TEojTK4)0xW4~>N-bxvXGBdd8U6tF{kLZL zBb*tkWn6#JpM2KXiMsWd|CJ0o!JGj38&Ot3_uNkVzCBt4|-+ya{z2VGIE#oc> zmd_d&0n7QxIb4+54As)tr|GL1_JuP;wd|!ISif4li-Db^X1F-D8LGv*1X#aXyi0=h z)(n@THbb>|mj>%s%U+iO*Z1FLspaf7Lp8Z%xCYqw-8ly; z2xo?B*~?B~{c7><40ev1;V#r>s21<8VEtvn8L)AKDI1~##dzW<&;EoZM8s>vn8v&0=T zoB(HrY8m%zuzc2d4p^>ccrLXWs-^F|rmto=5zY+NvX}G0`qkpS0PGw!!wad+P%Ykz z!1~qVy%?;wW_Ss;8LGwmJFtGW?DaBmegD0jTFzcGRFg}FH>h{W@J2W@RLi(i!17t+ zO<=j2;Z$lfR7>B@O<&FM7C19h%U*5;>sO2SHn4Nl3~#44L$!GC0P9za_m5z`HN!ip z%}_1gyTJO@ve&!8_5JrAYB_t&P)#lwJ_`1Cr`Gq%$KZUgRLi)JgXQCU3f%fTR_Pnu zyjT7WPM=zQ)4=laJqz}CvDW$i4yR8ozJGw_<9izW@5) zt!ABX!Ffj1;(Hq`pJ(t6Sk6!0JKm)>2etIQ2ez-Ck@usQNOKLgkI-_NP#?DdSO$tA;1J=)%XYlfZS z%up@kb^*(0ja|WVHN%;x%}_0UGdF!T!)|b9sFuBS2kTdhcNTE#yyZE~3TK9D@%8}g zSBrNxu-=;C?9^tc7VjKj{c73kT;TftJ2$nQy=JH;mkbvH`~F)qTold>)iQ1$uzc3o z7c5sZ>_=^eYUx|7>8lwo4rhjH*~=1O{c7$?w}&%BwT!z1SUzjq5iD0T+=!`-RPP%Yj)!1~qV{WVx`&2Ud@GgOQBH(>p0+3Vim`u;nDTFzcG zRFg}FqrkrZ)(j7aGefnEdjwcMYdjJxS2G+|{YjM@y<;=LTKUoCsR5?tSZe@`uEuNkVzCBvJ*zW>&9I2Fze)iUnQa$H&C zEnvBt;jPqWsFuFln!cLh?QmwOmc85o)~^=tAHmL1GrW`94AtVj3#?x)-amo$)(r2a zHbb>|?*Z#q%U+hQZ3^?1(uKRX|R7sK=nNX zr%x@uXTkFEJqPyh5~#j^!s%0s?|HC%d@q6hI|r)oWjK9m@x20;kMA9D>!1I{JGgmI ze;3X(q88tKVEH_Q_rY?0^8Wh)wK=Gz??bSC^^AN(?Y`CWto{YouNLpeV7>K>d_wKs z)#CjWtY0nOe}nbbGx8a=`&WzibFhB3?Daq3`u_VRwVb`45jDAF*p=*B|NL*sa3-)B zs%6}n!SY#SH?UmIusgLGs-!`Z3LP%Yj$!1~qV zofE9LW;hqM8LGuQH(0+~_Bs!^zW>fkEoZM8s>vn8zTnnB|I2-Fa}N7~%}_1lE(Vs* z8W#u4)eM)QHbb@aE!p(d43~m4L$&N>X|R5^cz+Idj+)^z)Mls_@3LV1YVj@y)>|`N zp4tr6;#~o(UoCrG30&WQe?cv0uNkVzCBt>Vt$+TPdoo-XY=&wXcMw=UYa9%gs~N6G zZH8*;8`AXE4A+M1F(Lzcz+3Yj+)_y)Mls_?@+LQwRkrI>#Z67irNg-;@udm zUoCsx6kOkbH=~xb*9_Ijww_DR! zGu$1{4AruiJ;3_a;{7$)IckP`Qk$V#yuShKSBrNqu-=;C-qdEO7VikKezoj%UvPc@ z-H%$%UNcmaONK{+TmSqo_hdL4Y=&wX_qSmAtnnzYT+Q%kYBN+z-!V;J&G1+_GgQl7 zjsxpgi}!f2bJPsSP@ADzyeEM5tHpaFSZ~d6EVUV`#XAnHUoCq*8C>6gPob8x*9_I< zlHo)#+%)dV@O-cts%6{@!17t+g6yemzw5adJ(pAadpPoUF7<|!_wRa0U!SJWKd+L$zF_&UxiD~ zJj9-_ap?WQW4Y&dggXA}aPs-x=4*f*U*60zj`#d-^EJWFV~+MahwB)?-H+?AXKk(x zxLk+W-=*!}i#LR85SPF2*tvZE{`V*AH=6T0mOlNpme2419>=rQeu5C;1eq5V9 zTX1d5<=Vtq+i>ttYirAGYq&vN=BA$9wgIcT7RPri$E>w{ayyCK^f;fn+3VVN~(DiaQBnk z_NSI}JaN``5ZJX%qc>lZ2UB;; zfc-l5Oq#E~S<~@oKdq_ld)OGbv5Zl_hn)Z?pS(^4>y>v+e19{`8jlP(^vlvu1nzjJGj(xt}X?Sg>yWgo8Rj%gOi`q z?DKN4bI4yuk9)rYEbrO*J$FCuS^i2cIrl8iTvvgeE5D=Hxvqwj&s^7lU8lUyB;$C` z8NL?m+~#S&b7WoDfz@5tWbS@km;59yIoBo5oKwJ_pA)#8Gd+S|*zr6Q*7C{Xc5ufm?tp9K+kU41k=oql^Q`Xz%bQWo(mh~#*Pp(7!Se3Q zIsIL-4}kq$vYs)Y^+fQ4aPnj5v3>|F=hvJ2dziZ)Gh7n)M{9_T$?8TK)&u zzqwqSI9WUoc1@$`&3a#;&U)3e-WS1YS?^0=xvcjUuxs#ZJnMaxTHfBw`8wD++kW=7 z{XL8`s57T}=6nOJmO0-9%Vo~D!OqG5wa@tug}l9)^L?;$w*A~_oAU$e%&DF^KLo2~ z&X2%yne$_?bNYUseSShMZ*RUYKgxTy&&3Q}lRch!pN-GK=aKb&4)#3y9P&N*OKSNO zxO}ExbNAyjRr`ud&auQ9>szpUtg-iLd;K1)mc9M}mdjqvNvb& zu&(sLc^2(;pIMuCzgKux&p@}wz3Zv#?uy?GJQvBJ8=QPHm<8;|4Ai=F$(eyT8T0^~ zK|N!$!6gIrWH38eEg8%KmP-b6fn7JlwLkB<;pFX221n2%pI$S_+Po)&v*@$OT=cl^ zdM-R4=^c;mIr3{(pX1n@?=#-hdnUTi%I~H69LICh6OSJ=>kBt8wVauWXKS9#{9vEm z(p-*p5qIZGulLL~0o@+w*XKCSwGekd&b2Jsg4A-(B~GRbgB`=qxaND?B5?ja-D=_b zHFaO^chc9B{wKlmz6VX>d!XMJpQ4r@03OISTf|V3BcfjG2)w^(J)r(6#z3+h?Q$6c@A8b}?>H82Y@3Y8$ zmjK(RmcAvyK5Mm{ho!-u2j@@U&*3}|YT=e^>UthNhm)`8;R|Z{j_2V^xSR()8TV`O z5aw3T+P?w2cC|eFZyQc6-1lJ5gIx0Y0W4q7gNIPwF)~Ld4<=VV4_(ZG%f5`)m0G@@ zhncD6>v-L$rL*YmIvwY+^f537LPYxchyI9$#{f4H0n_4KX|c1-oGZw;_nsiki% zu)NPA`(2aF?Ne*-gY#Ld^EvSwQw7QJr8opXA`h|JrA2%b2&!l*c>ch&%+kf^7duC zEy42jJPfCnuj6fP4_7@8+fvKhm-Da#*u7@|JAuRHJnRgY^PryIUBHg1p7retHY>IC z?GBdrS!BOM$lN})^sO()rIvT=4Z!{!@q5QwT=w{`vJiLQHN59t2K= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -void main() -{ - uint oc0 = gl_WorkGroupID.z * BM; - uint oy0 = gl_WorkGroupID.y * BH; - uint ox0 = gl_WorkGroupID.x * BW; - uint K = KH * KW * p.ic; - uint tid = gl_LocalInvocationID.x; - uint tm = tid & 15u, tn = tid >> 4; // tm: 4 oc each, tn: 8 pixels each - - vec4 acc[8] = vec4[8](vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0)); - - bool reflect = (p.flags & 2u) != 0u; - uint bk = tid & 31u; // fixed k lane for Bs loads (256 % 32 == 0) - - for (uint k0 = 0u; k0 < K; k0 += BK) - { - // As[k][m] = W[k0+k][oc0+m] - for (uint e = tid; e < BK * BM; e += 256u) - { - uint k = e >> 6, m = e & 63u; - uint gm = oc0 + m, gk = k0 + k; - Asv[e >> 2][e & 3u] = (gm < p.oc && gk < K) ? float(w[gk * p.oc + gm]) : 0.0; - } - // Bs[n][k]: a warp reads 32 consecutive k = channels of one pixel tap - uint gk = k0 + bk; - bool kok = gk < K; - uint tap = kok ? gk / p.ic : 0u; - uint ch = gk - tap * p.ic; - uint ky = tap / KW, kx = tap - ky * KW; - for (uint n = tid >> 5; n < BN; n += 8u) - { - float v = 0.0; - uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u); - if (kok && oy < p.oh && ox < p.ow) - { - int sy = int(oy * SH + ky) - int(p.pt); - int sx = int(ox * SW + kx) - int(p.pl); - if (reflect) - { - if (uint(sy) >= p.ih) sy = mirror(sy, int(p.ih)); - if (uint(sx) >= p.iw) sx = mirror(sx, int(p.iw)); - v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]); - } - else if (uint(sy) < p.ih && uint(sx) < p.iw) - v = float(x[(uint(sy) * p.iw + uint(sx)) * p.ic + ch]); - } - uint si = n * BKP + bk; - Bsv[si >> 2][si & 3u] = v; - } - barrier(); - - [[unroll]] for (uint k = 0u; k < BK; k += 4u) - { - vec4 a0 = Asv[k * 16u + tm]; - vec4 a1 = Asv[(k + 1u) * 16u + tm]; - vec4 a2 = Asv[(k + 2u) * 16u + tm]; - vec4 a3 = Asv[(k + 3u) * 16u + tm]; - [[unroll]] for (int j = 0; j < 8; j++) - { - vec4 bv = Bsv[(tn * 8u + uint(j)) * (BKP / 4u) + (k >> 2)]; - acc[j] += a0 * bv.x + a1 * bv.y + a2 * bv.z + a3 * bv.w; - } - } - barrier(); - } - - bool hasBias = (p.flags & 1u) != 0u; - bool relu = (p.flags & 4u) != 0u; - bool hasAdd = (p.flags & 8u) != 0u; - bool gelu = (p.flags & 16u) != 0u; - uint gm0 = oc0 + tm * 4u; - // constant component indices only: keeps acc[] in registers (no local-memory spill) - vec4 bias = vec4( - hasBias && gm0 < p.oc ? b[gm0] : 0.0, - hasBias && gm0 + 1u < p.oc ? b[gm0 + 1u] : 0.0, - hasBias && gm0 + 2u < p.oc ? b[gm0 + 2u] : 0.0, - hasBias && gm0 + 3u < p.oc ? b[gm0 + 3u] : 0.0); - bool full = gm0 + 3u < p.oc; - [[unroll]] for (int j = 0; j < 8; j++) - { - uint n = tn * 8u + uint(j); - uint oy = oy0 + (n >> 4), ox = ox0 + (n & 15u); - if (oy >= p.oh || ox >= p.ow || gm0 >= p.oc) continue; - uint base = (oy * p.ow + ox) * p.oc + gm0; - vec4 v = acc[j] + bias; - if (hasAdd) - { - v.x += float(r[base]); - if (full) { v.y += float(r[base + 1u]); v.z += float(r[base + 2u]); v.w += float(r[base + 3u]); } - else - { - if (gm0 + 1u < p.oc) v.y += float(r[base + 1u]); - if (gm0 + 2u < p.oc) v.z += float(r[base + 2u]); - } - } - if (relu) v = max(v, vec4(0.0)); - if (gelu) v = vec4(0.5 * v.x * (1.0 + ferf(v.x * 0.7071067811865475)), - 0.5 * v.y * (1.0 + ferf(v.y * 0.7071067811865475)), - 0.5 * v.z * (1.0 + ferf(v.z * 0.7071067811865475)), - 0.5 * v.w * (1.0 + ferf(v.w * 0.7071067811865475))); - o[base] = ET(v.x); - if (gm0 + 1u < p.oc) o[base + 1u] = ET(v.y); - if (gm0 + 2u < p.oc) o[base + 2u] = ET(v.z); - if (full) o[base + 3u] = ET(v.w); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.comp deleted file mode 100644 index 85421b3..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.comp +++ /dev/null @@ -1,135 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -#extension GL_EXT_control_flow_attributes : require -// Small-ic conv (stem: ic < 8, stride 1) with a fused 2x2/2 max pool, NHWC: -// o[py][px][n] = [relu] max_{dy,dx<2} [relu](b[n] + sum_k x[2py+dy, 2px+dx; k] w[k][n]) -// One thread per (pooled pixel, 8 output channels); weights staged in shared -// memory as vec4 over n ([K][oc/4], K * oc <= 4096). The full-resolution conv -// output is never stored. fp32 math. -// flags: bit0 bias, bit1 reflect pad, bit2 conv relu, bit3 pool relu -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; // [ih][iw][ic] -layout(set = 0, binding = 1) readonly buffer W { ET w[]; }; // [K][oc], k = (ky*kw + kx)*ic + ci -layout(set = 0, binding = 2) readonly buffer B { float b[]; }; // [oc] (dummy if none) -layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; // [ph][pw][oc] - -layout(push_constant) uniform PC { - uint ic, ih, iw, oc, ph, pw, kh, kw, pt, pl, flags; - float padv; -} p; - -shared vec4 ws[1024]; - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -void main() -{ - uint tid = gl_LocalInvocationID.x; - uint K = p.kh * p.kw * p.ic, oc4 = p.oc >> 2; - for (uint e = tid; e < K * p.oc; e += 256u) ws[e >> 2][e & 3u] = float(w[e]); - barrier(); - - uint groups = p.oc >> 3; - uint idx = gl_GlobalInvocationID.x; - if (idx >= p.ph * p.pw * groups) return; - uint g = idx % groups, pp = idx / groups; - uint pxx = pp % p.pw, pyy = pp / p.pw; - bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u; - uint n0 = g * 8u; - vec4 b0 = vec4(0.0), b1 = vec4(0.0); - if ((p.flags & 1u) != 0u) - { - b0 = vec4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]); - b1 = vec4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]); - } - vec4 m0 = vec4(-3.4e38), m1 = vec4(-3.4e38); - uint oh = p.ph * 2u, ow = p.pw * 2u; // conv output = input size (stride 1, 'same') -#ifdef PIXELS_INNER - // the four pool pixels share each weight load (wide-wave GPUs: shared/L1 loads, not - // FMAs, bound the per-pixel loop); same per-pixel summation order - vec4 a[8]; - [[unroll]] for (uint d = 0u; d < 4u; d++) { a[2u * d] = b0; a[2u * d + 1u] = b1; } - for (uint ky = 0u; ky < p.kh; ky++) - { - uint rb[2]; - bool rv[2]; - [[unroll]] for (uint dy = 0u; dy < 2u; dy++) - { - int iy = int(pyy * 2u + dy + ky) - int(p.pt); - rv[dy] = iy >= 0 && iy < int(p.ih); - if (!rv[dy] && reflect) { iy = mirror(iy, int(p.ih)); rv[dy] = true; } - rb[dy] = uint(clamp(iy, 0, int(p.ih) - 1)) * p.iw; - } - for (uint kx = 0u; kx < p.kw; kx++) - { - uint base[4]; - bool valid[4]; - [[unroll]] for (uint dx = 0u; dx < 2u; dx++) - { - int ix = int(pxx * 2u + dx + kx) - int(p.pl); - bool vx = ix >= 0 && ix < int(p.iw); - if (!vx && reflect) { ix = mirror(ix, int(p.iw)); vx = true; } - uint cx = uint(clamp(ix, 0, int(p.iw) - 1)); - [[unroll]] for (uint dy = 0u; dy < 2u; dy++) - { - valid[dy * 2u + dx] = rv[dy] && vx; - base[dy * 2u + dx] = (rb[dy] + cx) * p.ic; - } - } - uint kb = (ky * p.kw + kx) * p.ic; - for (uint ci = 0u; ci < p.ic; ci++) - { - uint wr = (kb + ci) * oc4 + g * 2u; - vec4 w0 = ws[wr], w1 = ws[wr + 1u]; - [[unroll]] for (uint d = 0u; d < 4u; d++) - { - float xv = valid[d] ? float(x[base[d] + ci]) : p.padv; - a[2u * d] += xv * w0; - a[2u * d + 1u] += xv * w1; - } - } - } - } - [[unroll]] for (uint d = 0u; d < 4u; d++) - { - vec4 a0 = a[2u * d], a1 = a[2u * d + 1u]; - if (crelu) { a0 = max(a0, vec4(0.0)); a1 = max(a1, vec4(0.0)); } - m0 = max(m0, a0); m1 = max(m1, a1); - } -#else - [[unroll]] for (uint d = 0u; d < 4u; d++) - { - uint oy = pyy * 2u + (d >> 1), ox = pxx * 2u + (d & 1u); - vec4 a0 = b0, a1 = b1; - for (uint ky = 0u; ky < p.kh; ky++) - { - int iy = int(oy + ky) - int(p.pt); - bool vy = iy >= 0 && iy < int(p.ih); - if (!vy && reflect) { iy = mirror(iy, int(p.ih)); vy = true; } - for (uint kx = 0u; kx < p.kw; kx++) - { - int ix = int(ox + kx) - int(p.pl); - bool vx = ix >= 0 && ix < int(p.iw); - if (!vx && reflect) { ix = mirror(ix, int(p.iw)); vx = true; } - bool valid = vy && vx; - uint base = valid ? (uint(iy) * p.iw + uint(ix)) * p.ic : 0u; - uint kb = (ky * p.kw + kx) * p.ic; - for (uint ci = 0u; ci < p.ic; ci++) - { - float xv = valid ? float(x[base + ci]) : p.padv; - uint wr = (kb + ci) * oc4 + g * 2u; - a0 += xv * ws[wr]; - a1 += xv * ws[wr + 1u]; - } - } - } - if (crelu) { a0 = max(a0, vec4(0.0)); a1 = max(a1, vec4(0.0)); } - m0 = max(m0, a0); m1 = max(m1, a1); - } -#endif - if ((p.flags & 8u) != 0u) { m0 = max(m0, vec4(0.0)); m1 = max(m1, vec4(0.0)); } - uint ob = pp * p.oc + n0; - [[unroll]] for (uint l = 0u; l < 4u; l++) { o[ob + l] = ET(m0[l]); o[ob + 4u + l] = ET(m1[l]); } -} \ No newline at end of file diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool.spv deleted file mode 100644 index c90b3a692d561fa62452600b8717af6ce950f825..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8404 zcma)=d$5*O6~;dq8S7yTM(}oOb>D?NWS-T>$$R_p~(Xn)L#>$y9e#Xf4 ztT?-q>s@ii$n~!{`<6ST;(8SB@QORAa6>E3dd9X_T(827thj>%=%Z(~Oz&f;QL z>)PW?X1kQNV;0?87`2gpwD}nCqyI8i?dB2>M>1#iqYxkM?u+$(7xitdb?t}HQ(Nql z+R^LU4;Q<+#MU-GdR_YqG*#F7B6@0xeNroWU27dac~9~1j*wO@^}5zy(Cb<+qNkSF zXE5S9ieA@x86WMwhj=)+Tcy{v-o)-q^&dzT>kP+RfAOK@u$OiO*T%6$?_2ak@ak9b z1HnG=^Xw$g@!)pEXApVDvC3Ji9eq5jHRLB2-ES>*^__tD=zk)sHu=NIGO5Kk8pHU? z{SM{-U+IYL-Z+UEzXjv%LA{B{<(;O0J7&_oF}2vN6Z;IX-(B?6z`mpJd;zPq{9d#h zcOfD%l>aOQ>LcenE@IVpA!0x3i;+71B6RZ~$$$MG%|8t6_oi;1OIW>A*%h9c&o<6SxJMic$QQh0nvt6ztBj~TDN0rAdBzD?lx zJEtv3&tkW)?;t*-nYJ_AipbTHT=&DmUF0f70Wg!pTRBHOC+t^!<*ZN584D=&?-)dj{>_yD0s8*Qne7 zt*rJhXaDNnEpo|oJG$Sy{&#@QX)g86=+4_&TE}}Rb-ZWP^?QlcoO0Gt_nwg(&71iB zdaqo`>fDv_OwF@mejm^4J{V{D8rlK;eck~3jlYGi-E|6He(N7VybrWbB9rrS-Hw)Fq`OfP(VPA)z(Sd(WaSpItQn0FRpoP6R=0XGrvxWt_bmQUPlu(9%qJ009a zyay6D7cBp1sW%TSHwxL$d!NDT`@8;|6J<~Hx^}#;Gg*OtCX%}6gRO0j`^f3}`3Pc+ zwpW?gS**tRypFD2UEW&a^s@wPto{6~^!U-jUBo(tefG{+j(Cqa6a9TZ&&?6aJHt4sbBU~_nG>KWMIZt|{VRlH}?=O@9wv$J*Ho|CJQRY-kKt|@%> zHS8?&X4Nl&^;MxxCL^V0mX8zi)u`+l0h#Ggw}~%;qt9y(<#?1F*cY&g_S*KF&t1oVTA8yEPO07&x)o6Z=!Jw#4oLo7zJMh558%Y^1}FB9U~QSj zpTNF{{g_v~-%ZYht9R!GtRL>$@$-S-l5w7w*v_nMPZpbbUIBZyT;qF0w?<-A;LoDYEY)s|QbcuyFc@31SJvG>CzmS3@3&1iEjy;$wTHFG(K7-wv68(4c+ zwgu?USlg|Y?RfOF(VdI7?Zg-igTB zqd4`C0ypt3$>tuG>HoY?m_ue!W?|DwA(5BKasurt$VA#ya5c@6|?cki;FhZW9VzFvA8 zqz|Iav-oaS^Z3}G_QMf*`w{yNBU$5P{19XqqL26*V(sxruzmOr#%R~iZ&iCccoeHW zg6jnBEChAy9PWd^uOE`Nv1 zcYZQhJ~1=Fa@ogOVDrf5T+Rl|=bkwY>|>AG<{)zRC{F#k;3j^apVXfRC!hNB!E*U7 zJ_NRwynB(nuI`0|NUeM9$vD>HS=A6TiCm z^Wfyu`}trwzqixb2cO(C$Dy0kHTTT%U}JL6j4d(tqV4^NoN;2`-@NMb>f?&;n)7`L z*qQ0G44I8&o)f^@>w9Km;p`>%%m?B0)#h28!)hKM`_n!Nk+&bQ?|lkue2i}*(-3{c zxo4(=?ZbC4M!SBwXJ&w%wLRr5o(va0+$=cxx92iv=HU#iXMO#xV_o0Jx~}yseJAzI z+BLZ@1v@+U!z-vo~h605_In%?ap&4x{ov0b}_ON zv2U?w&--W@_$u<}yk3glgs;!*WpMKOxobJtIQitcTuh$aD_5ZFC!e?#VEI$Y@0{$> ze(bB(<(-B7T}@8o?Ad!$-{kug*q@KQ_fq$2IC<~0_^n2lPu(?O>$+OUyVv{H``6kJ zAU9Mt?_2L*ZTD3+`&$cdU)mllw%p5W(Y0IGZ{hl4M{Vtk_UjON`w_c$x3I>?y}KFt z3ZjoV@A@UM^RY&L_xLj0I>fW4-TQJ9_%^UM=WKp+nbUp0gY{0NiQnz4iCJIxI>uOM zX3o!1*Sz+mulds3MloV<*@wH(<^5YxxVzE)UD}`F6MGMwzS`_>GwU~z#rSBmKlkig zh5tN0>RZs|?CaXXxnJ&|w&dRmHjlp}B>#8O&95!_zXz62{_lh3>inJ+&yF_d-+nr@8Ozx?8G;}N9vQ`hxg)>-xq&{&qnkuNd9i~ z6r8-@wR3w0?Bm?DJ&nj&Pn_O%f%(_jcRjafE1P|1Zpq`C*$l<5k9`_vjo603I}>fz z)0S`PdGI>;xkzU4dpP-f;pX#A_}%^k-8??IH(r5z85sy?jDO#h(>J;Hg0=a3esb+Y zw_l&++7Fjp#+XY^-{jiEzM0E!Fu7iYvoGk*Tra^Tmoes&(>J;P3O1L2`$?|9!P!@x z>+f*MWsJGx^i8f;!RGQlO0Ioy_QlfK*M5}bGN$#8tokO`8(?$!cctVy0B2t;ow?pZ zNiJi|C8uw4^&rMv7gf1>!r2%9I&<|xNiJi|C8uw4wSmp$->lMCA2|EsUuUkqD9L4v Tx#aZqd&=K;T@Ntn3CMo|#YtnQ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_h.spv deleted file mode 100644 index 7435cd67a1d886b9a6c2ca686c4c2ec98e8cb34f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8600 zcma)=eXN&d8OI+uM<7LzKoUg;qJoKNEMEYjAHD%Xf}++8@nwdXwP`I| z>N~c7)77Whr`fle)77+}Ax+<=56Jb*Aq}>EZGGn41#>4ZTYkyJj~;uBj)R&`fWCv9 z{>@&j-NwXS=a$!tGU@GNX%pY#*mcpv?jvT8S%cnp#`s~>{+X!pHX-@T}BWVN>QG1rr< zo%hh)#;CouwfWSwH=(Dt*xH?abT{gC?XUIu-+kVWo?2p`>@#{@>rQ-9OFX6%(%y%9 zUF)yt^-SMDPc5;}2*h&~y{`2pKH8m;cnr9!((77%N$gDZA4(PLjKN!f@&4qnm(d8R znj?!osOZD+8eH*1!9MZx>?F@5@My&6L*zM@RnA(Y(T`)bhWyl``>my}z9%3)`cG!n zCO@1k)7pH~jN&WzJCy%_r4!rTe2f^s1>@~Oy@kkSPcy)snRGWN6`OTpKNakE7yaX4 z_voG%v0BSBtKGN@5Q&le=R%-9a_;dWR(&r->_`1#q>f*VZhmL5gw=cEV6fk&`Y5p9 zp1L_NXB~*hnN!_wO-{QPq1agUYgnyk-|2ZdxaqS;zZTW^9Q$?P0od~%dphpDGg*nA zJvhr%h<93Mb`N;(qC2AvNc`0wLIxoE+t(wA&jcp#{2oQ*YDsU8!P8oMMjKg;cScXL zs=Mc>Se=2Kdr;qmWJXUHT+iecxa`3mw;}E=b$5VsE{yvNa@X@WyqBI|FPwZN^Ei-( z-R}|f>GyU#dTcYno=@Mo_tX-w`|%!9cRzQq+Pj?lQTN`FOP;&Yy$keT2R5gDsy~SC z{GF|JyrWXbJ4ju>zp zKVZNAZglOgGw|{rn1Xm$XrD$V=RXyZ*XJbGlaW?i-{W&i;p1~!@i~<>d1iv;Zz7L* zXCcPPCvG;lg?RrZ?sTwx;^u&jl~3Fm;1=Q?k+^wa`OT%?e6ZYjHJ<~w(sb<8`R_r8o(zZIpPdo<^A z)_NxPyPdq=M~e4Q`o13Q44k#|_uSlo^dR-QSyA}BqZ`p%$Vw#d=q9jyd{%>7$SNd0 zH-qK#9&TylrjZZ#1+cum{h+*?FM?Mf+C2ZavHJL4w0#MY^Sy}Ozu$v>Ut*@f_w(d=G5BL9Cgx$@JBh zSbp7Z)}kAm?{F_TWBn!)+Ye4(ZO&yNt9`g;E*B8vjO}eb>)^_^2;CWL`+8+_F6W~= z7j2t~F)ni%2G)KGHs>;^*iqYa(Y`MtZ$IMnd;r)z8-G2q{ZL1s%jfPp2;74A=P=Kh z{fq?5C+1+V+)C~}bBqF;NB+Tb7mfkTd*66ZjRpI7PiZ>@k+Vl}>K_Jf;pclw{lnqp zQ-2&-?%`5vJlI_yu!M9w&| z`#-wq^6JBj?&>^zXP*W;Gkq2#6Ohbv1X#Q8F7I<>;q2wU(%Zps`fBqm9>r=NAN$ij z3X!)zv3vM1YkZ6!gB*(JBkm#A9>;?1!#x zW**MKde+z9I@WbR)^)9C>7LXxYuDtu6zuGLAJ**krB8EtcH-NE-S^;GHO@Tuvpdh_ zZ0=6ao4*5jKCL&I)f}$o@cUdsjQ53kpF&P7Ht+cL*u00d-NRbjym!2ZwB258xo2j; znRg~Re7954jrH-}YM+A0+mAT+%&B1SN8@wuPeYf_w=xUdBA36@3jJ;@^jK~=$cK_y8msg)!bl05ktH91opR18Gk<4>C zSbKfX%qX0_l(1L^LiQa;*&+Aoi^7*;zYOrzg$+HY>-?>+oqw6Q1xNE`k{;k0|*`xi~SFOuC3;Vl~ zoW|L+_olwd*8}$FBk#S`T>&TWeHOnP(dAQj71+A2*75H3zV-gK_7li0mCgIs`&Zi| zmCgRX0&id1HWpj%<=fD;Ti0*l=3+-}zZdPR5qbL&`|iHV8Xw=?8ssiSA942j4Y2dE zhQE7ULC{*bI}y*CcJIq2;BSJpzUgco`ZF)-$q)*-OXx_Y2fb^zK${0 z8ERicALg}3ea)9%*Mofz_L}!{Kf3&UV#94fUxV(?^oe}{PG4>I_aN&-$i?_*vp?VA z!-fAcKI%V2m$R?W7tZ(R`_z{FKL(qp^LunB|4-1(uPyl>1Is7><6yZuzh}p@q|G^b zj-0J)>UmC_gZH;SIgg$L^%F~sYhwIa!84th{A@9`#0)Mmu8G+U_WMZ8z~VEA)pPbN ztB-etf9H4xUCw^R1IxGZ9GHLYJ7E#|9tFqtd}Z@z7j5n#d0g|2U4&g9>lIn5^gBf1l)j7fwF8-vcMNIm|7mUvj^}do{P;XmY;^XW!7hx&IC)pWOdY z$HBhMVQxA7lKY=vbH7sMehbdN>)boxyDn-DU?tXCgUFYtPE}z^3z{zb6bIa+M-1~vey|~K#K{)5l zzuvxwqR1!rFmQ64!`yQECHHW!xp!2#4}i08{`KY_fg+#W2ZEE^9OjnO&+oa5P5TYH K?&i&%fczIxd3CS= diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.comp deleted file mode 100644 index d904a8b..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.comp +++ /dev/null @@ -1,104 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -#extension GL_EXT_control_flow_attributes : require -// convpool.comp specialised for 3x3 kernels (ic <= 7): the 4x4 input window is read one row -// at a time into registers and each staged weight pair is applied to both pool columns, so -// every input element is loaded once (16 vs 36 taps) and shared-memory weight reads halve. -// Same thread mapping (pooled pixel, 8 output channels) and the same per-output accumulation -// order (bias, then k = (ky*3 + kx)*ic + ci ascending) as convpool.comp: -// o[py][px][n] = [relu] max_{dy,dx<2} [relu](b[n] + sum_k x[2py+dy, 2px+dx; k] w[k][n]) -// flags: bit0 bias, bit1 reflect pad, bit2 conv relu, bit3 pool relu -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; // [ih][iw][ic] -layout(set = 0, binding = 1) readonly buffer W { ET w[]; }; // [K][oc], k = (ky*3 + kx)*ic + ci -layout(set = 0, binding = 2) readonly buffer B { float b[]; }; // [oc] (dummy if none) -layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; // [ph][pw][oc] - -layout(push_constant) uniform PC { - uint ic, ih, iw, oc, ph, pw, kh, kw, pt, pl, flags; - float padv; -} p; - -shared vec4 ws[1024]; - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -void main() -{ - uint tid = gl_LocalInvocationID.x; - uint K = 9u * p.ic, oc4 = p.oc >> 2; - for (uint e = tid; e < K * p.oc; e += 256u) ws[e >> 2][e & 3u] = float(w[e]); - barrier(); - - uint groups = p.oc >> 3; - uint idx = gl_GlobalInvocationID.x; - if (idx >= p.ph * p.pw * groups) return; - uint g = idx % groups, pp = idx / groups; - uint pxx = pp % p.pw, pyy = pp / p.pw; - bool reflect = (p.flags & 2u) != 0u, crelu = (p.flags & 4u) != 0u; - int ro[4], co[4]; // window rows (pre-multiplied by iw) / columns - bool rv[4], cv[4]; - [[unroll]] for (uint r = 0u; r < 4u; r++) - { - int iy = int(pyy * 2u + r) - int(p.pt); - bool vy = iy >= 0 && iy < int(p.ih); - if (!vy && reflect) { iy = mirror(iy, int(p.ih)); vy = true; } - rv[r] = vy; ro[r] = vy ? iy * int(p.iw) : 0; - int ix = int(pxx * 2u + r) - int(p.pl); - bool vx = ix >= 0 && ix < int(p.iw); - if (!vx && reflect) { ix = mirror(ix, int(p.iw)); vx = true; } - cv[r] = vx; co[r] = vx ? ix : 0; - } - uint n0 = g * 8u; - vec4 b0 = vec4(0.0), b1 = vec4(0.0); - if ((p.flags & 1u) != 0u) - { - b0 = vec4(b[n0], b[n0 + 1u], b[n0 + 2u], b[n0 + 3u]); - b1 = vec4(b[n0 + 4u], b[n0 + 5u], b[n0 + 6u], b[n0 + 7u]); - } - vec4 a0[4], a1[4]; - [[unroll]] for (uint d = 0u; d < 4u; d++) { a0[d] = b0; a1[d] = b1; } - [[unroll]] for (uint r = 0u; r < 4u; r++) - { - float xr[4][8]; - [[unroll]] for (uint c = 0u; c < 4u; c++) - { - bool v = rv[r] && cv[c]; - uint base = uint(ro[r] + co[c]) * p.ic; - [[unroll]] for (uint ci = 0u; ci < 8u; ci++) - xr[c][ci] = ci < p.ic ? (v ? float(x[base + ci]) : p.padv) : 0.0; - } - [[unroll]] for (uint dy = 0u; dy < 2u; dy++) - { - if (r < dy || r - dy > 2u) continue; - uint ky = r - dy; - [[unroll]] for (uint kx = 0u; kx < 3u; kx++) - { - uint kb = (ky * 3u + kx) * p.ic; - [[unroll]] for (uint ci = 0u; ci < 8u; ci++) - { - if (ci >= p.ic) break; - uint wr = (kb + ci) * oc4 + g * 2u; - vec4 w0 = ws[wr], w1 = ws[wr + 1u]; - [[unroll]] for (uint dx = 0u; dx < 2u; dx++) - { - float xv = xr[kx + dx][ci]; - a0[dy * 2u + dx] += xv * w0; - a1[dy * 2u + dx] += xv * w1; - } - } - } - } - } - vec4 m0 = vec4(-3.4e38), m1 = vec4(-3.4e38); - [[unroll]] for (uint d = 0u; d < 4u; d++) - { - if (crelu) { a0[d] = max(a0[d], vec4(0.0)); a1[d] = max(a1[d], vec4(0.0)); } - m0 = max(m0, a0[d]); m1 = max(m1, a1[d]); - } - if ((p.flags & 8u) != 0u) { m0 = max(m0, vec4(0.0)); m1 = max(m1, vec4(0.0)); } - uint ob = pp * p.oc + n0; - [[unroll]] for (uint l = 0u; l < 4u; l++) { o[ob + l] = ET(m0[l]); o[ob + 4u + l] = ET(m1[l]); } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3.spv deleted file mode 100644 index 6073da8632d7215ead8ccb27c740513eef681a7c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21556 zcma*vd9-F_btdo&RSXJJAexRM2!#w9jBx;pnZhC{G6+P$G^VdiC4-t?abzqR?8NRx zBA_yACnYk^X!lA&m}PWzw~kB{bw7K2_kPZM&OKGB zn|Uj5yxPpXm1fqMdDYBsSDl$$>&&b?vr^C+GpkPIZhP#v#~=R1ClC3TWY`;I)R-4-1Fxc!TZo|R$imB~Q zgYA`5+vbDK+;X?b!o@+6_SNHV;r8Ij6LMJaqrEecW>BLE~<}vza=r<8{R1K^3|q~doPbZ?)8o6-3xYguj=Do-%L#R zf;U=`GTDnh?)AwO9{2iG^zH?_UYF$@RUh~IOk()W0o!Zdpf3z|@&8r)I2FHcc8Qw! zd6RhfwTaQohFNJda(WN*Hn{szGZN7B_fB+@&DO&z-Di;aKj6nD-eqU({~eU1vAS?PEBC!(%A`2Re3 z_4s?QJg-qF`FJ=wOHK9Y$7QXOW&g7xovZQeoDg8A9o+Y`a=Y4?6z2CjjHE5v6lkT0w);c>!$6xhbqnoSx?$PyW=KX{9)tL{N`q=ru z6sZq$(oc(IYy9cKYO(9@oT)yUJ#%dIUrb#amsXjCFt61=_hBNm4EF96DsRsw0e~r``{R;-&{rJ$!^bZd@{?C}|&t~^8 zY9`kU!HXjQb9DWXi__IMv2B~>eaXK=dh$-%E{k2vE|Kq^VDJ9l#VncHPfm=ST_?Vo z8TRJAXM$I}+gkg*v9aGY&D%YgZS$<>(%YVqdL*9?6KX${e)XhpA7FcBb>9yL+gpt* zQ`0lOf0i7+XEM{?k#bzmN9U)r+Y4?#e-0!D={5BM16>G<81_X?b_H&rb%MdAF?I%g+RR zpN{wPv(edmPmY=374O;JlVf9BVPA2-kDtbP_crE)VD|F9ELa|UZ6^jV%hKEB(|d7J zupaoF$x|X-?gQV+S#0hDY<}mK&i+?Zj=mVIj@nmpt5Ip05KvFRJ`{?|{iKY#sh z{~LnYyZ?>BY*$V9x+z!>?09Zn63+`FZ_ctWj>OyZ!t}y-YZm_n(|Wf}ZSJ$)?}&8i zo$vN6Hoe2`{mu!V+xuOyvA6fTgW1&mcJ70#@8&b3t4Urn?d9xXIrxxs?lea)eCK4b z$%D?;hwELZv|&!<^^99Sw8=3Fu!}(`?-8-)5|xfy?rA#v3$Ad+~y*bM{^Zk2Y*7vhK$9|U3#p3pnU>E+`9uEFVmKorl)gMjp3jX?>KNcH%&+3nZ z*~<6h!TM!4yS>y^f7IDK-}w>^tnY!7pa@9k$X+Gc#;o&vGv2 zMR)0)Z{;jDy~FK&{shnMeU;eQ+xx1)Z0i0XXVBGm*mcp>B>N7#K3GoQVK+{5^ul*T z7MncS{OZ!#={HSvvS({S&I-=NtQ~xRWcj=`n16hS-9EMHrSGsiViU{f9ri$^IxhY3 z-TRP4>QQ&7uRR%d%@medg@vH=h$km?Vi}!UtUYi z@__^U*}wwTk|UR9qL*v3wbq$(cSCIUY;4aA+1THY-+gda<*Czm*l+RJ8Z4k{5hx3K3%N5h?#Y#y4vocJnxLx#O}{S&7+Go$HnoRlk7K-Zz{3Qo|p@! zF&9PlJ@fj+d@;H|)AoCA^cV%JXusLSun z^-PeS8NB8AO#bf2Ih5zT)c2d5SAQ?Wx9!yD{QLW)1GD`3-x>6?pDw=s+35a0pnkf& z>c1xsm;8lU^66v#z0u{@PapFip5{Ae^66v#Bh&o)>0|y+r}?wfeEOLGFVp<`>GJV- zNc!?@k+^zCT#sgUv*Pxh!Am3i=cglr`JF%S_QRs{=fBA_%YSGVySgx*yq`W5>>EqI z`|aoNr(^-N));T6RyP29pn#n%L2CJzbv90I*cS3aE zOzjIj%VJ}w@bITJq$Z_wvXqCV0i) z4>sq@seR0mi(~KJ=W<`wwZB;P#F|HaGmJWWbDvMHeD&7P5^Dy3*LiK$8Cf{8JR@v! z%Hah$&kJKKhu1`R;gIj5tP8XBLFz*coe!s1MK%YgmqovL8bg+wOD1?OH*(exw<;|$}IKmtG96;b2awbVDa+tYWDK_t>E&i zFTAqJDX-TDd+(IjyP}sPz8kY{$TBaf4>5E;yxtMn9K7Bd{ibOQSzd3R;JLis5*xd^ z>ZmKfynZJbuX-%6x5ZXo`OE9=!F*#aHOtkV!D7_6uinOa%+=VtgT>3otJ%xzcZ18T zzVON>r@Y=5JR4tm{c&`c^UC)?*8N%LCG{bO&WG0@MK%Yo4@G})8bg-XA58FEUVj)H zySnPAE5E!x9E{i5Y*1bwiLJcym)A#w`NmpmmaE5t#i(yzy^ZsjtFeyqIsOudj;!%4rN) zUe}o5xxB6!8@syds4Ks`t`&^eowLg8g4oI{e|cRym~X75X1RKGuo(62tG96;b2aug z!Q$oP)$HZao0T7F&7cFRz;i^NqFCELU#{7Nfp>^)}99 zuEuT=EM7id&0b#L8eCrWg;zE?<#p@e-Q!ylVsP~C=;erSo2*4y<|XwZhR!Evab$Dw zxSe9-_d&o_j`Tc;PS)1cNUvjVf^^+e}Ay=^@Ulj zvrla7`fLuVPy6f_tft<@swdZX_6LH;Z}SgE_if(#V)l=X9pCf=g5|OI+x$bp?EN-B zFqmyg;@$sG1iQTBe1{(z{g5p4k$M+H=aciX$mW>g@T|kK#E?BRODA~lnfYjJ?CPo` zM}E)8k-gHu^DH<{<4UhR!GFj{H7jpAW{9 ze#_yB(aRx!IXo$tZ>*(ec{wFmjQaM~+c=N88vCDu#mmQ`*~{Ty2A4y9dB)h};N!f= z^5OgX^ysH$nT@ok7&@Pvb0V9A!>?wYktK#KhhLuHxg35aHgMt%G0ZJftkjs04%c=*T^@Uqz0)m2B1{Bn3jFb-$4LC@ip(aRx! zIlL;EZ>*(edATN7jQaM~+c=N88vCta@$zwK_Hy{`;Bu%h=a5YfKJJQi^}G4T=r?4U zjkKp2I-i_7BAbK5TeEJ;5<`~5nk&`G3wh_Z{s}XYV6&?;^pJe?B(#g!R1h2IAoK9kH;ci{YJh&`h8huBkd`M z&L`*5$mZbihglC~i6P73gA+WL!ym-PuC6+A!~a;WI6o#1kdI07qPLctBxG`cxcb<~w#4%Z0A;rQ*lW^CnTR6I zT#a2PSiF23n!OypI=CG6+m}s_UVM{o5ZrIyg~8>A{f${{W`*(N+jqlY-@blMwa!Md zvFo!rq(1Glaj=?t7ptCJ-?5tnkKewVM)&R8`eHVVjUC_g&4cB!_uKc)!R-C^-6EK6 zN#foAoq}Ec_T4i2yRytj>Rk+-PtFdJ%`w9^SzBd^A$w-Fp5VD>W>IYH>Z&70zWcK@ zMYjv?`Os_6$M(T|W1jrp^Bsf5sAUg*jJ=zou{#Hgm+yR-v*%-Ru=9cEe*3b?G5ZH2 zUHJ0tyGQi*WtoGtrx-e)oIiwP%(XvV83|!E^cAJ2rN8)sZ8=&)7b}cv_gA z%i+G!%OQU`+%K4Ktfgjo`9QE3_3f*-aUOFucK=}U@^NVPa(F;+InS!oN-# zqkk^TY@|KK(D~$?7}*>g9-nngmKd@e9y`HvIXo^lc6HT}BflJ;5RAjwY)}rDMK6c^ ztd}A#&%gaf@V$`><-o|;%)!0*l#mmQ`*~{SQw*I?&RLPo!Qq#)PRkNQmc!F0crJ%$#Kx|!I&$Qf!>yZ z`Q+Rd*&H0+oOMH%7_uDRIKgu{yeT$zb=8q0zZ~8YjKlHU_txm;kiQ&$Czx-nrDl1# zJy?wT_SM@skGUFqXRvtrI5c}XyeqgI_S=_D4n7`@EFZpo?~Q&>mf1*qilOt#c{s8; zID9bczAQ0hIlO;@=W_T!Z0zc)BS(HY{6R1d$8X;sMlXl_irJG3wh_ zZ{s}XYV41K#mmQ`*~{T$!R4^uzHD;zvLIwU5!`RzUj&yQ_9wI0%nIYjx9?NIzI|td zTj%GovFo!rq(1HQ%V0J2E>=CczGI&b9>0B`iSFCC^~F3J8#}(~zY3Pe-f!RMg4z4+ z`|Dt~CGvCs7f8(I-R19nSBXA9%Y3BX#nAcWyehIeW_U&PRi`mz&&+BQJon739vi#5 z>ZmK<{qgO)MsUxEUVA>)jGb@Hliz#3RTF$I_k>r zGqzDMo))I(a=0+Ia>!o}HxA|-YpGdYHVGD^zJ2vJ&SS2|ZWb(FJ`T-pfB%MM^Wbu* zFF!BYZmKf z94-pR;rQ*lO>E_mzZ`BG%s19jv%G8{EJl6%>TR6IT#el^SiF23n!OzE6kHDb?aL+y zAAb^AK79M`9{s&pW+Uw>hR!Ev-^k|RaIfg^o5qmkaE}R|%i*4}v8$_&y7J56-oZGW z%?3S(?~kn<@|VMXg89Z;YL=J%g2kwBU%idFT%d zXQO{6%WR}Q#nAcWEQ@Ro4v&rgxoHep4v(JTxf~u78@syds4Kr59v6(m@!R+K*vcV) zIXoelZ>*(edHH;>81?O|w{aeGHTI-n@$zwK_HuY~a5?O^FPj{EoE2Ff2Xu<2>eS?AgKM<>S!oG;cLcZ2rLp<0zB4vKj(iC5FU-;H$HSIpg6Z1#oKGb5e7eS9zY z(dj&5zCVp2%jM7F$ECct1>YB0J{}C_pN$`v_eST>f0JJL@5y4fH{5Ufhl1@dx99JX z=0XOA@EY$Ai@(+vES4_Ne|_gMS*?9)A|h|GG#${_jZs{5R>5|0h}O>cagU z?k9rf+8fVLMR(zu@5wASd9eOkhrM$>9c&J@I@hz&*>_L9+MbKrSU%78 zi;=&{I#3^xp2q`%e>=4|Z{_U8zc8zL^J0_7*Sz_`?0w!|9qf5?5A@sTecfQb_WIi3 zK1ZjfFFm|IHhqY-=X$~G1^e9#@!j(c!R()(=B*#hCg0C$KDO4~Bv_wg-A$u^_+@oB zi;cZ?HxFi$uPz^3Il3h1kVn6 zdk%Sg%@f0Cj_yyk&c4Cp{`*Bw%4Gkog83V}fAF~H0WXW?H~Yc!QXp4n|I_GfW`|qv zz~Fmw|2zAKqjzt<&LD=bd9%Uy4tXCL^7xu3hR^KXpKP7Og2(-rM(_T3;BV{^!Q-As z4zYOPH~X=XwPNt;bDePNAKQzok0v=^F9~+$dGsRkjK|NF??q4{$%T% z5IpX`EPD6H1Ak*r3Lf`7d5FaWzuCVWInMrHqO+MDZoN~3e>%+Y*U`H-UuO`**Syn$ z|7FNKeaPc$o)|u}cYm^V&I}&+KP!6o#{++3&kY{;Ja34_1Al*C^}{&_eqZ%yW-hyia}|F2tB$o;~V& zAwISF?8n!ir-uihn*8@>^__QQZ0xVke*5HSrQhK?F1k7{KePS)AquCH<@AXJ2b=5u3hZda2f1V{0urYO#rJ zt+xfMwewKx?Xl@=to4rAT1$>vY+_q$+hDbPKeeyzW7Aj6FZH!UY^^0nEjF>OwNtQK zyAFLVj!j=Vq0tPV6}E1`r0Qp eeMP)fYu_lXB}XkbvCi%nQ^PYrz8Iz#W&M9VC`rr! diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/convpool_k3_h.spv deleted file mode 100644 index fe50d315f6a608663ed82274fbd8e0fe864e4774..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22248 zcma*vdEEBpm_P9Q>3LcSNyU^FLfe?MX%VU??bI-p3QvnvO32WH6lJLynPJ9gw4hB< zDQ#%OB(#vyVyKD88nPz+KJV+fJ~{9H`h8!Wj?b~4$MN}|*Z2Nj_v^XMUF4aI&&-`O z^SGI%XVzMLW^yevv&hVxAaa*wX5w3HqHnX~%XfV4VQ<;*xf^V_uD~bE%$>xGed5gG zGmA#f%Q|H51K+^${H(*Xo|Js{l$udUytPfNVjh>Z;zXZ2^Pb3hi!?$raQLB4>mb$Pa16cW_!wDTWo53>R>~UxRnN*{n%C=Y)ejUs}HuNrnWT( zo4Mt#lZA_YBkgOJZNlw-Q@m|_u&e!4m+un4eA*nf{C{7r*;rf3winOrX8z zP4u}lYeippALdPP%#ZHP=Nk83C${bluNMEpIp)pK$Gy*tZm)&@xie=)?_RL0b5+y z%!`wwo_sygx5#4C!%d?b!^Yr9-wOJOsm@Ol> znqWT{{uVQTYJX2+1B~}Tz`(vPiMCm+ltf#wzluL2IF9xtlrxrf)CFc@9mM%*?W)PF~JL;!@WoEjBTNP;r)Nt zG{&>JF~ml+!xsV&M}?+{Zo!k z3|2?&^c8o=Hxf$Q|HXgoEO%n$GmOk^BjpaL51YEOvF_ z-iOZxn_1sIV;2XrtNHfSTP*nVS?uNH(g~iKxg@fjTo%mUoXdmRjtw7bT#=;?`}tFj zzBIM_thnb_MY`N`zALlX^bL3aYbMxtwZH9uZ7_TH|8g+fh10#h608SyJhv{1=O;#f zHOsy@5^vA5(+l5qS^Q^B>s>#!ogS(8Z$!HE&iC~!Hoe2`{hJd!yZ0MnV{h*_2D7RA z#oPy1pVg;CSChPG+RJIda_}MN6Vn{M@SUE;CJ#2hx^#B>C#O1z5BF?|{Gh^_m?eWR zk1U^O2J^djy`SeyZF)I#+S{jN6U*m(UJF^o#j(|1+adI){Vbu2 z#qB-8F8s6I9sJ`gGr&Eo_fGIa{`xupVQlO@t3L{6E8q78>zCc^_EJ~e z2D9~yJQ%DFd(ZqIg4y+r+doCRaLf0{EH)g#-T$Epp3UuFVq@?A4+pdLGxyhEd$HrP zbxB-0507No7r)}|*=N|CISc)LhUHgeGi;|PM(>LTyY$XCFN;m@aC={Df@k->cx>$L z-FG{-$$86taP=AX<>+dXeTIECSk4m?C+E6pj$ZiwCyPxUJUL5Kot^%*sZRE6Etj)` zGcn5te?PK(ej}LQz3ctFaca{`pJ6w}CYH}L><5wRxb(+=a~8Y)U^9F#vN7^+$+|5| z4BThft-<;*gB*Tw{Y>2+>=~w~p2a(2tG%|nVq<@7Ej7yr4(w-t@%FK=dF)G$T$+h~ z&(u%0)(TVZ+z)#;wr7TH?8@=G56-GQb@~ijGaq#4?UwkQU+2s->`Aft-qX)MU93FM z_T{F!+U}t|&y3|`_x-4Obg||*FMe~9edqB`CDz#!bIvs8+{iv>R!Yo?(S675``ojF zPmOK$$iB}#JD7dr*qwnjf?duD-*d9q^Zj>65X{%yje_NRRyOzf(VNTP+>L|zn!9PR+yz7KX3?9=-`p1j^X;6wv3ue- z4>pH>)a7s87X_<(#!&ag(OZ|lb+-uSYwrADxm#s*o~?tM%ir9W1oJg_yI{H7W;J*F z;O6o-cZXoU=DsvouHTU~_hrG&mKL4-zUej5_sVhx{p_cU_xbSqsC}~*41T)3>h~d%?~KW(kNI<=%dek4 z<}W$Tch2O~$NZ(H`SsJs{3lQI7fkc%WB#(!{QBwg@%Xy*<+DZNYQ5CBFQ08(;P&~! z2SxV#(;>n9&Yx%dfzkQ%-{dpPe?S(yx^SOQhX;GdlJ9=|`Flic>^}QPMY??U*^kU( zQwMIHcLaOR>9KX*8J*os%^}TXpQD4-)Q{NK3l%fRM)%ItzC0`69UHqn#T*yx8QDF> z93P$CIj6rjSRT8d>$rShFngbiCj_(2kFT75Ji5AUcsM<}3lDs!WnGem2UyG*k&Q9i zr?M`{5(E33`Tfdgg3rnFoU}K-_6h!MFkgSaa(*xy{?t}SJ@flL;YAaicY(-$Rxh5~ z#~ish_bh%S_gP*2h*eLl`P4VlsIxaWBTn+wTO~`ZnfxyFvMj%!!+R-R;3*?57XCj`Nwbv3CZGmyc&$ zl;^vG%X6RcY;wx;kAfGZ_wxLk=q~4(@5fp9WtpGUhZs5^o*#&84xWD&{U_5HvOM2E z!Lxb(X>9E3s-v#_^8E8)JTJ%wJnEXIEN(Cav# zIUD=iVDa+tjEnO8U~qZXr}NAvr#$~P*w41|{J42ha^aaTV&;)Z^OO1zL+8Wul0y!j z=S810jUmhPA`?8D=ee=5tE-N>^2_t0!FcZbpYyy}Y~`81JTD&1SDu#$W*ckUquebO zEXIEN(Cav#IUD=KG zLkyiy&NCvLgXgD5|GQ}nS)Ny%;MqK{6dSv`>ZmKfJg*##XFc{juM%5%<}c5y2J@BY z)q>f^+V&`S&kPo0KYi$RoX?z%eO9n|`FO@fd46_qdDbVM+2oYxb%VE!Z+?iv+2+y9 z8Q*$Y&&x7DsSh!9J~^93HV4leMPGj!Lzd?aCU`c_8^*@2t~%<^2_ti z!FbkVdEO!?ezRv9as3Iix=AvwyIfdKasn zT<`e@1RtAP>iceRV07ONT3^gTv9aTy{`z2f?EP+Va4>tn8yp(UHb3#s!@GlBo{7E( z93K5`S>_}4E{4u0=a|UmnBnbNM`Vd1dxnmj;Mr&BsMy%mRY#8eo|kt9_q^z}=jG^N zzMhwN1+$HH)aw~IHdu_>_SDxni__`DTdA`=f5JGgTD`FeJD!|S^oZQf@kyh@3FC~tBxG`{S1C27;pM5r~eVX zobs2`{|x3UrymVw8*AI6Jbf%!jQ#YX*Ks~`Huk@R#mmPjF3Rc0gUhKteFoX&;OC;q z^5gyf?C57@nU%Ds7&@Pv3nQC@)AO><$r3}B(@#(EY);ROja^-Jjk`QUQe_j@)u`1wX; z`SC7)b@Z#U%u3o*44qHT^^wiN=~uF@$r3}B(`zSqHm6^Xja^-Jcmlq&>yZ`Q+Ra*&LjHFYAshF=RQtbAo4cdRJ`h>Z&70emVW$V4RNc z?ca}HPWj8}4}$s1>D|F>V{Lnsr~eBUV?TZ9b)3(fjs0P;c=sMK3CG9DO&L`(jk}ACV(h06y^iylv$1~(7B3&CxG1L&2ba^nqqE7u z&$5gDjh}U6TO#`6S!N~eDTdA`XPL;6)5k|&avDRH)1@YOHm8q^ja^-J)RkXOmk!41 z_&)xG*vct?IelU+UJGAYU*9=q;`7n z9{u9z$EKe8-o3Yo?%lif#cUZHJO1fg1>6U;V0@y^4Z!LGi0?;QPQ zS>_}4E{4u0=arGoF~cjecF7V$_6+^w1kXN0FOQ8~U3KKhcYpkjXV>7K7rpkp>=w+| z^Rjy|+gL}vo`F4r#i(sheT}o2sj+(ni|y*@T}b=8q0zn{TFg7G%Kd%q!iIpr^> zhX(VN(>DgQjkWDjp57cR#(w(H>o}h|8~c`E@$zwsi*owU!R55?-fVL4b3$bK@$UVu z=tpOnm9(cAI-i{PM>Yqi@5wqQOAJ{~kDcJzoW46Yc6HT}Bfp#;7mQP9py%}X=;f5Z zoW3`hubjRwm~E_WkMi_^U@`X7hhE3|%-PrzgT>3oDK5(CzXX?4eR@x1lY^g6M!K9+ z@7^C3pJi6ko?_^Ha!!wI4o?3&>%X$ZkmdAa6Fi&KlVf97R~%jw61ak?NIl+#n9 zms9?7dTKCVIXx|yZLDpN^7M&dG4|7kUdQ>&+1O77iYqiKg_x(OAJ{~|8Ih4b9!%V?CPo`M}9f|Q7}%& zcklb6ms9?7`r}}}a{7~Cwz0N7%F|DS#n?|DdL8F8XJdaJEM7iNaZyfx5nN9D?#(6# zKXbis<#JBFd;gI=%dDh5#n30d`%^OGdqkg@7~J>d-q-t+&WK*ja{G3A@ymW<%8AKyIA$)M$9}l z|8s2WsqfwUY0eT-^Ip`|A?6t^FQ*~`|iC`Fniy>=5a~ zpLg#UM}JY4xk!78q4UYvF0wiJo1e8smKd`9Z8^cS`P(Wsc6HT}Bfp=)t%LD4zI(qU zdO77Ur`rVcmD6p5*~Z%TC{No5i?N?R^g7OG&c^N-EM7iNaZyfp3NEL8_hyrWpM4|C zk9Y6gqwkhwR??ng=zMbaj%*H2_sV)@mKd^}?lHl$Io&fhc6HT}Bfp%!Dj28ZyZ5W3 zms9?7`kG+Aa=IXxZLDpN^0ZH|82jl%uj72?Z0vr);^pHM7v=P|!R55?-fVL4b5vyc z@$UV`=!a&Rm9(cAI-i^)BAbKLw`9F3OAJ{~-#o#yIXx^kc6HT}Bfp&fb1+U9WP^V8 z-x|G~@|V-M1@o2D!-Ltz+V&_EN zPS3~&<@8^pms9?7`oUnna(YrQ+gRHk<>}vo#n?|DdL8F8XJbDcEM7iNaZye`5?oI8 ziBmQ?_&GPyg;Vd|r$s+C%dDh5#nAcWoD0&*t=#v9YVGjvV>r z^o(Gfj_=-QMlYxQ<@BszzH)kYFxyz$9_8uN!D8&E5511_nX|Fy1&f!DQ(TnO&jgp# zzI(ID!Oxc?%a3>O%c5VJWmeLjV(5Hwu8C|8PQRFSd6pQmoL(`(vpM}jZ0zc)BS(HY z{ZcSa$9L~5qnA_ua(Y!TUpc)xm~E_WkMeYFuo(O4L$Bj}=4|X&g2l_nDK5(CSA)xG z-@V!7;ODN$^5fn6=IA$NnU%Ds7&@PvJ0hEd)9++`D@zPnPQN|DvpKybHgh|J2WWmpwKNz9zo@JD{%w^ZR+{-^X7aoj?Cgo*(>IWwEOZ zH}|?={{~%t{|4=Av9Y^XUypQo4zXXK#ikDI@0#>?LvZVy7n}F`8)IYFm${lpnoG=0 z!S>RVcs1?&tw@)B#oV05W?xu6Gt$}H$8EvwL(J{d7_wYGkXdmlZ~frABFo43gZUT4 zkIOrw^XI=wFZ_38vD+K&cZz$0?Ju|I@!sg{YBuh^=BdU+b`Uu0I8v zL#@vBkoc_aQ?Is%qx+jIXP~)%jZG|{&-$FySR`w&#PIoi?jAgMYH!|R(fJp?(=W_h ze45ACyd{F!`*~YF*yqhX(62auE3iT^UweI8a6d=?n!fa~a%}n#tJkNe&MLwF_C|d7 zTs4^em}%Z>!EEyVZ0BQZ-8F;tIo4e(`hv&QT{|}R)?FundG+5U&Lm9v7P^y!R8m=`F|D6-aa1)W|OZ^KDO3en(Ab8{GF3$RdZf9jX5yVXY(cTxqLRyirhLnn||SP zzkTp)lJodHQ=QozqDP(lJu2U-QE(-OuX#HM?>pq}GUV|!PYj>gyFb}Fy9ST@?-o5N zll?ah=5Oqt!Q-BLJtmgl?1$!igsZdfADzwYaO)i)F25M+><32g-h7=w3}5pO7B}P_ zGUV|!PYj>gyFb}FZw?;!KP-Cp#{++3-^w@a`L-bz5Bz3-U!<$EA03^|>~QNH6YTHy zI{UHFyEk8F5X0BJI-A+y*85nnzd7s-Cr9tze4RlIU-M20_P2=5J9Wt8Yn~WBvv+^8bv_k5 z?tez~?vDrl#-1HK?s?7-iwFMxz4G2U2mW4p#T4t`EAJB=keFqFa{qUK!bx7|bo+r-<7h=pW&mQ$XJ3h7f?8n!ir-ual|L@r@tIxbc zV`E<_`|XmSm3|H%6I~sbckz89kByDp&v3sFeRuFtS^lO_4bL++ajkt^FrV+!t^I-M z`ggVV39+%a_KCr*t%llc;#zy%%%`@Wjjg?2Z2FGr(c0_B#@^Z+1h=*tYO{%J?Tv!f z-fF15acue?Yi|-8duwkR+}diW%_gq3w+vR>`&av(ADg~oezfncV`Fdamjt)A8fvqN zYwc}=)!ugKd;8e*J=WeKHul!uF}StWP@7F$Yri5`?FB>MyT+#Ph)4V0Eed;U?;hOR zYN*X7uC@0FR@-;4axAv=pTU!ma*~IzmoRpeAgXF`xa~o&dEAy$8poe{Q&je3jIG2>1{uOwCOs8iWU0wIWKFV6QrU({5~&!U55_*DB*s{( zv1X#l7GpP%)mj?%+OA&fRMWi!<2Og2 zFj~bg7!w$4;4e?8nt`-;w8CoBg|U7^ZmI3c+@7%~(ZoKz+NKqpzH-Kloj!6a7o4k; z>soO7$gNRuu3N5K!F8;-9~E4uirb*z%%^XUf?K8HHZ8c$71z7qtXtnL8TR4?X7gH0 zE4@9j!h50<`(!KWM=io68B`J&&r!HeAN41%x^8q4>6nD{upZ^bK869(7=c* za;x(xb1ygZ5YE^~j??tklCj+|U#pVS$-%=Ha6$tCXD=F+G`xy*Gmfy|G)SiRua2K&U$y^}cGf_pH0dJv~Cv-?wiUF7X6>{`|Dz-&(WT`Jk%o}BUZ+mYd; z{Z7p4CjrN#l^wjOn;4XkwLiGFvm zbw=J7Y>n1@1hcu^!>Q>=u*3%Z-%&tq%W^IpRxE|$W7^VM6WaHa|QOuqj8-o3< zDsKe#x2SB)lbE|QbKP0VDDtzd2UVzytg-Mg}tPweqv_q+Bp z!TQ^8W8a8uYJMiRMzfmW_u}Tf0aEa<+84h zY_KlK+IIrGM*C)5&(Os6j8xX{SY~6$8CTgeQf^B+=~?5M=zew|ICkKzL1EqY_}lst z#&>B9`};G7eNeWCgA3Un%_?MjGPjbyV6A5voabw>=h<(Ojqf~wES_sSGd%m$_alcr z-i0Br%|PZsjCw=|e@0hb)6?zS>a{1QBOky>-owG>HpX0H zx<7|A^ilUY{W_3YAD`vO>XqfqCC++Afc14fi>oyrUvVRu2e3~(6Hj5B%&;fgTi+<~ z7>2rW$nM8eD^9<$4ZiKnHji<4<-Lz%)^1{z&l-(6p1JIa>wSy3{$3rPt6BFsV0&P% z?Z5lxT*d@Oxo;*`eBROd$aTgfM&8k6uzYMT1lJiCFk&+WET8vqQ3Ka%^5Ld|S{VuqaWMQr{4MwI2JRz125Jdt7S3oGu*f{WdCaPn>IZQJ8B z3NAI?1XpLwVx-2|VEOEWo56DS{1O(+LQP`4UVt+_&x(x7vJZ=`i^A8_jz!9)yMY*u(~0f3Em$sGOWY( z7*~Dbz63U|{ZHJN!HKIrabE$ei|?yoeU~ue`x-dD>f^fWtI5UEAJrj_vRcZLy4f1$fm$cPO&4)Lq=98&>#DZqf}Y{C?M@8(jFE-lW^B z@Eg;l8&vq6*rXd+_#Id1wq@PVvVMPW*034a`f|Ul1y;WsI`5aR$m)IkO{(w0koPw! zwvXK^**SOHOI03i4rQ#%NS+^q)%)8rPxngC)X1ZL9frJl#5r3w1kb=u-|VrC;pB}O zt|zj*^_)YFj@WO4ET6M+Q*a&L?@afg{n-pGAD^FqsLuYJ=# zIcK8V0om_5d*vOitiAd5-3cuB=StogEO$$l|CeAn?+0t$73^cJ>iRL{tW}&^`-AJ) zm9^I4T+7YX+AfFFR-Jo%7PE1DT%Y|Xi)L#$o&Fq@8cmD_%KHLp(^3Ab~nSR&<^O;|J z^O)Cq%>U@>X-fF9KSEI^PRZQjbY5}t^MHY#4PvL z{&4cyTSLHdse33`PM@y)E7P9+3Yq`awqlsib!uk}&xT=OZIbr@a2?)0=l6RcoP5r! zgTVU9>(>|CgTek=D!)Nx?X2B%(w;i!Ec5)-*0pNWt+MIPtbAA{JKHbsL+>u{GHotn zKBCZhcX^koo7SZB?(!~EH>pYI-Q``T?(8O=cb9jWy75gq?=J5$b)%bf-d)~h>P9u` zyt};1)E!&sd>1d%5AQK`y};I&`{h`$`d0M*_J$*?_wl!<{&0r8y%yWY<0{$NKALAq zl}DRJj8Tl_ISH&j_t^23o~e;X{YZwqdBoWlqrul&cRvf^$05sS zAD#xT6Kgzk_TlMZ`S_dxmdpM>6Kov$oMUH$<=yw5MZW|4*fVwKFyyR7oc!m4>)4gQ zgNbnR$$uVLF6ZbZu({;zed0RXd+(z2!REDZ+M6@?(gk4kxtFGZ<#I1g1^c*mbr&(@ zT)Q}Hp9ZdDSFU|JoP5@PF<8!?{Db%4lY8lQWMew#Ub+*kPwu6;m5*ytH-{mopV<11 zt1PcPuacc}1}+2JGi}~sEM}yicZ1dY?(#nGt2oz^d+B~SZPoePc!JqDKCVywe1^R1 z5nK0T%(2mb0b?OU8*%QXhrq7G8uU@GUGAktV0-PFvKJqQ3m@)LIQiyS#!Nr#f%(j@ zy?M-QJ?3>TdudI|>9uoWy$f#ij@g{C&7N_N?R)5b5AId{jB_hDht66+M&g?LK4kfvUGu?mt|{l<0@g!OIf?bxw@t8oK&CnT?1!b{@XrvT?;3lx~>Dum33W@Y+c5*E@kaf*9~Cn zN?hxj0VkjJ%mmAob=`z4U)D7n*}9TTSv&Wj_lSGcIdQ#TtTFez_mDQnGpB!#gZ(#+ zcU%0PfRp!*h}~1j@~+2x&w!2P>^}Az!S4dU6O7-P`T0WUcY)st>ROs~eiuxxbRSSp z>F0NW-wFD?-K6uo!0!Zgk2mT3F7P`+-NGiF-vxdrsM~-V%Y1$p_?@6`twQH}TZ-)d zRM)f8YruE+c$Cg{VLf0>YsiuMV23ef4J9?FGDWx_&4-p zsB_P~&Ag0p40h_=%f6d;Dn2>iM^^8?jNgBd^-~wW55V&IoqP&*zvp-A8TA>myyyCt z%s$pE{{>?u>au2Weiz??=Q0w<{(g@vpEb6_PtLyhx1Q11viGHJM~u-Ayh_EFKKi=9)xU|}xYnVq@qSM{`@1^W9&2-6 z#kqgnPwHZ~9@w6icJ4vaj+?C;y(ySmst0=Cbkoo9q+hPw3mIk0{9H|2Wm^Yd`>_IW9@ zkA0SZiSYu%o;jz_FM{jXm3{svoV;}!M<4yu=a<3uIl1lgD{%7Z^S{7yWuKQIr_bus z=U2h%(&yK}a%Fzkkv{($tc`1Ot=5zNy#cn*+N95(zn;hHV)q`{K9_c$;hyR0(&vxB z_IV^Vrq3V4$=l~|nSJcD{O62M81~FLef|_&$FA)2XK?b?X&im@PoKX4+vnuA&tJmH zr_W!3<;p%UM^2yBr_W!5)uqqhfaS{kt|NW^4y=u9a;?^r{!wnD&)THV-jCjw>SEUk zY@bUz?^y3#b>@B^y|p{1F7GL8$~QpnZSNs{j;MT`>0H@#W%m8%n{uSU}LK@UzvX(a`Jyj9Bq?-_a>b=tnEmez{XZ*zB2!5$jNV=+GcHM z6gulPhqX;*E^E6GS^^^`prTIgI)dXPBIsq<)R)W*K(XO8HOLT7As=2Mq%--E#Z_v!r^IdcwyllQ*| z4y9(l(N07*j*sWF--ai_$>$sLWbg=v-;l=e-)(Z*CH82rI{({fVvj?1{XU639!@^7 zPXi~mF^nyzU1G0Cea80NF0t2#bKUkhu{VH|PwWlBiERvH%W0R`8-tC#q=?-U&UKfu zH-VE+>|Wr+Hiog~v`g$Qz{d7_CF|Y_&UHhtSa%;d`NZBDoY=-Nww!i}y$#sduNSfV z!ny7;_I7aciM>5Iv5jGDIqed=AK2L5|5azV%>EF`NSRoPHbZsTTZ*g9t1Y_ pvLg2GaITwaMeIEg coalesced. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) readonly buffer W { ET w[]; }; -layout(set = 0, binding = 2) readonly buffer B { float b[]; }; -layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl; - uint flags; // bit0 hasBias, bit1 reflect, bit2 relu, bit4 gelu - float padVal; -} p; - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -void main() -{ - uint id = gl_GlobalInvocationID.x; - if (id >= p.c * p.oh * p.ow) return; - uint ch = id % p.c, px = id / p.c; - uint ox = px % p.ow, oy = px / p.ow; - bool refl = (p.flags & 2u) != 0u; - float acc = ((p.flags & 1u) != 0u) ? b[ch] : 0.0; - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - float wv = float(w[(ky * p.kw + kx) * p.c + ch]); - // branch-free border: Adreno 750 mis-executed the former `acc += pad; continue;` form - bool oob = uint(sy) >= p.ih || uint(sx) >= p.iw; - int yy = refl ? mirror(sy, int(p.ih)) : clamp(sy, 0, int(p.ih) - 1); - int xx = refl ? mirror(sx, int(p.iw)) : clamp(sx, 0, int(p.iw) - 1); - float xv = float(x[(uint(yy) * p.iw + uint(xx)) * p.c + ch]); - acc += wv * (oob && !refl ? p.padVal : xv); - } - } - if ((p.flags & 4u) != 0u) acc = max(acc, 0.0); - if ((p.flags & 16u) != 0u) acc = 0.5 * acc * (1.0 + ferf(acc * 0.7071067811865475)); - o[id] = ET(acc); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise.spv deleted file mode 100644 index ec4f43b7f474eb64f99bd153c7d4937619080e14..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6284 zcmb7{f2`MK6~{l^d$}0D(g}&H@LqIF1DR#06M@hzB5&%FIX1QCMhUY^5=f$m2>oI$ zs=1k)VK@myB2lB5Vx^YikD+aq%^)GjUvp`ZfPmWT^}Wyg?iv5}?4F(Xd7tx~bDkfc z&*yvF+fE+Tv=3;8G{c(VZB1(&*R(YQKtr0&mY&vgY0voi3ucY~)Pyq(7~ZtE;*9wq zF$2*ZNbgm1=IZE0j)j*yPJ%#Vm1(g0haewm+3n4Iw52~wCb8q>KGn}TSI(S?bB^3G z6=!F1Lo3cXa>rJjJ-)Hi|| zBhlf_>4-fZY))-+oQpVOpVzXM{xDXg-e5c>vgT1GDsNcRs z`>oCW_6+Qm>wnZgeOKti{T2IBu(KTf zXJ6Vr=jqEjH()o&ICjrF>vJyWxhiLI3ZA#UIlLf0I1sU5q$SKJ%7<&F$UJ zyqm%DnYR?|T=`#+%lFeQhdCsqG`L}t(jX%I1I>+3; z`xCnv>|AwX-yzm`=MUqeZYe(R!a;b~pl>EvU-ob-*gf<;lRbO{EWfGj*Kfe)kl)W* zevLMV!@bb|D@5MD#L52{*!+(o$^ST5K6mc-V9!^6X*sti!1C5zOby>@Pa^Wk{S?^T z+mPgb8Z4hV&w%BYk=Gp0BIc2I-?yXfS)ILf|J+a4F$4X_;&U&LV7r(4E-SwD^Jh5w z@g1Cg{sNZIUcLY}hrE0FJlgyY_fr3Jh`fD?lmBmE^ZQOu{vBZX?B!0d&yRfe@+Gjm zb+a$Ku;r8cA7FFuMv{9ESUz+1g5|Q8{{-(sv z`fo?%?N96))}RyP{5z045o5$lm}`%#!1m!9oTJ}3@2mdRV9(Q@a{uoFhYxo*ocz(b z%$fag53FZ>*|*JE9Ey_c>@dwub%lMA}Fcbu04K zPCk8)0n3?pJKB+Zyb0T!>fGZ8z|P4%{w3I9FZv!tyC*mXaJPK@(^hdhoLBhEd3 z3~V2+!8!Vk%RT-B*uA!=+=C~;;lpi%lRr9_IkO+`f%U9!ymhSWdaSFi_tG_KXRp=C z)lv4&^RQ-Oy+`Kq?&Qv#20r@Ed>l?bcV;YDF6$WwmaFgj>Dc_W?z-pd*%@b#sjE(p z1L?~??c2HbWIew>6Tm(T>G3S=qkB9XPCnnjbHL7%Po8tZ*2y<Ag_axvn$h+T6J z`tyiuz7U;S7Zu(;HIH+gTif#PvETJh*6tkh+NZJKBEP;bfDe?ITM9P=&fjbL%;|3_ zYpAn_m*Mlbhi7lh_7dZ7ENy?wI!B#3Gl@S~&DmUH{EeIc8Lq&;3j1<&{%5!nEI$|C zz55c_;oj-{A|hu!@zApFtHAs!0J}tJ z)6;CQ!=ChAjmX)PIQxAKcn)H{aC3{@i{4W1%L`z6dtT31#IyPX(@4Mx^|6VwK;dX(i6DPO4#O(&l_rRyd9&CAYq{d#b{3TV~ z%h>Y9CEqJx`59H*YuNI2zSptkXI619?(1OBTAz0=cR`&ycNpwF@E#iPJ;`tMMc{>q Z@0oeXWbVD3bIi9G?cGwp%ZkoH{s-K-%0mDE diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.comp deleted file mode 100644 index b158de4..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.comp +++ /dev/null @@ -1,58 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// depthwise.comp with 4 channels per invocation (c % 4 == 0), same per-element math. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET4 x[]; }; -layout(set = 0, binding = 1) readonly buffer W { ET4 w[]; }; -layout(set = 0, binding = 2) readonly buffer B { vec4 b[]; }; -layout(set = 0, binding = 3) writeonly buffer O { ET4 o[]; }; - -layout(push_constant) uniform PC { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl; - uint flags; // bit0 hasBias, bit1 reflect, bit2 relu, bit4 gelu - float padVal; -} p; - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -void main() -{ - uint id = gl_GlobalInvocationID.x; - uint c4 = p.c >> 2; - if (id >= c4 * p.oh * p.ow) return; - uint ch = id % c4, px = id / c4; - uint ox = px % p.ow, oy = px / p.ow; - bool refl = (p.flags & 2u) != 0u; - vec4 acc = ((p.flags & 1u) != 0u) ? b[ch] : vec4(0.0); - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - vec4 wv = vec4(w[(ky * p.kw + kx) * c4 + ch]); - // branch-free border: Adreno 750 mis-executed the former `acc += pad; continue;` form - bool oob = uint(sy) >= p.ih || uint(sx) >= p.iw; - int yy = refl ? mirror(sy, int(p.ih)) : clamp(sy, 0, int(p.ih) - 1); - int xx = refl ? mirror(sx, int(p.iw)) : clamp(sx, 0, int(p.iw) - 1); - vec4 xv = vec4(x[(uint(yy) * p.iw + uint(xx)) * c4 + ch]); - acc += wv * (oob && !refl ? vec4(p.padVal) : xv); - } - } - if ((p.flags & 4u) != 0u) acc = max(acc, vec4(0.0)); - if ((p.flags & 16u) != 0u) - for (int l = 0; l < 4; l++) acc[l] = 0.5 * acc[l] * (1.0 + ferf(acc[l] * 0.7071067811865475)); - o[id] = ET4(acc); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/depthwise4.spv deleted file mode 100644 index ff663e4c8405bd7e0f9e4867338b238846e9c660..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6600 zcmb7|e~j049ml`i9S6uAf^sNnjsx3V;pLVvL&1zTWSvT7rE@biH!(Y`A?a<12MWq8 zsjYw7)Zla?l8B)e=%(2a=kKL7O@}(OUnp4`3PtvKeDD3dy?6ih-o5sEzMk*bulJA7 z=llI$Ys-u=jn+|(@r};L(JhVUnAB)#i~@~sv^VvF-mmrcEWPT|o|9*tq{DZ>e(b$&c@Y}0OSEJa)TOfrcbGltYKXRMq#<7bTA z*ow0=xsHl6Ms8xo*|Xf_ifb?2(G@qQaQ2$I)-!fm#f>Z6@fCMi;ZCSH*KO>~inCWa zH`o2S7i};0yMVpkf*#tRwkF1Gq(|o*#yj-iigw;y;%>zF*weud?Y-!_);rj#C3d7% z?7G&zGH))iwLGiXb*;}aQP(;dJGI1))QVl#nvGA^CGH*yY1Y!NYpubqYu$*QT4Kjk z#50Xu*IJ8@^R7eO4Q{RMy4Eesd#?H)NfqmKlpj`;O}t)XrF9K;d(w5B!iXzaN|_UWn=~VSADGdk+rV=VITo=_kYS=V7<^ubStY7lWrEj-#-zK+Cx=uKT-a*C>B#vHJ>V zKl&~~9Qt30)?fZO-aNi5#Q%>B?R#6}D&l-*&0%frWrefW4+=K7y?B1h5qpn40QTJy z`&O{`Aie(_?Af^XLA15?H~$@I&s;wK8^NyclT^94%pC{b0@gO}K6D2nXW!aB<8sdb zvS9Phq=Nb(#CuUodVUn%cVX@sALHL(pOGJ9>#sh8D4&(@A$^GRbID;J-$&&2 z@s3}Oc<+6$#OIpAKaP(v{fJx-lDKQZ#`#PnZUtEWh7xxj*jV|*tppo4fFy1eSUz#r zgN>EnLawgn+)(e%x&Ae1$3fP019~+g=N^a;pxq;F`L(5|o5AjZeXS|nx{8b6PvPY4 zA$~urxUAzeO zE;?u2cCdNW#(8JPGiT4{x`1ds7EhS{I7$}zXwVFH^B1QuQ$Oy6K^2dueZSR*3H`gh%KMoe*&BPZ6vw> z43 zgNSqXwitZ~@%*x$|AJl5WF+hP7%ZQ6mG61uvx)XZ}VqfB% z*CeoOGv*PnBcDUJ7k~A(@+_|h8{?iCGYEEA$GHuNoOQ(ZZC-78?K_ICb{#%fhjZ?p zjXo2>+tK%znslgi$C9@dPncf?%Yi`<7#fV~gt^%!h9&%1+fhpzKjZ2lT2BA%J&qM!AX zQ=R%pP}3gl$yn>T@6*wa+|h3APb2mu&a*NDJn~sN9!@@ee+n#T-Z#;X+~eP3n^T>8 zyc29p?(r_L!(NCtr57%If^ZMl;zYO*s+f(kr zE8y_qUWJn%naiBn5BI=&*4N)U)^$DBRo8pznzXam>g4Jwd*^vrGrry@juK>2V5u*{6LQYfskm^XC+>&q8|q zJod;Q=fKJ5Gw@Wfaq`JC7i^t;CeFjwPd;&90L%ANH-2BlmajiM=VQ-CT$l6t?EDhg zxqRlG3EqIu#`3ORi0x;Oymzk`?eKm&_hsa(i02}9&1a*}L0t1$Xlr$X7ZrXHp9O>1 zU%{5E#r?L1`OI6}^6tN}N3k!)Thle@yPDe0eI4v~0)4X1Z@}FNx3c(r6I)K-d^eV} z-fsM?ug;#H&z#>`yfgab``}B(_Is`|>cm_?jNg9}v%UEE{WrfKzJ-s!A1*}a_rpbC z`6ck~`6Xb7d+yxDh@ADrS>w0CuCb%+(WTh@HBLaB^S40O>YVFH9(8(J47R7s(CO)N zuzY&D0_?CS=e~o;*^}7ZeZ^Mix#?g(yo36zMwTFXZ!QHpKft{2!z;@?X0wj-`pa8K z?0uWS{MF#*+KJD7y9Q3)d*S`-2QNe9ep0^M44xCa#OPzk%iFfH(2UXs*53 z@~464A{~W)7mh!=ABsuszeCQ4bKW~>j`zUEI(Jjy{!uvR!o3e(Sp2UqejmWe_rj;f zer$Pjq{fF}`9+oA0c?5wlJ6i`{@lv%KiKkhzC+mZ=T&|mW6SGz9V>TjEu4dEO>b@3 z;j&kw;c#gjgE$}GcCa~}Pk&>;?v-=#8wZy6EVAB}VEvqn-zqrI-Z}5T=Pl=R>h~P) zq4zG&srS|UoxnO)FenEU3gG9ze0O zC`;Sgx(G3WsepMY$Of^Qm1s9;nGQ82YpKypRFLZR`km)}_Kg2}c0T*OKkw&rdoJJa z@Ao_HZD$Q@+J`hFo9w7#tL(v_`!cSbW$ccr>Xm~l&G$zqknF;IsNaXaE-QFxg+rkB85=Ez&d%hzD$W?W(G_RUa$_s5vv8+W+_1vgYwB9h*xrgeu5hofxDkb$SaGh~*hv*< zuX1j#`*SDSUhH=!d%XocussmKr*R{Ti zomyhYIK(rJUDsNRkMpiWJOSKZ*>$a(nfF}vKancdnSi(c;vRC?OD}?Ja}qP%#qMQp z6xe$v_m<-8z0`Ls8G0(-8jkq&fvur!{A9!tdrHgd-x+)AfIU6-`Pf5uY@8N*7WSEs z-!M2o8~f>I$Dln&`{2EU_B`yhn{OJ7zZko7*{W%-c|N!oah#0(8MK`H;<^{0U8DSa zioLLK_M`72#G(IUwEptrdGq+L5dS|iuGUntnz_Tu?1N9;Xz zKiGFm?C*lT2kHF>V9&<2uSZ*3fAjwk?U~ERecQY^-^FrzDT>3~l}1jW$NkJleh^ zGX|f9RoE|foIBmVeE*~`-!IzM`5M|D zohrS+>a}UI? zqTM5H`7f8AZUnmr_O+&PU$40M-3%vh5Apj>#brI;gL6H;C$gUJgXJGA`ELcw`R=&A ztY;ngR>ZT3-zKnr-iP>Y2FvT`y6-?cT(@(#BljY%Q#^zF<@@4pWE(OLan1I&9(@nu zocm)gDOR+>n^88C-@;mKDi$Tn|lwE+>e0e6Z0Fe+$!>#@)EslKpxbEN|Vc{TXce+m`2;U0K4`kVybiN34UyaVjK=e@OX zTMK6|w-s(1SYPLSj&`BVy>w05*=u!jom}?L^RQ-o zy(8xGUgSQE2YVmV>+8UBo_81DEL~?WHh;}q5zowX(a-wHsZRYU;{5!Do^*!EOID5%G-Urs#Iq&fSw0Ru%=lmZL zdHWH&?!TbpWBjAYjkG(T*gOkslc{^Ax>v;!Q zuDz=D;r=LBht~x!QPG9zE-^SXL_5A#q0`^%*k5jRa?Qt5Md_EJWgN>6< zo*7{4z|`?2Ng&(fLLlM&bDd_GGr06Uk@(pg}+d~2KfyLchm;eB;(4)PJia}v7;bJ6n<_uwM5wY$L|D*PNi8`fig7+bCu_um@k zGjDCndmhG~!9E#pP1m6B_0)FmqhP-y=wofUkHKv~uPiGw+}z-tFy{Bp2s zH$M068aR3Hh4*g-_-aHxJ}bes&uVPBRme|>>mlwSSbj2iGoO^^`WLqR6!27}tMD(v z@kjRqF{%9$x?4tKeD1e|7OY3@1MuJ~dv$mN!Rg90kkI zsr>$nEw5klv7L?lyvnZ)PQK39jxB$2tUcYNuxoaDa?OM~@2<&j#tC4WHG^ZiX z$F~b?PUq8KH@LOW==dEEmiH{O-ZfzToQvNLaGt$$-haEm_O`g--Tko0o wIQLqeXYCtc@0;&Yf1})lEtlUi-vqBkyzBmN!@6?%ne!I3cUXOtT{sWRX^-Hdg4(Aj%?am z@y6~;%pi0-(tXjTm+R<8_JfxrO=A*cm1(g0ham^I?6zhu+7?bBlh`r7SM_nul{07J zoFg}+;_OUrc*QwKZoi7NXSvZ8*HO5yRovjh*=y=r&$(kOZfM~StGIm%cSOaxZs$&@ zID3_IbKReZ(e`5B)7b0P=-&NlZ(+ zSMVwL#E#U8UDw)C{N@r{%d?7I*ZMY&y4KOy*0q+{ky^3qTE`HRb%{HBr?hHm*R`(2 zu4~Q1PA#$H>xgF>yRNl>82zq8+zD>0?7CJD{$cox-=8Yh=_J~C@fdR0%UFca<^Vh+ ziv7*v^B(FO&5SYV@aA;H?gE=r+Z@Lsj@T!(te#)Tp44lf7<(%Ap!E+;j(sZjxGg{H z^H0a#)~xTdXK4RtZJ&KE_ARS!?VCRnyJP;6$*yr0cr4=hD)wx&ock~q`!cj^ls~@M z-G#FsV_Po}wDDIIpZuY`X?!<`zl`+mb9-|Y^M)eku(tNqg|pT*1)JMmJg@nPy~n;D z>@ykrZm{Q`-tPtHY|Qfj;(8Aui_f&Z9132I*0$~=XrF&M`_f)hIQ@?m?A+9S2JHJQ z_6D%;so0yqt~K^w!LBFv-@!vFdn?#?mG|;rXy23eydAA=&p!Lw_VjPGJQJU-Xj#!W<0 z-#OU-ZJ&6eeff?^U%mshtur5O4|4XU?K>cSS!WS;gG^v&X9Ih{a-O+z3@7AWus4VI zXg7@KwhL>q^BJFlz4J@3^{b~4<=wv$=|=QVB9nFJAo9j|XRk%P$G#^LGq3Q8Sx{oU z&x!d7SpG)xnD;uwdGeXJ5NvMmeda9!%V%B>*tznLlFN73V#Gds5Z8JG+VKf%(6){DlY4}2hR2Q+-5z?!15bQ{$GIQ ze3n<2^(+U=dk%?v5Nw=howyZXdE;F7Dzw9O>syIDiMUSj6z-1C_v6SK#4~iw_U7Gv z0@3IGn9F(2uWkAN@$?)2fA-Kh=Jvag*tKBisuTMevBo?9D_qod#phkv3-22AT>#dX zJ$x4I9{LW-9zF+_f4uD1^I&tx?_@1cq0QlNFZBNkk+&~#^1lE!{{|%ae+!n+o%C+{*#@+)I7u7hn4M8=U?4eojBHf#tK8uY=7Y?_R!wHowEY)c^k%VjU$1;35R zr~Zdv&&>6u{zqVW>nHAGZ25Ue;&y`Nmm)q(yU@E4zePSv`t8kk{2oN#Tx_3}PYP$R ztviqHS!aEpfnDEdBclAoeBB`Hcd* zHs?GFcI12Y^AfNAWqIyz13Sk(bIu)LhjsMbj>uU@Y~SY9me;gR=B$hXD=%XcMsTDeLhEzpv~j3KmE%PdHWH&hE?dqIDa{EKVpn{ zA#?5VKCpeb2IuHE&ikwXL9pj(PkA0zfWwDd2`Aq_mpQW^?t%5JZ@hJ^>w2uKuJ_V4 zX=kt1$u+3#o#$cA#E!%^m-iy~;XttWA-x`iE$4X;=NF~xJQ$n5=19ad^IVLxesZc) zzk{0gU{B7qp8Gxy?Z_SN#Qp|iPvSf)x-?T1e{po)Jk+&bQ>)wn`jPrkwY(k6?=N@ka+lOm#j(+2Ek6!|NkL@Y<;AL?5 zaDRf6@1M(@*$?-?de%4II@WbP)>YSg>6)~&*XraNTK3NKux4VtN9OYGbrgvHh-^ktv+?Oc1Zp5LEi!9EM=@jKZ4 zdpr(KKHtRS!OoLUo=ITqLi+}wmRNQ6=yR|!_ToJAq^C>4_B0Efo_++DPfwSD9rmPeHX>(F z;_UYo;BLfv;eK4~Ip}rezH9-@+w&^EDxTGw*zzZUCn3WN{}vp7RPLRVyjvls!s*Z1 zz5{lyzMIQ=Z!4U>aPNXoB~I>!68BHA{B-!#*p4l4j?{P$EI*@)+kq`_T=KmSmOraBF;j&k|;P`6}Li8tgH`tu|)88Jjd!;XN zpMvE*i>&u*uyOhlcMaIH*5{qeT~O!FLAUO~#c;-ZPkf*FTm5>l{`{t2h`j*uJv0ZI U%yS@TocR}{y=&^dtngUm3kmYb`2YX_ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.comp deleted file mode 100644 index 3eb4856..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.comp +++ /dev/null @@ -1,33 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Elementwise binary + fused activation, general rank<=4 broadcast. -// dst flat index decomposed by out dims (d0..d3); operand index = sum(dk * stride_k). -// op: 0 add 1 sub 2 mul 3 div; act: 0 none 1 relu 2 sigmoid -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer A { ET a[]; }; -layout(set = 0, binding = 1) readonly buffer B { ET b[]; }; -layout(set = 0, binding = 2) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint n, s0, s1, s2; // out strides (s3 = 1 implied) - uint as0, as1, as2, as3; - uint bs0, bs1, bs2, bs3; - uint op, act; -} p; - -void main() -{ - uint i = gl_GlobalInvocationID.x; - if (i >= p.n) return; - uint d0 = i / p.s0, r0 = i % p.s0; - uint d1 = r0 / p.s1, r1 = r0 % p.s1; - uint d2 = r1 / p.s2, d3 = r1 % p.s2; - float va = float(a[d0 * p.as0 + d1 * p.as1 + d2 * p.as2 + d3 * p.as3]); - float vb = float(b[d0 * p.bs0 + d1 * p.bs1 + d2 * p.bs2 + d3 * p.bs3]); - float v = p.op == 0u ? va + vb : p.op == 1u ? va - vb : p.op == 2u ? va * vb : va / vb; - if (p.act == 1u) v = max(v, 0.0); - else if (p.act == 2u) v = 1.0 / (1.0 + exp(-v)); - o[i] = ET(v); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2.spv deleted file mode 100644 index fc007c59567a1ae01c04abba15c2d36901cb402b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3500 zcmbW2+l!V}6vp3gzDcVY9nDjw$u~1y2u?$gnGrdt$yiJWvck@c4VIKfY1ok&OGCjZ zC@=~Ny69kNP)TQ@w4089O>`pY_xol)urKPS4QuwZp0(Dq_FnJ%e$(z5p4+rrO!eZtd&zUoHcUiR-7l3>#sO#%BadQfH zVa3fYoM%m6`&rwmxbq5kNyVLCxYZTsy{#RoIL|8QaMNR6;nuP{vrX)t&%2B>|IO^P zGwbbQ+$QHX=2FvT{uXxQYKaFiSNx$_x9d-W>s~wX(@X41ulRMZsS;OPY_Ap;e+cZ- z|B}7#^%Z`4iCxPv=Mul}b-KjW65GrA_;s%{@XlYoMP!*b$ZXo|;w5N!mJa4xhVOHg zGpB>^GxaX=R~KJRxizKU=jk%v=V(lgRlI?|-{Svc>Rmvsz16Y5{-VNpe&6Q~mhbP64yV5^%UP(zk~U1Snv5yV6OP5y57u~_wpOb zeEkOWz1P?5_LTEp`hElTIljgBAR|oC`<%h#JfF6XX?rf8fvb)E4&yWU4ezWZ>o2gn z-(kkxH2T2+fm6$GAzGJ$<)bwUwpKn` zmsec0u7p#|?v^G>+w6?;j<@Xb<>%j5{%YCi~dr$e)-2k?30!!VE zVENSD1olqyGezrWuzJR#wGFJ6-)OXM1G=TIo;$Jhd=M<3p1Z)#P(F1JgRS#7 zB6YjL@~PVcc82mf>my+Gj794)uv-4UL~ActK3e<0&QLyDkAu}S7Of}1YVF6O^(0t6 zT2FzUp?tKS2CHW*GkgX-g*ijx>N&$_F=OdD1GeWhmYxT|GuUz7o#pJ$VRHSL-}NDO zm*2IqgV+(w?^t|*I(r?)d}j9aTer>}bEEwtSnZ>j+Ap)a)He1K_9|wb`1zv!3b@v` z&Kz^2{Tf*9W0=}+u)EYY_B!?!W}Wy@(S8$LYg=cIxra;6+y9Yc?yvlZ8)N?-Q%i00 zt#=MNt9x>P0DC9zXq`Is%$!|ve*`;QXKkH2d7kfreV+b*%JX~=ET6f&50=Z!KLER& zi?QR_hnRO3_m}&e1h=r&m}fYFc}DBawI)420^1|A{{-x^hp~?_IqSsJ@XqX0OznPt z3-mw3?70FnmUruO@F~oAp2-)5Gv>LX@1EKH3|8N>ns4v=4&EiXKY_ioGq6sbd|%&y bPh);x1K0-MFF9+}JIn6f-2dhV>#=_T`Q{OF diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2_h.spv deleted file mode 100644 index 87a2dfe23509da41645d4dcc6c97574e6c75473d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3568 zcmbW2`)k)#9LB%9G%Kgm<|VUabNw)IjUY24a;UYf(n{6^yO^R4EoQ~6u$#+E83jf` zfl^S=4_b)@m2@9;v0u9WEzymj*Xz6U!1AcXHVjVg*)h2C@=M)0x6#&2aBmlT&SGu{6Wey} zc40P{hpvWALlgIk5meX3h0VCFxu+FAic@nxybm~{nX9H}G;`F>t~4uC>#Q_$)aF&1 zHLESCw2q>kUum<7X018b^UUq3v~!AfaiyJGwEjx--sTQenzgFg+?+9|Xe*iR*?MN{ z^Ufpm|C+)_S(7_bcVayq?jL4$T`zera3o$n6E@@R@cOJB#52#Qy&-qxti<(M<7HiM zxo7$O64z&aj$WVj1#!-jJC*?ZlDIx=x~%Ih_bl@h*Jpi;ZvXY`W|#YV=}jB6d?6mz z(gPfeiG8kW?&%@+nYIA+t4ge=+KQ6z^K`i1=jfUq%XkBQzvcf2`khCvXY1qn#@$7; ze&6RFka~<40*CpFm~*EE+}d7P_rK+>V9mZ{eHGk3EaLAFb5G&xnEkdcLSN5pybSJp zXzaPyGTRq5>oy)PTKq?e+d<-uaKF>UV{rSHcr)DZQ~xc@`g^u@Y%kn8d`FGlcQ3R1 z)U3mJylC;=N$fYCcmi(E5>LW?4-)T%`wk@D5BCgX>pcQms`x0pwZ!)QIPg7jzkNLg z9Endiqp2b9<@b{Q`aKwXuP>QBQ_XuBPlLLSZ-^~qfU3NY--DX-ZB-*Ome`&zJwfY<(KNX1@XEbu70rG=LWd< zR8QWGaPu~TwEnauP z_1X;LbthasUU$Lmg?ha1hU@3rTAu6{=6k>}@Lh9VKfj-?z_m%@A?9shD@YBy;MTAc zq=x(8>Z##=xII-*-fp;g6CimHz}1uYAl#m+r}l^7`neXbJ#f7yLA)M@tH%S zJ_7W9nc1PYYcGLUfqC)+#rqX_?QNcW+#By>aJ`QLyekk3H`FmH&X=|9ha9-tITwKBQLX?EL}moxG!Y`qVwMciH;t9Rh^B%kf^aE=+1+3A0_nMP4K7e~h zdjApJ;Tf)d2-M7z??bm|9|OHR`K>Vi1bF6B;9B0TPvNJ5>v<-h70os4iobJu_cL67 zYjwY8*LUzP+4~dRJKF>E^zl1d$NUv=J>TEg@M+-p*9F$_zNxuS|1-?q)%kC3v>N;a Dq4^qA diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.comp deleted file mode 100644 index de8d905..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.comp +++ /dev/null @@ -1,26 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Elementwise binary + fused activation, contiguous same-shape fast path (4 floats/thread). -// op: 0 add 1 sub 2 mul 3 div; act: 0 none 1 relu 2 sigmoid -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer A { ET4 a[]; }; -layout(set = 0, binding = 1) readonly buffer B { ET4 b[]; }; -layout(set = 0, binding = 2) writeonly buffer O { ET4 o[]; }; - -layout(push_constant) uniform PC { uint nv4; uint op, act; } p; - -void main() -{ - uint i = gl_GlobalInvocationID.x; - if (i >= p.nv4) return; - vec4 va = vec4(a[i]), vb = vec4(b[i]), v; - if (p.op == 0u) v = va + vb; - else if (p.op == 1u) v = va - vb; - else if (p.op == 2u) v = va * vb; - else v = va / vb; - if (p.act == 1u) v = max(v, vec4(0.0)); - else if (p.act == 2u) v = 1.0 / (1.0 + exp(-v)); - o[i] = ET4(v); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v.spv deleted file mode 100644 index da05819d4170cf56a133ff1e393f82dd3fb99a27..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2336 zcmZ{kX-id66o!vp(@IUvsj^J7sPJ}aPz$Wopv)2qw23URq>vmSwJ!}S5%gR7(7%f& zf}ZExz2YFW;bpyRUVEQ)Z>e^mAuH8njahTHomEd;R-4s88?*XcPEOBESJu`SDnr9( zjo6o!@<3x+h}ng#1IzP^D>@oMD|T&d8VEF2Ok&k94(EC)kEw;J4>!sXRty}l6-T{2(NBxkK-01{9{nnEA15fDI_oxpdi@nO|(M#@$ zUZEFzRg&LYa(l)5s26)Jp%;5Cqem~f=OFN2p%;6tCcnMp_A)>8Vy|`VejDqx6J=aK zZ&O0bJE`F;eZbR=z6Ytzm_GEqNbjP3H0jpVHkk68;2z_fk@~FBn|44RFlIlx_1q6@ zs<$VbGj+iGK=e@W1RnFdkTGWmH}QU5Ox&CMtM@nxS9iyb;PYLow(2gaTkAN|`F&$& zQFmXo=|7pcxz=%4qD~pEZrmB9IohnF?zh+GJ9&rmpeE_Q$Jh>i3f=D#@v{ZpyF3P- za@LT~FbCIGgt^8pfGpb@bLJ&j%H(yUSXBRQc z7TlgijPU)vPaW{MQ$MiyAM+h;LnG%m+@5cMe&qyF&B%Sgebax1O3u^*v>W3tbOZOt z9f=r!uiC#7<9-|h=4rRL|5u)0)YR7lwAo+&6X|^F+9&AjE}R1HhrO(4Ki~E=&=+z3 zxmLi=xO3R_#q1a0&hBo+>_c$vU()QuaBaSUvtLAdoL%1txD1?CzD=GpjRNPgr}r_> z7-OUMINaKPOKV?4daON;oB%g~dGe2`_H}r%wt2=F8?|r2tvv;-J&p8OTiKO3TwA>PBe=(R(f1HM1>RZSltlh-EtG@*H>;d{>w-(`BK!1FbSIMT&xuU)@ z-fk0aeP=b^-o-n3mx%oU_s+h7dDe;F*K7DH=mh=_ud-j-%(31E(z_}DaDz+WFMN%h A5&!@I diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/ew2v_h.spv deleted file mode 100644 index c10857836231c8217a039b683d9b3e29c9cadceb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2420 zcmZ{kX-id66o$86(@M?Esj|$nS+OFuK`N!e3Y##HY9xuU+WT zOYVtYp%;7Y$^F)r+biBjz1Zssda>72^ynq`tO3pyda>7R?zflRUgn2h?DZPE-^O|^ zL>ae{w<#gzZPf5C-N4g`-l-Md4Wfs78}OLlj*NS@GKurGbK|beUzuYAT-_bh=ew)~&ggrp>syb{ zUDMVDcbC-dvlZ!V+PtT_yQEG3j+~oo9rq~exF_ny`9m{Dn|0LpB1>HGs08d}4OU^z8v=w2lvG+hqHXBTkJ8y) z*a_SZds)wZzHKkiH;eAK>C0`Aw-?=g_&09=o4&{!glmi0_Q9Ra-HF-u!?l0PXFCAb z<{LTNFw)~}`VN8-;4Jca^1R~_;63f>yyh7*f{fb7;MVqATl)mkV{LuM!D(Qgd?BxW z3SO*jo-xKo?Xz%eyZ6>UkMvku-#Ks*m?vM%Yfr$7waqie*pGS4rT@ej`z8PVyn|cI z+QysjJH%a;5jzieCTBFyI>k5h?IN}e_iZb0ow9oIJuhSTdrpG*o>$=7<6W-8wZ)rX zhkJY%eb>NE;LP%de4c4IKh?Ayl))76j^-O{PV~49w@1AFUAV^{`tAU2=E>h;cV`}e zGHBrUp#BipvlHlx-I{?f0{!t#9_KcF-Ye=WOO`CDnpF=vU@)r~B2fqO;z@w@F diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.comp deleted file mode 100644 index e37fd69..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.comp +++ /dev/null @@ -1,88 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Tiled GEMM: C[M,N] = A[M,K] * B + bias[N]; B is [K,N] or [N,K] when transB. -// Tile 64x64, K-chunk 16, 256 threads -> 4x4 each. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer A { ET a[]; }; -layout(set = 0, binding = 1) readonly buffer B { ET b[]; }; -layout(set = 0, binding = 2) readonly buffer BS { float bs[]; }; // bias[N] (dummy if none) -layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint m, n, k; - uint flags; // bit0 transB, bit1 hasBias, bit2 relu, bit3 gelu -} p; - -const uint BM = 64u, BN = 64u, BK = 16u; -shared float As[BK * BM]; -shared float Bs[BK * BN]; - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -void main() -{ - uint m0 = gl_WorkGroupID.y * BM; - uint n0 = gl_WorkGroupID.x * BN; - uint tid = gl_LocalInvocationID.x; - uint ty = tid >> 4, tx = tid & 15u; - bool transB = (p.flags & 1u) != 0u; - - float acc[4][4]; - for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) acc[i][j] = 0.0; - - for (uint k0 = 0u; k0 < p.k; k0 += BK) - { - for (uint e = tid; e < BK * BM; e += 256u) - { - uint kk = e >> 6, mm = e & 63u; - uint gm = m0 + mm, gk = k0 + kk; - As[e] = (gm < p.m && gk < p.k) ? float(a[gm * p.k + gk]) : 0.0; - } - for (uint e = tid; e < BK * BN; e += 256u) - { - uint kk = e >> 6, nn = e & 63u; - uint gn = n0 + nn, gk = k0 + kk; - float v = 0.0; - if (gn < p.n && gk < p.k) - v = float(transB ? b[gn * p.k + gk] : b[gk * p.n + gn]); - Bs[e] = v; - } - barrier(); - for (uint kk = 0u; kk < BK; kk++) - { - float av[4], bv[4]; - for (int i = 0; i < 4; i++) av[i] = As[kk * BM + ty * 4u + uint(i)]; - for (int j = 0; j < 4; j++) bv[j] = Bs[kk * BN + tx * 4u + uint(j)]; - for (int i = 0; i < 4; i++) - for (int j = 0; j < 4; j++) - acc[i][j] += av[i] * bv[j]; - } - barrier(); - } - - bool hasBias = (p.flags & 2u) != 0u; - for (int i = 0; i < 4; i++) - { - uint gm = m0 + ty * 4u + uint(i); - if (gm >= p.m) continue; - for (int j = 0; j < 4; j++) - { - uint gn = n0 + tx * 4u + uint(j); - if (gn >= p.n) continue; - float v = acc[i][j]; - if (hasBias) v += bs[gn]; - if ((p.flags & 4u) != 0u) v = max(v, 0.0); - if ((p.flags & 8u) != 0u) v = 0.5 * v * (1.0 + ferf(v * 0.7071067811865475)); - o[gm * p.n + gn] = ET(v); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm.spv deleted file mode 100644 index ff9d30db8e75142a584e4d70c68a3619a65c7e0d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7040 zcmZvf53HAE702KEhYNJCTp&dRy>THIlu()U&j}a&b4jr@|18`zQ7N&^5p5#kQ zNKF%a_B6j&6Q)6o11?slT=7*eLg>)@7=q*-g!LV^F8M|=XuV1 z-sk>kYn?Q-YHO)FtK+I^tyS|GRkcJxt8wmDt_!jD4s-|Nx_-{G6Md{!dlO?;V-a)e^Kre8@w?Fa%_Z(dd}2>L zW;X33;CiipW2ctbC$(bNYaK0qbBnE&9%|QXJ&dDXYYTR2iG5NlcD>d%Vsb8VFY&Io zwsyVNAF%7Sp2JQpvCm0}J&awiwVxRM<`(yY+iG^b)?e^D8^*tvD%R;G+IaC8kb9ZX zly)YvYKq*4$gduO_#DrQZglwOY*x#6#U9;^T0S$j{TQ zuus)~$NXK{7#Z@9@zf%(A6u^)~bv9xC(AK zqTf2&Uo2emUX7h|&M!Xm#D6{5Cv}&A-N(uN{}#0GR8D&-T29;gx1yZ^`Q&SWTWaSI`O&`tnTc5ML9{;CSZ@>By~%rC?ahTdg;VYT8{Y%oiPm=AZ_wUJ za<0?%K9SSEr(oA=??YQhJAHWuT(wk{b&eFDb-YWoUH2~9oO0ID_U@3&e!IzG?&LiI zyWXEM*u#+cd%z=!H~(a?{q`NWkK9ACxUFbw%2`+2dqpmJcVl~ZYkSVU*gON@ z(?0A&?bBykCwGu_ykoSj^E}#|a@NuIj*&Z!tbJ(j6np3QqkNou%JbR-A7}G0jDOqA z;Y=++;qw^5?6~t)IDO^7~>n;Gh&TlpA=7Qz3 z?n1C!eceUae$VxF7h}usF7+-4%lTdJE!-7_n}>D}^U>EJezR92*~7J9`K-GREa&+) zvCey80V3}nj?lk_X#1#si*9+R3~#L}zi)fH0{JSkwXDAhtl#~;P0zkk{3zzPz0!XJ zB5z)?bruy{ojOxX9b@i5zK*2M&0zhhv$*(C%{uyxm$#017PI5~z8!HE+^0EaFfDTa zjakk)JfrpWHPH3_nNQm@n^&DR_QIT5liqn&*W6Xss6C_e^B&^uwK0pCzt!lx!@I%0 z!!=0W;aaf#meTL3qNjeZO<$2!D#*IVOpw2w86 z*^WGgSVNp|{|T^rvqsMOBv{_s&gmZXUSt>IIrV$i4)pI3eeTbi=C@AWmUq9#e~|vU z#yyNcjJ22h&`%?q5#NvBfwuAH^8K4zoqakN#(MV}`|#usJ-V@vgVVztBF` zFy;^iMmcS$;q7%sqG}W36p$`k_wjWMZ9n zd+a*%WbQ|^&fiSVk8`hWyt$k|&!x^8oLgg`fios^z90J_J@@yP;_oc|W6|bN=Q|k7 zdhdGIEP@}8WIo;p*6;5!@7Hm~PiXU<>OTpQH?P<_y~S2r=ThQNCXanJ<`c+7B!4$Q z0M>sQe(Ovse#~Ya{l?2%N1S(i8rVJd6B}+CoWDtyAnDbI;f&R{j2hnU9|c=CF{fi2 zC!crnG4N($^<^H;0LvRQ6yJ2PzuU~^d($??yq?|j>T7`O{qa27pF(>sb^3J*@%GD@ z&mx~j(yy6d{pr`4#ZP#%U;2%gH?KIoIU78DT#SXINkQbpcpD{hAB*cdfbXm$otKm*>^jU{Ce_cpmMG(4I@3e)SM-nroSs|* zc81(%`gSe0d}0=W?VB|c^CfKgJDIf(3Vj*uI{EyzydLatOLN!PE`-z0DetQ=zvf?MSIcxfFMdZyZ&bPS&Jow(a6K)P`_2oNU3ARSgx(eGkd1qxl@!tXu z{yq9_IQfT&@i*fduxsW0d%YIzxw^h_1}xgTSuJ!{180&o>~ugA&d2;KR*UrBmMaawsG?5&;8)RzpH)-0f9@^09s%pixBMvB{PONAxzxTV?}`!Z#TfT$ymj(zYzO;o_%|Wn#^Z4EejDz2 zC)me%(YFJUbI;j)3K-)Z*U7mapeKiY-627WZFldE;*9 zlRtsw{r>V@z6jor_+9Gv@28x(>-U29gL(X2cmV8u&;T2gyVSc+`!=+5tInE(VDFJe QS+lgnEJOc?j(!sPAN%M79{>OV diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot.comp deleted file mode 100644 index 3ef44c9..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot.comp +++ /dev/null @@ -1,89 +0,0 @@ -#version 450 -#extension GL_EXT_shader_16bit_storage : require -#extension GL_EXT_control_flow_attributes : require -// GEMM C[M,N] = A[M,K] * B[K,N] + bias[N] for the conv_dot devices (wide waves, no -// cooperative matrices): conv_dot.comp's register tiling for a 1x1 "conv" without -// shared memory. Each invocation computes TM rows x TN*4 columns from f16vec4 loads -// (n4 fastest across the wave: A loads broadcast, B loads coalesce). Requires fp16 -// storage, !transB, K % 4 == 0 and N % (4*TN) == 0. fp32 accumulation, K in order. -// Push constants as gemm.comp (flags: bit0 transB (unsupported), bit1 bias, bit2 relu, -// bit3 gelu). -layout(local_size_x = 64) in; - -layout(set = 0, binding = 0) readonly buffer A { f16vec4 a[]; }; -layout(set = 0, binding = 1) readonly buffer B { f16vec4 b[]; }; -layout(set = 0, binding = 2) readonly buffer BS { float bs[]; }; -layout(set = 0, binding = 3) writeonly buffer O { f16vec4 o[]; }; - -layout(push_constant) uniform PC { - uint m, n, k; - uint flags; -} p; - -#ifndef TM -#define TM 4 -#endif -#ifndef TN -#define TN 2 -#endif - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * exp(-ax * ax)) * s; -} - -void main() -{ - uint ng = p.n / uint(4 * TN); - uint gid = gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * gl_WorkGroupSize.x; - uint mg = (p.m + uint(TM) - 1u) / uint(TM); - if (gid >= ng * mg) return; - uint g = gid % ng, m0 = (gid / ng) * uint(TM); - uint k4 = p.k >> 2, n4s = p.n >> 2, n40 = g * uint(TN); - - vec4 acc[TM * TN]; - [[unroll]] for (int i = 0; i < TM * TN; i++) acc[i] = vec4(0.0); - uint arow[TM]; - [[unroll]] for (int i = 0; i < TM; i++) arow[i] = min(m0 + uint(i), p.m - 1u) * k4; - - for (uint kk = 0u; kk < k4; kk++) - { - vec4 av[TM]; - [[unroll]] for (int i = 0; i < TM; i++) av[i] = vec4(a[arow[i] + kk]); - uint brow = kk * 4u * n4s + n40; - [[unroll]] for (int j = 0; j < TN; j++) - { - vec4 w0 = vec4(b[brow + uint(j)]); - vec4 w1 = vec4(b[brow + n4s + uint(j)]); - vec4 w2 = vec4(b[brow + 2u * n4s + uint(j)]); - vec4 w3 = vec4(b[brow + 3u * n4s + uint(j)]); - [[unroll]] for (int i = 0; i < TM; i++) - acc[i * TN + j] += av[i].x * w0 + av[i].y * w1 + av[i].z * w2 + av[i].w * w3; - } - } - - bool hasBias = (p.flags & 2u) != 0u, relu = (p.flags & 4u) != 0u, gelu = (p.flags & 8u) != 0u; - [[unroll]] for (int j = 0; j < TN; j++) - { - uint n4 = n40 + uint(j), c0 = n4 * 4u; - vec4 bias = hasBias ? vec4(bs[c0], bs[c0 + 1u], bs[c0 + 2u], bs[c0 + 3u]) : vec4(0.0); - [[unroll]] for (int i = 0; i < TM; i++) - { - uint row = m0 + uint(i); - if (row < p.m) - { - vec4 v = acc[i * TN + j] + bias; - if (relu) v = max(v, vec4(0.0)); - if (gelu) v = vec4(0.5 * v.x * (1.0 + ferf(v.x * 0.7071067811865475)), - 0.5 * v.y * (1.0 + ferf(v.y * 0.7071067811865475)), - 0.5 * v.z * (1.0 + ferf(v.z * 0.7071067811865475)), - 0.5 * v.w * (1.0 + ferf(v.w * 0.7071067811865475))); - o[row * n4s + n4] = f16vec4(v); - } - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/gemm_dot_h.spv deleted file mode 100644 index c20ca7c8d745059e78b2d4e0a9e8c8d384bae622..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26404 zcmZ9U1+-n&)rB8KNFYRTCyx+{yF<`~AOVu#v_S9#r@?|IxI;=?tQ08_++B(Xch}O= z7AX?k;s3suGv8SMy@NB(Tyw6yw#|Ltej(6h+5ywGb(yAZhPD~o&g{|F@#)t#ZQC>; z-y{09p>^+whYs6p*pg!p*k#G(S6JG_-fdkv^3CnjHeFjc-d%b2-f_%63LEom!PA%a z@|0>@i^qJQp|m%imS=rhi~S6k`s(D&m9s|jY$NB8!#NtcUOYp|w*MJ;=4oQbVmwC^ zKgzp1k2M_2W4x!1rE3Q!?8vpgnjh=?G5-nPsU>b#;A7v$3v@(N??8Y>!U`uw%ER=uHn?U_O%;=eYswab9dtOCH9lP64$=Qkdt|d+sU`ReH+)lo+7UE zdYU+WiT%vUQQLQ+J&2wcXVj4>X=~YiGO&@#$Ai z>}oJ2s~{Q+t2tWcKoM;zbv^(aoA)7a+sIkR*fT7bdG}n5yc%0~G;eFjS=ZR}nepw};OGbI&;xD_du%`WZmNS0O zsm0O%0&@Y}%_tyRQS&{SZzPZMJoZELW-FSaX^fvi^egx~OcZk(} zuT7TTU5`h<59S#}CHH?QkNolE7;n&U>+`mqo*d4jZ>^hlBN{$An>9I(H94Du<&(1| zSpU{KTM^42+w{LR*mm;awgt<(9^;$5-+~>3TIRPs*!g(|GKL+%@@cmdSWbUr&N~y! zkK#$6yMtW|wd?59@4C@E_N(Ta@4?&853Hry7#=y-UHm<7_r+NLq-N~v!)~R5*)H)licWSM3;8IIHwax{rrPg_1xzxH0tcPl? z%i&T>J+&r*)l%yUuw2g7>&)3Yqj-mV6D;ppPM>dsTC?bXtDaj<=`&7EzRgmbQHX}c8I_G)RnEZDYPO55e(oQGQ4t^l^Z zS~$Fbh@G!m z+TIJcy;|Dd2exgU@BPHiS1oNH0NY+IoZo+bAF8F^b70S}pUn4pV&|)tyce6iI^UPz zoUdB?co}SawX}T|Y#(*LuMs<6wX}U5Y?bX8l1or2oTE1(#^F8c$!APEb7fc72?*jFF7xVzD<-4FKST41C zf&K2QwPu7%E%nsu4OUC7K47`jni=f(T&*<=TxzMO)~sN))S3+}ms)dy{Z6a3=7vix z_0*aNtd?5yg5~0cg}~0+@3wG@5Ig>b;Ih_>8uO?nZwatttKV5m5}T`*yk)@lTfd){ zB{o+reRv1*PNbH$Yl9u%E$nf)!NiX5_Tq&h#OA6cZ+)=itKTsj5Sy!(ykTI+SHDj- zCN@_s2PQK=C+T^WY@`l67o0t2@DB^vI&GUD|U5Wj8FSh?(cn;(7 zUM%)I$v;oni)ReacH~;ad9g)pdpBCvXJ2CVoORpo$CI||X}f=;rR`W^+m5Gh`aOU+ zZPnBEK(Jcc9t^f^PoA_r1e~_&X*&+AHoW;==}@rqa6HykcW+%A*TMCuv3!2dIvFq6 z-nE*@xXsP)A*X@;z0TiLUF$Q5{kYa@r}N0!hd91F2kgDa+O|8NIPKi~^LXTJC-!$n z?|m15`KN7R9&0!r$DLf?8TaYbHphN#XC3>xh_@fdsDCcxk+Uyxeh&T(Yz;rh;a|OQ zDV%@Kq!#XqCic&p?r41eAy~dQZLXxHZ9gKG?+2cVr&q&&497o=W@?+%{Sbb+_`zwliLk!P4$egJJ_{S zOI{DKynB)PUJW)+EqT|#xz}p(!*yW&z^gOwdIJ5RmbN!FarHxA+R0Zx^n;Uc^~20f zANs*IsXHsYeo)WYX9GKSwe0=u4W|}vPOyHE%lga(mT!koAM+5)TO)nU3zn~bn4ehQ zywvLtmal$TfLOlP8$c{y{VVX$+}*cJtciysz)V~INapq|{t!PZpI_?7^> zR%*#x3M}tlWWKjj&^)!|-3I4gtHlp@fb|3abmrX&rytbP_MRrLeprb{^3@M3!^yY$ zVO2O%JH8)mle(+J>j(9WeGu5Ot7Y%kXgIZSYk~EHT-Il8uzWjw`dF7(-Wus+Fj&6& zVLf7b^HOgpSibsU17i7FZ$o1F>W5*(^5(@4n}VHd#x?>RE`AsZ=UVywpq|{#z}8gH z_%;W-R%*%H5-jgtWWM)N&^)!|-4EwptHlowg7t&HHzn^OIQ^g&ZgLY>KWtAs`Ra!q z;N)BVup=C)9e%J)>h27$AJjAUUBHfAEqlLf!>NVa9jqVZvOc51^6l{HV-I3^Yow1o z!SdA)dlSo>mwNkv<*Og|BbKlA_9vFFei%zEZ(jUx5ZJk9Y=?lu#Si1)Tr2&cp4>yh z)>O~<4gJTJhkLK0q0(;<<9ja*qx^KHinouHccgo!B+;UUn_mKi4^#w|A@?c>A&4M%2HKSk89h z+~=+bBX`{AtT7Sn*pllzYjh*n9Q(DMb?mEu8RKw@-2^XZU*eq4o59xblYP5|*z>6t z?l!P-J)gG|%h&UH2eEwX`MeX(Ir;P3HmQ3T*z>6#FWn8+OKLfb_cWYZxO>5#Pr3N@ z?_l|M`1ElfvAi|X$NgaWdOjZ@mNzf;9t6wR^Z5|5e69B|v3xzBj}Xh7m-9Ip>|8Up zN5SE8J|BZ~tvsLV$$cDbP4$fL39xIWmb@py^6o|E`xMwbwd6ewcCXdqhiAb0!Tyu? zEU|u23->(OxccD*V)^Qa7m4Lt{qPc;edq_9`!U$K`r#8|`Ra#HiRD}U@EKhEV4KvP0@e@e8T;p8$F7#W|Dxg4!hH$W z4{}+bufX!v4__0@TO)mZ1D3CT_?B4Sywv*+EMNWbJ+XYP_XDwf^}~S70T>PM(++V@gRL}VMQP8ngYRQ|%pA9_jMdmwglc$!vE@1asEq>?<)(^Zo z^STk}2eq`F4s2Zg(1TdM`k^PWe5)U(hl?LvAlVyH#1ni`e7Dg`C4yQV)^Qa*@)%MiyvkOJJ-yA z4sf{mVNSUCK|Q&1fvu^Y@y!i(t<;h?4_My4$b9Doo2Qn%`M~bATJCf6gZ(+;cZYs4 z$Gb}I8oqN^X$R{a?=QByiFfXE3lMw9s`t48aPkuvNA7bA5?d$txq;x^=gjv$Cug4H z$i2*W?sLn6{refY&)rDu$9tLDay)Y0mBg-z_qmn9{(GyF8Mk+=)rkG%KDP?7obAN9 z&#ek}fBf?~YfR+r*pllzYqUDp9Q(DMb?obE-hS-M``jR6Ir|dle69huhM(-)n#7(@ zwQy^LjqCYbhgiOz&vl9AThHfUIOpWgZ`-8q5U}S{JziQ5te4bs7Kb*RTDbMWo=>^> zbpx<`JAC@skXYUt>0=|Xd_AATh~>>oy^X>0^?YtZEMMzwN-ST`=Wt?q^Kw2%fSqf` zHWC~z=W{bS*UF#Y>dD<4Y)$oyZws(%rIx%c!Se1!=DQWxJhkL)4R){9;)iX(`oaE_ zw=J=LPz(1vuyOUncEs}458D&VxB6iRIQ!5Kwn^PlVEv$;vF`|W>}uKjof=Lp+|FSA zAeZ&o1uWkVpFVaambXUw*bOXS{jfW+ym_fN8Z2M^FosyZ*4u+vzWQNLVtMo8hrPhg zHDlWw94>y?2hO$959-O?7i>-SjBh`%Yo(UF{lW6?Mdtf^uz7098w+-?)#8T(!1{r& zu+F>#iS>h8+8zuxu6{U#SibsU9I9}b0!A8eDlhk^Bjdd7Y@*s-f+?~iCWwQxs* z^@Cj2=P0mz^}~2#d26JP31Io^hojYb%uBsv!1C1($Ex$xddCsVS3ew2EN@=?a01x5 zX8tFF!^ICL!Nm{i$vqitP4$fL6tHWhmb_EJ^6o|Edm7k0wd9=+cCXdqhcm$Xfqy#l z&Lq$eYH9liuyOUnS;X?y4}T<sOV<*OeqAeJ{T_5KQ$uYS0YSiaV~h*-Y*;bLNW^Wuj~z|J-E z{~I`5{BS8;{GguP%fQxD&-g9}yH;w+n*^43FEZaNz~-qX?@F+Ht(N=TRp708{O-^% z=6F}Rnzwfi-?^(?1J*mM@z`ze&ugY}YH&f*IVrxxx-u;)`Q zetiin-wvNXUM7~eM*4UKEML#(tHko=rQT~``FcKICzh}E-XNB*=krZsdGm5U-vT?= zjO}f3xSY>-;9M(zeyb<$XMFF1T`RTZ{Sz$jUSz)i0-L9ny!XKFwOah}Z?Jx_ z|K$CLSU;$Rdmn6E{qO;?eD%YJ#PY3v_z2EE^n-0u_kUphpq{aR40h~l+51l#PA%N0 zVErJM_4y1e-wvNXrVz_pBYk`hmal&Jf>_?X)cX=FU;XeEv3#xfHL-m4!#Bk8=EV=+ zf}Lx|_8mA}{O~=TYo#C5llueMn(7(fk6_nIEqOnI<=u|U$I55I!- zL%q+rY5GAeZKs7Zu72nOCtv;0l~}&j58dG62iv4>cd&j?&)BB}J9f3~eUFAy3)d5@ zALOz=(}U%!A7&tyw?_Kt1(vUVn2}iCywvLrmal&3Lo8qG^(B_Ce&|OmZ(jT`6WF?nUN12iQEd(H# zhSLvfX*)02xcXr}V)^Qa`HAIQ{m>sSey~mIwuAM9dd9v0*s-f+?*}xTTDS$l`av%1 zGY~9a{jdYA z!QtYErQqTR_2e!Mwx)WlV0rf<^IZ;Xo?7yj2fNp5xzDWt_UDM-9s0!_ z?<%=#_|9ErC9vM{{$e}tA-T`3Oza)2-se_57uP2q%H#dXx3$eNR&zXilD7eP3~&Ga zuQl^`12+V#WegjE)g43H4kJ!m^|aj>td_Q$fNeXzwB3|AZPn9uI9M%hM}TeHlQE{> zk;G}Mp0=BT)w0){gPn)tv95Yu2iGHey(QRu*JdK)bj@7XcX<18UH!X?qlo2PN3s6% z?z}75zXNR@+l}V!CwJ%Fh~;c2&fR%;Fl9UL&epJR$CzB-8RHnRIreKi>)6+8y#3gh zcjrBbv!RP#PY4*h5N%fCte+GQup@| zeiy3e+>Ql%Zq@R=cR<6bg*y=JccEOq8x8`?x5KB8gNfy>kvSibt<&&2Yz-r2?66*)Gw7m~(T>WrAv3&Kz1H|&Jes~Zr zey~mIJ_Obe>KXgPV8^bOy?>j$~4&!b@Z>W9aO<*kuE9tX=;KRiJ!Z(iy> z36`&Zc#2rQ)_a;*zWU)AVtMo8hiAdgHS>QC94>x%9xi@RPwop~YpQ2_FM?ewwdB17 zmUk~Q-gC}>_H^8lb zSMg0a`5`>HJHG|CPVUZcgL8K_-@CJ%d5+^9-j2g}#~rL2lj7sOXcAv)AHk1?Ge_+F zy}Nw|_P-C?vT64PZ$G)aO(B-Eoj7;5&%y46f9`3GiM$<0a(!n#z66_NzqYfEW8A5X z(YxDM@N)Jg&RP5#Yz;rzqi={ki)!J%0~^<~_&u?FJ&QjO%eS7zAK{!6uZ}jU`xAs` zQ9Zu-8LV&Aa_)XW5i~<*OfNC6;gX!)$Q&L027Z zQg?QE{h*$)&jEIhYT5fa8%{0UTwwhmm-U$&EZ+{FKIS2ow?_Jy7c4)p$MZhS3fL3EN@=?FaYdaGqwf6;o^sZaITepP*3hcU~8&pe7^y^R%*#x z7%cBzWWI}l%~MO>qG0!0Eq+)GtRL##ZE-mLpq92vf{m*mmLis~eps4VzSR%Qz{L-? zN!?|^`awNoUk>cp)w1`?H=J6y6~OvIF6*-*SibsUC1QDNq>q)s@&jp~v8+NYZ(i!H z3YM>aSdCb|)?1xezWQMhvAlWl!x~`cnm*SAhl?N9f{P#2le;$9n(7(fI$+mIEqUvL z<=uB8YwQ$41`av%1GXgAM{VIdRu_y zs~@%`map};B9^ay*qT`0y!c@ouyakH+k(Tz55I+rAJmikJFqp?GrsM>u9aHywg=0* z7n$!4VDr?HHwx@ttL5&tBe)Nb-yQnJ9PcVS@%FCaJ9m|x!FtF0i|xFJR8wd8k zozLCvaAH5XyB$g_XFGB3Zij*03-4~$n8@34B-eM=;|Q=h_G>%qIL428`>`+YZbuT! z*_SwH@hGr0{A7>D6MGib!W|7Zu4nNWV)=R&k0q9GJ&VV|IVZNOqfP1_5ARu2k8e%@ z>l?M4yAvBuE!;_9&!Swsc`{hO-J0x!|6T4W#PZfiAE$!l2bR3kh~>>oz0<++^(>x2 zEMM!LNi1K_;zVM3^Kusd0Cuh!+gadnIg5XUbFKVYt)ARJfvu^Y@%TB#-PY_R;X zJelt~VDr?HcP`kyR*N6b1M3I-Pu}^&`avz+1z_XqhrbfbS3g`xEZ^#fi{R`-KiDR9 zF9z!e^^E-zuwz%t-v6!P)WTf~)(>)7pUc4V?eOX2a$o8p`GFW91G;s@KL z?mb}rpq{bc3wG>k+55jYoLad1!1_Th>vKO?zWU(-VtH$%j|ai>18JYJJVY#SUg|vz zmal$zgjl}Tn@lWU{qQKUym|4%V_@f+J|72%iyxkViyzdJ`y|+!>KWftVAo15c~67o z-HXik8L)Y3$$J*;UaQ3q&w=$ry}Lb6tRK|U_C>I9^}|cV^3@M76U(>y;T5>}!8WP; zDp)_LXY8+m9lKig{`H1a3-<>oy?=n^ zs~_Gamap~xNi1Le@GoL{^Wum1z|J*&{u>-Fe)tbu{GguP|AMWlp7Fg8cCFNs_W@Ym zy~un&1e>RpypO={wOa0O{{x@P<9COCF~_^g$Gp93_|9GB6R_SnoX2)Y^5pLJDY*6T zAbtiXU+->Hh^>>m+vnii-OTsyCTE`G_=30N@a?_ma4!UWycgYziGP8YGe_+Fy}NaR z^ZzT9yIc3t&i@{`E3usI#JRh51G^XA-K;T@x8q2z@2tmkU~}x(cGhu>2l4ikyIT)p zIr|dlEcOIj!%y~TdScI_TDV?d<9ZfnB$lsdu{W`N>sjmr=bZdmZJX5X3-&Ck$2a}J z`bI71Zl;D)3pX>^vnUsD&H|Qihfg1~63bg7ear@yA6W8cCzdxa_2vM}*Rwb$v3#vJ z7qNUji*pmpo0qdV57@b8Z1aM{2g|z`nePH% z^VE_z0PJ3?#SaUD^@IHRLPz(1PuyOUn!o>2`4~r1XxB6jGIQ!5Kwn^Q^!1_Tw zV_zKX*wwQ4OEjEXxFx~*K`!gF6j;6;K7A}rEN_kUu?$$g`e9jOdGk_lIk0^7!}7%P zwcZNE^3@M363d$xKdc0Ht{L0P;BfK7DsZlqeo#;Ds$grXXMC%HT`RTZtqzuVFEZaj zVDr?Hw+7g~R*N6j1nY--cUy~CKd7bcI$-1KhjoeNs~-jv%eVSr2wePNo77zotRK`f z_Mu?Mu9m%Dzv0xvZ2;B}a#^1Z!SdA)8xhM}BYg}5%U3^aOe}9+>TLp+uYTB+SiaU9 zPAp&jFoIa#y!c@x*tur@n}Nf{51Yfq59-O?0&Gq7jBiV@Yo(UFt-$i`MdrIT*gUo5 zZ3A|%)#8V3!TO=z-F{1~AJo!zJFs!}!}i4T)ek!m%eVSr6kPmZo7CMAtRK`f_MO0v zT`haRbHk~H+Xbv2bG)k@#M`@u@7z@m2J4;fJhtn>le^m?;MTuyI1Wy}-rWu*wodMD ghkq6~~|5g+*iA1yn+Vm0hAVg<(?6#^FGym0{&${XdWPfqpZ_`coO{o6 z?|q*g+gc|KtJ+$s;nh2xL)f3_Q1M^RAQghie0aDu=vd_wpMzmU9WXN4r|#D=lL`2)DruoR_uDM z2Z_nL#6842-rCyrTD!38>v{n@wZuM0Bla+Mz1EAw=r^~x2i#V(>$U!d-@Re{5md2G z57EYpM}wTp_@)eDM^;UdJAwS_PQ>R(Ms%UWH}__>dVB0q&8XEgV%v|Q+wPbiyBoXr z57!U+o%dg=ZG-ki?N=Tcw2#N$u=%FJ@sqIIufBHrQOs)^cs%0sF6^0TIoCBFdltF{ znTecM?Ae8LKE|G2Y~$w?EFXU#cxKH$1MHK0o;zzep9Sdg#df^*Ld5-)e4hr}zizs^ z1Z}OlIFEDSyhrp~NBi8uCGUCIS?AK?Gf(^%fqhcv-S-3t3#&Fu&(!0*6;nJZQb9Yts!S!ZSNns6UjOg?R{e} zJzL5t++nWYp7^+LUx)E`&m8Wzm5A%swg;1tx;>}ZvlumpxDO+<5%W)D%rx{z5cxI4 zXwNNNA3Ebc26mk1IOFDlaSZyDM?s`t_@@0+gHs`BjH+cn6g$d)qx z3t;`uZy!Cor1(+Qo7nmMSpQRI`qLpUfhm& zp7e#=40e6yNMG&&+ZXp>`f@K=KJ(oIHlO@|dB69gee8+8`w%(jD$cl{gRN;l?NegZ z_NfcK6*0#xi1CiM#xK!6)-Yxp@+4vnanAmuVCQCytn)Wud272*`_YdfI}q2Y-?e&Q zK91;Hj@^!aqHxaFIatrSbz9!~8$X49I>tG5BF5TVf3KcK?m~QDo(pZ`&EtB7 zpBTFnZH)WL825qpbH!F?pN%7aKDIGq*z+%-)BoRt^=F^$!q)F&U-dta$XiF8efARA zvGsknyKtM(ne!g7_wVi+_jj=S%pBQg{{Xws8c6opKf&^u@4vw2lXnlijP`M^`u>f` zIahJUy$ZJG2E;xkMxA~38rU4J(RjyOV;|bb8piBJ~%)844%w{IVC@u@7Qjgy_#+90u0s8mwu4zdMq@18g4qll+}<^7bwHyRegA zfAWt2>#OItjy>`$yq|aG99)yJ*3P*Y1=jD_^hcd@Fd1Jbwmmo2yg3J>vHiw!KY0$c zjW?J3(RHe`Cik(iJK>CRzZ5+i=WWuJJvtW7{t6{>r5=R+Byq}??1bZ2jCWAUxlPsC&3x3ZzVOn`=^4fo0wCujg!wi`5@T4NuT?}yX;i3 zyfMSDKLqx>&Ro7XZDY*q+Fh@{2DsiI*P}fR?Yh+I*97A2moc9}rX%Us>0tfo*NoyP zyxA}P#><;moZj?-5AD}S;TAKt-mf#@jMbNZ%>!FE{W=rdIQjJJ<6!r&zVz!XuzdP; zHh2zVF8ifzO#0<|^);ANy+5u;djZ;Ysnf6b5O2SXS&n=PNxv3@^`~Epil6Xizw|Fe zLQmLREfK3IS1oKyU%qu+RW>xk2n&w<@T&NF@cJhpsdmVxb? zH4<|%w*1%FYwpcUz>bs8x96qcmH5nEAA1?RzD5~)Iatmdz88IRc`vTl@eOj-`{R1F z?YZkxXN>Q{x{Wd>@6KL3=2*02)Y(^8g59HyGA8#{H~jy**VYkd4s~*@AlBd3UbJ@uyYb|un;SFGGWUcG5jgxn;EG2#e_|Wgw8{y>dCZ>y+Z-5;u z@9+1`Xdi#S_1%QXIZv_uaIM<%+8c|lPJc#~{ur|dxfMx&z6;i${(QUmQ&<1D5P9o} z)1U8y551>0!TDXKFa7x;*c$22kFbrCPk(L)ANrm3V>tQr=MJ!AofO#=;U?}Zb9VCB{tVq^e>PH5%-^SORjCjm$Up3 zw)y3qS8}O+Pu>+rG8bc*$H301ADuZp4wlcHo&@_iCw)&Ka?VMdd-7@U zQ;7A#JzMN&&|AwHc^NG4oHx@o`}PX9{7CR9q@(cvf#V;Q2gW4tYmj5$^ymHT1v^&X zT56ehU*YtHdmY@3PwuNF?hUYf4}5C8g)MK6)YuP}pID1~8(ZGEjcaD zF0$U|z{crI-1Bh0Yklsy>;-l9+-Tz62kt}T-6y$oUIKeZdI#h_`6F1~^XGTJe-r!( zY@YmW@n`H85zndd{+`MiSHCB`FU;$A;vTT~Lj!EgMr`-3ccFHp*y@b&_s4ssQO4x= MYBl@;I(#bfKZ?r~bN~PV diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.comp deleted file mode 100644 index 09e4b94..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.comp +++ /dev/null @@ -1,41 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// LayerNorm over last dim: one workgroup (256 threads) per row. -// o[r,i] = (x[r,i] - mean) * rsqrt(var + eps) * w[i] + b[i] -// Single reduction pass: sum and sum-of-squares accumulate together in vec2 lanes. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) readonly buffer W { float w[]; }; -layout(set = 0, binding = 2) readonly buffer B { float b[]; }; -layout(set = 0, binding = 3) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { uint len; float eps; } p; - -shared vec2 red[256]; - -void main() -{ - uint row = gl_WorkGroupID.x; - uint tid = gl_LocalInvocationID.x; - uint base = row * p.len; - - float s = 0.0, s2 = 0.0; - for (uint i = tid; i < p.len; i += 256u) - { - float v = float(x[base + i]); - s += v; s2 += v * v; - } - red[tid] = vec2(s, s2); - barrier(); - for (uint st = 128u; st > 0u; st >>= 1) - { - if (tid < st) red[tid] += red[tid + st]; - barrier(); - } - float mean = red[0].x / float(p.len); - float inv = inversesqrt(red[0].y / float(p.len) - mean * mean + p.eps); - for (uint i = tid; i < p.len; i += 256u) - o[base + i] = ET((float(x[base + i]) - mean) * inv * w[i] + b[i]); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm.spv deleted file mode 100644 index 6f5d4a94cb4382580ce8a26715fb6495f8e47e44..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2932 zcmZvc`D@Q{9LGP~cg|sE%spRID4J{T$Pu%0Hb*HWlOxA!!Vk*kDE#CqKagBuEGD-f z{U!bhXDN@z_xpb7lf3J>*L{9ppA{;n49E%rv`;?VmZC068z*cAdOSENu+xn)? z6YWjS^JnWZEGv{8^{L4QWc{F3;OO2%$23%fS}>e=_3|dtU$mgrM~q9dKFZ@uav>WH ztpw}HF7KOCCVzG2QfCcw&Vly-D;Ki4@Z}s;B_1&w1I($-7dcy)sToZuXP`NzSaZes3q@~ z2)Pm7r`!Y0*LsE=wd8$KE988w-efnIytRxEIbZ7ydVh2GREL*-li7lMkRL@3dnp3n zXyidqb@~*M2Sc4j{n8|xQ{9XdKLp;V|4^tlbA-RQr>>r?_0S?{z}LIeudJ&o&21g^ z#<~Z4S7)!@auJxz+REOu&)O59(Z6={#y&5*F`8UIxFXS!o1F08s z8**2&yN|;FbE%&HEUcIB_hioA4e!3RyO(Q;cQ1ZZ%Jy{w>JHSo7iGUGb;jL?n!~sU zP-WvDLiJH+oU-3X#Pz^OoVTuQ-1Fq4&N$`Xq?^gb*t56oUY$oQ;QxSoa$a8n{6ORX z&XxT2E{6ec*ZRxw^!93i`tTV6Ukma+b;#=V@x~?s@6y|gJPk>I0Uz^D0_Il#lRlcY=CoqGj{y3Sxt zf6x4`>~S->)%()fnyi#}U9zTFKV4Wji z2k0fYd)x`Mdk5}uJ5;;RJ!;|q~##Kny5 z=$uiTyLT7+f%=dSz&pD;)_xGE(=Ya-&OJn3>HqfRjP@n{Og{JIZk)ycE%{wKyLWO7 zdK}oNcM{)DJ$why)lA$CItd>)cN$(>#GZjyXKdWuS$K1b z^QlG`yuRApYd7>9@Edlf$bUZR-J`u-09`=+5av{O5mbO8(C$7iLEV`&ZtEkwzvXuD z8e9f(A6MbE$2WQ<*&(GH(XPLG^UBAK+=8D3jE`^hHheR1_u6BgJMh|K4|n0!xtrM6 zJ$Uub75N^+KLX~nPUL$6uPtJq!ruexBlZP+FZeh1B|2@f+dg>X)ko|r_?sZ+@OIVd U8}qwMds+?hyLVT=`DGS?KQ!sx4*&oF diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/layernorm_h.spv deleted file mode 100644 index 38e49d028fbb7269f0facf9e7908ee31e98075c0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3016 zcmZwHX^WO+6bJBo7PWGnEX~qsOsJrkvYfDN!ANE*A#G@bpoA6-6Uz$gMQCb8^{Qo{ z7Zs5ym6f#79`s5260H#Q|IKrqxQQG1-& z57W`Q1Lro`9rAhe@<_KjbU2tvbiA`+K>pQHbu+i-lhwI3-^&5a;FB!uNB%9(hpjXoq*!1ag(!&o$TVV&ml+2}R!2TPl+K-r$Y+G=g?>wu#iYBW`7a6YxW6=%O^mv4_vlt7 z>Z;I*U_5v2oqGe_=zQPu(DOA1wbQB9E>8qv$xXJ;j@)ZQtA2KIYaVB%4C zTbqu3N8|Za{M*BeSMqH5(~_^`J0c%!cC*fvtu_G@2}?K zeUa7mMyMIkshd34bo`Em7sEWg6*9l$q5SChk-e3g=Le0i{N72$@eau1eA@2Oi9_}d z#JM!vqz8L%&iaF^@*iPt`giR1AJHrb-I`xJyDYHR;#`-T_HubZUw6(7KNj@wtc*AcuI}mE9N4Ok>F{dsefzUh^Nz&wUTzD-=j*$8OFIMa z%WSUC29VwRB{h0iFdK+@XQ=*BGdI^C0`E(ny}>=f$;39-djocRVXk+FvODI=epf&* zF226Wec{!_|M%3jE7ak~c7H&}4_|Y5Fnsw{j}JwsM>gNWEFKQ%OWqw`?`F*YNI>Ua z&4tb!DlhtfbJC-_(4P~}oXkcqdxHKf^=@zc4?PyB)85ouvKNmBZ2M9>`N_bpzL?nL zrvi8AE7sHDE7pPV`gHf1fUSI=Z92ZSbBDr<35Q#b=fbd6! zJsPQiGbZaUJ5lcw07&q@V=!PeI2|U)PB4go_##9Td#z&J9dPfOfN3J zcI3_Q-Yoxmvu}l;3e28eKN~~e4%q6=zSDFymv)wnI_ Ne;c#>J6~>d@DDeo=br!o diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.comp deleted file mode 100644 index 96dce91..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.comp +++ /dev/null @@ -1,37 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// NHWC maxpool + optional relu (channel-fastest flat index). -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl; - uint relu; -} p; - -void main() -{ - uint id = gl_GlobalInvocationID.x; - if (id >= p.c * p.oh * p.ow) return; - uint ch = id % p.c, px = id / p.c; - uint ox = px % p.ow, oy = px / p.ow; - float m = -3.4e38; - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - if (uint(sy) >= p.ih) continue; - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - if (uint(sx) >= p.iw) continue; - m = max(m, float(x[(uint(sy) * p.iw + uint(sx)) * p.c + ch])); - } - } - if (m == -3.4e38) m = 0.0; - if (p.relu != 0u && m < 0.0) m = 0.0; - o[id] = ET(m); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool.spv deleted file mode 100644 index 38f34c4a3cbcf16ed7737a540261205502f70e4f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3116 zcmb7_-HVk~6vp3~V`Dj^qm83xIh}F5kP7Q!luAaOz{ZQI$SI@gi)w-_vPA8JrW`|! zAP6EWLIlmQP=`J=0)ytp!1{YCO~ih`bM^!8Z5tl;v!1>7+H0@9-g6qAL-VUfN7Y>| ztp2RpV@cInbwInTuGTy@J~6&-&)%tZ!|U(SV^P&;IqK7k&kfA;z-!O%+~q^S}YpAAA6WjV3Uwvb-jj48L+1~?q=)aKJHDe6WrS_8i|G+pm zv&S>t@44E2MKf=ErU{aVc0X{qe;IS`VxF#B2k#Yo72G#pN;L+Vm$7<3a}($z9|6ZM z?Z2KqTe0oi*yC{fGxh}Bb7N1!>p8xRZ7s6y$7(Of-Uqklv8Um_)hcqcPt9onZXarU z*Tc-dftt0_whz@@KT^1Rjq7|IIC9PjxbNyY_Gr#I1J^drS!QFZd5*R{sAe4R^&{}Y zv43itd~02;y4aKYsEZ9$)BE`j4&Uc8iZ%WR>%1c4D%^8^2Ch4g;$^=r0_N`eNXetO z==#_T%RwKoN8;l*s~(?~aDD8P_}mItKS&(!=Wmev)U$5@?mqh|`)-G;XI~TUUiDdG z4Kl9==Gg?^bq%xQU+&=A5K!~J@+-{NLtFi$lG7b<4*=J!#|2>BT{{X~OI;p>TbHdMbs2-Je_v{_4Q>qezq$7) zvoRdj!SyXb-Mreo{iYA08Gjrk{=;zf)cp~-cT}G#wRse-?zz*P;cvrspq|_xgByDS zB=+NQ_3YUJSNnjt#&`l4M_nK5^CVDD+^68iodk*dG+aG>@eJI3>h{HR%ns|}+Ot5- zddPh@Yo@KP{X(&w^Nw?H-%_97!Al_T_#51JbnPqPT5{eAH)nsllJhHY^}OS&aAT3s+CUqcgaQCVEjt7_>*2A?Kpk_VfzMD1E zR?po(hT9Lu$=yGJs~bOlpTgDs9f;p&aCLtpM(M~S%%21QANqmo=I?j@1#qpWeD`0% zJHT2n2VAe`nEr9E{WJ$=fcwlpIoMCW(Ilw%=2WrMyUyv=i*S3@{+6bBo8u**ejK=#xBMFJTb=-U%ad^R($H2?qr diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.comp deleted file mode 100644 index 649b334..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.comp +++ /dev/null @@ -1,39 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// maxpool.comp with 4 channels per invocation (c % 4 == 0): same per-element math, -// a quarter of the index arithmetic and load instructions (bandwidth-bound iGPUs). -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET4 x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET4 o[]; }; - -layout(push_constant) uniform PC { - uint c, ih, iw, oh, ow; - uint kh, kw, sh, sw, pt, pl; - uint relu; -} p; - -void main() -{ - uint id = gl_GlobalInvocationID.x; - uint c4 = p.c >> 2; - if (id >= c4 * p.oh * p.ow) return; - uint ch = id % c4, px = id / c4; - uint ox = px % p.ow, oy = px / p.ow; - vec4 m = vec4(-3.4e38); - for (uint ky = 0u; ky < p.kh; ky++) - { - int sy = int(oy * p.sh + ky) - int(p.pt); - if (uint(sy) >= p.ih) continue; - for (uint kx = 0u; kx < p.kw; kx++) - { - int sx = int(ox * p.sw + kx) - int(p.pl); - if (uint(sx) >= p.iw) continue; - m = max(m, vec4(x[(uint(sy) * p.iw + uint(sx)) * c4 + ch])); - } - } - if (m.x == -3.4e38) m = vec4(0.0); - if (p.relu != 0u) m = mix(m, vec4(0.0), lessThan(m, vec4(0.0))); // as the scalar m < 0 test - o[id] = ET4(m); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4.spv deleted file mode 100644 index 59aff0e3150a2ee7f619497d8b4a54a9025b6927..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3332 zcmb7`-HX*#6o!8@M`x;0N7K=gW;)|&37Hg>1UlhhHGRy5CVf+BMI~r~8f6(yCY9X; z5m^yJftndR*RhjR8>d$8*!jrHYp%V@6?2+aXQ68bSu+zm18m#8o1zQ=3LKnZ@bdYFWS;dyP#;xE6sbmcLlH(FJnE|+$O2(E7)!=W^_1ilYE?W zxZmOWSFz3;OFjx5@t1Vnj{hmVKI=35oF#W$47_Lj`m9rB-q>=_@?GNBXMKt8U5z)N zQ#@ysd|O!g5HZZ94IGQ`t)ZH0+W6Mb`0A^RZ%nl-%6{M3;riKF=ZrBzm)cA6{{!P( z$R5vhzvt@DFPeGVGi{JO^oM}M{R^?Vi+MV49lUq^VYv5b|JHCux(&% z#2oZ~+L@2UK)aO;@)?Qnkw z@pr*}uVM1Gr_FZ+ZqMplm;G4ZOU=6K+p}uUA1K_t#`TVefFpN23|Fg>9FCydAJ*$Q zb{Vi<=49_DCwp7pGf!eYU(KBK?QJ#Vdgsr9#fbk!*CnUEPR;Cf*QkqCRNA-pY=`gs z2g*$B?|5ev8Gpk)=PTg6c7j#*@f_e?oF6ZF3>Mur_Shm|U)p#61e+DLH3Qo)w6FYybaVpCYEn-88CO}y#I2n<1g;z z+~q*c8p!{|S|fe+{UxU>;l?qqmx^|ErDfeJG<9>xy496t{B`)o*Zx|5Kj+{Rz%zdU zHv#9Y<5}QaICl^@m%7{vw=Vw1QkUD{>L*GKrr^d<|BZX!h&6`8Iyip=P&Y4m;%|T( z-`{NF-wsz#-8aI$qxy8I%^h%c&)v-#euwS^>WO_f+}Qs96Z;;xdiLB4SKCWmW84Rf zqwX5(b3af|+y~*t^*fTdo8ao{i_LKNZ36bi!&ry)aPA?XWEu{*MGG5+Pvd; zaNp83KY}eF@Axg;cXaMu;9PRv1~=!eAUXRFje6ekNw_i8eaFYK#&DRo^N#^_^OPt4 z4!H5RgT&tnSI;{>1-F0I^Nvr$)jcc-EyH{j}vL2C3ST-|TP1RZ$*`xfx;%Mfth{Kv6x1LtOy-~BuA9^h}=-f_OpG5zCS z`^h!l!+qwT9PB6G$eyoz(_YX|?`qSlerL`B``fs#O+3FdXF=kvgIhDd5gYg|uED+s z-0wJ9zBTW|)z^ac-0?W}C!k*AD0T8qKbQH#SYzewKY;u8M?l{GC|vzmIqO5Xn!hRE z{v)izw|DLsP&04&Y4(}V3E;V&nLIuzx|(y=EPJ)7**9h0I=IKUdGpioQ@}SL1lN<3 RntP0Q25TRM5ySuh diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool4_h.spv deleted file mode 100644 index e577d02454dcb7fc50b0f2add33ad3a320febddf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3400 zcmb7`?Q7Rn7{}z!612b>Wf?A$+R8UU zMD~bK(9|rdHEbJVxYx|Ce zotO(2psNw4GKg!%2&(hq!lvKRT+y4T+=#>c79FkDw?_GT+efFqo$oxw3RjO+@cNEH1F-+Rlr(IVLjKFBh>YIY^xS? znjE)EKF&GZ?{NJgtnEj=vS&R`Ykl ztz+hQ!|mDld*Qy9c|V8sTQl!TtiE|KXKo6s)(5wr^{v?pSl?33n(N!oYR(@n+`XRZ z{oe(S+;uns*aLr{_%^@e&0-yY!#(FK;JkL2RrdIN;9Z;_DtYu2-8J^# zQee;8ms#WQg! zP(MK|-{2Bp?#_AtL9F91?&aL2K+PJ+|HRG~Uwx|NbUEBO=JjmRuB>TUw+2n!9I|e0 zO*8%me9zPVTE4B*@L}MYXTS}>xlZgiz_)Pj2yiZSxe0DvHiFdUX1MwXr3RyLW2pbe zy|2d_!(knqzYeIImpt*u;KuiRPW)To>Z$u?xOY^aD7Cp2uI{<}IK$tl+kkpv-wrqS z7LeF?z}2(oPPp0u;u_;FU>tSVSf9Irdg9&#H||!DxZB|B>5J`f_iY3A#eGFT9 zpk_VfzMD1ESJ!`__}aYV4{+boH9vwKAn*7++;?>DE#O>o-UT=3ogg{;&yITD@e#N& z)P2W?u*PthxAPAIb@P-b{y5zDyFudbfve{oABEe$>Uqb<;Od^6H+dYcp4d;ojlCBn z_LFe+?AZrb%R4>=H;#JVaX(x=aVOx$_4gxjpN3C>yyF45`_z5MgII_4aPAqPWEu{&)r{y+Yhbyt>3Sg@YRi@VNUZ=l|*_J)3XS)1NH13wMy zcjLM?@s7a#mJ@FS+#33uF~+yK4*NE6zvEQ-?Rf{Tz8-Aijwi7{0rfhjshfBDxy(;v zjg?+F2DcadJDXlO4p%=`&Uz27)(z~1_puIp!MPJa&AjC^>@%Maf#-T=@;Fs=HRr5Z v_G(kJ@5;P&aF20)+ez%F!1?sVXYd(dPxOFm$x+R9#-GKSoAw{N;2Q8B^yeEj diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/maxpool_h.spv deleted file mode 100644 index f8fa66d59e20ae158dca47cfd56897144e46a9f7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3168 zcmb7^+izAy6va>b)q-*<2rb^IUnvhn4f;@xmsBhXwhtA9RfzQ#wI-kuH3cuRN;S3O zgNca|FEI@XC;=ii5w*TRqA!rB|Bd$;{r$dgRy?mxva;7+duGqfo_$WEb8uPJ=%`jy ztE!)?_E=kWRvl37uC*1h?v_6~J~6&^`oPrIp_^}V=IW}^THxFs=3Ij90?$6aZ@+;R zU=6w&X)2RASB#<>7uU4>Mr%$de2h%3`S3yDh~`=~b4GKG+VYy_$<$WXG}oxDscD{B zZGBDaF4`3}ZCTMgYxZ?N*EVa~rA6CZ(=IF8mYU|hU3&wt7H?tQ*Sm~T*ZIQ7h{+u} zPkh%qoPQW=++6Yj;E2De?Y8_c;Pt)E;J54ERmmM!0`D2WzSm-jn_KQ)zDxZ2Uf-g7 zSM#l77xx+9ybY|pmmHp@2^<^nt)ZH8n)uew{OWfU-<)dKm-W80!}+VQ#>~-Am)cA6 z{{!<}&Kmc0z5D8~E1KuFXPO}A(C-Bf*Y{y_7td+jI(V=68{zilda5yi?IU_Swh5dg z9|j8-5B-xhJMit>_~UTu21HZ!Hq{V_jbJ55lc^;xlmH%CpU3{eCvW z?OFYP_&iqMd%unKJ=Cn5zCEdC{7B)hbr0`&3^=mqaky{nKK5(&u^;u#^Ci}tYVMJAMtkeEe_PF5g{ut48*xbJWExRMxw80f%q(JBs!G4R1-2@fX~Ez5~X!Bh0e* zRs+v&e7KyWr|8bxa9}-paab z;p$n}gu7P#47modn}FwR0`IyR>-dv97#jp?-dFwy)_Ulxzf;b19o#&gYrbeV*0jtU zLR0q)nRi=FGygVx^J{;h%k7o(z;D7mFM#d9nDsaZth=!VU@UdH8*W{8fYfCauD)1m zFa|e=`mfx31Zxh5bufMxQ1@JI-hR`2(9Ay$l7A;$J$1hq?j6-=c!`0p-uQ~Pr^Qb$=`aB5KllLLGc_%^gJ`7h+UpxYLow|MT z7}jAuj6Dj}tcTopvu670`cD*Jn|J&H?pr$NB6teq9lwYBj>bL(#&YI;aL>FK(ep^}OQ?@FAd{cbtZ+yKmm)09-w} zUxb_cAV}_);ObfPGF&b1_zK)S>Uqak;p)lz8r-}yAbDShtLGgL!Cj~BJHCN+SPx^f zK+Sr{eK%{Sub#WV3%4K4le@nMS2us=y$@ISKahDJz}5XnjL?xsupffMpcfeT{Bzik zfU#+Ozx$7iwzB*OKEdw*TY$Y}yng2NlWXlS=gb1vd7qrg{_@QxLA_U}ioYFe4YcXq z3-I&6emAdklWzfj4kX_;xa&u-XUl))DBShK;3ReMj6Z^-;21ELxBU$6+a3pb+t19nVNcyP& diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.comp deleted file mode 100644 index c015c7a..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.comp +++ /dev/null @@ -1,47 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Rank-4 pad on all dims (spatial in practice), constant or reflect. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint id0, id1, id2, id3; // in dims - uint od0, od1, od2, od3; // out dims - uint pb0, pb1, pb2, pb3; // pad before per dim - uint mode; // 0 const 1 reflect - float value; -} p; - -int mirror(int i, int d) { if (d <= 1) return 0; while (i < 0 || i >= d) i = i < 0 ? -i : 2 * d - 2 - i; return i; } -int coord(uint oidx, uint pb, uint idim, out bool miss) -{ - int s = int(oidx) - int(pb); - if (uint(s) >= idim) - { - if (p.mode == 0u) { miss = true; return 0; } - s = mirror(s, int(idim)); - } - miss = false; return s; -} - -void main() -{ - uint i = gl_GlobalInvocationID.x; - uint n = p.od0 * p.od1 * p.od2 * p.od3; - if (i >= n) return; - uint s2 = p.od3, s1 = p.od2 * s2, s0 = p.od1 * s1; - uint d0 = i / s0, r0 = i % s0; - uint d1 = r0 / s1, r1 = r0 % s1; - uint d2 = r1 / s2, d3 = r1 % s2; - bool m0, m1, m2, m3; - int c0 = coord(d0, p.pb0, p.id0, m0); - int c1 = coord(d1, p.pb1, p.id1, m1); - int c2 = coord(d2, p.pb2, p.id2, m2); - int c3 = coord(d3, p.pb3, p.id3, m3); - if (m0 || m1 || m2 || m3) { o[i] = ET(p.value); return; } - uint si2 = p.id3, si1 = p.id2 * si2, si0 = p.id1 * si1; - o[i] = x[uint(c0) * si0 + uint(c1) * si1 + uint(c2) * si2 + uint(c3)]; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad.spv deleted file mode 100644 index b5e56ec387b2ea3b487dbdc651de233de1f75ac7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6260 zcmb8zZ_L+a6$bDhJO~IPl8RvIKw8r@Oh<^KCRu_A*800OTOr7>vNF__nmK1RU(~Wz zn>Aa^W@)a*S~*?I{@A}Ydr_uao4HIgHC%7X%(Sl0!}qf9o_Bq>!@16N&V9~(pZh+~ z15C|cHE%LCYqD^%XmbAS$@I5mGJ7&BXyIh;wBESoEn8OazHGBZ`FSFA}1CnN8_x)%^z`719#Af zvqsz@BhDUihmE+o1Gj9%%^SFvjX3qJT`}Sg9JrT{xCH}u?1)=9a3_p7dzDW!d$vUb zw8g0MMriYJ3@9*K9X&iI1JR%>TPYs{=$D{e>f>%cN-^YIYnAkr4 z7ox|t{u8^kU|)Mydt7T)67$OitL3XbuC*+F=d`y&Q$?MX+3!>|e0Xx$%Zf-sr)|Ar z`wmOg;^^atIC;d?J}TH(`{ZDGm!`s|=oOKp^HRJ+dH-M8pJUEsbJltj@~qEZ*yjyg zYj26|?pMUWb?~XjProVRJ1%>3rng33Ip7O|k4W6MXtDYu&xO&mB6xM#+Xs$+$ADW$ zFLy<}qwec{!RDj(2ZP1c-W_Z%YTpt(e`N0scJ})32{sG0?+bRvwSOIK|3~Fd?^+L! z4)$)@_WkE*y}~&c+q=az&lADUCH~21d+2@a3l_^($D3{)ZdwJ$@#8S)Lf*bHVQX$k_b!hD7ZMcTvPW z@}Du(-8t}Lwgz7u@y2#V8uQ+PZ_N9L7y&Cy(>^p=}+$>pm1Lx3|!Bmj>gz z?!&>>;=As$VC%e@uDd)K-*s06TZ{iz_Ta65G~#@GdURE^?_W7P-<1)ZKEcmM>mwWA z+}8)&kKE;M2zEBUPY!4Pbg+7SHw@g3Bd&2@2#2?}aW{>)#@!qauWsY^jJWpa8{zcF z47ESE1>+m@&0sNpb~WaXV7%U(opby8eqJ^7fxbHZnj7DV!OmFwq}b|Qm$UMp99s-t z%%X5>g7LkRwZS;G)!h*6b0>VKL~!n48YlPG_}7KA|1FX3|I~2!+Y{ryPYcF*-*TTG z?30^seFP^r?7p3wjqkb64DSAR1^e2U%|kr>ouTfQU~85qPTli^ed_XUjo{RQ)s>fx zXTNc<>A&YEqkgW*QobbOp*zZSs>N|N)aQSi5?c1TndV}lZ z_dC}6^80n1=d}3T7oVJbozL%u_rCW=`o8ZB#y94!U@_iVW9|;dyR%OvpRaHA(%AY! z_igVE_VcIq<+0W2JzNo648Ak`kzjmh_{w0M+UkBH*yla*T@}Hp4R`-n2hW`0kA=f` zhOY_6b%w7EZf?GhM{sh(?!md)_@3*!;O_5^U|;+4nIWF;41YG*n$Ga&f_=`!cVh&n z4y>-cY&`q(gH8ACoSu8uJ27Vj-xl2&{$eoyc!qBpID6>~e<@fjpBcVATAuEY|H~1) z{lM(Py|{wt9^5ixLQ_^ZM8;SARBi|Y*E5^RR;sqf@#!7uW|tlt_AKQot{?S~$y zr@nY~)O8+p>2WWeiQQh)&2{F`JNKbxWBc~##=bUw_h1&RlSkgZ9a^jz-VnbT_P)&U zIL}$}xi3CB`TD-^4ex#XyK3k2r@{Ef{5)8Uem3S8!FYG}tK{?bt$sJQzR-Q!cLv{_ znA&&6R;PD!cWg2E&hYnx@txrx1mo0J_m{yw@15_S2u^Ld`~P9^%o+YsIDBXL$HBPH z@V&v!&G(ZCPHxyeI5!*LbNwv1`}=7GyFiX&zbms7s06mt1B-X z&wg;Q>As!U<(~CU%-O;FqC3OC59S}w@IwP|UOV zZjAMhL>`NXfjh&G2HS@-Si>)_GyHh4H)l_MCw~bpU+&3p_?fxnY(MlsJ@v(_qptI) zOOJc$OzifWZm!o4y>lOGHnwk%ZtNT4cMoR4I(g*n+o8pp;f?W|VeiWfkMo=ppZnsI zldtdlx8c3-`y!ps`-Aa~c_3Jfe{(kG!C<^QTb!J}zSU=wUtj3H?dO8`B&PQBvDN84 zybxOqzB8N@6MScQwmK2D)m=2y_1^iWA~?0-?tf13%o#o)9KJI=HyGC$o*(R!n{QqO zCpYXKoSTjBxfTR>e`|t$?aR_3p6(1U3$~^+d}OdsUB2ZJoI0?&^0M*lmku`Fx3ei& z@5G!NIVsW^K026xJi{vo&R#mh#{`SzGs7oG%hUbwuZrO92X-$fMmNU#S457Bh=Du9 z#|GPnGg!kft}}dmuoan4nH%Oob87msHeVob<}kpb?I?0or&FE)6KOx zSnu42nvLz-qZ@l(uzN5I*2yDp-wrL-3~voK!`_z}9_QH-PE9^J`Fc+eh4=SxzsJix z91g#6#QiZgULE)Jr(mDA#`j1BrysES@qa>3Ma<6=5%a$(`mYiEhWx8%U-Z)v|F>k1 zJ0oW%KTe$UF3x9pa7OH diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/pad_h.spv deleted file mode 100644 index 683332e89ab45ef317009b4aa5f093f78422a8b0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6296 zcmb8zd&u8)6$kL&_RL+Ja~n>bI%PHkQ!~va-nKNeJ=4t#2^B~&H%%?OO}m(xi4^`3 zR3;QP$VQ?m#t0HCx@rkw|0G&an6-3yX%ciXv3kFrec$or`L8b=&ilO2`JD6loX_Vx z&$CU$?2?weQWlq|8Zh*ax~5w+`CjTN}M0r`{TEt$ncn6m7qHpodQncsf4VcUVNe+G_a3?}_G@3qCrs?>_d~uZr#C z=NQ+T6MNr%&6~i!_N?}}R-V<2UoQCQeM@F?v&Xd#kL{fHc2KIQb9DAQ6%8Mp9QLw0 zlF%7juh_m7iCP+c{17LPxY|bs`)a>FSl&ZYVLEzs^-smD)W9Pzy-dvm5sBgYN+vfx7#wXEA^qdG@UG~<2 zVADVXZUseq&oV zH(DG{9k%z1Q%C-lv8}ItSTO2@bZkqs_bdiaPEBXMw|NntbN@4pe*YuZ5sC4=9PIvA z#pb8aNz}G*S46f(_}2~lxN_jdY!1FU;=Q@s##}S-jd|}7#h&Rcio4Bt;OG;J$UyYi#Q*j9(_F8_ph9t z?}i9YpWv6G^^uKl?puTHNA7aB1v?wxr-w6tHdsBr+Xn9QBd&3Ggu`3gxG#^m#_b4) zSGRF@j=1*c?r{3EGt&Ot6O3=ny}@Gq9Ba&d!FatnFX#64{k(4I1ASxqH9x+SgPpPV zDY4bLC1>Sd8(R!s%#v`Y2IG4trv>BGR`=XspF80@J%V!waQA;ku>Eg}bpL0D!+&$Q z@3Vq&-nZQAgMD)It&8B~hTXSwv++IG*}>i4HNn32Wz!H(e|xBVVX!raB~IOogMI4q zT@=Bo1FI`98_&LEu<5_$r=)(KmVwthF{cN=J9^hp^U`2`_xY`Xd)vU-%UuKa_F%Dm zf6sodik8P`fBf%=;Oz%?hAW~QWBp~3cSXd&H)X9ozBAZ9oWUA?apr`7OR(RS_SAQB zd2so1+rr^z=aRGi&;#|<7q5=G&Z90p?xi!a+iSYH&KP>*%!G8YK{zzf3UB8`P>jscZR%SJ+84&|_hQA(cAI@M6zqrouH-gQuJ@uX36EZFc0L~t#y95Y!D95YF^>e}-Pxnb=j&U2 zAhy2HecKNP??_DT@5NT9_wfDLV(^{e9|Yq&!#@hfsjcoWf_>gQ-;X0Wwc+mnC&9C4 z_^09Uo#BUqah>6x1vfX}!x5a^uzPTBHooV2B)I$gYp}0mZ5je+b@Ds8}9z+2hX121>x|W;f2At z&hY-h&CRzcf|DC|56;cT_gn`AcYmh_``VY4LpAs!wgX5Zsxgh$KNN0F;F#mXl*9@G!bcT-!7RzUb*G9|Zvp@bev`;Lh-I!S>+{*6@q#48Jzm4BJ!R$%(<`%bgSsKRcJ4?S~$yr@nY~)O8+p z>2WWeiQQh)&9y05@7#x)jqTf`8@o9;X=Y}@I(g*n+o8pp;fsRJu=iz#$9XOcrzW4A ze7&dNh4**3-{a+;42R!1;+~3)SI0g5G1%uW`Th{W=?83no{N4yVt)Kzk@=sFej$S2 z9(#B6Uk2`+{3~d0@QaZ>5qrKevOe{2;++48%(*)1xo>~}zZCKN&-|48cd)-}`TBEV zzpV9di^eR7jT76LMX~+u(wN1uaq`?e)H@)ywR~N7VDR*4-Llv?d+NI7v90Cnx`Tq< sQDaub7Kd+74i27*=n21G>fIs1e2qIacy{k_;>Nwx!&fGACZ|RI3ri!FjsO4v diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.comp deleted file mode 100644 index 56a9127..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.comp +++ /dev/null @@ -1,13 +0,0 @@ -#version 450 -#extension GL_KHR_shader_subgroup_basic : require - -// Device probe for pipelines created with a requiredSubgroupSize (32, or 16 for SG16): every -// invocation reports subgroup size | count << 8 | id << 16 | lane << 24 (VkDevice.ProbeSubgroup). -layout(local_size_x = 256) in; -layout(set = 0, binding = 0) writeonly buffer O { uint o[]; }; - -void main() -{ - o[gl_LocalInvocationID.x] = gl_SubgroupSize | (gl_NumSubgroups << 8) - | (gl_SubgroupID << 16) | (gl_SubgroupInvocationID << 24); -} \ No newline at end of file diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/sgprobe.spv deleted file mode 100644 index 913cb72682a35cbe097a5f3c08b11a578fc06b2c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1080 zcmZ9K%PvDv6oz+u>ei~F?w2}sscXalkqC)V4Lb7#5)%U^JRdr*UDyfm0=_n1)T&kuDs`3WeNaF?X4;}`mZ|_g1=ND&2&ZJCH_*$u+#?Wiv z1FiS0C=75Rz5q+!^$Z?>*Pwh!$72IP{_pE|@ItAY;ti$+v>9>}8 z_t!V%+auiB8o;^i<(wXK)%EQbJWGp6k3R3BuUB&OpkD+^+f}wcJY%Jezo8R~>YBA}|anq_NtqCwF&|Z|p4a81H}8r`>qD`_vZP9Y((Mtbmx;cy~JL--O#E^7DUw z|Eo3ZzlZcb_V<3uePF@h{wwRB;PqbqA7iwOOWv1z_Z@oN#TORe_Z?qnoCof~xo*ln z?b06HcjMiIFQa?+4{A7<|67~0f0nj2bo2B@-UhlmiM$TFHun;FTj=KMi@Yv+4kFJt gr_Fsuo^RY-eRoXd{(2zZjCZ= p.n) return; - uint d0 = i / p.os0, r0 = i % p.os0; - uint d1 = r0 / p.os1, r1 = r0 % p.os1; - uint d2 = r1 / p.os2, d3 = r1 % p.os2; - o[i] = x[(p.st0 + d0 * p.sp0) * p.is0 + (p.st1 + d1 * p.sp1) * p.is1 - + (p.st2 + d2 * p.sp2) * p.is2 + (p.st3 + d3 * p.sp3) * p.is3]; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice.spv deleted file mode 100644 index 459b1b826203f31af0d9a472f17211e601bbc341..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2668 zcmbu9Yj0LX6opUwmI4+ig#vQTTk3}bwOJaOj4 z=$*SWqq{%eX~v4C)k!p`pPWU^J=pB@)$0cOu$Ay~0}>>ft4+ch*EV(hRwt($JcUvx zA3TgX!nsyX&2X-fTT*d$Cbz8OTqC!#;_O*&u;O|P_hH2?E}XrluJv5ouDGRz`>5hR zDBMWJxo_8QuQ+>^^KesRdEs_f+={~Oskr{a?PvCE4>G$C_dLma8fRWO+nx@_ZPsJf zG3Kzg!~6+m<7$bAu&&DA3U(Mj%v{$xf}dJqN6s>SU2Ceu)e>9F5x=fA5AWI8+iI#< zXNZ1V%;Gg@^fI?G$2xr9iJUoYeBXzAk>6K*HRZOI^}Y*-`Mv{VYHVp%b?&D4|CqX~ z(69&Zx_#)cE}Z-DPPVc1rN0JqxPC2j_BP1wJrnzX-@0wYJfFnJz@Eoio}V$dvCZIv z%=%lvhXm$l0s@MB!ZL9mu*PV13t3mK-_P>HCh=N-syj-qZLe!AmOtOK_e4 z4Zh!P^1lar4)M=}eP{7!z`m>aSHRwZ__JWYsra|Se(&*r1$#f@-v|4>xxYDPdvou` zt@9LXy59NE7F@pd-cFzMB}ZNxV>jK*o`J*n@(RX1{)6{ilJPItS}!o;&f{dcSNpd| z`~qv@5(yN`{f-Q~#~U<_ZN%iPFMiEzfBN#NH3YU+ z8%wRNVEK7!c}~Nawd7wGZlvO(wH;0^?{KtsfaRk#3U;l0w02cov_64T%ljWK|D*EJ z+6#8Ae6$W!T(pkCspb8R)^V_Way|!}a}rC=DX{$0vggxa&q98#a9@DcG8U~fV72@n zqV*M6K3ZRcT`Qk;--2D|cawGBf#s9)1K1qDrR4kwmXG#Xu;(nFKF@*GG8U~1V72@X zqcsh_fJN&f*tPP}x&*d=W6`<{R%-@})=yyhpEDiSu9#N!1B?$33jb~v~Gdz-&nNnfYtJU60N&n`Q-ctHphQVa_)iUQ}1`M_fJ0c z{s60GELso1YTd`8^(R<9T7Q9ED<7?gVEZ>V&X+mI{0N)E{8t!P&wt!w%vkcCfG05X W`mqE2Gvv&x{|){l-ttr2gZ&3X35_TK diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/slice_h.spv deleted file mode 100644 index 6a0e3a4a6c66b392f6fc0de15481e3cb3f4aef9a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2676 zcmbu9Yj2fB6oudRXelBTC{P3;r<4yQKtm!F3{WVIwA@rK3gM1aL%~*o2IQg?#R`9h zzXwW4e4fLsI3N7RB>T+TYwx}G%)B$F)jhPRX>~PAnx)N)=G|D)bT?f<=aVZMVm%#y z;`r&~!*}PVhIj7TVdk=?)hRHykDL#gKfq?LP0t!wf-Q%aL#9#0Tx|l@xb|tsZ*_9I z!ILO;^1(xxBb>EzYKF5$uD9Y`ncT+}XN}zQigV3!t17OiaBC}WQQ=%``r6OhcEv3& z+~$h=sBl{<&V5_Et>Rp(oQInpOAEKN;+7R|PsQ~WZXdH}dyv_Ecs3J!r?G-3iHRMb zVD^b`Z-@Eg%*NFc4`N-FzX|Lx{snX0>oC53?Im{PEaTU`CQDo`vArDe>s|}+o}Fv! zr;B|CS#OJ3yc&%j<~HUS!1tTTnbXGid$aIk?HF(!uhkk$I+=q9vjb&Z>t1*Z5YnZdQRqWm~asB^Sx0RUZllTbO^H{_00cPLC z2Jk^<{f*!;W_^1dV)jhrT&w-=x=eYeR!5B415Ujh5g;!lD7R`G9uy#w)Qz`m*Ycfh{)`1ioxkNA(kzBl(b&+OXV zyK(zG!WIu}Zgyf(sax|ux#hu`HjjC*{A_m?E&4cK1KG2_l-WVu(@ z?;4HoFY9oP^5%F~1~BiJcPKf&OZk`Ncz@Pm*2&x3|2xM&Xc}9O$=P50H?!;0mrt)j zu)W$?dTj#BFVM?#8p7-)|DtePDlS^v;MDRCM{7G+K3ctD(K-!Q z%l8niGhq2>eFL^uK6Pio*72$qlbC9vl#pLJdat7R-&SHWue z4x@Dqd=-n30k4ca=}SAHixF zi`Fbytr;v@KY`_=^)uL7`Don+yMANQx(imz|C4CVf#s8PA8d~QG0FJ_ET4W4z}`Rk z^m_byV&-@gd#Qd)?uAcvKzhlOd_Xl_! YGp`RD src idx via permuted in-strides. -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { - uint n; - uint os0, os1, os2; // out strides (os3 = 1) - uint ps0, ps1, ps2, ps3; // in strides in out-dim order: inStr[perm[k]] -} p; - -void main() -{ - uint i = gl_GlobalInvocationID.x; - if (i >= p.n) return; - uint d0 = i / p.os0, r0 = i % p.os0; - uint d1 = r0 / p.os1, r1 = r0 % p.os1; - uint d2 = r1 / p.os2, d3 = r1 % p.os2; - o[i] = x[d0 * p.ps0 + d1 * p.ps1 + d2 * p.ps2 + d3 * p.ps3]; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose.spv deleted file mode 100644 index 7d1764974b7e927aa11f83f32eeb373e17a7d65c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1900 zcmaKrS!?~Kr#aX-X-idpgVv2K` z?6-m!&!ADmH!#;6@d3P?H4Wl}c+Vn#DaUHcE#`jj#bx~{-kciqbbI&W|CxP^qG37T9+k@jbBLGUCTz-?RE# zc(t5u-ntzumHx)BhGAMF>-XBltdZB2nO4TT2bbUFJB)kyM)X>e^$YB*H<)>Shbr%X z4D*ecznu3mo_TA$t2xa3^$w%PcQ5~y8t?BIW}m#X{lB_?qG|3pCg*(d54`&`mXBFY zu(KLi%vu1;e`c27sf9U9{!8Xg6}Zq^f>X;k6k4aj@}boR+bbVhXA4|tU4T=|_ZM0# zVENFx2)0)~v{nmTXx)HQ%l8vneX#tee4d+N&nX{$x54)L&Z6%QSU&phf<2S`$E>vp zR?l2$-3P1X8x5@oVENE`2=+|!q4fx?p1II^0#?iS9a>Mp@}cz%?6;5)t><9%%yqcg qE&Mjt!+dMz)$>2{0y7tNFTv}Wb-smF-kh9u#kb`f`^}$f1^WX5HC#vl diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/transpose_h.spv deleted file mode 100644 index be61c6c67ba6c9ba7821a27feb291d2b11fea79a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1908 zcmaKrTThcg5Qeucf{L8vc`NA6(3xih&V1!s@kXu-K>xru`7&)l(s8^~O5uYGBS*g9vo`3|j&JM2vCI*z%= zh@IoIeid)t?;xJVDn&d8cA39{FK4Y0JJ(raSL`+7a#knj)e<|)6>&Lh58gX*Z{tjH z&Mf<_;l-0^^yBN8Ynu2FUe202@nO7Yk-w5-HRa}WzxU#@eh6<)jTySVd-4CwzN6GQ z!+Pf!k7v$(dZ%?P<{D38F8inOvG)m1=b7E#zxw9MdH%>Rf<5mPzt`}-sng&UyzvaU zg*SHAb-ZVmbFap$nX}J3j=nXpv3>1aBWItncc@nEgdpV83O=kHNlY z^|$eAIorH-yI89HjbHV`v_RJHwTD?FuPrjIigyn#zsq+R_wbGAwIu5o*jaBd^ZFK5 z-v2P>8!>-1?_(tM)_7ObnD^@)Mvd=Y{wp=!-wDh~mTxGu&VuDbs|mJOKD5pkTxeZ_Q_J@k zT1#O0(7Ft^S3b0s3of*7!Kvl@39WUo{HJ`L4Y22wkG?x#`+R57w+WVyzPn)0B>ypM zZGqJ@7h3nhYWYS(>j79kv>t*zlYD4B0;^{(w4Q*~@_mQaQ?Pt!Jp=nK@~ diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.comp b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.comp deleted file mode 100644 index ade3618..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.comp +++ /dev/null @@ -1,44 +0,0 @@ -#version 450 -#extension GL_GOOGLE_include_directive : require -#include "et.glsl" -// Elementwise unary: mode 0 relu, 1 sigmoid, 2 erf, 3 gelu, 4 copy -layout(local_size_x = 256) in; - -layout(set = 0, binding = 0) readonly buffer X { ET x[]; }; -layout(set = 0, binding = 1) writeonly buffer O { ET o[]; }; - -layout(push_constant) uniform PC { uint n, mode; } p; - -float fexp(float v) -{ - v = clamp(v, -87.3, 88.7); - float fx = v * 1.4426950408889634 + 0.5; - float nf = floor(fx); - float r = v - nf * 0.693359375 - nf * -0.00021219444005469058; - float poly = (((((1.9875691500e-4 * r + 1.3981999507e-3) * r + 8.3334519073e-3) * r - + 4.1665795894e-2) * r + 1.6666665459e-1) * r + 5.0000001201e-1) * r * r + r + 1.0; - return poly * uintBitsToFloat((uint(int(nf)) + 127u) << 23); -} - -float ferf(float v) -{ - float s = v < 0.0 ? -1.0 : 1.0; - float ax = abs(v); - float t = 1.0 / (1.0 + 0.3275911 * ax); - float poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t; - return (1.0 - poly * fexp(-ax * ax)) * s; -} - -void main() -{ - uint i = gl_GlobalInvocationID.x; - if (i >= p.n) return; - float v = float(x[i]); - float r; - if (p.mode == 0u) r = max(v, 0.0); - else if (p.mode == 1u) r = 1.0 / (1.0 + fexp(-v)); - else if (p.mode == 2u) r = ferf(v); - else if (p.mode == 3u) r = 0.5 * v * (1.0 + ferf(v * 0.7071067811865475)); - else r = v; - o[i] = ET(r); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary.spv deleted file mode 100644 index 72742dd51638f3f9fa7724a6af6f1d7658afb2f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4712 zcmZ9Pd5D%(6vkg2*VNR?EHWLnP{ta}(o9qNESXFTW5g(|&|I2IL0V{$+#QwFG=sEq zDMg)fN^-ZwJyWo#gi8PGpC|&u%Cw|@zt8u4T;?)|`#jG%_uS>Z_wY?wjQC4j2v&NaQRo1N7SBQ%yO5X&VGOlK-j`>(S zbhY*h1RASVq3Umibt>|*>{a^4*ga$t+qz%sN}6NU%xR7{>6?{!S+){vn)=fUk!3CyM_es2&a{J7un2P zO5C;2nfd7Mi0=WPTr=v9;1%!_ckLVyyb|7GVT(J1_kr))( z!1{-Q$}!JFUX^pY;{N2ru^Blxegx(oB40Jwb`>Wynm+U~MRn%q!N*}fQXiP}`MKtL zjIG6N#xI~ZUcEEicOcgEFQ#vashdx}B-hO6xh%zO#=k~yyyLHDRq?9p$r8uR-&m-(E}GhKt}i#4qSyQZ~Rtm$2_dR@Mz_rPlI%{6VHx49;L>#@z4 zYY?~2niuEV1m@VYo|yAiW3Jcn#yTeE*aCJAzlU>tNN;lvef8K5%yHrw`F^&7&D}a{ zQJnK=tk{sQ;F^u>D4fz>Oph&vBfPvb7Y)hn@x`xmU9#$AM~_rW6W zKd^cl=Z4k$C2@^C1T2l?XF#Uzc}2daaCPHCYYtaW$F_v4r`lz3^#Wt3@?D3~w}E?p z)8W3Q%i(G>Idgeiu-b6&pR^MK1ALcmmE^?2izm9plv%ZJT`F-nzralI(fWjRiMtuDp608Bt4~Yfdc)O?i+6r2*uBQu`hr7?w?7n(zvA21-|+3< zjb=>x_J_gM)3;v*=dbwo^~ap|g7u|ue+1l|`XcT=uyN_zzaMU_zK9zI_U-G7^L+rU zp1%ERxOy5l8m^wc{V{O$H10vTdiwSsf~%)-55v{dw?7uHp2j@_S5M#mV{mojVsGQX z>gm|W;p(Y20j^%)9OCUyhI@YT_MZT&#oMm|tEF%MNw|9Y_Md{Q$C{o7JD+-Jw}ZWJ z`r_{P2Nzh6zK9zDR`;D+4}9ys>wWZ54-SE?2M00h!4Z0!^+4ZY>}$+%;>djz?0H%f zLOY)GWAsrso?~wHv~E0)rrxk_ynx1Ev2Hjf^1g&<-OwL*?qzTRO<&ZA>A9vav{%5^ z4YfF{8DRCaZp?zKnpILEC##R z*#A;+Xi+!bM&qwoH}p5G8>`WbN$bWMxO!SQ*1`EJ)(!nJ=X$Wdv~FyGn^RxJZ3G*a z){RYYWA#N`9oV{|FV6RUuzFfIw!qcXxO%vHS~os|tEX{W;p%DK_!zF9#%+VEr*&fo zTs@8330F_+#%{Q}ak00(VD)tDr*QRD`wXsL;2ffE?1y`PQ8x~N)uL`31goWW;}Bdu zts95o>anIHVCPd0Z5i16rZ4X9a&Uq5=!>`&V0GVVe5(`06fk`e_Z8eapwIeZ?XXT* zFH)}lXa4*87~>shEwOe)Y}6+Ei2agSYnZ=79A}>Nca6VqBK9bLfB)2Djx&$38|Yn= qeZ+o&e-nH?<~Z}j{~+IkPhv~4Mc80!qMBpOcM5Di`-{As{@DK+L*d~7 diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary_h.spv b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders/unary_h.spv deleted file mode 100644 index b6a9285350ac9b29d437a016fc299ee587dd0846..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4764 zcmZ9PYlxLq6o!wEmnp~6)TW@k7Rp$IRt~1AoI*y^LK!g%yNMU3(h&QRbaA|nTAG?t zTBKv9Inv}{dCM%tTc%P`31#-9pHT=_rX}?}$8Q}rete7lzVBLl?aR0KnlmLWd$rA# zw8*u~wa=Ywkt;kMb1id1+p!&U_}UbB_2`<>l{254RC(8cegdIfF+|fS^DUtjUk`FF-Lbyp6L-MYu4y_D+&!N%rTO0AUEqTocMS_( z4(BQIZnC=mMef|^)jhU5;(NoV){eP5cm@3AeLDvQuY{K_F1;uC9q|2IDux8Ff;To+ z3=N)#GvKf;%zrqjJLZ1Kt20hhSf6|(Hap|SkHTC-Vq@BFw>lmvGtgb@r!7USMLh<9*8ylOKGpd)XgVfmTBg5UtYs}j9*S`y!|~hZ_;{B zZU%dI-ufY5+ev61%ndwc$cs^gp?5FP7 zt7&~Ml2hL*OwIAd7igWITs_Yk{TAymzmH>?&+**9^_aeyaQCHt(bEXm*v_P#%ct69{OO8>)mwz_BYl(F~)~r$ME|& zMl-FCW9Zw4?ZWIQo}H~{JJ{T1xzfUzzk(gJFQ(6PZVg<(T3r8cm@!?jIhpo*rdvnK zh%)XhW}Z?^fAl{G*4GW2m-YVxY)*X<_ZQf>URcDP2OFy|;x2%DWBOt~7s2WkSj7Dc zR!`$D!PP6Vi2Dz$p2oQlb!&0NwRCY<8plt7oO)Ff*BY*##1C5uAm;1yzdLd3g(;w@&1)N9I7wfn+)AWVb8$6LX zwK$u7!0MCHV~h&8x;bKuzF_s*B<^;&x^a=O60Gju$;j6auAb)W4_BX&#N7#3H!kjZ z71*^#u7Tjt;_eSe<1O5M{T;yR-LFP7CcXP3;OgbX#-85~=PlfQ{W0bPV14P`e-Lg? zeGxYXY+QQx$HI-(7jZRU@4mj+-*I5|^zM&`tEX`f!PV2d|1exVje7*Hp5FZlaP>6q zQMh_~_aBF=r*Tif)!naH$3(cgaiL8DtEYXZz|~W2DqKBJzPS5O!QH>O`%i<_;_g2K zR!i^xG`MeaP>6bD{%F+ZY+hX8y9tBIoP$v`d5NOi@LE6jki!Y^j}#w8qkbM>&6DSdRjL& z!g&jILw}698LTg@8}Gx-sW0Lh!N#R^V=LTPeG&Ho*t(%F_O}VFp4N?RaP>5<8Lpnz zjqPytH0~p~dRjMjz}3^ZopANEZtRAur*V7W>S^8B3s*NT*0v9c(NPT3R=bz}3^baTKl|bNT}8cMLUMn7)Yn8g3oXXMM4DSSPF(DOdk9`+a?a{`Rw$G++@MwaGSOj}vPR^LL2- z%#;4EX-;Cl#J>~14YQwljNMG@oNOcSG5n42O_=@6 - { - if (name != LibName) - return IntPtr.Zero; - foreach (string candidate in s_candidates) - if (NativeLibrary.TryLoad(candidate, out IntPtr h)) - { - LoadedLibrary = candidate; - return h; - } - throw new DllNotFoundException( - $"Vulkan loader not found (tried {string.Join(", ", s_candidates)})"); - }); - } - - ///

File name the resolver loaded, or null before the first call. - public static string? LoadedLibrary { get; private set; } - - public const ulong WholeSize = ~0UL; - public const uint QueueFamilyIgnored = ~0u; - - public static void Check(VkResult r, string what) - { - if (r != VkResult.Success) - throw new InvalidOperationException($"Vulkan {what} failed: {r}"); - } - - // ---- structs (field order must match the Vulkan spec) ---- - - [StructLayout(LayoutKind.Sequential)] - public struct VkApplicationInfo - { - public uint SType; public void* PNext; public void* PAppName; - public uint AppVersion; public void* PEngineName; public uint EngineVersion; public uint ApiVersion; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkInstanceCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public VkApplicationInfo* PAppInfo; - public uint EnabledLayerCount; public void* PpEnabledLayerNames; - public uint EnabledExtensionCount; public void* PpEnabledExtensionNames; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceProperties - { - public uint ApiVersion, DriverVersion, VendorID, DeviceID, DeviceType; - public fixed byte DeviceName[256]; - public fixed byte PipelineCacheUUID[16]; - public fixed byte LimitsAndSparse[1024]; // VkPhysicalDeviceLimits then Sparse; timestampPeriod float @428 - public float TimestampPeriodNs - { - get { fixed (byte* p = LimitsAndSparse) return *(float*)(p + 428); } - } - // limits start 4 bytes in (8-byte aligned natively); maxComputeSharedMemorySize @216 - public uint MaxComputeSharedMemorySize - { - get { fixed (byte* p = LimitsAndSparse) return *(uint*)(p + 4 + 216); } - } - public uint MaxStorageBufferRange - { - get { fixed (byte* p = LimitsAndSparse) return *(uint*)(p + 4 + 28); } - } - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceProperties2 - { - public uint SType; public void* PNext; - public VkPhysicalDeviceProperties Properties; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceSubgroupSizeControlProperties - { - public uint SType; public void* PNext; - public uint MinSubgroupSize, MaxSubgroupSize; - public uint MaxComputeWorkgroupSubgroups, RequiredSubgroupSizeStages; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceSubgroupProperties - { - public uint SType; public void* PNext; - public uint SubgroupSize; - public uint SupportedStages; // VkShaderStageFlags - public uint SupportedOperations; // VkSubgroupFeatureFlags - public uint QuadOperationsInAllStages; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMemoryType { public uint PropertyFlags; public uint HeapIndex; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMemoryHeap { public ulong Size; public uint Flags; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceMemoryProperties - { - public uint MemoryTypeCount; - public fixed byte MemoryTypes[32 * 8]; // VkMemoryType[32] - public uint MemoryHeapCount; - public fixed byte MemoryHeaps[16 * 16]; // VkMemoryHeap is 16 bytes (8-aligned), not 12 - - public VkMemoryType TypeAt(int i) - { - fixed (byte* p = MemoryTypes) return ((VkMemoryType*)p)[i]; - } - public VkMemoryHeap HeapAt(int i) - { - // native VkMemoryHeap: { VkDeviceSize size; VkMemoryHeapFlags flags; } = 16 bytes - fixed (byte* p = MemoryHeaps) - { - byte* h = p + i * 16; - return new() { Size = *(ulong*)h, Flags = *(uint*)(h + 8) }; - } - } - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkQueueFamilyProperties - { - public uint QueueFlags, QueueCount, TimestampValidBits; - public uint MinImageTransferGranularityX, MinImageTransferGranularityY, MinImageTransferGranularityZ; - } - - /// VkPhysicalDeviceFeatures as raw uint[55]; shaderInt16 is index 41. - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceFeatures - { - public fixed uint F[55]; - public uint ShaderInt16 { get => F[41]; set => F[41] = value; } - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceFeatures2 - { - public uint SType; public void* PNext; - public VkPhysicalDeviceFeatures Features; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDevice16BitStorageFeatures - { - public uint SType; public void* PNext; - public uint StorageBuffer16BitAccess, UniformAndStorageBuffer16BitAccess, StoragePushConstant16, StorageInputOutput16; - } - - // VK_KHR_shader_float16_int8 (Vulkan 1.2: shaderFloat16/shaderInt8) - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceVulkanMemoryModelFeatures - { - public uint SType; public void* PNext; - public uint VulkanMemoryModel, VulkanMemoryModelDeviceScope, VulkanMemoryModelAvailabilityVisibilityChains; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceShaderFloat16Int8Features - { - public uint SType; public void* PNext; - public uint ShaderFloat16, ShaderInt8; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDeviceQueueCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint QueueFamilyIndex, QueueCount; public float* PQueuePriorities; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDeviceCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint QueueCreateInfoCount; public VkDeviceQueueCreateInfo* PQueueCreateInfos; - public uint EnabledLayerCount; public void* PpEnabledLayerNames; - public uint EnabledExtensionCount; public void* PpEnabledExtensionNames; - public void* PEnabledFeatures; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkBufferCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public ulong Size; public uint Usage; public uint SharingMode; - public uint QueueFamilyIndexCount; public uint* PQueueFamilyIndices; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMemoryRequirements { public ulong Size; public ulong Alignment; public uint MemoryTypeBits; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMemoryAllocateInfo - { - public uint SType; public void* PNext; - public ulong AllocationSize; public uint MemoryTypeIndex; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMappedMemoryRange - { - public uint SType; public void* PNext; - public IntPtr Memory; public ulong Offset; public ulong Size; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkShaderModuleCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public nuint CodeSize; public uint* PCode; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPushConstantRange { public uint StageFlags; public uint Offset; public uint Size; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPipelineLayoutCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint SetLayoutCount; public IntPtr* PSetLayouts; - public uint PushConstantRangeCount; public VkPushConstantRange* PPushConstantRanges; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPipelineShaderStageCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint Stage; public IntPtr Module; public byte* PName; public void* PSpecializationInfo; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceCooperativeMatrixFeaturesKHR - { - public uint SType; public void* PNext; - public uint CooperativeMatrix; public uint CooperativeMatrixRobustBufferAccess; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkCooperativeMatrixPropertiesKHR - { - public uint SType; public void* PNext; - public uint MSize, NSize, KSize; - public uint AType, BType, CType, ResultType; - public uint SaturatingAccumulation, Scope; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPhysicalDeviceSubgroupSizeControlFeaturesEXT - { - public uint SType; public void* PNext; - public uint SubgroupSizeControl; public uint ComputeFullSubgroups; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkPipelineShaderStageRequiredSubgroupSizeCreateInfo - { - public uint SType; public void* PNext; public uint RequiredSubgroupSize; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkComputePipelineCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public VkPipelineShaderStageCreateInfo Stage; - public IntPtr Layout; public IntPtr BasePipelineHandle; public int BasePipelineIndex; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorSetLayoutBinding - { - public uint Binding; public uint DescriptorType; public uint DescriptorCount; - public uint StageFlags; public IntPtr* PImmutableSamplers; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorSetLayoutCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint BindingCount; public VkDescriptorSetLayoutBinding* PBindings; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorPoolSize { public uint Type; public uint DescriptorCount; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorPoolCreateInfo - { - public uint SType; public void* PNext; public uint Flags; - public uint MaxSets; public uint PoolSizeCount; public VkDescriptorPoolSize* PPoolSizes; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorSetAllocateInfo - { - public uint SType; public void* PNext; public IntPtr DescriptorPool; - public uint DescriptorSetCount; public IntPtr* PSetLayouts; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDescriptorBufferInfo { public IntPtr Buffer; public ulong Offset; public ulong Range; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkExtensionProperties - { - public unsafe fixed byte ExtensionName[256]; - public uint SpecVersion; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkWriteDescriptorSet - { - public uint SType; public void* PNext; public IntPtr DstSet; - public uint DstBinding; public uint DstArrayElement; public uint DescriptorCount; - public uint DescriptorType; public void* PImageInfo; public VkDescriptorBufferInfo* PBufferInfo; - public void* PTexelBufferView; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkDeviceQueueGlobalPriorityCreateInfo - { - public uint SType; public void* PNext; public uint GlobalPriority; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkCommandPoolCreateInfo - { - public uint SType; public void* PNext; public uint Flags; public uint QueueFamilyIndex; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkCommandBufferAllocateInfo - { - public uint SType; public void* PNext; public IntPtr CommandPool; - public uint Level; public uint CommandBufferCount; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkCommandBufferBeginInfo - { - public uint SType; public void* PNext; public uint Flags; public void* PInheritanceInfo; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkSubmitInfo - { - public uint SType; public void* PNext; - public uint WaitSemaphoreCount; public IntPtr* PWaitSemaphores; public uint* PWaitDstStageMask; - public uint CommandBufferCount; public IntPtr* PCommandBuffers; - public uint SignalSemaphoreCount; public IntPtr* PSignalSemaphores; - } - - [StructLayout(LayoutKind.Sequential)] - public struct VkFenceCreateInfo { public uint SType; public void* PNext; public uint Flags; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkMemoryBarrier - { - public uint SType; public void* PNext; public uint SrcAccessMask; public uint DstAccessMask; - } - - // ---- entry points ---- - - [LibraryImport(LibName)] public static partial VkResult vkCreateInstance(VkInstanceCreateInfo* ci, void* alloc, out IntPtr instance); - [LibraryImport(LibName)] public static partial void vkDestroyInstance(IntPtr instance, void* alloc); - [LibraryImport(LibName)] public static partial VkResult vkEnumeratePhysicalDevices(IntPtr instance, uint* count, IntPtr* devices); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceProperties(IntPtr device, VkPhysicalDeviceProperties* props); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceProperties2(IntPtr device, VkPhysicalDeviceProperties2* props); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceMemoryProperties(IntPtr device, VkPhysicalDeviceMemoryProperties* props); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceQueueFamilyProperties(IntPtr device, uint* count, VkQueueFamilyProperties* props); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceFeatures(IntPtr device, VkPhysicalDeviceFeatures* features); - [LibraryImport(LibName)] public static partial void vkGetPhysicalDeviceFeatures2(IntPtr device, VkPhysicalDeviceFeatures2* features); - [LibraryImport(LibName)] public static partial VkResult vkCreateDevice(IntPtr physicalDevice, VkDeviceCreateInfo* ci, void* alloc, out IntPtr device); - [LibraryImport(LibName)] public static partial void vkDestroyDevice(IntPtr device, void* alloc); - [LibraryImport(LibName)] public static partial void vkGetDeviceQueue(IntPtr device, uint family, uint index, out IntPtr queue); - - [LibraryImport(LibName)] public static partial VkResult vkCreateBuffer(IntPtr device, VkBufferCreateInfo* ci, void* alloc, out IntPtr buffer); - [LibraryImport(LibName)] public static partial void vkDestroyBuffer(IntPtr device, IntPtr buffer, void* alloc); - [LibraryImport(LibName)] public static partial void vkFreeCommandBuffers(IntPtr device, IntPtr pool, uint count, IntPtr* buffers); - [LibraryImport(LibName)] public static partial VkResult vkFreeDescriptorSets(IntPtr device, IntPtr pool, uint count, IntPtr* sets); - [LibraryImport(LibName)] public static partial void vkDestroyCommandPool(IntPtr device, IntPtr pool, void* alloc); - [LibraryImport(LibName)] public static partial void vkGetBufferMemoryRequirements(IntPtr device, IntPtr buffer, VkMemoryRequirements* req); - [LibraryImport(LibName)] public static partial VkResult vkAllocateMemory(IntPtr device, VkMemoryAllocateInfo* ai, void* alloc, out IntPtr memory); - [LibraryImport(LibName)] public static partial void vkFreeMemory(IntPtr device, IntPtr memory, void* alloc); - [LibraryImport(LibName)] public static partial VkResult vkBindBufferMemory(IntPtr device, IntPtr buffer, IntPtr memory, ulong offset); - [LibraryImport(LibName)] public static partial VkResult vkMapMemory(IntPtr device, IntPtr memory, ulong offset, ulong size, uint flags, void** data); - [LibraryImport(LibName)] public static partial void vkUnmapMemory(IntPtr device, IntPtr memory); - [LibraryImport(LibName)] public static partial VkResult vkFlushMappedMemoryRanges(IntPtr device, uint count, VkMappedMemoryRange* ranges); - [LibraryImport(LibName)] public static partial VkResult vkInvalidateMappedMemoryRanges(IntPtr device, uint count, VkMappedMemoryRange* ranges); - - [LibraryImport(LibName)] public static partial VkResult vkCreateShaderModule(IntPtr device, VkShaderModuleCreateInfo* ci, void* alloc, out IntPtr module); - [LibraryImport(LibName)] public static partial void vkDestroyShaderModule(IntPtr device, IntPtr module, void* alloc); - [LibraryImport(LibName)] public static partial VkResult vkCreatePipelineLayout(IntPtr device, VkPipelineLayoutCreateInfo* ci, void* alloc, out IntPtr layout); - [LibraryImport(LibName)] public static partial VkResult vkCreateComputePipelines(IntPtr device, IntPtr cache, uint count, VkComputePipelineCreateInfo* ci, void* alloc, IntPtr* pipelines); - [LibraryImport(LibName)] public static partial void vkDestroyPipeline(IntPtr device, IntPtr pipeline, void* alloc); - - [LibraryImport(LibName)] public static partial VkResult vkCreateDescriptorSetLayout(IntPtr device, VkDescriptorSetLayoutCreateInfo* ci, void* alloc, out IntPtr setLayout); - [LibraryImport(LibName)] public static partial void vkDestroyPipelineLayout(IntPtr device, IntPtr layout, void* alloc); - [LibraryImport(LibName)] public static partial void vkDestroyDescriptorSetLayout(IntPtr device, IntPtr setLayout, void* alloc); - [LibraryImport(LibName)] public static partial VkResult vkCreateDescriptorPool(IntPtr device, VkDescriptorPoolCreateInfo* ci, void* alloc, out IntPtr pool); - [LibraryImport(LibName)] public static partial VkResult vkAllocateDescriptorSets(IntPtr device, VkDescriptorSetAllocateInfo* ai, IntPtr* sets); - [LibraryImport(LibName)] public static partial VkResult vkResetDescriptorPool(IntPtr device, IntPtr pool, uint flags); - [LibraryImport(LibName)] public static partial void vkDestroyDescriptorPool(IntPtr device, IntPtr pool, void* alloc); - [LibraryImport(LibName)] public static partial void vkUpdateDescriptorSets(IntPtr device, uint writeCount, VkWriteDescriptorSet* writes, uint copyCount, void* copies); - [LibraryImport(LibName)] public static partial VkResult vkEnumerateDeviceExtensionProperties(IntPtr physicalDevice, byte* layerName, uint* count, VkExtensionProperties* props); - [LibraryImport(LibName)] public static partial IntPtr vkGetDeviceProcAddr(IntPtr device, byte* name); - [LibraryImport(LibName)] public static partial IntPtr vkGetInstanceProcAddr(IntPtr instance, byte* name); - - [LibraryImport(LibName)] public static partial VkResult vkCreateCommandPool(IntPtr device, VkCommandPoolCreateInfo* ci, void* alloc, out IntPtr pool); - [LibraryImport(LibName)] public static partial VkResult vkAllocateCommandBuffers(IntPtr device, VkCommandBufferAllocateInfo* ai, IntPtr* buffers); - [LibraryImport(LibName)] public static partial VkResult vkResetCommandBuffer(IntPtr cmd, uint flags); - [LibraryImport(LibName)] public static partial VkResult vkBeginCommandBuffer(IntPtr cmd, VkCommandBufferBeginInfo* bi); - [LibraryImport(LibName)] public static partial VkResult vkEndCommandBuffer(IntPtr cmd); - [LibraryImport(LibName)] public static partial void vkCmdBindPipeline(IntPtr cmd, uint bindPoint, IntPtr pipeline); - [LibraryImport(LibName)] public static partial void vkCmdBindDescriptorSets(IntPtr cmd, uint bindPoint, IntPtr layout, uint firstSet, uint count, IntPtr* sets, uint dynCount, uint* dynOffsets); - [LibraryImport(LibName)] public static partial void vkCmdPushConstants(IntPtr cmd, IntPtr layout, uint stageFlags, uint offset, uint size, void* values); - [LibraryImport(LibName)] public static partial void vkCmdDispatch(IntPtr cmd, uint x, uint y, uint z); - [LibraryImport(LibName)] public static partial void vkCmdPipelineBarrier(IntPtr cmd, uint srcStage, uint dstStage, uint depFlags, - uint memBarrierCount, VkMemoryBarrier* memBarriers, uint bufBarrierCount, void* bufBarriers, uint imgBarrierCount, void* imgBarriers); - [LibraryImport(LibName)] public static partial void vkCmdFillBuffer(IntPtr cmd, IntPtr buffer, ulong offset, ulong size, uint data); - [LibraryImport(LibName)] public static partial void vkCmdCopyBuffer(IntPtr cmd, IntPtr src, IntPtr dst, uint count, VkBufferCopy* regions); - - [StructLayout(LayoutKind.Sequential)] - public struct VkQueryPoolCreateInfo { public uint SType; public void* PNext; public uint Flags; public uint QueryType; public uint QueryCount; public uint PipelineStatistics; } - [LibraryImport(LibName)] public static partial VkResult vkCreateQueryPool(IntPtr device, VkQueryPoolCreateInfo* ci, void* alloc, out IntPtr pool); - [LibraryImport(LibName)] public static partial void vkDestroyQueryPool(IntPtr device, IntPtr pool, void* alloc); - [LibraryImport(LibName)] public static partial void vkCmdResetQueryPool(IntPtr cmd, IntPtr pool, uint first, uint count); - [LibraryImport(LibName)] public static partial void vkCmdWriteTimestamp(IntPtr cmd, uint stage, IntPtr pool, uint query); - [LibraryImport(LibName)] public static partial VkResult vkGetQueryPoolResults(IntPtr device, IntPtr pool, uint first, uint count, nuint dataSize, void* data, ulong stride, uint flags); - - [StructLayout(LayoutKind.Sequential)] - public struct VkBufferCopy { public ulong SrcOffset, DstOffset, Size; } - - [StructLayout(LayoutKind.Sequential)] - public struct VkBufferMemoryBarrier - { - public uint SType; public void* PNext; - public uint SrcAccessMask, DstAccessMask; - public uint SrcQueueFamilyIndex, DstQueueFamilyIndex; - public IntPtr Buffer; public ulong Offset, Size; - } - - [LibraryImport(LibName)] public static partial VkResult vkQueueSubmit(IntPtr queue, uint submitCount, VkSubmitInfo* submits, IntPtr fence); - [LibraryImport(LibName)] public static partial VkResult vkCreateFence(IntPtr device, VkFenceCreateInfo* ci, void* alloc, out IntPtr fence); - [LibraryImport(LibName)] public static partial VkResult vkWaitForFences(IntPtr device, uint count, IntPtr* fences, uint waitAll, ulong timeout); - [LibraryImport(LibName)] public static partial VkResult vkResetFences(IntPtr device, uint count, IntPtr* fences); - [LibraryImport(LibName)] public static partial void vkDestroyFence(IntPtr device, IntPtr fence, void* alloc); - [LibraryImport(LibName)] public static partial VkResult vkGetFenceStatus(IntPtr device, IntPtr fence); - [LibraryImport(LibName)] public static partial VkResult vkQueueWaitIdle(IntPtr queue); - [LibraryImport(LibName)] public static partial VkResult vkDeviceWaitIdle(IntPtr device); -} - -internal enum VkResult : int -{ - Success = 0, - NotReady = 1, - Timeout = 2, - ErrorOutOfHostMemory = -1, - ErrorOutOfDeviceMemory = -2, - ErrorInitializationFailed = -3, - ErrorDeviceLost = -4, - ErrorExtensionNotPresent = -7, - ErrorFeatureNotPresent = -8, - ErrorIncompatibleDriver = -9, -} - -internal static class VkConst -{ - // VkStructureType - public const uint StApplicationInfo = 0; - public const uint StInstanceCreateInfo = 1; - public const uint StDeviceQueueCreateInfo = 2; - public const uint StDeviceCreateInfo = 3; - public const uint StSubmitInfo = 4; - public const uint StMemoryAllocateInfo = 5; - public const uint StMappedMemoryRange = 6; - public const uint StFenceCreateInfo = 8; - public const uint StQueryPoolCreateInfo = 11; - public const uint StBufferCreateInfo = 12; - public const uint StShaderModuleCreateInfo = 16; - public const uint StPipelineShaderStageCreateInfo = 18; - public const uint StComputePipelineCreateInfo = 29; - public const uint StPipelineLayoutCreateInfo = 30; - public const uint StDescriptorSetLayoutCreateInfo = 32; - public const uint StDescriptorPoolCreateInfo = 33; - public const uint StDescriptorSetAllocateInfo = 34; - public const uint StWriteDescriptorSet = 35; - public const uint StCommandPoolCreateInfo = 39; - public const uint StCommandBufferAllocateInfo = 40; - public const uint StCommandBufferBeginInfo = 42; - public const uint StMemoryBarrier = 46; - public const uint StBufferMemoryBarrier = 44; - public const uint QueueFamilyIgnored = 0xFFFFFFFFu; - public const uint StPhysicalDeviceFeatures2 = 49; - public const uint StPhysicalDevice16BitStorageFeatures = 1000083000u; - public const uint StPhysicalDeviceShaderFloat16Int8Features = 1000082000u; - public const uint StPhysicalDeviceVulkanMemoryModelFeatures = 1000211000u; - public const uint StPhysicalDeviceCooperativeMatrixFeaturesKHR = 1000506000u; - public const uint StCooperativeMatrixPropertiesKHR = 1000506001u; - public const uint StPhysicalDeviceSubgroupProperties = 1000094000u; - public const uint StPhysicalDeviceSubgroupSizeControlPropertiesEXT = 1000225000u; - public const uint StPhysicalDeviceSubgroupSizeControlFeaturesEXT = 1000225002u; - public const uint StPhysicalDeviceProperties2 = 1000059000u; - public const uint StPipelineShaderStageRequiredSubgroupSizeCreateInfo = 1000225001u; - - // VkPhysicalDeviceType - public const uint PhysDeviceIntegrated = 1; - public const uint PhysDeviceDiscrete = 2; - public const uint PhysDeviceVirtualGpu = 3; - public const uint PhysDeviceCpu = 4; - - // VkQueueFlagBits - public const uint QueueGraphics = 0x1; - public const uint QueueCompute = 0x2; - public const uint QueueTransfer = 0x4; - - // VkBufferUsageFlagBits - public const uint BufferUsageTransferSrc = 0x1; - public const uint BufferUsageTransferDst = 0x2; - public const uint BufferUsageStorageBuffer = 0x20; - - // VkMemoryPropertyFlagBits - public const uint MemDeviceLocal = 0x1; - public const uint MemHostVisible = 0x2; - public const uint MemHostCoherent = 0x4; - public const uint MemHostCached = 0x8; - - // VkDescriptorType - public const uint DescStorageBuffer = 7; - public const uint DescUniformBuffer = 6; - - // VkShaderStageFlagBits / VkPipelineStageFlagBits - public const uint StageComputeShader = 0x20; - public const uint PipelineStageTransfer = 0x1000; - public const uint PipelineStageComputeShader = 0x800; - public const uint PipelineStageHost = 0x4000; - public const uint PipelineStageTopOfPipe = 0x1; - public const uint PipelineStageBottomOfPipe = 0x2000; - - // VkAccessFlagBits - public const uint AccessShaderRead = 0x20; - public const uint AccessShaderWrite = 0x40; - public const uint AccessTransferRead = 0x800; - public const uint AccessTransferWrite = 0x1000; - public const uint AccessHostRead = 0x2000; - public const uint AccessHostWrite = 0x4000; - public const uint StageTopOfPipe = 0x1; - public const uint StageAllCommands = 0x10000; - - // VkPipelineBindPoint - public const uint BindPointCompute = 1; - - // VkCommandBufferLevel - public const uint CmdBufferLevelPrimary = 0; - - // VkCommandPoolCreateFlagBits - public const uint CmdPoolResetCommandBuffer = 0x2; - - // VkSharingMode - public const uint SharingExclusive = 0; - - // VkQueryType / VkQueryResultFlagBits - public const uint QueryTypeTimestamp = 1; - public const uint QueryResult64 = 0x1; - public const uint QueryResultWait = 0x2; -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkBackend.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkBackend.cs deleted file mode 100644 index 661321e..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkBackend.cs +++ /dev/null @@ -1,91 +0,0 @@ -using System.Collections.Concurrent; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; - -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -/// -/// Process-wide Vulkan device + per-(model,size) compiled graph cache. -/// Constructed lazily on first Vulkan use. -/// -internal sealed unsafe class VkBackend : IDisposable -{ - internal readonly VkDevice Dev; - // (model, c, h, w) -> compiled graph (weights upload + dispatch list) - private readonly ConcurrentDictionary<(OnnxModel m, int c, int h, int w, bool physIo), VkGraphModel> _graphs = new(); - - private static readonly Lazy s_default = new(() => new VkBackend()); - internal static VkBackend Default => s_default.Value; - - /// - /// Plan options for GPU compiles: always channels-last (shaders address NHWC). - /// Standalone sessions keep logical NCHW I/O (boundary relayouts in the plan); - /// the fused pipeline keeps I/O physical and its glue shaders speak NHWC. - /// - internal static CompileOptions PlanOptions => new(Nhwc: true, PhysicalIo: false, Gpu: true); - internal static CompileOptions PipeOptions => new(Nhwc: true, PhysicalIo: true, Gpu: true); - - /// fp16 activation/weight storage (needs storageBuffer16BitAccess; GCDRNET_VK_F16=0 forces fp32). - internal bool F16 => Dev.StorageBuffer16 && Environment.GetEnvironmentVariable("GCDRNET_VK_F16") != "0"; - - internal VkBackend() - { - Dev = VkDevice.Create(); - } - - /// Standalone session for one model at [1,c,h,w] (test path). - public VkInferenceSession NewSession(OnnxModel m, int[] shape) - { - VkGraphModel g = AcquireGraph(m, shape, physIo: false); // caller ref: later evictions can't free it under us - try { return new VkInferenceSession(Dev, g.Plan, g); } - finally { g.Release(); } - } - - /// Compiled GPU graph for a model+shape (cached, bounded). - internal VkGraphModel GraphFor(OnnxModel m, int[] shape, bool physIo = false) - { - var key = (m, shape[1], shape[2], shape[3], physIo); - if (_graphs.TryGetValue(key, out var g)) return g; - // weights live in a per-model const store shared across sizes, so a graph is - // just a plan + dispatch list (no per-size device memory) - g = new VkGraphModel(Dev, GraphCompiler.Compile(m, [1, shape[1], shape[2], shape[3]], - physIo ? PipeOptions : PlanOptions), F16, constKey: m); - if (!_graphs.TryAdd(key, g)) { g.Dispose(); return _graphs[key]; } - // bound the cache: evict an arbitrary other graph beyond the cap - while (_graphs.Count > MaxGraphs) - { - var victim = _graphs.Keys.FirstOrDefault(k => !k.Equals(key)); - if (victim.Equals(default)) break; - if (_graphs.TryRemove(victim, out var old)) old.Release(); - } - return g; - } - - // graphs hold no device memory beyond the shared const store; the bound is for - // the managed plans (each holds its own copy of the model's const floats) - private const int MaxGraphs = 32; - - /// plus a caller reference. Retries when a concurrent - /// eviction freed the cached graph between lookup and acquire (never resurrects it). - internal VkGraphModel AcquireGraph(OnnxModel m, int[] shape, bool physIo) - { - while (true) - { - VkGraphModel g = GraphFor(m, shape, physIo); - if (g.TryAcquire()) return g; - } - } - - /// New size-agnostic GPU worker for a (gc, dr) model pair (stateful; callers pool them). - internal VkGcdrPipeline NewWorker(OnnxModel gc, OnnxModel dr) => new(this, gc, dr); - - /// New GPU worker pre-bound to one source size (debug/bench helper). - internal VkGcdrPipeline NewPipe(OnnxModel gc, OnnxModel dr, int ih, int iw) - { - var p = new VkGcdrPipeline(this, gc, dr); - try { p.Bind(ih, iw); return p; } - catch { p.Dispose(); throw; } - } - - public void Dispose() => Dev.Dispose(); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkConstStore.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkConstStore.cs deleted file mode 100644 index 5c46aeb..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkConstStore.cs +++ /dev/null @@ -1,91 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -/// -/// Device-side const tensors (backend-packed weights) plus a 64B zero buffer, -/// shared by every of one model: per-size plans copy -/// the same initializers, so instead of re-packing and re-uploading ~20 MB per -/// input size, a new graph whose const list is element-for-element identical -/// (same consumer packing, same contents) reuses the existing buffer. A graph -/// whose consts differ gets a private, unshared store. Ref-counted; the -/// buffers are freed when the last graph releases. -/// -internal sealed class VkConstStore -{ - private static readonly Dictionary<(VkDevice dev, object key, bool f16), VkConstStore> s_shared = new(); - - private readonly (VkDevice, object, bool)? _key; - private readonly float[][] _src; - private readonly int[] _kinds; - private int _refs = 1; - internal readonly VkBuffer Consts, Dummy; - internal readonly ulong[] Offsets; - internal readonly ulong Bytes; - - private VkConstStore(VkDevice dev, (VkDevice, object, bool)? key, float[][] src, int[] kinds, - ulong[] offsets, byte[] host) - { - _key = key; - _src = src; - _kinds = kinds; - Offsets = offsets; - Bytes = (ulong)host.LongLength; - Consts = dev.NewStorageBuffer(Bytes, hostVisible: dev.CoherentDeviceLocal); - Dummy = dev.NewStorageBuffer(64, hostVisible: dev.CoherentDeviceLocal); - unsafe - { - fixed (byte* hp = host) dev.Upload(Consts, hp, Bytes); - byte* z = stackalloc byte[64]; - new Span(z, 64).Clear(); - dev.Upload(Dummy, z, 64); - } - } - - /// - /// Returns a store holding (kinds = consumer packing ids), - /// shared under when an identical list was uploaded before. - /// (offsets, host bytes) only runs when a new store is built. - /// - internal static VkConstStore Acquire(VkDevice dev, object? key, bool f16, float[][] src, int[] kinds, - Func<(ulong[] offsets, byte[] host)> pack) - { - if (key is null) - { - var (o, h) = pack(); - return new VkConstStore(dev, null, src, kinds, o, h); - } - var k = (dev, key, f16); - lock (s_shared) - { - if (s_shared.TryGetValue(k, out VkConstStore? s) && s.Matches(src, kinds)) - { - s._refs++; - return s; - } - var (offs, host) = pack(); - bool share = s is null; // keep the first registered store on a mismatch - var ns = new VkConstStore(dev, share ? k : null, src, kinds, offs, host); - if (share) s_shared[k] = ns; - return ns; - } - } - - private bool Matches(float[][] src, int[] kinds) - { - if (src.Length != _src.Length || !kinds.AsSpan().SequenceEqual(_kinds)) return false; - for (int i = 0; i < src.Length; i++) - if (!ReferenceEquals(src[i], _src[i]) && !src[i].AsSpan().SequenceEqual(_src[i])) return false; - return true; - } - - internal void Release() - { - lock (s_shared) - { - if (--_refs > 0) return; - if (_key is { } k && s_shared.TryGetValue(k, out VkConstStore? cur) && cur == this) - s_shared.Remove(k); - } - Consts.Free(); - Dummy.Free(); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkDevice.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkDevice.cs deleted file mode 100644 index 34f6d51..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkDevice.cs +++ /dev/null @@ -1,938 +0,0 @@ -// Thin object wrappers over Vk.cs: instance/device/queue, buffers with -// memory-type selection (ReBAR fast path vs host-visible staging), shader -// modules from embedded SPIR-V, compute pipelines, descriptor sets, and -// one-shot command submission. Adapted from Sdcb.SimdPaddleOCR. -using System.Runtime.InteropServices; - -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -internal unsafe sealed class VkDevice : IDisposable -{ - public IntPtr Instance, PhysDevice, Device, Queue; - public string DeviceName = ""; - public uint VendorId; - public uint SubgroupSize; - public uint QueueFamily; - public Vk.VkPhysicalDeviceMemoryProperties MemProps; - public bool CoherentDeviceLocal; // DEVICE_LOCAL|HOST_VISIBLE|HOST_COHERENT exists (ReBAR) - public bool StorageBuffer16; // VK_KHR_16bit_storage: activations may be stored as float16 - public uint MaxSharedMemory; - public ulong MaxStorageRange = uint.MaxValue; - public bool PushDescriptors; - /// Tensor-core convs (conv_cm.comp; GCDRNET_VK_COOPMAT=0 disables): VK_KHR_cooperative_matrix - /// plus shaderFloat16, in one of two shapes. Default: a 16x16x16 fp16 x fp16 -> fp32 subgroup config - /// on 32-lane subgroups; devices whose default compute subgroup is not 32 lanes (RDNA wave64) qualify - /// only when VK_EXT_subgroup_size_control can pin it to 32 (). - /// Otherwise : the 8x16x16 config on subgroups pinned to 16 lanes (Intel Xe2). - /// A probe dispatch must confirm the pinned width; failing that the convs stay on conv_tile. - public bool CoopMat16; - /// The coopmat convs use the SG16 build (8x16x16 fragments, 16-lane subgroups; Intel Arc - /// XMX): taken only when the 16x16x16 / 32-lane path is unavailable (GCDRNET_VK_SG16=0 disables). - public bool CoopM8; - /// requiredSubgroupSize for the coopmat pipelines: 0 = device default (already 32), - /// 32 / 16 = pinned through VK_EXT_subgroup_size_control. - public uint CoopSubgroupSize; - /// Coopmat convs use the LITE variants (single-buffered 28 KB staging, scalar epilogue, - /// grouped raster): wave64-capable parts or less than the 48 KB shared memory the - /// double-buffered kernels take. - public bool CoopLite; - /// VK_EXT_subgroup_size_control range (1..128 when the extension is absent). - public uint SubgroupMin = 1, SubgroupMax = 128; - /// Memory-bound NHWC shaders (maxpool, bilinear, depthwise, shift) take 4 channels per - /// invocation when c % 4 == 0 — integrated GPUs, where they are bandwidth/issue bound, and Intel - /// GPUs (Arc B580: 10.4 -> 6.7 ms for the four; per-element index math is costly on SIMD16) - /// (GCDRNET_VK_VEC4=0 disables; other discrete GPUs keep the per-element shaders). - public bool Vec4Eltwise; - /// 3x3 stem conv+pool uses convpool_k3 (input window rows in registers): Intel GPUs - /// (Arc B580: 7.5 -> 4.9 ms for the two stems) (GCDRNET_VK_CONVPOOL3=0 disables). - public bool ConvPoolK3; - /// Register-tiled kernels for GPUs without cooperative matrices whose compute subgroups - /// are at least 64 lanes (Adreno 750: 64/128, 32 KB shared memory), where conv_tile / gemm's - /// shared-memory staging is load-bound: 3x3 convs on conv_dot.comp, Gemm on gemm_dot.comp, - /// the stem conv+pool with the four pool pixels per weight load (convpool_pi) and the UpHead - /// tail two pixels per invocation (uphead_sc2). fp16 storage only (GCDRNET_VK_DOT=0 disables). - public bool ConvDot; - private const uint VendorIntel = 0x8086; - /// Largest DEVICE_LOCAL heap (bytes). - public ulong DeviceLocalHeap; - /// Integrated GPU: "device-local" memory is system RAM shared with the CPU. - public bool UnifiedMemory; - private unsafe delegate* unmanaged[Cdecl] _pushDesc; - public double TimestampPeriodNs = 1; - private IntPtr _commandPool; - private IntPtr _descPool; - - // live vkAllocateMemory bytes across the process (diagnostic — buffers are pooled) - private static long _liveBytes, _peakBytes; - public static long LiveDeviceBytes => Interlocked.Read(ref _liveBytes); - public static long PeakDeviceBytes => Interlocked.Read(ref _peakBytes); - public static void ResetPeakDeviceBytes() => Interlocked.Exchange(ref _peakBytes, _liveBytes); - private static void TrackAlloc(long bytes) - { - long now = Interlocked.Add(ref _liveBytes, bytes), p; - while ((p = Interlocked.Read(ref _peakBytes)) < now) - Interlocked.CompareExchange(ref _peakBytes, now, p); - } - internal static void TrackFree(long bytes) => Interlocked.Add(ref _liveBytes, -bytes); - /// Guards the device-level command pool / descriptor pool (uploads). - public readonly object Sync = new(); - /// vkQueueSubmit on the single compute queue is externally synchronized. - public readonly object QueueLock = new(); - private readonly Dictionary<(string name, int bindings, int pcBytes, uint sg), VkPipeline> _pipes = new(); - - /// Process-wide pipeline cache: every graph/session shares one - /// VkPipeline per (shader, binding-count, push-const-size). Set layouts carry - /// PUSH_DESCRIPTOR_BIT when the extension is available. - public VkPipeline GetPipeline(string name, int bindings, int pcBytes, uint requiredSubgroupSize = 0) - { - var key = (name, bindings, pcBytes, requiredSubgroupSize); - lock (_pipes) - { - if (_pipes.TryGetValue(key, out VkPipeline? p)) return p; - p = NewPipeline(NewShaderModule(VkShaders.Load(name)), bindings, pcBytes, PushDescriptors, requiredSubgroupSize); - p.Name = name; - p.Bindings = bindings; - _pipes[key] = p; - return p; - } - } - - /// Pipeline of a conv_cm variant ("conv_cm", "conv_cm_n64", ...; fp16 storage): the - /// LITE build on devices (or when forces it), - /// the SG16 build (always LITE-style) on devices, with the subgroup pinned - /// to . Every variant keeps BM = 128 rows per workgroup. - public VkPipeline GetCoopPipeline(string variant, int bindings, int pcBytes, bool? lite = null) - { - if (!variant.StartsWith("conv_cm", StringComparison.Ordinal)) throw new ArgumentException(variant); - string suffix = variant["conv_cm".Length..]; - string name = CoopM8 ? "conv_cm_sg16" + suffix : (lite ?? CoopLite) ? "conv_cm_l" + suffix : variant; - return GetPipeline(name + "_h", bindings, pcBytes, CoopSubgroupSize); - } - - public static VkDevice Create(uint deviceIndex = 0) - { - Vk.RegisterResolver(); - var d = new VkDevice(); - - Vk.VkApplicationInfo app = new() - { - SType = VkConst.StApplicationInfo, - AppVersion = 1, EngineVersion = 1, ApiVersion = (1u << 22) | (1u << 12), // Vulkan 1.1 - }; - Vk.VkInstanceCreateInfo ici = new() { SType = VkConst.StInstanceCreateInfo, PAppInfo = &app }; - Vk.Check(Vk.vkCreateInstance(&ici, null, out d.Instance), "vkCreateInstance"); - - uint ndev = 0; - Vk.Check(Vk.vkEnumeratePhysicalDevices(d.Instance, &ndev, null), "enum physical devices"); - if (ndev == 0) throw new PlatformNotSupportedException("Vulkan: no physical devices (no Vulkan-capable GPU/driver)"); - IntPtr* devs = stackalloc IntPtr[(int)ndev]; - Vk.Check(Vk.vkEnumeratePhysicalDevices(d.Instance, &ndev, devs), "enum physical devices"); - - // Prefer discrete GPU; honor deviceIndex as "the nth discrete first, else nth overall". - int chosen = -1; - for (int pass = 0; pass < 2 && chosen < 0; pass++) - { - uint seen = 0; - for (int i = 0; i < (int)ndev; i++) - { - Vk.VkPhysicalDeviceProperties p; - Vk.vkGetPhysicalDeviceProperties(devs[i], &p); - bool want = pass == 0 ? p.DeviceType == VkConst.PhysDeviceDiscrete : p.DeviceType != VkConst.PhysDeviceCpu; - if (!want) continue; - if (seen++ == deviceIndex) { chosen = i; break; } - } - } - if (chosen < 0) throw new PlatformNotSupportedException($"Vulkan: device index {deviceIndex} out of range ({ndev} devices)"); - d.PhysDevice = devs[chosen]; - - Vk.VkPhysicalDeviceProperties props; - Vk.vkGetPhysicalDeviceProperties(d.PhysDevice, &props); - d.DeviceName = Marshal.PtrToStringAnsi((IntPtr)props.DeviceName) ?? "?"; - d.VendorId = props.VendorID; - d.TimestampPeriodNs = props.TimestampPeriodNs; - d.MaxSharedMemory = props.MaxComputeSharedMemorySize; - d.MaxStorageRange = props.MaxStorageBufferRange; - - uint nqf = 0; - Vk.vkGetPhysicalDeviceQueueFamilyProperties(d.PhysDevice, &nqf, null); - Vk.VkQueueFamilyProperties* qf = stackalloc Vk.VkQueueFamilyProperties[(int)nqf]; - Vk.vkGetPhysicalDeviceQueueFamilyProperties(d.PhysDevice, &nqf, qf); - if (Environment.GetEnvironmentVariable("GCDRNET_VK_DBG") == "1") - for (uint i = 0; i < nqf; i++) - Console.Error.WriteLine($"[vk] qf{i}: flags={qf[i].QueueFlags:x} count={qf[i].QueueCount} tsbits={qf[i].TimestampValidBits}"); - // Prefer a compute-only family (no GRAPHICS bit) for async-compute — - // but only if it supports timestamps; otherwise fall back to graphics+compute. - d.QueueFamily = uint.MaxValue; - for (uint i = 0; i < nqf; i++) - if ((qf[i].QueueFlags & VkConst.QueueCompute) != 0 && (qf[i].QueueFlags & 1u) == 0 - && qf[i].TimestampValidBits > 0) - { d.QueueFamily = i; break; } - for (uint i = 0; i < nqf && d.QueueFamily == uint.MaxValue; i++) - if ((qf[i].QueueFlags & VkConst.QueueCompute) != 0 && qf[i].TimestampValidBits > 0) - { d.QueueFamily = i; break; } - for (uint i = 0; i < nqf && d.QueueFamily == uint.MaxValue; i++) - if ((qf[i].QueueFlags & VkConst.QueueCompute) != 0) { d.QueueFamily = i; break; } - if (d.QueueFamily == uint.MaxValue) - throw new PlatformNotSupportedException("Vulkan: no compute queue family"); - - // Extension probe: VK_KHR_push_descriptor (inline descriptor writes, - // skips per-dispatch descriptor-set binds). - uint next = 0; - Vk.vkEnumerateDeviceExtensionProperties(d.PhysDevice, null, &next, null); - Vk.VkExtensionProperties* exts = stackalloc Vk.VkExtensionProperties[(int)next]; - Vk.vkEnumerateDeviceExtensionProperties(d.PhysDevice, null, &next, exts); - byte* wantPush = stackalloc byte[] { (byte)'V', (byte)'K', (byte)'_', (byte)'K', (byte)'H', (byte)'R', - (byte)'_', (byte)'p', (byte)'u', (byte)'s', (byte)'h', (byte)'_', (byte)'d', (byte)'e', (byte)'s', (byte)'c', - (byte)'r', (byte)'i', (byte)'p', (byte)'t', (byte)'o', (byte)'r', 0 }; - bool hasPush = false, hasCoop = false, hasF16Int8 = false, hasVmm = false, hasSgc = false; - for (int i = 0; i < (int)next; i++) - { - string en = new((sbyte*)exts[i].ExtensionName); - if (en == "VK_KHR_push_descriptor") hasPush = true; - else if (en == "VK_KHR_cooperative_matrix") hasCoop = true; - else if (en == "VK_KHR_shader_float16_int8") hasF16Int8 = true; - else if (en == "VK_KHR_vulkan_memory_model") hasVmm = true; - else if (en == "VK_EXT_subgroup_size_control") hasSgc = true; - } - if (Environment.GetEnvironmentVariable("GCDRNET_VK_NOPUSH") == "1") hasPush = false; - - float prio = 1f; - Vk.VkDeviceQueueCreateInfo qci = new() - { - SType = VkConst.StDeviceQueueCreateInfo, - QueueFamilyIndex = d.QueueFamily, QueueCount = 1, PQueuePriorities = &prio, - }; - Vk.VkPhysicalDeviceSubgroupProperties sgP = new() { SType = VkConst.StPhysicalDeviceSubgroupProperties }; - Vk.VkPhysicalDeviceSubgroupSizeControlProperties sgcP = new() { SType = VkConst.StPhysicalDeviceSubgroupSizeControlPropertiesEXT }; - if (hasSgc) sgP.PNext = &sgcP; - Vk.VkPhysicalDeviceProperties2 p2 = new() { SType = VkConst.StPhysicalDeviceProperties2, PNext = &sgP }; - Vk.vkGetPhysicalDeviceProperties2(d.PhysDevice, &p2); - d.SubgroupSize = sgP.SubgroupSize; - bool sgcCompute = false; - if (hasSgc) - { - d.SubgroupMin = sgcP.MinSubgroupSize; - d.SubgroupMax = sgcP.MaxSubgroupSize; - sgcCompute = (sgcP.RequiredSubgroupSizeStages & VkConst.StageComputeShader) != 0; - } - - // Query storageBuffer16BitAccess (VK 1.1 core) for fp16 activation storage. - Vk.VkPhysicalDevice16BitStorageFeatures feat16 = new() - { - SType = VkConst.StPhysicalDevice16BitStorageFeatures, - }; - Vk.VkPhysicalDeviceFeatures2 feats2 = new() - { - SType = VkConst.StPhysicalDeviceFeatures2, - PNext = &feat16, - }; - Vk.vkGetPhysicalDeviceFeatures2(d.PhysDevice, &feats2); - d.StorageBuffer16 = feat16.StorageBuffer16BitAccess != 0; - - // tensor-core path: coopmat + shaderFloat16 on top of 16-bit storage, 32-lane subgroups; - // coopmat SPIR-V uses the Vulkan memory model, so that feature is required as well - Vk.VkPhysicalDeviceShaderFloat16Int8Features f16q = new() { SType = VkConst.StPhysicalDeviceShaderFloat16Int8Features }; - Vk.VkPhysicalDeviceCooperativeMatrixFeaturesKHR coopQ = new() { SType = VkConst.StPhysicalDeviceCooperativeMatrixFeaturesKHR }; - Vk.VkPhysicalDeviceVulkanMemoryModelFeatures vmmQ = new() { SType = VkConst.StPhysicalDeviceVulkanMemoryModelFeatures }; - Vk.VkPhysicalDeviceSubgroupSizeControlFeaturesEXT sgcQ = new() { SType = VkConst.StPhysicalDeviceSubgroupSizeControlFeaturesEXT }; - // a default other than 32 lanes (RDNA: wave64) needs the compute subgroup pinned to 32 - bool pinSg32 = d.SubgroupSize != 32 && hasSgc && sgcCompute && d.SubgroupMin <= 32 && d.SubgroupMax >= 32 - && Environment.GetEnvironmentVariable("GCDRNET_VK_SG32") != "0"; - // the SG16 build (8x16x16, Intel Xe2 XMX) needs 16 lanes in the pinnable compute range - bool canSg16 = hasSgc && sgcCompute && d.SubgroupMin <= 16 && d.SubgroupMax >= 16 - && Environment.GetEnvironmentVariable("GCDRNET_VK_SG16") != "0"; - bool wantCm = hasCoop && hasF16Int8 && hasVmm && d.StorageBuffer16 && (d.SubgroupSize == 32 || pinSg32 || canSg16) - && Environment.GetEnvironmentVariable("GCDRNET_VK_COOPMAT") != "0"; - bool pinSg16 = false; - if (wantCm) - { - f16q.PNext = &coopQ; - coopQ.PNext = &vmmQ; - if (pinSg32 || canSg16) vmmQ.PNext = &sgcQ; - Vk.VkPhysicalDeviceFeatures2 fq = new() { SType = VkConst.StPhysicalDeviceFeatures2, PNext = &f16q }; - Vk.vkGetPhysicalDeviceFeatures2(d.PhysDevice, &fq); - bool feats = f16q.ShaderFloat16 != 0 && coopQ.CooperativeMatrix != 0 && vmmQ.VulkanMemoryModel != 0; - CoopShapes(d, out bool m16, out bool m8); - // 16x16x16 on 32 lanes first (NVIDIA, RDNA pinned to 32); 8x16x16 on 16 lanes only without it - bool cm32 = feats && (d.SubgroupSize == 32 || pinSg32) && (!pinSg32 || sgcQ.SubgroupSizeControl != 0) && m16; - pinSg16 = !cm32 && feats && canSg16 && sgcQ.SubgroupSizeControl != 0 && m8; - wantCm = cm32 || pinSg16; - pinSg32 &= cm32; - } - else pinSg32 = false; - bool pinSg = pinSg32 || pinSg16; - - byte** extsToEnable = stackalloc byte*[5]; - uint nExt = 0; - if (hasPush) extsToEnable[nExt++] = wantPush; - byte* pCm = stackalloc byte[32], pF16 = stackalloc byte[32], pVmm = stackalloc byte[32], pSgc = stackalloc byte[32]; - System.Text.Encoding.ASCII.GetBytes("VK_KHR_cooperative_matrix\0", new Span(pCm, 32)); - System.Text.Encoding.ASCII.GetBytes("VK_KHR_shader_float16_int8\0", new Span(pF16, 32)); - System.Text.Encoding.ASCII.GetBytes("VK_KHR_vulkan_memory_model\0", new Span(pVmm, 32)); - System.Text.Encoding.ASCII.GetBytes("VK_EXT_subgroup_size_control\0", new Span(pSgc, 32)); - if (wantCm) { extsToEnable[nExt++] = pCm; extsToEnable[nExt++] = pF16; extsToEnable[nExt++] = pVmm; } - if (pinSg) extsToEnable[nExt++] = pSgc; - Vk.VkPhysicalDeviceShaderFloat16Int8Features f16en = new() - { - SType = VkConst.StPhysicalDeviceShaderFloat16Int8Features, ShaderFloat16 = 1, - }; - Vk.VkPhysicalDeviceCooperativeMatrixFeaturesKHR coopEn = new() - { - SType = VkConst.StPhysicalDeviceCooperativeMatrixFeaturesKHR, CooperativeMatrix = 1, - }; - Vk.VkPhysicalDeviceVulkanMemoryModelFeatures vmmEn = new() - { - SType = VkConst.StPhysicalDeviceVulkanMemoryModelFeatures, VulkanMemoryModel = 1, - }; - Vk.VkPhysicalDeviceSubgroupSizeControlFeaturesEXT sgcEn = new() - { - SType = VkConst.StPhysicalDeviceSubgroupSizeControlFeaturesEXT, SubgroupSizeControl = 1, - ComputeFullSubgroups = sgcQ.ComputeFullSubgroups, - }; - if (pinSg) vmmEn.PNext = &sgcEn; - coopEn.PNext = &vmmEn; - f16en.PNext = &coopEn; - feat16.PNext = wantCm ? &f16en : null; - - // To enable 16-bit storage we chain feat16 into VkDeviceCreateInfo.pNext - // (VK 1.1 core: no extension needed). PEnabledFeatures must be null when - // using Features2 pNext; we inline a zero VkPhysicalDeviceFeatures in feat2. - Vk.VkPhysicalDeviceFeatures2 feat2enable = new() - { - SType = VkConst.StPhysicalDeviceFeatures2, - PNext = d.StorageBuffer16 ? &feat16 : null, - }; - Vk.VkDeviceCreateInfo dci = new() - { - SType = VkConst.StDeviceCreateInfo, - QueueCreateInfoCount = 1, PQueueCreateInfos = &qci, - PEnabledFeatures = null, - PNext = &feat2enable, - EnabledExtensionCount = nExt, - PpEnabledExtensionNames = nExt > 0 ? extsToEnable : null, - }; - Vk.Check(Vk.vkCreateDevice(d.PhysDevice, &dci, null, out d.Device), "vkCreateDevice"); - d.CoopMat16 = wantCm; - d.CoopM8 = pinSg16; - d.CoopSubgroupSize = pinSg32 ? 32u : pinSg16 ? 16u : 0u; - d._fullSubgroups = pinSg && sgcQ.ComputeFullSubgroups != 0; - d.CoopLite = wantCm && !pinSg16 && ((hasSgc ? d.SubgroupMax : d.SubgroupSize) >= 64 || d.MaxSharedMemory < 48 * 1024) - && Environment.GetEnvironmentVariable("GCDRNET_VK_LITE") != "0"; - Vk.vkGetDeviceQueue(d.Device, d.QueueFamily, 0, out d.Queue); - - if (hasPush) - { - byte* fnName = stackalloc byte[] { (byte)'v', (byte)'k', (byte)'C', (byte)'m', (byte)'d', - (byte)'P', (byte)'u', (byte)'s', (byte)'h', (byte)'D', (byte)'e', (byte)'s', (byte)'c', - (byte)'r', (byte)'i', (byte)'p', (byte)'t', (byte)'o', (byte)'r', (byte)'S', (byte)'e', - (byte)'t', (byte)'K', (byte)'H', (byte)'R', 0 }; - IntPtr fn = Vk.vkGetDeviceProcAddr(d.Device, fnName); - if (fn != IntPtr.Zero) - { - d.PushDescriptors = true; - d._pushDesc = (delegate* unmanaged[Cdecl])fn; - } - } - - fixed (Vk.VkPhysicalDeviceMemoryProperties* mp = &d.MemProps) - Vk.vkGetPhysicalDeviceMemoryProperties(d.PhysDevice, mp); - for (int i = 0; i < (int)d.MemProps.MemoryTypeCount; i++) - { - uint f = d.MemProps.TypeAt(i).PropertyFlags; - if ((f & (VkConst.MemDeviceLocal | VkConst.MemHostVisible | VkConst.MemHostCoherent)) == (VkConst.MemDeviceLocal | VkConst.MemHostVisible | VkConst.MemHostCoherent)) - { - d.CoherentDeviceLocal = true; - break; - } - } - d.UnifiedMemory = props.DeviceType == VkConst.PhysDeviceIntegrated; - d.Vec4Eltwise = (d.UnifiedMemory || d.VendorId == VendorIntel) && Environment.GetEnvironmentVariable("GCDRNET_VK_VEC4") != "0"; - d.ConvPoolK3 = d.VendorId == VendorIntel && Environment.GetEnvironmentVariable("GCDRNET_VK_CONVPOOL3") != "0"; - for (int i = 0; i < (int)d.MemProps.MemoryHeapCount; i++) - { - var hp = d.MemProps.HeapAt(i); - if ((hp.Flags & 1u) != 0) d.DeviceLocalHeap = Math.Max(d.DeviceLocalHeap, hp.Size); // HEAP_DEVICE_LOCAL - } - - Vk.VkCommandPoolCreateInfo cpci = new() - { - SType = VkConst.StCommandPoolCreateInfo, - Flags = VkConst.CmdPoolResetCommandBuffer, - QueueFamilyIndex = d.QueueFamily, - }; - Vk.Check(Vk.vkCreateCommandPool(d.Device, &cpci, null, out d._commandPool), "vkCreateCommandPool"); - - Vk.VkDescriptorPoolSize psz = new() { Type = VkConst.DescStorageBuffer, DescriptorCount = 1 << 20 }; - Vk.VkDescriptorPoolCreateInfo dpci = new() - { - SType = VkConst.StDescriptorPoolCreateInfo, - Flags = 1, MaxSets = 1 << 16, PoolSizeCount = 1, PPoolSizes = &psz, - }; - Vk.Check(Vk.vkCreateDescriptorPool(d.Device, &dpci, null, out d._descPool), "vkCreateDescriptorPool"); - - // never trust a pinned subgroup size blindly: the coopmat lane mapping is hard-wired to - // the pinned width (32, or 16 for SG16), so anything else would silently compute garbage - if (pinSg && !d.ProbeSubgroup(d.CoopSubgroupSize)) - { - d.CoopMat16 = false; - d.CoopM8 = false; - d.CoopLite = false; - d.CoopSubgroupSize = 0; - } - d.ConvDot = !d.CoopMat16 && hasSgc && d.SubgroupMin >= 64 && d.StorageBuffer16 - && Environment.GetEnvironmentVariable("GCDRNET_VK_DOT") != "0"; - if (Environment.GetEnvironmentVariable("GCDRNET_VK_VERBOSE") == "1") - { - Console.Error.WriteLine($"[vk] device={d.DeviceName} type={props.DeviceType} vendor=0x{d.VendorId:x4} " + - $"api={props.ApiVersion >> 22}.{(props.ApiVersion >> 12) & 0x3ff}.{props.ApiVersion & 0xfff} driver=0x{props.DriverVersion:x8} qfamily={d.QueueFamily} " + - $"sg={d.SubgroupSize} sgRange={d.SubgroupMin}-{d.SubgroupMax} sgcCompute={sgcCompute} push={d.PushDescriptors} " + - $"coopmat={d.CoopMat16} coopSg={(d.CoopSubgroupSize == 0 ? "default" : d.CoopSubgroupSize.ToString())} " + - $"m8={d.CoopM8} fullSg={d._fullSubgroups} lite={d.CoopLite} vec4={d.Vec4Eltwise} cp3={d.ConvPoolK3} " + - $"dot={d.ConvDot} lds={d.MaxSharedMemory} sb16={d.StorageBuffer16}"); - for (uint i = 0; i < nqf; i++) - Console.Error.WriteLine($"[vk] qf{i}: flags=0x{qf[i].QueueFlags:x} count={qf[i].QueueCount} tsbits={qf[i].TimestampValidBits}" - + (i == d.QueueFamily ? " <- selected" : "")); - if (hasCoop) - foreach (var cp in CoopProps(d)) - Console.Error.WriteLine($"[vk] coopmat {cp.MSize}x{cp.NSize}x{cp.KSize} a={cp.AType} b={cp.BType} c={cp.CType} r={cp.ResultType} scope={cp.Scope}"); - for (int i = 0; i < (int)d.MemProps.MemoryHeapCount; i++) - { - var hp = d.MemProps.HeapAt(i); - Console.Error.WriteLine($"[vk] heap{i}: {hp.Size >> 20} MB flags={hp.Flags:x}"); - } - for (int i = 0; i < (int)d.MemProps.MemoryTypeCount; i++) - Console.Error.WriteLine($"[vk] type{i}: heap{d.MemProps.TypeAt(i).HeapIndex} flags={d.MemProps.TypeAt(i).PropertyFlags:x}"); - } - return d; - } - - private bool _fullSubgroups; - - /// Runs sgprobe.comp (256 threads) with requiredSubgroupSize = and - /// checks every invocation reports -lane subgroups, 256 / lanes per workgroup. - private bool ProbeSubgroup(uint lanes) - { - VkPipeline? pipe = null; - VkBuffer? buf = null; - uint count = 256 / lanes; - try - { - pipe = NewPipeline(NewShaderModule(VkShaders.Load("sgprobe")), 1, 0, PushDescriptors, lanes); - buf = NewStorageBuffer(256 * 4, hostVisible: true, preferHost: true); - IntPtr set = IntPtr.Zero, pool = IntPtr.Zero; - lock (Sync) - { - IntPtr cmd = NewCommandBuffer(); - IntPtr fence = NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, pipe.Pipeline); - Vk.VkDescriptorBufferInfo bi = new() { Buffer = buf.Buffer, Offset = 0, Range = 256 * 4 }; - Vk.VkWriteDescriptorSet w = new() - { - SType = VkConst.StWriteDescriptorSet, DstBinding = 0, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, PBufferInfo = &bi, - }; - if (PushDescriptors) CmdPushDescriptors(cmd, pipe.Layout, &w, 1); - else - { - pool = NewDescriptorPool(1); - set = AllocateDescriptorSet(pool, pipe.SetLayout); - BindBuffer(set, 0, buf, 0, 256 * 4); - Vk.vkCmdBindDescriptorSets(cmd, VkConst.BindPointCompute, pipe.Layout, 0, 1, &set, 0, null); - } - Vk.vkCmdDispatch(cmd, 1, 1, 1); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - Submit(cmd, fence); - WaitFence(fence); - Vk.vkDestroyFence(Device, fence, null); - FreeCommandBuffer(cmd); - if (pool != IntPtr.Zero) Vk.vkDestroyDescriptorPool(Device, pool, null); - } - uint* r = (uint*)buf.Map(); - buf.Invalidate(0, 256 * 4); - bool ok = true; - for (int i = 0; i < 256; i++) - ok &= (r[i] & 0xff) == lanes && ((r[i] >> 8) & 0xff) == count && ((r[i] >> 16) & 0xff) < count && (r[i] >> 24) < lanes; - buf.Unmap(); - return ok; - } - catch (InvalidOperationException) { return false; } - finally - { - buf?.Free(); - if (pipe is not null) - { - Vk.vkDestroyPipeline(Device, pipe.Pipeline, null); - Vk.vkDestroyPipelineLayout(Device, pipe.Layout, null); - Vk.vkDestroyDescriptorSetLayout(Device, pipe.SetLayout, null); - } - } - } - - /// Which fp16 x fp16 -> fp32 (C and result) subgroup coopmat shapes the device offers: - /// 16x16x16 (the default conv_cm builds) and 8x16x16 (SG16). - private static void CoopShapes(VkDevice d, out bool m16, out bool m8) - { - m16 = m8 = false; - // component types: 0 = float16, 1 = float32; scope 3 = subgroup - foreach (var p in CoopProps(d)) - if (p.NSize == 16 && p.KSize == 16 && p.AType == 0 && p.BType == 0 - && p.CType == 1 && p.ResultType == 1 && p.Scope == 3) - { - m16 |= p.MSize == 16; - m8 |= p.MSize == 8; - } - } - - /// vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR (empty when unavailable). - private static Vk.VkCooperativeMatrixPropertiesKHR[] CoopProps(VkDevice d) - { - byte[] name = System.Text.Encoding.ASCII.GetBytes("vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR\0"); - IntPtr fn; - fixed (byte* pn = name) fn = Vk.vkGetInstanceProcAddr(d.Instance, pn); - if (fn == IntPtr.Zero) return []; - var query = (delegate* unmanaged[Cdecl])fn; - uint np = 0; - if (query(d.PhysDevice, &np, null) != VkResult.Success || np == 0) return []; - var props = new Vk.VkCooperativeMatrixPropertiesKHR[np]; - for (int i = 0; i < props.Length; i++) props[i].SType = VkConst.StCooperativeMatrixPropertiesKHR; - fixed (Vk.VkCooperativeMatrixPropertiesKHR* pp = props) - if (query(d.PhysDevice, &np, pp) != VkResult.Success) return []; - return props; - } - - /// Allocate a storage buffer. hostVisible=false prefers DEVICE_LOCAL; - /// hostVisible=true prefers DEVICE_LOCAL|HOST_VISIBLE|HOST_COHERENT (ReBAR zero-copy), falling - /// back to HOST_VISIBLE|HOST_COHERENT. preferHost skips DEVICE_LOCAL types — for buffers the - /// CPU reads (uncached WC reads from VRAM are ~15MB/s, catastrophic). - public VkBuffer NewStorageBuffer(ulong bytes, bool hostVisible, bool preferHost = false) - { - var b = new VkBuffer { Dev = Device, Size = bytes }; - Vk.VkBufferCreateInfo bci = new() - { - SType = VkConst.StBufferCreateInfo, - Size = bytes, - Usage = VkConst.BufferUsageStorageBuffer | VkConst.BufferUsageTransferSrc | VkConst.BufferUsageTransferDst, - SharingMode = VkConst.SharingExclusive, - }; - Vk.Check(Vk.vkCreateBuffer(Device, &bci, null, out b.Buffer), "vkCreateBuffer"); - Vk.VkMemoryRequirements req; - Vk.vkGetBufferMemoryRequirements(Device, b.Buffer, &req); - - uint memType = uint.MaxValue; - for (int i = 0; i < (int)MemProps.MemoryTypeCount; i++) - { - if ((req.MemoryTypeBits & (1u << i)) == 0) continue; - uint f = MemProps.TypeAt(i).PropertyFlags; - bool ok; - if (!hostVisible) - ok = (f & VkConst.MemDeviceLocal) != 0; - else if (preferHost) - ok = (f & (VkConst.MemHostVisible | VkConst.MemHostCoherent)) == (VkConst.MemHostVisible | VkConst.MemHostCoherent) - && (f & VkConst.MemDeviceLocal) == 0 - && (f & VkConst.MemHostCached) != 0; - else - ok = (f & VkConst.MemHostVisible) != 0 && (f & VkConst.MemHostCoherent) != 0; - if (ok && (memType == uint.MaxValue || (!preferHost && (f & VkConst.MemDeviceLocal) != 0))) - { - memType = (uint)i; - if (preferHost || (f & VkConst.MemDeviceLocal) != 0) break; - } - } - if (memType == uint.MaxValue && preferHost) - for (int i = 0; i < (int)MemProps.MemoryTypeCount && memType == uint.MaxValue; i++) - { - if ((req.MemoryTypeBits & (1u << i)) == 0) continue; - uint f = MemProps.TypeAt(i).PropertyFlags; - if ((f & (VkConst.MemHostVisible | VkConst.MemHostCoherent)) - == (VkConst.MemHostVisible | VkConst.MemHostCoherent) - && (f & VkConst.MemDeviceLocal) == 0) - memType = (uint)i; - } - if (memType == uint.MaxValue && preferHost) - { - for (int i = 0; i < (int)MemProps.MemoryTypeCount && memType == uint.MaxValue; i++) - { - if ((req.MemoryTypeBits & (1u << i)) == 0) continue; - uint f = MemProps.TypeAt(i).PropertyFlags; - if ((f & (VkConst.MemHostVisible | VkConst.MemHostCoherent | VkConst.MemHostCached)) - == (VkConst.MemHostVisible | VkConst.MemHostCoherent | VkConst.MemHostCached)) - memType = (uint)i; - } - for (int i = 0; i < (int)MemProps.MemoryTypeCount && memType == uint.MaxValue; i++) - { - if ((req.MemoryTypeBits & (1u << i)) == 0) continue; - uint f = MemProps.TypeAt(i).PropertyFlags; - if ((f & (VkConst.MemHostVisible | VkConst.MemHostCoherent)) - == (VkConst.MemHostVisible | VkConst.MemHostCoherent)) - memType = (uint)i; - } - } - if (memType == uint.MaxValue) - throw new PlatformNotSupportedException($"Vulkan: no suitable memory type for {bytes}-byte buffer (hostVisible={hostVisible})"); - - Vk.VkMemoryAllocateInfo mai = new() - { - SType = VkConst.StMemoryAllocateInfo, - AllocationSize = req.Size, MemoryTypeIndex = memType, - }; - Vk.Check(Vk.vkAllocateMemory(Device, &mai, null, out b.Memory), "vkAllocateMemory"); - b.AllocSize = req.Size; - TrackAlloc((long)req.Size); - Vk.Check(Vk.vkBindBufferMemory(Device, b.Buffer, b.Memory, 0), "vkBindBufferMemory"); - b.Flags = MemProps.TypeAt((int)memType).PropertyFlags; - return b; - } - - public IntPtr NewShaderModule(byte[] spirv) - { - if ((spirv.Length & 3) != 0) throw new ArgumentException("SPIR-V size must be a multiple of 4"); - fixed (byte* p = spirv) - { - Vk.VkShaderModuleCreateInfo ci = new() - { - SType = VkConst.StShaderModuleCreateInfo, - CodeSize = (nuint)spirv.Length, PCode = (uint*)p, - }; - Vk.Check(Vk.vkCreateShaderModule(Device, &ci, null, out IntPtr m), "vkCreateShaderModule"); - return m; - } - } - - /// One set layout with `bindings` storage buffers (0..bindings-1) + push constants. - /// requiredSubgroupSize != 0 pins the compute subgroup width (VK_EXT_subgroup_size_control). - public VkPipeline NewPipeline(IntPtr shaderModule, int bindings, int pushConstBytes, bool pushLayout, - uint requiredSubgroupSize = 0) - { - var dslb = stackalloc Vk.VkDescriptorSetLayoutBinding[bindings]; - for (int i = 0; i < bindings; i++) - dslb[i] = new Vk.VkDescriptorSetLayoutBinding - { - Binding = (uint)i, DescriptorType = VkConst.DescStorageBuffer, - DescriptorCount = 1, StageFlags = VkConst.StageComputeShader, - }; - IntPtr setLayout; - Vk.VkDescriptorSetLayoutCreateInfo dslci = new() - { - SType = VkConst.StDescriptorSetLayoutCreateInfo, - BindingCount = (uint)bindings, PBindings = dslb, - Flags = pushLayout ? 1u : 0u, // VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR - }; - Vk.Check(Vk.vkCreateDescriptorSetLayout(Device, &dslci, null, out setLayout), "vkCreateDescriptorSetLayout"); - - Vk.VkPushConstantRange pcr = new() - { - StageFlags = VkConst.StageComputeShader, Offset = 0, Size = (uint)pushConstBytes, - }; - IntPtr layout; - Vk.VkPipelineLayoutCreateInfo plci = new() - { - SType = VkConst.StPipelineLayoutCreateInfo, - SetLayoutCount = 1, PSetLayouts = &setLayout, - PushConstantRangeCount = pushConstBytes > 0 ? 1u : 0u, - PPushConstantRanges = pushConstBytes > 0 ? &pcr : null, - }; - Vk.Check(Vk.vkCreatePipelineLayout(Device, &plci, null, out layout), "vkCreatePipelineLayout"); - - byte* name = stackalloc byte[5] { (byte)'m', (byte)'a', (byte)'i', (byte)'n', 0 }; - Vk.VkPipelineShaderStageRequiredSubgroupSizeCreateInfo sgc = new() - { - SType = VkConst.StPipelineShaderStageRequiredSubgroupSizeCreateInfo, - RequiredSubgroupSize = requiredSubgroupSize, - }; - Vk.VkComputePipelineCreateInfo cpci = new() - { - SType = VkConst.StComputePipelineCreateInfo, - Stage = new Vk.VkPipelineShaderStageCreateInfo - { - SType = VkConst.StPipelineShaderStageCreateInfo, - Stage = VkConst.StageComputeShader, Module = shaderModule, PName = name, - PNext = requiredSubgroupSize != 0 ? &sgc : null, - // REQUIRE_FULL_SUBGROUPS: every subgroup of a pinned pipeline has all its lanes - Flags = requiredSubgroupSize != 0 && _fullSubgroups ? 2u : 0u, - }, - Layout = layout, - }; - IntPtr pipeline; - Vk.Check(Vk.vkCreateComputePipelines(Device, IntPtr.Zero, 1, &cpci, null, &pipeline), "vkCreateComputePipelines"); - return new VkPipeline { Pipeline = pipeline, Layout = layout, SetLayout = setLayout, PushLayout = pushLayout }; - } - - public IntPtr NewDescriptorPool(uint maxSets) - { - Vk.VkDescriptorPoolSize psz = new() { Type = VkConst.DescStorageBuffer, DescriptorCount = maxSets * 16 }; - Vk.VkDescriptorPoolCreateInfo dpci = new() - { - SType = VkConst.StDescriptorPoolCreateInfo, - Flags = 0, MaxSets = maxSets, PoolSizeCount = 1, PPoolSizes = &psz, - }; - Vk.Check(Vk.vkCreateDescriptorPool(Device, &dpci, null, out IntPtr pool), "vkCreateDescriptorPool"); - return pool; - } - - public IntPtr AllocateDescriptorSet(IntPtr pool, IntPtr setLayout) - { - IntPtr set; - Vk.VkDescriptorSetAllocateInfo ai = new() - { - SType = VkConst.StDescriptorSetAllocateInfo, - DescriptorPool = pool, DescriptorSetCount = 1, PSetLayouts = &setLayout, - }; - Vk.Check(Vk.vkAllocateDescriptorSets(Device, &ai, &set), "vkAllocateDescriptorSets"); - return set; - } - - public IntPtr NewDescriptorSet(IntPtr setLayout) - { - IntPtr set; - Vk.VkDescriptorSetAllocateInfo ai = new() - { - SType = VkConst.StDescriptorSetAllocateInfo, - DescriptorPool = _descPool, DescriptorSetCount = 1, PSetLayouts = &setLayout, - }; - Vk.Check(Vk.vkAllocateDescriptorSets(Device, &ai, &set), "vkAllocateDescriptorSets"); - return set; - } - - /// VK_KHR_push_descriptor: write bindings inline into the command buffer - /// (DstSet in each write is ignored). - public void CmdPushDescriptors(IntPtr cmd, IntPtr layout, Vk.VkWriteDescriptorSet* writes, uint count) - => _pushDesc(cmd, VkConst.BindPointCompute, layout, 0, count, writes); - - /// Record the standard write->read barrier between two compute dispatches. - public void CmdComputeBarrier(IntPtr cmd) - { - Vk.VkMemoryBarrier mb = new() - { - SType = VkConst.StMemoryBarrier, - SrcAccessMask = VkConst.AccessShaderWrite, DstAccessMask = VkConst.AccessShaderRead, - }; - Vk.vkCmdPipelineBarrier(cmd, VkConst.PipelineStageComputeShader, VkConst.PipelineStageComputeShader, 0, - 1, &mb, 0, null, 0, null); - } - - /// Upload raw bytes into a buffer (host memcpy when mappable, else staging + copy cmd). - public void Upload(VkBuffer buf, void* src, ulong bytes) - { - if ((buf.Flags & VkConst.MemHostVisible) != 0) - { - void* p = buf.Map(); - Buffer.MemoryCopy(src, p, bytes, bytes); - buf.Flush(0, bytes); - buf.Unmap(); - return; - } - VkBuffer staging = NewStorageBuffer(bytes, hostVisible: true); - void* sp = staging.Map(); - Buffer.MemoryCopy(src, sp, bytes, bytes); - staging.Flush(0, bytes); - staging.Unmap(); - lock (Sync) - { - IntPtr cmd = NewCommandBuffer(); - IntPtr fence = NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - Vk.VkBufferCopy r = new() { Size = bytes }; - Vk.vkCmdCopyBuffer(cmd, staging.Buffer, buf.Buffer, 1, &r); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - Submit(cmd, fence); - WaitFence(fence); - Vk.vkDestroyFence(Device, fence, null); - FreeCommandBuffer(cmd); - } - staging.Free(); - } - - /// Download raw bytes from a buffer into dst (host memcpy or staging). - public void Download(VkBuffer buf, void* dst, ulong bytes, ulong srcOffset = 0) - { - if ((buf.Flags & VkConst.MemHostVisible) != 0) - { - void* p = buf.Map(); - buf.Invalidate(srcOffset, bytes); - Buffer.MemoryCopy((byte*)p + srcOffset, dst, bytes, bytes); - buf.Unmap(); - return; - } - VkBuffer staging = NewStorageBuffer(bytes, hostVisible: true, preferHost: true); - lock (Sync) - { - IntPtr cmd = NewCommandBuffer(); - IntPtr fence = NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - Vk.VkBufferCopy r = new() { Size = bytes, SrcOffset = srcOffset }; - Vk.vkCmdCopyBuffer(cmd, buf.Buffer, staging.Buffer, 1, &r); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - Submit(cmd, fence); - WaitFence(fence); - Vk.vkDestroyFence(Device, fence, null); - FreeCommandBuffer(cmd); - } - void* sp = staging.Map(); - staging.Invalidate(0, bytes); - Buffer.MemoryCopy(sp, dst, bytes, bytes); - staging.Unmap(); - staging.Free(); - } - - public void BindBuffer(IntPtr set, uint binding, VkBuffer buf, ulong offset, ulong size) - { - Vk.VkDescriptorBufferInfo bi = new() { Buffer = buf.Buffer, Offset = offset, Range = size }; - Vk.VkWriteDescriptorSet w = new() - { - SType = VkConst.StWriteDescriptorSet, DstSet = set, DstBinding = binding, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, PBufferInfo = &bi, - }; - Vk.vkUpdateDescriptorSets(Device, 1, &w, 0, null); - } - - public IntPtr NewCommandBuffer() - { - Vk.VkCommandBufferAllocateInfo ai = new() - { - SType = VkConst.StCommandBufferAllocateInfo, - CommandPool = _commandPool, Level = VkConst.CmdBufferLevelPrimary, CommandBufferCount = 1, - }; - IntPtr cmd; - Vk.Check(Vk.vkAllocateCommandBuffers(Device, &ai, &cmd), "vkAllocateCommandBuffers"); - return cmd; - } - - public void FreeCommandBuffer(IntPtr cmd) - { - if (cmd != IntPtr.Zero) Vk.vkFreeCommandBuffers(Device, _commandPool, 1, &cmd); - } - - /// Private resettable command pool (command pools are externally synchronized: - /// a worker that re-records on its own thread must not share the device pool). - public IntPtr NewCommandPool() - { - Vk.VkCommandPoolCreateInfo ci = new() - { - SType = VkConst.StCommandPoolCreateInfo, - Flags = VkConst.CmdPoolResetCommandBuffer, - QueueFamilyIndex = QueueFamily, - }; - Vk.Check(Vk.vkCreateCommandPool(Device, &ci, null, out IntPtr pool), "vkCreateCommandPool"); - return pool; - } - - public IntPtr AllocateCommandBuffer(IntPtr pool) - { - Vk.VkCommandBufferAllocateInfo ai = new() - { - SType = VkConst.StCommandBufferAllocateInfo, - CommandPool = pool, Level = VkConst.CmdBufferLevelPrimary, CommandBufferCount = 1, - }; - IntPtr cmd; - Vk.Check(Vk.vkAllocateCommandBuffers(Device, &ai, &cmd), "vkAllocateCommandBuffers"); - return cmd; - } - - /// Destroys a private command pool (frees its command buffers). - public void DestroyCommandPool(IntPtr pool) - { - if (pool != IntPtr.Zero) Vk.vkDestroyCommandPool(Device, pool, null); - } - - public IntPtr NewFence(bool signaled = false) - { - Vk.VkFenceCreateInfo ci = new() { SType = VkConst.StFenceCreateInfo, Flags = signaled ? 1u : 0u }; - Vk.Check(Vk.vkCreateFence(Device, &ci, null, out IntPtr f), "vkCreateFence"); - return f; - } - - public void Submit(IntPtr cmd, IntPtr fence) - { - Vk.VkSubmitInfo si = new() - { - SType = VkConst.StSubmitInfo, - CommandBufferCount = 1, PCommandBuffers = &cmd, - }; - VkResult r; - lock (QueueLock) r = Vk.vkQueueSubmit(Queue, 1, &si, fence); - Vk.Check(r, "vkQueueSubmit"); - } - - public void DestroyFence(IntPtr fence) - { - if (fence != IntPtr.Zero) Vk.vkDestroyFence(Device, fence, null); - } - - public void WaitFence(IntPtr fence) - { - Vk.Check(Vk.vkWaitForFences(Device, 1, &fence, 1, ulong.MaxValue), "vkWaitForFences"); - Vk.Check(Vk.vkResetFences(Device, 1, &fence), "vkResetFences"); - } - - public void Dispose() - { - if (Device != IntPtr.Zero) - { - Vk.vkDeviceWaitIdle(Device); - if (_descPool != IntPtr.Zero) { Vk.vkDestroyDescriptorPool(Device, _descPool, null); _descPool = IntPtr.Zero; } - if (_commandPool != IntPtr.Zero) { Vk.vkDestroyCommandPool(Device, _commandPool, null); _commandPool = IntPtr.Zero; } - lock (_pipes) - foreach (VkPipeline p in _pipes.Values) - { - Vk.vkDestroyPipeline(Device, p.Pipeline, null); - Vk.vkDestroyPipelineLayout(Device, p.Layout, null); - Vk.vkDestroyDescriptorSetLayout(Device, p.SetLayout, null); - } - Vk.vkDestroyDevice(Device, null); Device = IntPtr.Zero; - } - if (Instance != IntPtr.Zero) { Vk.vkDestroyInstance(Instance, null); Instance = IntPtr.Zero; } - } -} - -internal sealed class VkPipeline -{ - public IntPtr Pipeline, Layout, SetLayout; - public string Name = ""; - public int Bindings; - public bool PushLayout; -} - -internal unsafe sealed class VkBuffer -{ - public IntPtr Dev, Buffer, Memory; - public ulong Size, AllocSize; - public uint Flags; - - public bool HostCoherent => (Flags & VkConst.MemHostCoherent) != 0; - - public void* Map() - { - void* p; - Vk.Check(Vk.vkMapMemory(Dev, Memory, 0, Size, 0, &p), "vkMapMemory"); - return p; - } - public void Unmap() => Vk.vkUnmapMemory(Dev, Memory); - - public void Flush(ulong offset, ulong size) - { - if (HostCoherent) return; - Vk.VkMappedMemoryRange r = new() { SType = VkConst.StMappedMemoryRange, Memory = Memory, Offset = offset, Size = size }; - Vk.vkFlushMappedMemoryRanges(Dev, 1, &r); - } - public void Invalidate(ulong offset, ulong size) - { - if (HostCoherent) return; - Vk.VkMappedMemoryRange r = new() { SType = VkConst.StMappedMemoryRange, Memory = Memory, Offset = offset, Size = size }; - Vk.vkInvalidateMappedMemoryRanges(Dev, 1, &r); - } - - public void Free() - { - if (Buffer != IntPtr.Zero) { Vk.vkDestroyBuffer(Dev, Buffer, null); Buffer = IntPtr.Zero; } - if (Memory != IntPtr.Zero) - { - Vk.vkFreeMemory(Dev, Memory, null); Memory = IntPtr.Zero; - VkDevice.TrackFree((long)AllocSize); - AllocSize = 0; - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkGraph.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkGraph.cs deleted file mode 100644 index d686644..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkGraph.cs +++ /dev/null @@ -1,738 +0,0 @@ -using System.Runtime.InteropServices; -using Sdcb.SimdGcdrNet.Engine; - -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -/// Binding slots resolved per-session: which underlying buffer. -internal enum VkSlot : int { Arena = 0, Consts = 1, Dummy = 2, StageIn = 3, StageOut = 4, Direct = 5 } - -internal readonly struct VkBind -{ - public readonly VkSlot Slot; - public readonly VkBuffer? Buf; // when Slot == Direct - public readonly ulong Offset, Size; - public VkBind(VkSlot s, ulong off, ulong size) { Slot = s; Offset = off; Size = size; } - public VkBind(VkBuffer b, ulong off, ulong size) { Slot = VkSlot.Direct; Buf = b; Offset = off; Size = size; } -} - -/// One recorded compute dispatch. -internal sealed class VkDispatch -{ - public required VkPipeline Pipe; - public required VkBind[] Binds; - public required byte[] Pc; - public uint Gx, Gy = 1, Gz = 1; - public string Name = ""; -} - -/// -/// Per-(device, ExecPlan) half of the Vulkan backend: backend-private constant -/// packing/upload and the node list compiled to records. -/// Sessions own the activation arena and resolve slot ids to live buffers. -/// -/// Plans are layout-planned (NHWC, see ); every -/// addressing decision reads . With -/// activations and weights are stored as float16 (16-bit storage only — all -/// shader math, reductions included, is fp32); conv bias and LayerNorm params -/// stay fp32. Const bytes therefore differ per backend and per precision. -/// -internal sealed class VkGraphModel : IDisposable -{ - private readonly VkDevice _dev; - private readonly ExecPlan _plan; - private int _refs = 1; // the graph cache holds the first reference - private readonly VkConstStore _store; - internal readonly VkBuffer Consts; // all const tensors, backend-packed (shared store) - private readonly Dictionary<(int v, CK k), ulong> _constOff = new(); - internal readonly VkBuffer Dummy; // 64B zero buffer for unused bindings - internal readonly VkDispatch[] Dispatches; - internal readonly ulong ConstBytes; - /// Activations + weights stored as float16 (fp32 math in every shader). - internal readonly bool F16; - /// Bytes per stored activation element. - internal int Es => F16 ? 2 : 4; - - /// How a const is laid out on the device (decided by its consumer). - private enum CK { Raw, F32, ConvW, DwW, ConvWT, GemmNK } - - /// Vulkan device the graph is lowered for. - /// Compiled execution plan to lower. - /// Store activations and (non-fp32) weights as fp16; math stays fp32. - /// Sharing key for the const store (the source model): graphs of the - /// same model at other sizes reuse its uploaded weights. Null = private store. - internal VkGraphModel(VkDevice dev, ExecPlan plan, bool f16, object? constKey = null) - { - _dev = dev; - _plan = plan; - F16 = f16; - if (!plan.Nhwc) throw new NotSupportedException("Vulkan backend requires a layout-planned (NHWC) plan."); - - // ---- consts: per-consumer packing ---- - var need = new List<(int v, CK k)>(); - void Need(int v, CK k) - { - if (v < 0 || plan.Values[v].Floats is null) return; - if (!need.Contains((v, k))) need.Add((v, k)); - } - foreach (ExecNode n in plan.Nodes) - { - switch (n.P) - { - case ConvP cp: - Need(n.In[0], CK.Raw); - Need(n.In[1], cp.Group > 1 ? CK.DwW : UseCoopConv(dev, f16, plan, n, cp) ? CK.ConvWT : CK.ConvW); - Need(cp.BiasValue, CK.F32); - Need(cp.AddValue, CK.Raw); - break; - case GemmP gp: - Need(n.In[0], CK.Raw); Need(n.In[1], UseCoopGemm(dev, f16, plan, n, gp) ? CK.GemmNK : CK.Raw); - Need(gp.BiasValue, CK.F32); - break; - case LnP: - Need(n.In[0], CK.Raw); Need(n.In[1], CK.F32); Need(n.In[2], CK.F32); - break; - case UpHeadP up: - Need(n.In[2], CK.Raw); - if (n.In.Length > 3) Need(n.In[3], CK.F32); - if (up.SkipConv is not null) { Need(up.SkipW, CK.ConvW); Need(up.SkipB, CK.F32); } - break; - default: - foreach (int iv in n.In) Need(iv, CK.Raw); - break; - } - } - foreach (var (v, k) in need) - { - CValue cv = plan.Values[v]; - if (k == CK.Raw && cv.Layout is { } l && !Engine.Layout.IsIdentity(l)) - throw new NotSupportedException($"Vulkan: const '{cv.Name}' with non-identity layout"); - } - float[][] srcs = need.Select(x => plan.Values[x.v].Floats!).ToArray(); - int[] kinds = need.Select(x => (int)x.k).ToArray(); - _store = VkConstStore.Acquire(dev, constKey, f16, srcs, kinds, () => - { - ulong total = 0; - var offs = new ulong[need.Count]; - var items = new List<(ulong off, float[] data, bool half)>(); - for (int i = 0; i < need.Count; i++) - { - var (v, k) = need[i]; - CValue cv = plan.Values[v]; - float[] data = k switch - { - CK.ConvW => PackConv(cv), - CK.ConvWT => PackConvT(cv), - CK.GemmNK => PackGemmNK(cv, plan.Nodes.First(m => m.P is GemmP && m.In[1] == v)), - CK.DwW => PackDw(cv), - _ => cv.Floats!, - }; - bool half = f16 && k != CK.F32; - total = (total + 63) & ~63UL; - offs[i] = total; - items.Add((total, data, half)); - total += (ulong)data.Length * (half ? 2UL : 4UL); - } - byte[] host = new byte[Math.Max((total + 63) & ~63UL, 64)]; - foreach (var (off, data, half) in items) - { - if (half) - { - Span dst = MemoryMarshal.Cast(host.AsSpan(checked((int)off), data.Length * 2)); - for (int i = 0; i < data.Length; i++) dst[i] = (Half)data[i]; - } - else - MemoryMarshal.AsBytes(data.AsSpan()).CopyTo(host.AsSpan(checked((int)off))); - } - return (offs, host); - }); - for (int i = 0; i < need.Count; i++) _constOff[need[i]] = _store.Offsets[i]; - Consts = _store.Consts; - Dummy = _store.Dummy; - ConstBytes = _store.Bytes; - - var disps = new List(); - NodeRanges = new (int start, int count)[plan.Nodes.Length]; - for (int i = 0; i < plan.Nodes.Length; i++) - { - int start = disps.Count; - disps.AddRange(Emit(plan.Nodes[i])); - NodeRanges[i] = (start, disps.Count - start); - } - Dispatches = disps.ToArray(); - } - - // ONNX [OC][IC][KH][KW] -> [K][OC], k = (ky*KW + kx)*IC + ic (matches conv_tile.comp) - private static float[] PackConv(CValue w) - { - int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3]; - float[] src = w.Floats!, dst = new float[src.Length]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < taps; t++) - dst[(t * ic + c) * oc + o] = src[(o * ic + c) * taps + t]; - return dst; - } - - // ONNX [OC][IC][KH][KW] -> [OC][Kpad], k = (ky*KW + kx)*IC + ic, Kpad = K rounded up to 8 - // (zero-filled; matches conv_cm.comp) - private static float[] PackConvT(CValue w) - { - int oc = w.Shape[0], ic = w.Shape[1], taps = w.Shape[2] * w.Shape[3], k = (taps * ic + 7) & ~7; - float[] src = w.Floats!, dst = new float[oc * k]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < taps; t++) - dst[o * k + t * ic + c] = src[(o * ic + c) * taps + t]; - return dst; - } - - /// Tensor-core conv eligibility (conv_cm.comp): fp16 storage on a coopmat device, - /// dense, channel counts in 16 B quads, an epilogue it implements, M within the grid limit. - private static bool UseCoopConv(VkDevice dev, bool f16, ExecPlan plan, ExecNode n, ConvP cp) - { - if (!dev.CoopMat16 || dev.ConvDot || !f16 || cp.Group != 1 || cp.Pool is not null || cp.Dilations.Any(d => d != 1)) return false; - if (cp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false; - if (cp.PadMode == PadMode.Constant && cp.PadValue != 0f) return false; - int[] si = plan.Values[n.In[0]].PhysShape, so = plan.Values[n.Out[0]].PhysShape; // [1,h,w,c] - long m = (long)so[1] * so[2]; - return so[3] % 8 == 0 && (m + 127) / 128 <= 65535; // cin % 8 != 0 -> scalar-gather stem variant - } - - // Gemm B -> [N][K] (k fastest), the conv_cm weight layout - private static float[] PackGemmNK(CValue b, ExecNode n) - { - var gp = (GemmP)n.P!; - if (gp.TransB) return b.Floats!; // already [N][K] - int k = b.Shape[0], nn = b.Shape[1]; - float[] src = b.Floats!, dst = new float[src.Length]; - for (int i = 0; i < k; i++) - for (int j = 0; j < nn; j++) dst[j * k + i] = src[i * nn + j]; - return dst; - } - - /// Tensor-core Gemm (a 1x1 conv over M "pixels" through conv_cm.comp): fp16 storage - /// on a coopmat device, const B, K/N in 16 B quads, relu/gelu/none epilogue. - private static bool UseCoopGemm(VkDevice dev, bool f16, ExecPlan plan, ExecNode n, GemmP gp) - { - if (!dev.CoopMat16 || dev.ConvDot || !f16 || plan.Values[n.In[1]].Floats is null || plan.Values[n.In[1]].Shape.Length != 2) return false; - // B packed per value: only when no other node reads it (its transB could differ) - if (plan.Nodes.Count(m => m.In.Contains(n.In[1])) != 1) return false; - if (gp.Act is not (ActivationKind.None or ActivationKind.Relu or ActivationKind.Gelu)) return false; - CValue a = plan.Values[n.In[0]]; - int k = a.Shape[^1], nn = plan.Values[n.Out[0]].Shape[^1]; - if (a.PhysShape[^1] != k) return false; - long m = a.ElementCount / k; - return k % 8 == 0 && nn % 8 == 0 && (m + 127) / 128 <= 65535; - } - - // ONNX [C][1][KH][KW] -> [tap][C] - private static float[] PackDw(CValue w) - { - int c = w.Shape[0], taps = w.Shape[2] * w.Shape[3]; - float[] src = w.Floats!, dst = new float[src.Length]; - for (int ch = 0; ch < c; ch++) - for (int t = 0; t < taps; t++) dst[t * c + ch] = src[ch * taps + t]; - return dst; - } - - /// Dispatch index range per source ExecNode (for per-node debug runs). - internal readonly (int start, int count)[] NodeRanges; - - // ---- value → bind resolution (byte offsets/sizes: all long math) ---- - private VkBind Bind(int valueId, CK kind = CK.Raw) - { - CValue v = _plan.Values[valueId]; - if (v.Buf >= 0) - return new VkBind(VkSlot.Arena, (ulong)_plan.BufOffset[v.Buf] * (ulong)Es, - (ulong)Math.Max(v.ElementCount, 1) * (ulong)Es); - if (_constOff.TryGetValue((valueId, kind), out ulong off)) - { - ulong es = F16 && kind != CK.F32 ? 2UL : 4UL; - // ConvWT rows are K-padded to 8: bind the packed length, not the logical one - long elems = kind == CK.ConvWT - ? (long)v.Shape[0] * ((v.Shape[1] * v.Shape[2] * v.Shape[3] + 7) & ~7) - : Math.Max(v.ElementCount, 1); - return new VkBind(VkSlot.Consts, off, (ulong)elems * es); - } - if (v.Floats != null) - throw new InvalidOperationException($"const value '{v.Name}' not packed as {kind}"); - throw new InvalidOperationException($"value '{v.Name}' has no GPU storage (ints must be folded)"); - } - - private static byte[] PcOf(params uint[] ints) - { - byte[] b = new byte[ints.Length * 4]; - Buffer.BlockCopy(ints, 0, b, 0, b.Length); - return b; - } - - private static byte[] PcOfF(uint[] ints, params float[] floats) - { - byte[] b = new byte[(ints.Length + floats.Length) * 4]; - Buffer.BlockCopy(ints, 0, b, 0, ints.Length * 4); - Buffer.BlockCopy(floats, 0, b, ints.Length * 4, floats.Length * 4); - return b; - } - - private static uint[] U(params int[] v) { uint[] u = new uint[v.Length]; for (int i = 0; i < v.Length; i++) u[i] = (uint)v[i]; return u; } - - private static uint G(uint n) => (n + 255) / 256; - - /// - /// Broadcast strides (padded rank-4, storage order of the output) of operand - /// onto output . Output physical dim i - /// holds logical dim o.Layout[i]; the operand's logical dim is that minus the - /// numpy right-alignment offset. - /// - private static int[] PhysBroadcastStrides(CValue a, CValue o) - { - int rank = o.Shape.Length, off = rank - a.Shape.Length; - if (rank > 4) throw new NotSupportedException("Vulkan elementwise: rank > 4"); - int[] lo = o.Layout ?? Engine.Layout.Identity(rank); - int[] la = a.Layout ?? Engine.Layout.Identity(a.Shape.Length); - int[] ap = a.PhysShape; - int[] aStr = new int[ap.Length]; - for (int i = ap.Length - 1, acc = 1; i >= 0; i--) { aStr[i] = acc; acc *= ap[i]; } - int[] r4 = new int[4]; - for (int i = 0; i < rank; i++) - { - int d = lo[i] - off; - r4[4 - rank + i] = d < 0 || a.Shape[d] == 1 ? 0 : aStr[Array.IndexOf(la, d)]; - } - return r4; - } - - private string Sh(string name) => F16 ? name + "_h" : name; - - private IEnumerable Emit(ExecNode n) - { - CValue[] V = _plan.Values; - // all addressing on storage-order shapes - int[] shapeIn(int k) => V[n.In[k]].PhysShape; - int[] shapeOut() => V[n.Out[0]].PhysShape; - int nOut() => checked((int)V[n.Out[0]].ElementCount); - - switch (n.Op) - { - case ExecOp.Copy: - yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)nOut(), 4), G((uint)nOut())); - break; - - case ExecOp.Relu or ExecOp.Sigmoid or ExecOp.Erf or ExecOp.Gelu: - { - uint mode = n.Op switch - { - ExecOp.Relu => 0, ExecOp.Sigmoid => 1, ExecOp.Erf => 2, _ => 3, - }; - yield return ND("unary", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)nOut(), mode), G((uint)nOut())); - break; - } - - case ExecOp.Elementwise: - { - var p = (EwP)n.P!; - CValue a = V[n.In[0]], bv = V[n.In[1]], o = V[n.Out[0]]; - int[] os = Pad4(shapeOut()); - int[] ostr = Strides(os); - int[] astr = PhysBroadcastStrides(a, o); - int[] bstr = PhysBroadcastStrides(bv, o); - uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, _ => 0u }; - // same-storage fast path: vec4 (n % 4 guard — no OOB tail writes) - bool dense = astr.SequenceEqual(ostr) && bstr.SequenceEqual(ostr); - if (nOut() % 4 == 0 && dense) - { - yield return ND("ew2v", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])], - PcOf((uint)(nOut() / 4), (uint)p.Op, act), G((uint)(nOut() / 4))); - break; - } - yield return ND("ew2", [Bind(n.In[0]), Bind(n.In[1]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - astr[0], astr[1], astr[2], astr[3], - bstr[0], bstr[1], bstr[2], bstr[3], (int)p.Op, (int)act)), - G((uint)nOut())); - break; - } - - case ExecOp.Conv: - { - var p = (ConvP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - if (p.Pool is { } pool) - { - // small-ic stem conv + 2x2/2 max pool (GraphCompiler.GpuSmallConv) - int[] ci = shapeIn(0), co = shapeOut(), cw = V[n.In[1]].Shape; - if (!GraphCompiler.GpuSmallConv(p, cw, ci[3]) || p.AddValue >= 0 || ci[1] != co[1] * 2 || ci[2] != co[2] * 2) - throw new NotSupportedException("Vulkan convpool: unsupported conv"); - uint pf = (p.BiasValue >= 0 ? 1u : 0u) | (p.PadMode == PadMode.Reflect ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) | (pool.FuseRelu ? 8u : 0u); - bool k3 = _dev.ConvPoolK3 && p.Kernel[0] == 3 && p.Kernel[1] == 3; - string cpName = k3 ? "convpool_k3" : _dev.ConvDot && F16 && cw[0] % 8 == 0 ? "convpool_pi" : "convpool"; - yield return ND(cpName, - [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOfF(U(ci[3], ci[1], ci[2], cw[0], co[1], co[2], p.Kernel[0], p.Kernel[1], p.Pads[0], p.Pads[1], (int)pf), - p.PadMode == PadMode.Constant ? p.PadValue : 0f), - G((uint)co[1] * (uint)co[2] * (uint)(cw[0] / 8)), name: "convpool"); - break; - } - if (p.AddValue >= 0) RequireNhwc(p.AddValue); - int[] si = shapeIn(0), sw_ = V[n.In[1]].Shape, so = shapeOut(); // [1,h,w,c] - uint ih = (uint)si[1], iw = (uint)si[2], ic = (uint)si[3]; - uint oc = (uint)sw_[0], oh = (uint)so[1], ow = (uint)so[2]; - if (p.PadMode == PadMode.Constant && p.PadValue != 0f) - throw new NotSupportedException("Vulkan conv: nonzero constant pad"); - if (p.Dilations.Any(d => d != 1)) - throw new NotSupportedException("Vulkan conv: dilation"); - if (p.Act == ActivationKind.Sigmoid) - throw new NotSupportedException("Vulkan conv: sigmoid epilogue"); - uint flags = (p.BiasValue >= 0 ? 1u : 0u) - | (p.PadMode == PadMode.Reflect ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) - | (p.AddValue >= 0 ? 8u : 0u) - | (p.Act == ActivationKind.Gelu ? 16u : 0u); - if (p.Group > 1) - { - if (p.Group != ic || oc != ic) throw new NotSupportedException("grouped non-depthwise conv"); - if (p.AddValue >= 0) throw new NotSupportedException("depthwise conv + residual"); - bool v4 = _dev.Vec4Eltwise && ic % 4 == 0; - yield return ND(v4 ? "depthwise4" : "depthwise", - [Bind(n.In[0]), Bind(n.In[1], CK.DwW), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOfF(U((int)ic, (int)ih, (int)iw, (int)oh, (int)ow, p.Kernel[0], p.Kernel[1], - p.Strides[0], p.Strides[1], p.Pads[0], p.Pads[1], (int)(flags & ~8u)), p.PadValue), - G(ic * oh * ow / (v4 ? 4u : 1u)), name: "depthwise"); - break; - } - if (UseCoopConv(_dev, F16, _plan, n, p)) - { - // BM = 128 rows (pixels) in every variant; BN by oc divisibility - (string cm, uint bn) = ic % 8 != 0 ? ("conv_cm_n32s", 32u) - : oc % 128 == 0 ? ("conv_cm", 128u) : oc % 64 == 0 ? ("conv_cm_n64", 64u) : ("conv_cm_n32", 32u); - uint m = oh * ow, kr = (uint)(p.Kernel[0] * p.Kernel[1]) * ic, kd = (kr + 7u) & ~7u; - uint cflags = (p.BiasValue >= 0 ? 1u : 0u) | (p.AddValue >= 0 ? 2u : 0u) - | (p.PadMode == PadMode.Reflect ? 4u : 0u) - | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4); - yield return ND(cm, - [Bind(n.In[0]), Bind(n.In[1], CK.ConvWT), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])], - PcOf(m, oc, kd, cflags, ow, iw, ih, ic, (uint)p.Kernel[1], - (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], kr), - (m + 127) / 128, (oc + bn - 1) / bn, name: "conv_cm"); - break; - } - string convShader = - p.Kernel[0] == 3 && p.Kernel[1] == 3 && p.Strides[0] == 1 && p.Strides[1] == 1 ? "conv_k33s1" : - p.Kernel[0] == 3 && p.Kernel[1] == 3 && p.Strides[0] == 2 && p.Strides[1] == 2 ? "conv_k33s2" : - p.Kernel[0] == 1 && p.Kernel[1] == 1 && p.Strides[0] == 1 && p.Strides[1] == 1 ? "conv_k11s1" : - "conv_gen"; - if (_dev.ConvDot && F16 && convShader is "conv_k33s1" or "conv_k33s2" && ic % 4 == 0 && oc % 4 == 0) - { - // Adreno 750 sweep (ms, 1600x1152 gc+dr): stride 1 at 4 px x 8 oc, stride 2 at 2 px x 8 oc - uint tm = p.Strides[1] == 1 ? 4u : 2u, tn = oc % 8 == 0 ? 2u : 1u; - uint threads = oc / (4 * tn) * ((ow + tm - 1) / tm) * oh; - uint wgs = (threads + 63) / 64, gx = Math.Min(wgs, 32768u); - yield return ND($"conv_dot_s{p.Strides[1]}_{tm}x{tn}", - [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])], - PcOf(ic, ih, iw, oc, oh, ow, (uint)p.Kernel[0], (uint)p.Kernel[1], - (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], - flags, 0), - gx, (wgs + gx - 1) / gx, name: "conv"); - break; - } - yield return ND(convShader, - [Bind(n.In[0]), Bind(n.In[1], CK.ConvW), B(p.BiasValue, CK.F32), B(p.AddValue), Bind(n.Out[0])], - PcOf(ic, ih, iw, oc, oh, ow, (uint)p.Kernel[0], (uint)p.Kernel[1], - (uint)p.Strides[0], (uint)p.Strides[1], (uint)p.Pads[0], (uint)p.Pads[1], - flags, 0), - (ow + 15) / 16, (oh + 7) / 8, (oc + 63) / 64, name: "conv"); - break; - } - - case ExecOp.MaxPool: - { - var p = (PoolP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0), so = shapeOut(); - bool v4 = _dev.Vec4Eltwise && si[3] % 4 == 0; - yield return ND(v4 ? "maxpool4" : "maxpool", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf((uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], - (uint)p.Kernel[0], (uint)p.Kernel[1], (uint)p.Strides[0], (uint)p.Strides[1], - (uint)p.Pads[0], (uint)p.Pads[1], p.FuseRelu ? 1u : 0u), - G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "maxpool"); - break; - } - - case ExecOp.LayerNorm: - { - var p = (LnP)n.P!; - uint len = (uint)V[n.In[0]].Shape[^1]; - if ((uint)shapeIn(0)[^1] != len) throw new NotSupportedException("Vulkan LayerNorm: last dim not innermost"); - uint rows = (uint)(V[n.In[0]].ElementCount / len); - yield return ND("layernorm", - [Bind(n.In[0]), Bind(n.In[1], CK.F32), Bind(n.In[2], CK.F32), Bind(n.Out[0])], - PcOfF([len], p.Eps), rows, name: "layernorm"); - break; - } - - case ExecOp.Gemm: - { - var p = (GemmP)n.P!; - uint k = (uint)V[n.In[0]].Shape[^1]; - if ((uint)shapeIn(0)[^1] != k) throw new NotSupportedException("Vulkan Gemm: K not innermost"); - uint m = (uint)(V[n.In[0]].ElementCount / k); - uint nn = (uint)V[n.Out[0]].Shape[^1]; - if (UseCoopGemm(_dev, F16, _plan, n, p)) - { - (string cm, uint bn) = nn % 128 == 0 ? ("conv_cm", 128u) : nn % 64 == 0 ? ("conv_cm_n64", 64u) : ("conv_cm_n32", 32u); - uint cflags = (p.BiasValue >= 0 ? 1u : 0u) - | ((p.Act == ActivationKind.Relu ? 1u : p.Act == ActivationKind.Gelu ? 2u : 0u) << 4); - // [M,K] x [N,K]^T as a 1x1 conv over a 1 x M image - yield return ND(cm, - [Bind(n.In[0]), Bind(n.In[1], CK.GemmNK), B(p.BiasValue, CK.F32), B(-1), Bind(n.Out[0])], - PcOf(m, nn, k, cflags, m, m, 1u, k, 1u, 1u, 1u, 0u, 0u, k), - (m + 127) / 128, (nn + bn - 1) / bn, name: "gemm_cm"); - break; - } - uint gflags = (p.TransB ? 1u : 0u) | (p.BiasValue >= 0 ? 2u : 0u) - | (p.Act == ActivationKind.Relu ? 4u : 0u) - | (p.Act == ActivationKind.Gelu ? 8u : 0u); - if (_dev.ConvDot && F16 && !p.TransB && k % 4 == 0 && nn % 8 == 0) - { - // 4 rows x 8 columns per invocation - uint gth = nn / 8 * ((m + 3) / 4); - uint gwg = (gth + 63) / 64, ggx = Math.Min(gwg, 32768u); - yield return ND("gemm_dot", - [Bind(n.In[0]), Bind(n.In[1]), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOf(m, nn, k, gflags), ggx, (gwg + ggx - 1) / ggx, name: "gemm"); - break; - } - yield return ND("gemm", - [Bind(n.In[0]), Bind(n.In[1]), B(p.BiasValue, CK.F32), Bind(n.Out[0])], - PcOf(m, nn, k, gflags), - (nn + 63) / 64, (m + 63) / 64, name: "gemm"); - break; - } - - case ExecOp.ResizeBilinear: - { - var p = (ResizeP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - bool hasAdd = n.In.Length > 1; - if (hasAdd) RequireNhwc(n.In[1]); - int[] si = shapeIn(0), so = shapeOut(); - uint flags = (p.FuseRelu ? 1u : 0u) | (hasAdd ? 2u : 0u) | (p.AddRelu ? 4u : 0u); - bool v4 = _dev.Vec4Eltwise && si[3] % 4 == 0; - yield return ND(v4 ? "bilinear4" : "bilinear", [Bind(n.In[0]), hasAdd ? Bind(n.In[1]) : B(-1), Bind(n.Out[0])], - PcOfF([(uint)si[3], (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], flags], - p.Scales[2], p.Scales[3]), - G((uint)so[1] * (uint)so[2] * (uint)so[3] / (v4 ? 4u : 1u)), name: "bilinear"); - break; - } - - case ExecOp.UpHead: - { - var p = (UpHeadP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.In[1]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0), so = shapeOut(); - uint c = (uint)si[3], oc = (uint)so[3]; - if (oc > 8 || c > 128 || c % 4 != 0) throw new NotSupportedException($"Vulkan UpHead: c={c} oc={oc}"); - bool hasBias = n.In.Length > 3; - uint act = p.Act switch { ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, ActivationKind.None => 0u, - _ => throw new NotSupportedException($"Vulkan UpHead: {p.Act}") }; - uint flags = (p.ResizeRelu ? 1u : 0u) | (p.SkipRelu ? 2u : 0u) | (hasBias ? 4u : 0u) | (act << 3); - if (p.SkipConv is { } sc) - { - // skip recomputed from the stem conv's input (GraphCompiler.GpuSmallConv, c <= 32) - int[] xi = shapeIn(1), cw = V[p.SkipW].Shape; - if (!GraphCompiler.GpuSmallConv(sc, cw, xi[3]) || cw[0] != c || c > 32 || xi[1] != so[1] || xi[2] != so[2]) - throw new NotSupportedException("Vulkan UpHead: unsupported skip conv"); - uint cf = (sc.BiasValue >= 0 ? 1u : 0u) | (sc.PadMode == PadMode.Reflect ? 2u : 0u) - | (sc.Act == ActivationKind.Relu ? 4u : 0u); - uint np = _dev.ConvDot && F16 ? 2u : 1u; - yield return ND(np > 1 ? $"uphead_sc{np}" : "uphead_sc", - [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0]), - Bind(p.SkipW, CK.ConvW), B(p.SkipB, CK.F32)], - PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], - p.Scales[2], p.Scales[3]).Concat(PcOfF(U(xi[3], sc.Kernel[0], sc.Kernel[1], sc.Pads[0], sc.Pads[1], (int)cf), - sc.PadMode == PadMode.Constant ? sc.PadValue : 0f)).ToArray(), - G((uint)so[1] * (((uint)so[2] + np - 1) / np)), name: "uphead_sc"); - break; - } - yield return ND("uphead", - [Bind(n.In[0]), Bind(n.In[1]), Bind(n.In[2]), hasBias ? Bind(n.In[3], CK.F32) : B(-1, CK.F32), Bind(n.Out[0])], - PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], - p.Scales[2], p.Scales[3]), - G((uint)so[1] * (uint)so[2])); - break; - } - - case ExecOp.Pad: - { - // generic rank-4 pad in storage order (pads permuted to physical dims) - var p = (PadP)n.P!; - CValue x = V[n.In[0]]; - if (x.Shape.Length != 4) throw new NotSupportedException("Vulkan pad: rank != 4"); - int[] l = x.Layout ?? Engine.Layout.Identity(4); - int[] id4 = shapeIn(0), od4 = shapeOut(); - int[] pb4 = new int[4]; - for (int d = 0; d < 4; d++) pb4[d] = p.Pads[l[d]]; - yield return ND("pad", [Bind(n.In[0]), Bind(n.Out[0])], - PcOfF(U(id4[0], id4[1], id4[2], id4[3], od4[0], od4[1], od4[2], od4[3], - pb4[0], pb4[1], pb4[2], pb4[3], p.Mode == PadMode.Reflect ? 1 : 0), - p.Value), - G((uint)V[n.Out[0]].ElementCount)); - break; - } - - case ExecOp.Transpose: - { - // only planner relayouts reach here; the perm is physical - var p = (TransposeP)n.P!; - int[] is4 = Pad4(shapeIn(0)); - int[] os4 = Pad4(shapeOut()); - int[] ostr = Strides(os4), istr = Strides(is4); - int[] perm4 = PermuteTo4(p.Perm, shapeIn(0).Length); - yield return ND("transpose", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - istr[perm4[0]], istr[perm4[1]], istr[perm4[2]], istr[perm4[3]])), - G((uint)nOut())); - break; - } - - case ExecOp.Slice: - { - // SliceP.Axes are physical after the layout pass - var p = (SliceP)n.P!; - int[] is4 = Pad4(shapeIn(0)), os4 = Pad4(shapeOut()); - int[] ostr = Strides(os4), istr = Strides(is4); - uint[] st = [0, 0, 0, 0], sp = [1, 1, 1, 1]; - int shift = 4 - shapeIn(0).Length; - for (int i = 0; i < p.Axes.Length; i++) - { - st[p.Axes[i] + shift] = (uint)p.Starts[i]; - sp[p.Axes[i] + shift] = (uint)p.Steps[i]; - } - yield return ND("slice", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(U(nOut(), ostr[0], ostr[1], ostr[2], - (int)st[0], (int)st[1], (int)st[2], (int)st[3], - (int)sp[0], (int)sp[1], (int)sp[2], (int)sp[3], - istr[0], istr[1], istr[2], istr[3])), - G((uint)nOut())); - break; - } - - case ExecOp.Concat: - { - // ConcatP.Axis is physical after the layout pass - var p = (ConcatP)n.P!; - int[] os = shapeOut(); - int axis = p.Axis; - int inner = 1, dstAxis = os[axis]; - for (int d = axis + 1; d < os.Length; d++) inner *= os[d]; - int axisOff = 0; - foreach (int iv in n.In) - { - int[] ishape = V[iv].PhysShape; - int cnt = checked((int)V[iv].ElementCount); - if (cnt > 0) - { - yield return ND("concat_ax", [Bind(iv), Bind(n.Out[0])], - PcOf((uint)cnt, (uint)inner, (uint)ishape[axis], (uint)dstAxis, (uint)axisOff), - G((uint)cnt)); - } - axisOff += ishape[axis]; - } - break; - } - - case ExecOp.ChannelShift: - { - var p = (ShiftP)n.P!; - RequireNhwc(n.In[0]); RequireNhwc(n.Out[0]); - int[] si = shapeIn(0); // [1,h,w,c] - var ge = p.GroupEnds; var of = p.Offsets; - if (ge.Length > 5) throw new NotSupportedException("Vulkan shift: > 5 groups"); - var pc = new List { (uint)si[3], (uint)si[1], (uint)si[2], (uint)p.Axis }; - for (int i = 0; i < 5; i++) pc.Add((uint)(i < ge.Length ? ge[i] : si[3])); - for (int i = 0; i < 5; i++) pc.Add((uint)(i < of.Length ? of[i] : 0)); - bool v4 = _dev.Vec4Eltwise && si[3] % 4 == 0 && ge.All(e => e % 4 == 0); - yield return ND(v4 ? "shift4" : "shift", [Bind(n.In[0]), Bind(n.Out[0])], - PcOf(pc.ToArray()), G((uint)nOut() / (v4 ? 4u : 1u)), name: "shift"); - break; - } - - default: - throw new NotSupportedException($"Vulkan: ExecOp {n.Op} not implemented"); - } - - VkDispatch ND(string shader, VkBind[] binds, byte[] pc, uint gx, uint gy = 1, uint gz = 1, string? name = null) - { - return new VkDispatch - { - Pipe = shader.StartsWith("conv_cm", StringComparison.Ordinal) - ? _dev.GetCoopPipeline(shader, binds.Length, pc.Length) - : _dev.GetPipeline(Sh(shader), binds.Length, pc.Length), - Pc = pc, Binds = binds, Gx = gx, Gy = gy, Gz = gz, - Name = $"{shader}({n.Name})", - }; - } - - VkBind B(int valueId, CK kind = CK.Raw) => valueId >= 0 ? Bind(valueId, kind) : new VkBind(VkSlot.Dummy, 0, 64); - } - - private void RequireNhwc(int v) - { - CValue cv = _plan.Values[v]; - if (cv.Shape.Length != 4 || !Engine.Layout.Equivalent(cv.Shape, cv.Layout ?? Engine.Layout.Identity(4), Engine.Layout.Nhwc)) - throw new NotSupportedException($"Vulkan: '{cv.Name}' expected NHWC storage, got {Engine.Layout.Str(cv.Layout)}"); - } - - // pad a shape to rank 4 with leading 1s - private static int[] Pad4(int[] s) - { - if (s.Length == 4) return s; - int[] r = [1, 1, 1, 1]; - s.CopyTo(r, 4 - s.Length); - return r; - } - - private static int[] Strides(int[] shape4) - { - int[] s = new int[4]; s[3] = 1; - for (int i = 2; i >= 0; i--) s[i] = s[i + 1] * shape4[i + 1]; - return s; - } - - // lift an original-rank perm into padded rank-4 (leading dims map to themselves) - private static int[] PermuteTo4(int[] perm, int rank) - { - int shift = 4 - rank; - int[] p4 = [0, 1, 2, 3]; - for (int i = 0; i < rank; i++) p4[i + shift] = perm[i] + shift; - return p4; - } - - /// Sessions hold a reference so cache eviction can't free live const/dummy buffers. - public void Acquire() - { - if (Interlocked.Increment(ref _refs) <= 1) - throw new ObjectDisposedException(nameof(VkGraphModel), "const buffers already freed"); - } - /// Acquire unless already freed (no resurrection; for racing cache readers). - public bool TryAcquire() - { - while (true) - { - int r = Volatile.Read(ref _refs); - if (r <= 0) return false; - if (Interlocked.CompareExchange(ref _refs, r + 1, r) == r) return true; - } - } - public void Release() - { - if (Interlocked.Decrement(ref _refs) == 0) _store.Release(); - } - - /// The plan this graph was compiled from (arena offsets, I/O values). - internal ExecPlan Plan => _plan; - public void Dispose() => Release(); -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs index dd6a5d7..83c6359 100644 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs +++ b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs @@ -1,5 +1,7 @@ -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Backends.Vulkan; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Onnx; namespace Sdcb.SimdGcdrNet.Backends.Vulkan; @@ -45,9 +47,9 @@ internal VkGcdrPipeline(VkBackend be, OnnxModel gc, OnnxModel dr) _drModel = dr; _f16 = be.F16; string sfx = _f16 ? "_h" : ""; - _pPre = _dev.GetPipeline("bgr2nhwc" + sfx, 2, 9 * 4); - _pDiv6 = _dev.GetPipeline("div6" + sfx, 3, 4); - _pPost = _dev.GetPipeline("nhwc2bgr" + sfx, 3, 9 * 4); + _pPre = _dev.GetPipeline("bgr2nhwc" + sfx, 2, 9 * 4, owner: GcdrExtensions.Owner); + _pDiv6 = _dev.GetPipeline("div6" + sfx, 3, 4, owner: GcdrExtensions.Owner); + _pPost = _dev.GetPipeline("nhwc2bgr" + sfx, 3, 9 * 4, owner: GcdrExtensions.Owner); _cmdPool = _dev.NewCommandPool(); try { @@ -80,9 +82,9 @@ internal void Bind(int ih, int iw, GcdrPixelFormat fmt = GcdrPixelFormat.Bgr24) if (_gcG is null || h != _h || w != _w) { // a new size compiles two plans + graphs: build them side by side - var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], physIo: true)); + var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], physIo: true, GcdrExtensions.Instance)); VkGraphModel gd; - try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], physIo: true); } + try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], physIo: true, GcdrExtensions.Instance); } catch { try { tg.GetAwaiter().GetResult().Release(); } catch { } throw; } VkGraphModel gg; try { gg = tg.GetAwaiter().GetResult(); } diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkSession.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkSession.cs deleted file mode 100644 index edfc0ae..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkSession.cs +++ /dev/null @@ -1,394 +0,0 @@ -using Sdcb.SimdGcdrNet.Engine; - -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -/// -/// GPU counterpart of : one activation arena in -/// device-local memory, persistent host-visible staging buffers, and a single -/// pre-recorded command buffer (staging copy-in → all dispatches → copy-out). -/// Not thread-safe; one per thread or serialize externally. -/// -public sealed unsafe class VkInferenceSession : IDisposable -{ - private readonly VkDevice _dev; - private readonly VkGraphModel _graph; - private readonly ExecPlan _plan; - - internal readonly VkBuffer Arena; // device-local activation arena - private readonly VkBuffer _stageIn; // host-visible float input staging - private readonly VkBuffer _stageOut; // host-visible float output staging - private float* _stageInMap, _stageOutMap; - private readonly float[] _inHost, _outHost; - private readonly IntPtr _cmd, _fence; - private readonly IntPtr[] _descSets; // per-dispatch sets when !PushDescriptors - private readonly ulong _inOff, _outOff; - private readonly int _inFloats, _outFloats; - private readonly int _es; // bytes per stored element (2 = fp16 storage) - internal bool F16 => _es == 2; - - internal VkInferenceSession(VkDevice dev, ExecPlan plan, VkGraphModel graph) - { - _dev = dev; - _plan = plan; - _graph = graph; - _graph.Acquire(); - _es = graph.Es; - - Arena = dev.NewStorageBuffer((ulong)plan.ArenaFloats * (ulong)_es + 64, hostVisible: false); - _inFloats = checked((int)plan.Values[plan.InputValue].ElementCount); - _outFloats = checked((int)plan.Values[plan.OutputValue].ElementCount); - _inOff = (ulong)plan.BufOffset[plan.Values[plan.InputValue].Buf] * (ulong)_es; - _outOff = (ulong)plan.BufOffset[plan.Values[plan.OutputValue].Buf] * (ulong)_es; - _stageIn = dev.NewStorageBuffer(InputBytes, hostVisible: true, preferHost: true); - _stageOut = dev.NewStorageBuffer(OutputBytes, hostVisible: true, preferHost: true); - _stageInMap = (float*)_stageIn.Map(); - _stageOutMap = (float*)_stageOut.Map(); - _inHost = new float[_inFloats]; - _outHost = new float[_outFloats]; - _fence = dev.NewFence(); - - if (!dev.PushDescriptors) - { - _descSets = new IntPtr[_graph.Dispatches.Length]; - for (int i = 0; i < _graph.Dispatches.Length; i++) - { - var d = _graph.Dispatches[i]; - IntPtr set = dev.NewDescriptorSet(d.Pipe.SetLayout); - for (uint b = 0; b < d.Binds.Length; b++) - { - (VkBuffer buf, _, _) = Resolve(d.Binds[b]); - dev.BindBuffer(set, b, buf, d.Binds[b].Offset, d.Binds[b].Size); - } - _descSets[i] = set; - } - } - else _descSets = []; - - _cmd = dev.NewCommandBuffer(); - Record(); - } - - internal VkBuffer ResolveBuf(VkBind b) => Resolve(b).buf; - internal ulong ResolveOff(VkBind b) => b.Offset; - - internal (VkBuffer buf, ulong off, ulong size) Resolve(VkBind b) => b.Slot switch - { - VkSlot.Arena => (Arena, b.Offset, b.Size), - VkSlot.Consts => (_graph.Consts, b.Offset, b.Size), - VkSlot.StageIn => (_stageIn, b.Offset, b.Size), - VkSlot.StageOut => (_stageOut, b.Offset, b.Size), - VkSlot.Direct => (b.Buf!, b.Offset, b.Size), - _ => (_graph.Dummy, b.Offset, b.Size), - }; - - /// Input tensor shape (e.g. [1,3,H,W]). - public int[] InputShape => _plan.Values[_plan.InputValue].Shape; - /// Output tensor shape (e.g. [1,3,H,W]). - public int[] OutputShape => _plan.Values[_plan.OutputValue].Shape; - - /// The writable input window (host copy staged on ). - public Span Input => _inHost; - /// The output tensor after . - public ReadOnlySpan Output => _outHost; - - internal ulong InputOffset => _inOff; - internal ulong OutputOffset => _outOff; - internal ulong InputBytes => (ulong)_inFloats * (ulong)_es; - internal ulong OutputBytes => (ulong)_outFloats * (ulong)_es; - internal int ElemBytes => _es; - - /// Host float -> staged storage elements (fp16 narrowing when F16). - internal static void ToStorage(ReadOnlySpan src, void* dst, bool f16) - { - if (!f16) { src.CopyTo(new Span(dst, src.Length)); return; } - var d = new Span(dst, src.Length); - for (int i = 0; i < src.Length; i++) d[i] = (Half)src[i]; - } - - /// Stored elements -> host float (fp16 widening when F16). - internal static void FromStorage(void* src, Span dst, bool f16) - { - if (!f16) { new ReadOnlySpan(src, dst.Length).CopyTo(dst); return; } - var sp = new ReadOnlySpan(src, dst.Length); - for (int i = 0; i < dst.Length; i++) dst[i] = (float)sp[i]; - } - - /// Downloads dst.Length stored elements at byte offset . - internal void DownloadElems(VkBuffer buf, ulong off, Span dst) - { - ulong bytes = (ulong)dst.Length * (ulong)_es; - byte[] tmp = new byte[bytes]; - fixed (byte* t = tmp) - { - _dev.Download(buf, t, bytes, off); - FromStorage(t, dst, F16); - } - } - - /// Record one dispatch (push-descriptor path only; caller emits barriers). - internal static void RecordOne(VkDevice dev, IntPtr cmd, VkPipeline pipe, - (VkBuffer buf, ulong off, ulong size)[] binds, byte[] pc, uint gx, uint gy, uint gz = 1) - { - Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, pipe.Pipeline); - var writes = stackalloc Vk.VkWriteDescriptorSet[8]; - var infos = stackalloc Vk.VkDescriptorBufferInfo[8]; - for (uint b = 0; b < binds.Length; b++) - { - infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = binds[b].buf.Buffer, Offset = binds[b].off, Range = binds[b].size }; - writes[b] = new Vk.VkWriteDescriptorSet - { - SType = VkConst.StWriteDescriptorSet, DstBinding = b, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, - PBufferInfo = infos + b, - }; - } - dev.CmdPushDescriptors(cmd, pipe.Layout, writes, (uint)binds.Length); - if (pc.Length > 0) - fixed (byte* p = pc) - Vk.vkCmdPushConstants(cmd, pipe.Layout, VkConst.StageComputeShader, 0, (uint)pc.Length, p); - Vk.vkCmdDispatch(cmd, gx, gy, gz); - } - - /// Record all dispatches with barriers into an open command buffer. - internal void RecordCompute(IntPtr cmd) - { - var writes = stackalloc Vk.VkWriteDescriptorSet[8]; - var infos = stackalloc Vk.VkDescriptorBufferInfo[8]; - bool first = true; - for (int i = 0; i < _graph.Dispatches.Length; i++) - { - var d = _graph.Dispatches[i]; - if (!first) _dev.CmdComputeBarrier(cmd); - first = false; - Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, d.Pipe.Pipeline); - if (_dev.PushDescriptors) - { - for (uint b = 0; b < d.Binds.Length; b++) - { - (VkBuffer buf, ulong off, ulong size) = Resolve(d.Binds[b]); - infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = buf.Buffer, Offset = off, Range = size }; - writes[b] = new Vk.VkWriteDescriptorSet - { - SType = VkConst.StWriteDescriptorSet, DstBinding = b, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, - PBufferInfo = infos + b, - }; - } - _dev.CmdPushDescriptors(cmd, d.Pipe.Layout, writes, (uint)d.Binds.Length); - } - else - { - IntPtr set = _descSets[i]; - Vk.vkCmdBindDescriptorSets(cmd, VkConst.BindPointCompute, d.Pipe.Layout, 0, 1, &set, 0, null); - } - if (d.Pc.Length > 0) - fixed (byte* pc = d.Pc) - Vk.vkCmdPushConstants(cmd, d.Pipe.Layout, VkConst.StageComputeShader, 0, (uint)d.Pc.Length, pc); - Vk.vkCmdDispatch(cmd, d.Gx, d.Gy, d.Gz); - } - } - - private void Record() - { - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(_cmd, &begin), "vkBeginCommandBuffer"); - // staging -> arena(input) - Vk.VkBufferCopy cin = new() { Size = InputBytes, DstOffset = _inOff }; - Vk.vkCmdCopyBuffer(_cmd, _stageIn.Buffer, Arena.Buffer, 1, &cin); - // barrier: transfer-write -> shader-read - Vk.VkMemoryBarrier mb = new() - { - SType = VkConst.StMemoryBarrier, - SrcAccessMask = VkConst.AccessTransferWrite, DstAccessMask = VkConst.AccessShaderRead, - }; - Vk.vkCmdPipelineBarrier(_cmd, VkConst.PipelineStageTransfer, VkConst.PipelineStageComputeShader, 0, - 1, &mb, 0, null, 0, null); - RecordCompute(_cmd); - // barrier: shader-write -> transfer-read - Vk.VkMemoryBarrier mb2 = new() - { - SType = VkConst.StMemoryBarrier, - SrcAccessMask = VkConst.AccessShaderWrite, DstAccessMask = VkConst.AccessTransferRead, - }; - Vk.vkCmdPipelineBarrier(_cmd, VkConst.PipelineStageComputeShader, VkConst.PipelineStageTransfer, 0, - 1, &mb2, 0, null, 0, null); - Vk.VkBufferCopy cout = new() { Size = OutputBytes, SrcOffset = _outOff }; - Vk.vkCmdCopyBuffer(_cmd, Arena.Buffer, _stageOut.Buffer, 1, &cout); - Vk.Check(Vk.vkEndCommandBuffer(_cmd), "vkEndCommandBuffer"); - } - - /// Debug/test: record a single dispatch list slice and run it. - internal void DebugRunRange(int dStart, int dCount) - { - IntPtr cmd = _dev.NewCommandBuffer(); - IntPtr fence = _dev.NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - var writes = stackalloc Vk.VkWriteDescriptorSet[8]; - var infos = stackalloc Vk.VkDescriptorBufferInfo[8]; - for (int i = dStart; i < dStart + dCount; i++) - { - var d = _graph.Dispatches[i]; - if (i > dStart) _dev.CmdComputeBarrier(cmd); - Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, d.Pipe.Pipeline); - for (uint b = 0; b < d.Binds.Length; b++) - { - (VkBuffer buf, ulong off, ulong size) = Resolve(d.Binds[b]); - infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = buf.Buffer, Offset = off, Range = size }; - writes[b] = new Vk.VkWriteDescriptorSet - { - SType = VkConst.StWriteDescriptorSet, DstBinding = b, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, - PBufferInfo = infos + b, - }; - } - _dev.CmdPushDescriptors(cmd, d.Pipe.Layout, writes, (uint)d.Binds.Length); - if (d.Pc.Length > 0) - fixed (byte* pc = d.Pc) - Vk.vkCmdPushConstants(cmd, d.Pipe.Layout, VkConst.StageComputeShader, 0, (uint)d.Pc.Length, pc); - Vk.vkCmdDispatch(cmd, d.Gx, d.Gy, d.Gz); - } - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - _dev.Submit(cmd, fence); - _dev.WaitFence(fence); - _dev.DestroyFence(fence); - _dev.FreeCommandBuffer(cmd); - } - - /// - /// Debug/test: time each dispatch by submitting it alone (barrier + dispatch) and - /// wall-timing the fenced submission. Includes ~20-40us launch overhead per dispatch; - /// accurate enough to rank ms-scale kernels. Returns name→best-of-2 ms. - /// - internal Dictionary DebugProfile() - { - int n = _graph.Dispatches.Length; - IntPtr cmd = _dev.NewCommandBuffer(); - IntPtr fence = _dev.NewFence(); - var writes = stackalloc Vk.VkWriteDescriptorSet[8]; - var infos = stackalloc Vk.VkDescriptorBufferInfo[8]; - var best = new double[n]; - Array.Fill(best, double.MaxValue); - void RecordDispatch(VkDispatch d) - { - _dev.CmdComputeBarrier(cmd); - Vk.vkCmdBindPipeline(cmd, VkConst.BindPointCompute, d.Pipe.Pipeline); - for (uint b = 0; b < d.Binds.Length; b++) - { - (VkBuffer buf, ulong off, ulong size) = Resolve(d.Binds[b]); - infos[b] = new Vk.VkDescriptorBufferInfo { Buffer = buf.Buffer, Offset = off, Range = size }; - writes[b] = new Vk.VkWriteDescriptorSet - { - SType = VkConst.StWriteDescriptorSet, DstBinding = b, - DescriptorCount = 1, DescriptorType = VkConst.DescStorageBuffer, - PBufferInfo = infos + b, - }; - } - _dev.CmdPushDescriptors(cmd, d.Pipe.Layout, writes, (uint)d.Binds.Length); - if (d.Pc.Length > 0) - fixed (byte* pc = d.Pc) - Vk.vkCmdPushConstants(cmd, d.Pipe.Layout, VkConst.StageComputeShader, 0, (uint)d.Pc.Length, pc); - Vk.vkCmdDispatch(cmd, d.Gx, d.Gy, d.Gz); - } - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - // warm pass to keep state identical to production (input already in arena) - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - foreach (var d in _graph.Dispatches) RecordDispatch(d); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - _dev.Submit(cmd, fence); - _dev.WaitFence(fence); - // timed passes: one dispatch per submit, min of 2 - var sw = new System.Diagnostics.Stopwatch(); - for (int rep = 0; rep < 2; rep++) - for (int i = 0; i < n; i++) - { - var d = _graph.Dispatches[i]; - Vk.vkResetCommandBuffer(cmd, 0); - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - RecordDispatch(d); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - sw.Restart(); - _dev.Submit(cmd, fence); - _dev.WaitFence(fence); - double ms = sw.Elapsed.TotalMilliseconds; - if (ms < best[i]) best[i] = ms; - } - _dev.DestroyFence(fence); - _dev.FreeCommandBuffer(cmd); - var agg = new Dictionary(); - PerDispatch = new (string, double)[n]; - for (int i = 0; i < n; i++) - { - string name = _graph.Dispatches[i].Name ?? $"d{i}"; - PerDispatch[i] = (name, best[i]); - name = name[..(name.IndexOf('(') is int cut && cut > 0 ? cut : name.Length)]; - agg[name] = agg.TryGetValue(name, out double v) ? v + best[i] : best[i]; - } - return agg; - } - - /// Per-dispatch timings from the last call. - internal (string Name, double Ms)[]? PerDispatch; - - /// Debug: the compiled plan backing this session. - internal ExecPlan DebugPlan => _plan; - - /// Debug: download the whole arena (ArenaFloats floats) into dst. - internal void DebugArena(Span dst) - { - DownloadElems(Arena, 0, dst.Slice(0, _plan.ArenaFloats)); - } - - /// Debug: graph consts buffer + the session's arena buffer. - internal (VkBuffer consts, VkBuffer arena) DebugBuffers => (_graph.Consts, Arena); - - /// Debug/test: download an activation value's region into dst. - internal void DebugDownload(int valueId, float[] dst) - { - CValue v = _plan.Values[valueId]; - DownloadElems(Arena, (ulong)_plan.BufOffset[v.Buf] * (ulong)_es, dst.AsSpan(0, checked((int)v.ElementCount))); - } - - /// Debug/test: copy host input into the arena's input region. - internal void DebugUploadInput() - { - ToStorage(_inHost, _stageInMap, F16); - _stageIn.Flush(0, InputBytes); - IntPtr cmd = _dev.NewCommandBuffer(); - IntPtr fence = _dev.NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "vkBeginCommandBuffer"); - Vk.VkBufferCopy cin = new() { Size = InputBytes, DstOffset = _inOff }; - Vk.vkCmdCopyBuffer(cmd, _stageIn.Buffer, Arena.Buffer, 1, &cin); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "vkEndCommandBuffer"); - _dev.Submit(cmd, fence); - _dev.WaitFence(fence); - _dev.DestroyFence(fence); - _dev.FreeCommandBuffer(cmd); - } - - /// Copy host input into staging and run the recorded pipeline. - public void Run() - { - ToStorage(_inHost, _stageInMap, F16); - _stageIn.Flush(0, InputBytes); - _dev.Submit(_cmd, _fence); - _dev.WaitFence(_fence); - _stageOut.Invalidate(0, OutputBytes); - FromStorage(_stageOutMap, _outHost, F16); - } - - /// Releases the session's device buffers, fence and command buffer, and its - /// reference to the shared graph. - public void Dispose() - { - if (_stageInMap != null) { _stageIn.Unmap(); _stageInMap = null; } - if (_stageOutMap != null) { _stageOut.Unmap(); _stageOutMap = null; } - _dev.DestroyFence(_fence); - _dev.FreeCommandBuffer(_cmd); - // descriptor sets live in the device pool; pool teardown frees them - Arena.Free(); _stageIn.Free(); _stageOut.Free(); - _graph.Release(); - } -} - diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkShaders.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkShaders.cs deleted file mode 100644 index 28c5b78..0000000 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkShaders.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Reflection; - -namespace Sdcb.SimdGcdrNet.Backends.Vulkan; - -/// Loads SPIR-V blobs embedded next to their .comp sources. -internal static class VkShaders -{ - private static readonly Assembly s_asm = typeof(VkShaders).Assembly; - private const string Prefix = "Sdcb.SimdGcdrNet.Backends.Vulkan.Shaders."; - - public static byte[] Load(string name) - { - using Stream? s = s_asm.GetManifestResourceStream(Prefix + name + ".spv"); - if (s == null) - throw new InvalidOperationException( - $"Embedded shader '{name}' not found. Available: {string.Join(", ", s_asm.GetManifestResourceNames().Where(n => n.StartsWith(Prefix)))}"); - byte[] b = new byte[s.Length]; - s.ReadExactly(b); - return b; - } -} diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat index 017cbf1..914ae88 100644 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat +++ b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/tools/build-shaders.bat @@ -1,8 +1,6 @@ @echo off -rem Dev-time only: compile .comp GLSL -> .spv next to each source (committed). -rem Not part of `dotnet build` - SPIR-V is embedded as a resource. -rem Every shader is built twice: name.spv (fp32 storage) and name_h.spv -rem (-DF16: 16-bit storage, fp32 math; see Shaders/et.glsl). +rem Dev-time only: compile the GCDR-specific .comp GLSL -> .spv next to each source (committed). +rem Every shader is built twice: name.spv (fp32 storage) and name_h.spv (-DF16, see Shaders/et.glsl). setlocal set "GLSLC=C:\VulkanSDK\1.4.357.0\Bin\glslc.exe" if not exist "%GLSLC%" for %%i in (glslc.exe) do set "GLSLC=%%~$PATH:i" @@ -13,53 +11,9 @@ if not exist "%GLSLC%" ( set "DIR=%~dp0..\Shaders" set "O=-O --target-env=vulkan1.1" for %%f in ("%DIR%\*.comp") do ( - if not "%%~nxf"=="conv_tile.comp" if not "%%~nxf"=="conv_cm.comp" if not "%%~nxf"=="sgprobe.comp" if not "%%~nxf"=="conv_dot.comp" if not "%%~nxf"=="gemm_dot.comp" ( - echo glslc %%~nxf - "%GLSLC%" %O% "%%f" -o "%%~dpnf.spv" || exit /b 1 - "%GLSLC%" %O% -DF16 "%%f" -o "%%~dpnf_h.spv" || exit /b 1 - ) + echo glslc %%~nxf + "%GLSLC%" %O% "%%f" -o "%%~dpnf.spv" || exit /b 1 + "%GLSLC%" %O% -DF16 "%%f" -o "%%~dpnf_h.spv" || exit /b 1 ) -rem conv_tile specialization variants (constant kernel/stride -> fast index math) -call :conv "" "" || exit /b 1 -call :conv -DF16 _h || exit /b 1 -rem tensor-core conv (fp16 storage only): BN=128 / 64 / 32 output-channel tiles -set "CM=%DIR%\conv_cm.comp" -"%GLSLC%" %O% "%CM%" -o "%DIR%\conv_cm_h.spv" || exit /b 1 -"%GLSLC%" %O% -DSGX=2u -DSGY=2u -DTSM=4u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_n64_h.spv" || exit /b 1 -"%GLSLC%" %O% -DSGX=1u -DSGY=4u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_n32_h.spv" || exit /b 1 -rem any-cin stems (k elements gathered one by one) -"%GLSLC%" %O% -DSCALARK -DSGX=1u -DSGY=4u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_n32s_h.spv" || exit /b 1 -rem LITE: the same four for wave64-capable / 32 KB shared-memory parts (subgroup pinned to 32); -rem more, narrower (64x16) subgroup strips per tile measured fastest on the Radeon 880M -"%GLSLC%" %O% -DLITE -DSGX=8u -DSGY=2u -DTSM=4u -DTSN=1u -DNTHR=512 "%CM%" -o "%DIR%\conv_cm_l_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSGX=4u -DSGY=2u -DTSM=4u -DTSN=1u -DNTHR=256 "%CM%" -o "%DIR%\conv_cm_l_n64_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSGX=2u -DSGY=2u -DTSM=4u -DTSN=1u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_l_n32_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSCALARK -DSGX=1u -DSGY=4u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_l_n32s_h.spv" || exit /b 1 -rem SG16: 8x16x16 fragments on 16-lane subgroups (Intel Xe2 XMX), LITE staging, BM = 128; -rem a 16x32 strip per subgroup, eight stacked along M, measured fastest on the Arc B580 -"%GLSLC%" %O% -DLITE -DSG16 -DSGX=4u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=512 "%CM%" -o "%DIR%\conv_cm_sg16_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSG16 -DSGX=2u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=256 "%CM%" -o "%DIR%\conv_cm_sg16_n64_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSG16 -DSGX=1u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_sg16_n32_h.spv" || exit /b 1 -"%GLSLC%" %O% -DLITE -DSG16 -DSCALARK -DSGX=1u -DSGY=8u -DTSM=2u -DTSN=2u -DNTHR=128 "%CM%" -o "%DIR%\conv_cm_sg16_n32s_h.spv" || exit /b 1 -rem wide-wave no-coopmat GPUs (Adreno): register-tiled direct kernels, fp16 storage only -set "CD=%DIR%\conv_dot.comp" -"%GLSLC%" %O% -DSW=1u -DTM=4 -DTN=2 "%CD%" -o "%DIR%\conv_dot_s1_4x2_h.spv" || exit /b 1 -"%GLSLC%" %O% -DSW=1u -DTM=4 -DTN=1 "%CD%" -o "%DIR%\conv_dot_s1_4x1_h.spv" || exit /b 1 -"%GLSLC%" %O% -DSW=2u -DTM=2 -DTN=2 "%CD%" -o "%DIR%\conv_dot_s2_2x2_h.spv" || exit /b 1 -"%GLSLC%" %O% -DSW=2u -DTM=2 -DTN=1 "%CD%" -o "%DIR%\conv_dot_s2_2x1_h.spv" || exit /b 1 -"%GLSLC%" %O% "%DIR%\gemm_dot.comp" -o "%DIR%\gemm_dot_h.spv" || exit /b 1 -"%GLSLC%" %O% -DF16 -DPIXELS_INNER "%DIR%\convpool.comp" -o "%DIR%\convpool_pi_h.spv" || exit /b 1 "%GLSLC%" %O% -DF16 -DNP=2u "%DIR%\uphead_sc.comp" -o "%DIR%\uphead_sc2_h.spv" || exit /b 1 -rem subgroup-size probe for pinned (requiredSubgroupSize = 32 / 16) pipelines -"%GLSLC%" %O% "%DIR%\sgprobe.comp" -o "%DIR%\sgprobe.spv" || exit /b 1 -exit /b 0 - -:conv -set "CT=%DIR%\conv_tile.comp" -set "F=%~1" -set "S=%~2" -"%GLSLC%" %O% %F% -DKH=3u -DKW=3u -DSH=1u -DSW=1u "%CT%" -o "%DIR%\conv_k33s1%S%.spv" || exit /b 1 -"%GLSLC%" %O% %F% -DKH=3u -DKW=3u -DSH=2u -DSW=2u "%CT%" -o "%DIR%\conv_k33s2%S%.spv" || exit /b 1 -"%GLSLC%" %O% %F% -DKH=1u -DKW=1u -DSH=1u -DSW=1u "%CT%" -o "%DIR%\conv_k11s1%S%.spv" || exit /b 1 -"%GLSLC%" %O% %F% "%CT%" -o "%DIR%\conv_gen%S%.spv" || exit /b 1 exit /b 0 diff --git a/src/Sdcb.SimdGcdrNet/Engine/CompiledModel.cs b/src/Sdcb.SimdGcdrNet/Engine/CompiledModel.cs deleted file mode 100644 index 59f24c9..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/CompiledModel.cs +++ /dev/null @@ -1,30 +0,0 @@ -using Sdcb.SimdGcdrNet.Onnx; - -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// An specialized to a concrete input shape: all -/// dynamic-shape plumbing folded, ops lowered to the exec set, buffers -/// planned. Immutable; share across sessions freely. -/// -public sealed class CompiledModel -{ - internal ExecPlan Plan { get; } - - private CompiledModel(ExecPlan plan) => Plan = plan; - - /// Compiles for a fixed input shape. - /// The ONNX model to compile. - /// Concrete input shape, e.g. [1, 3, H, W]. - public static CompiledModel Compile(OnnxModel model, ReadOnlySpan inputShape) => - new(GraphCompiler.Compile(model, inputShape)); - - /// Creates a new CPU with its own arena. - public InferenceSession CreateSession() => new(Plan); - - /// Number of executable nodes after folding and fusion. - public int NodeCount => Plan.Nodes.Length; - - /// Size of the activation arena, in floats, a session needs. - public int ArenaFloats => Plan.ArenaFloats; -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/CpuKernels.cs b/src/Sdcb.SimdGcdrNet/Engine/CpuKernels.cs deleted file mode 100644 index 7a6e336..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/CpuKernels.cs +++ /dev/null @@ -1,76 +0,0 @@ -using Sdcb.SimdGcdrNet.Kernels; - -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// CPU-private packed weights (conv panels + Winograd transforms, Gemm panels, -/// UpHead lanes) stored in the plan's op params. Per-size plans of one model -/// carry identical weights, so a shared across sizes packs -/// each layer once: entries are keyed by node identity and only reused when the -/// weights (and bias) are element-for-element equal. -/// -internal static class CpuKernels -{ - internal sealed class Cache - { - private readonly Dictionary<(ExecOp op, string name, int src), (float[] w, float[]? b, object kp)> _d = new(); - - internal object GetOrPack(ExecNode n, float[] w, float[]? b, Func pack) - { - var key = (n.Op, n.Name, n.SrcNode); - lock (_d) - if (_d.TryGetValue(key, out var e) && Same(e.w, w) && Same(e.b, b)) return e.kp; - object kp = pack(); - lock (_d) _d[key] = (w, b, kp); - return kp; - } - - private static bool Same(float[]? a, float[]? b) => - ReferenceEquals(a, b) || (a is not null && b is not null && a.AsSpan().SequenceEqual(b)); - } - - /// Fills missing kernel plans of an NHWC plan (idempotent; plans may be shared). - internal static void Pack(ExecPlan plan, Cache? cache) - { - if (!plan.Nhwc) return; - CValue[] V = plan.Values; - foreach (ExecNode n in plan.Nodes) - switch (n.P) - { - case GemmP gp when gp.KernelPlan is null && V[n.In[1]].Floats is { } bw: - { - int[] bs = V[n.In[1]].Shape; - int kk = gp.TransB ? bs[1] : bs[0], nn = gp.TransB ? bs[0] : bs[1]; - float[]? gb = gp.BiasValue >= 0 ? V[gp.BiasValue].Floats : null; - object kp = Get(cache, n, bw, gb, () => Nhwc.PackGemm(bw, kk, nn, gp.TransB, gb)); - Interlocked.CompareExchange(ref gp.KernelPlan, kp, null); - break; - } - case ConvP cp when cp.KernelPlan is null: - { - CValue w = V[n.In[1]]; - float[]? b = cp.BiasValue >= 0 ? V[cp.BiasValue].Floats : null; - int ic = V[n.In[0]].Shape[1]; - object kp = Get(cache, n, w.Floats!, b, () => - { - NhwcConvPlan p = Nhwc.Pack(w.Floats!, w.Shape, b, cp.Group, ic); - if (Nhwc.WinogradEligible(cp, w.Shape, ic)) p.WinoU = Nhwc.PackWinograd(w.Floats!, w.Shape[0], ic); - return p; - }); - Interlocked.CompareExchange(ref cp.KernelPlan, kp, null); - break; - } - case UpHeadP up when up.KernelPlan is null: - { - CValue w = V[n.In[2]]; - float[]? b = n.In.Length > 3 ? V[n.In[3]].Floats : null; - object kp = Get(cache, n, w.Floats!, b, () => Nhwc.PackUpHead(w.Floats!, w.Shape[0], w.Shape[1], b)); - Interlocked.CompareExchange(ref up.KernelPlan, kp, null); - break; - } - } - } - - private static object Get(Cache? cache, ExecNode n, float[] w, float[]? b, Func pack) => - cache is null ? pack() : cache.GetOrPack(n, w, b, pack); -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/ExecPlan.cs b/src/Sdcb.SimdGcdrNet/Engine/ExecPlan.cs deleted file mode 100644 index dca1a69..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/ExecPlan.cs +++ /dev/null @@ -1,208 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Engine; - -/// Executable op set — the lowered form the session runs. -internal enum ExecOp -{ - Conv, // NCHW conv, optional bias + fused activation - MaxPool, // NCHW pooling - Elementwise, // Add/Sub/Mul/Div with broadcast (incl. scalar const) - Relu, - Sigmoid, - Erf, // elementwise erf (normally fused into Gelu) - Gelu, // exact-erf GELU, elementwise - LayerNorm, // normalize over last dim - Gemm, // [.., M, K] x [K, N] + bias[N] - ResizeBilinear, // NCHW, half_pixel, per-axis scales - Pad, // constant / reflect on trailing dims - Transpose, // arbitrary permute (physical perm in layout-planned plans) - Slice, // strided subrange copy - Concat, // axis join - ChannelShift, // fused shift-MLP: per channel group, shift along H or W (zero-fill) - Copy, // plain copy (for Squeeze/Unsqueeze/identity leftovers) - UpHead, // fused NHWC tail: act(W·(resize(x) + [relu](skip)) + b), pointwise W with few outputs -} - -internal enum ActivationKind { None, Relu, Sigmoid, Gelu } - -internal enum PadMode { Constant, Reflect } - -internal enum EwOp { Add, Sub, Mul, Div } - -// ---- per-op parameter records (resolved at compile time) ---- - -internal sealed class ConvP -{ - public required int[] Pads; // [top, left, bottom, right] spatial only - public required int[] Strides; // [sh, sw] - public required int[] Dilations; // [dh, dw] - public required int[] Kernel; // [kh, kw] - public int Group = 1; - public ActivationKind Act; - public PadMode PadMode = PadMode.Constant; - public float PadValue; // constant-mode fill - public int BiasValue = -1; // value id of per-channel bias, -1 none - public int AddValue = -1; // value id of same-shape residual addend, -1 none - public object? KernelPlan; // packed weights / kernel choice, filled at plan build - // NHWC plans: fused 2x2/2 max pool epilogue — Out[0] is the pooled tensor - public PoolP? Pool; -} - -internal sealed class PoolP -{ - public required int[] Kernel; - public required int[] Strides; - public required int[] Pads; - public bool FuseRelu; -} - -internal sealed class EwP -{ - public required EwOp Op; - public ActivationKind Act; -} - -internal sealed class LnP -{ - public float Eps; -} - -internal sealed class GemmP -{ - public bool TransB; // B stored [N,K] when true (onnx transB=1) - public int BiasValue = -1; - public ActivationKind Act; - public object? KernelPlan; // backend-private packed B, filled at session build -} - -internal sealed class ResizeP -{ - public required float[] Scales; // per-dim, only [.., sh, sw] used - public bool FuseRelu; // relu on the resized value (before the optional add) - // optional same-shape addend in In[1] (NHWC plans): out = [relu](resize(x)) + [relu](In[1]) - public bool AddRelu; // relu on the addend -} - -/// -/// Fused NHWC network tail (In = [x, skip, w, bias?]): -/// out[o] = act(bias[o] + Σ_c w[o,c]·s[c]) with s = [relu](resize(x)) + [relu](skip). -/// Replaces Resize → (Relu) → Add → 1x1 Conv → (act) so the full-resolution -/// intermediates are never stored. -/// -internal sealed class UpHeadP -{ - public required float[] Scales; - public bool ResizeRelu; - public bool SkipRelu; - public ActivationKind Act; - public object? KernelPlan; // backend-private packed weights - // NHWC plans: In[1] is the input of this (still scheduled) conv and the skip is - // recomputed, so the full-resolution skip tensor need not stay live - public ConvP? SkipConv; - public int SkipW = -1, SkipB = -1; // the skip conv's weight / bias value ids -} - -internal sealed class PadP -{ - public required int[] Pads; // [2*rank] begin..end - public PadMode Mode = PadMode.Constant; - public float Value; -} - -internal sealed class TransposeP -{ - public required int[] Perm; -} - -internal sealed class SliceP -{ - public required int[] Starts; - public required int[] Ends; - public required int[] Axes; - public required int[] Steps; -} - -internal sealed class ConcatP -{ - public int Axis; -} - -internal sealed class ShiftP -{ - public int Axis; // logical spatial axis index (2 = H, 3 = W) - public required int[] GroupEnds; // cumulative channel-group end indices (len 5) - public required int[] Offsets; // per-group shift (len 5) -} - -/// A resolved tensor value: compile-time constant or activation. -internal sealed class CValue -{ - public int[] Shape = []; - public int DType; // onnx dtype: 1 float, 7 int64 - public long[]? Ints; // compile-time int content - public float[]? Floats; // compile-time float content (weights or small consts) - public int Buf = -1; // activation buffer id; >= 0 → runtime tensor - public string Name = ""; - /// Physical dim order: physical dim i holds logical dim Layout[i]. null = identity. - public int[]? Layout; - /// Zero-copy view source (reshape/squeeze/unsqueeze/identity); -1 none. - public int AliasOf = -1; - public bool IsInt => Ints != null; - public bool IsConst => Ints != null || Floats != null; - public bool IsActivation => Buf >= 0; - public long ElementCount => Shape.Aggregate(1L, (a, b) => a * b); - /// Shape in storage order — what kernels address by. - public int[] PhysShape => Layout is null ? Shape : Engine.Layout.Phys(Shape, Layout); -} - -/// A buffer reference into either the activation arena or a const array. -internal readonly unsafe struct FBuf -{ - public readonly float[]? Data; - public readonly float* Ptr; - public readonly int Offset; - public readonly int Length; - public FBuf(float[] data, int offset, int length) { Data = data; Offset = offset; Length = length; } - public FBuf(float* ptr, int length) { Ptr = ptr; Length = length; } - public Span Span => Data != null ? Data.AsSpan(Offset, Length) : new Span(Ptr, Length); -} - -/// One executable step: op + value ids. -internal sealed class ExecNode -{ - public required ExecOp Op; - public required int[] In; // value ids - public required int[] Out; - public object? P; - public int SrcNode = -1; // onnx node index (diagnostics) - public string Name = ""; -} - -/// Compiled graph ready for a session: exec nodes + buffer plan. -internal sealed class ExecPlan -{ - public required ExecNode[] Nodes; - public required CValue[] Values; - public required int[] BufSize; // floats per activation buffer - public required int[] BufOffset; // arena offset per buffer - public required int ArenaFloats; - public required int InputValue; - public required int OutputValue; - /// Rank-4 activations stored channels-last (layout pass ran). - public bool Nhwc; - /// Input/output stay in plan layout (no boundary relayout); else logical NCHW. - public bool PhysicalIo; -} - -/// Plan compilation options. -/// run the layout pass: rank-4 activations stored channels-last. -/// keep input/output in plan layout (no boundary relayout); else logical NCHW. -/// memory fusions (conv+pool, UpHead skip recompute; NHWC only). -/// plan for the Vulkan backend (restricts fusions to what its shaders implement). -internal sealed record CompileOptions(bool Nhwc, bool PhysicalIo = false, bool MemFuse = true, bool Gpu = false) -{ - /// GCDRNET_NHWC=0 forces the legacy all-NCHW plan (A/B oracle). - public static bool DefaultNhwc { get; } = Environment.GetEnvironmentVariable("GCDRNET_NHWC") != "0"; - public static CompileOptions Default => new(DefaultNhwc); - /// GCDRNET_NOFUSE_MEM=1 disables the CPU memory fusions (conv+pool, skip recompute). - public static bool MemFusions { get; } = Environment.GetEnvironmentVariable("GCDRNET_NOFUSE_MEM") != "1"; -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/GraphCompiler.cs b/src/Sdcb.SimdGcdrNet/Engine/GraphCompiler.cs deleted file mode 100644 index 662de6a..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/GraphCompiler.cs +++ /dev/null @@ -1,1529 +0,0 @@ -using Sdcb.SimdGcdrNet.Onnx; - -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// Compiles an into an for a -/// concrete input shape. All ONNX dynamic-shape plumbing (Shape/Squeeze/Div/ -/// Mod/Concat/Slice on int64 scalars) is evaluated at compile time; remaining -/// float ops are emitted with fully resolved shapes. -/// -internal static class GraphCompiler -{ - public static ExecPlan Compile(OnnxModel model, ReadOnlySpan inputShape) => - Compile(model, inputShape, CompileOptions.Default); - - public static ExecPlan Compile(OnnxModel model, ReadOnlySpan inputShape, CompileOptions opt) - { - var c = new Compiler(model); - return c.Run(inputShape, opt); - } - - /// Vulkan small-conv shaders (uphead skip recompute, convpool): dense stride-1 - /// conv, ic < 8, oc a multiple of 8 (<= 128), weights fit the 16 KB staging. - internal static bool GpuSmallConv(ConvP cp, int[] wShape, int ic) => - cp.Group == 1 && ic < 8 && wShape.Length == 4 && wShape[0] % 8 == 0 && wShape[0] <= 128 - && (long)wShape[0] * ic * wShape[2] * wShape[3] <= 4096 - && cp.Strides.All(s => s == 1) && cp.Dilations.All(d => d == 1) - && cp.Act is ActivationKind.None or ActivationKind.Relu; - - private sealed class Compiler - { - private readonly OnnxModel _m; - private readonly List _values = []; - private readonly Dictionary _byName = new(StringComparer.Ordinal); - private readonly List _nodes = []; - private readonly List _bufSize = []; - private int _inputValue = -1; - - public Compiler(OnnxModel m) => _m = m; - - private int AddValue(CValue v) - { - int id = _values.Count; - _values.Add(v); - if (v.Name.Length != 0) _byName[v.Name] = id; - return id; - } - - private int NewBuf(int size) - { - int id = _bufSize.Count; - _bufSize.Add(size); - return id; - } - - private CValue V(int id) => _values[id]; - - private int Input(int i, OnnxNodeData n) - { - string name = n.Inputs[i]; - if (name.Length == 0) return -1; - if (!_byName.TryGetValue(name, out int id)) - throw new InvalidDataException($"Node '{n.Name}' ({n.OpType}): input '{name}' not produced."); - return id; - } - - private int[] Inputs(OnnxNodeData n, int skip = 0) - { - int[] ids = new int[n.Inputs.Count - skip]; - for (int i = skip; i < n.Inputs.Count; i++) ids[i - skip] = Input(i, n); - return ids; - } - - private int _curNi; // onnx node index currently being emitted (for SrcNode) - - private int Emit(ExecOp op, int[] ins, CValue output, object? p, OnnxNodeData n) - { - int outId = AddValue(output); - _nodes.Add(new ExecNode { Op = op, In = ins, Out = [outId], P = p, SrcNode = _curNi, Name = n.Name }); - return outId; - } - - public ExecPlan Run(ReadOnlySpan inputShape, CompileOptions opt) - { - OnnxGraphData g = _m.Graph; - foreach (OnnxTensorData t in g.Initializers) - { - int[] dims = t.DimsArray(); - if (t.DataType == OnnxTensorData.TypeFloat) - AddValue(new CValue { Name = t.Name, Shape = dims, DType = 1, Floats = t.AsFloats() }); - else - AddValue(new CValue { Name = t.Name, Shape = dims, DType = t.DataType, Ints = t.AsInt64s() }); - } - if (g.Inputs.Count != 1) throw new NotSupportedException("Expected single graph input."); - { - string inName = g.Inputs[0].Name; - int elems = 1; - int[] shape = inputShape.ToArray(); - foreach (int d in shape) elems *= d; - int buf = NewBuf(elems); - _inputValue = AddValue(new CValue { Name = inName, Shape = shape, DType = 1, Buf = buf }); - } - - for (int ni = 0; ni < g.Nodes.Count; ni++) - { - OnnxNodeData n = g.Nodes[ni]; - _curNi = ni; - EmitNode(n); - } - - // graph outputs - if (g.Outputs.Count != 1) throw new NotSupportedException("Expected single graph output."); - int outputValue = _byName[g.Outputs[0].Name]; - - return FuseAndPlan(_nodes, _values, _bufSize, _inputValue, outputValue, opt); - } - - // ---------- fusion + buffer planning ---------- - private static ExecPlan FuseAndPlan(List nodes, List values, - List bufSize, int inputValue, int outputValue, CompileOptions opt) - { - // fusion passes operate on the logical graph and are layout-blind; - // physical layout is decided afterwards by LayoutPlanner - if (Environment.GetEnvironmentVariable("GCDRNET_NOFUSE") != "1") - { - Fuse(nodes, values); - // NHWC-only kernels (CPU channels-last + Vulkan); the NCHW oracle plan keeps the plain ops - if (opt.Nhwc && Environment.GetEnvironmentVariable("GCDRNET_NOFUSE_UP") != "1") - FuseUpsample(nodes, values, outputValue); - } - if (opt.Nhwc) - LayoutPlanner.Run(nodes, values, bufSize, inputValue, ref outputValue, opt.PhysicalIo); - if (opt.Nhwc && opt.MemFuse && CompileOptions.MemFusions - && Environment.GetEnvironmentVariable("GCDRNET_NOFUSE") != "1") - FuseMemory(nodes, values, outputValue, opt.Gpu); - - PlanArena(nodes, values, bufSize, inputValue, outputValue, - out int[] offsets, out int arenaFloats); - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "2") - { - string Vs(int v) => v < 0 ? "-" : $"v{v}/b{values[v].Buf}[{string.Join('x', values[v].Shape)}]{(values[v].Layout is { } l && !Layout.IsIdentity(l) ? Layout.Str(l) : "")}"; - for (int i = 0; i < nodes.Count; i++) - { - ExecNode m = nodes[i]; - string extra = m.P switch - { - TransposeP t => $" perm=[{string.Join(',', t.Perm)}]", - ConvP c => $" k={c.Kernel[0]}x{c.Kernel[1]} s={c.Strides[0]} g={c.Group} add={Vs(c.AddValue)} act={c.Act}{(c.Pool is { } pl ? $" +maxpool{(pl.FuseRelu ? "+relu" : "")}" : "")}", - UpHeadP u => $" rrelu={u.ResizeRelu} srelu={u.SkipRelu} act={u.Act}" + (u.SkipConv is { } sc ? $" skip=conv{sc.Kernel[0]}x{sc.Kernel[1]}(In[1]) sact={sc.Act}" : ""), - GemmP g => $" act={g.Act}", - ConcatP c => $" axis={c.Axis}", - ShiftP s => $" axis={s.Axis}", - EwP e => $" {e.Op} act={e.Act}", - _ => "", - }; - Console.Error.WriteLine($" #{i,-3} {m.Op,-14} {string.Join(' ', m.In.Select(Vs))} -> {string.Join(' ', m.Out.Select(Vs))}{extra}"); - } - } - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - { - var hist = nodes.GroupBy(m => m.Op).OrderByDescending(g => g.Count()); - Console.Error.WriteLine($"[plan] {nodes.Count} nodes, arena {arenaFloats * 4.0 / 1e6:F1} MB"); - foreach (var h in hist) Console.Error.WriteLine($" {h.Key,-14} x{h.Count()}"); - } - return new ExecPlan - { - Nodes = nodes.ToArray(), - Values = values.ToArray(), - BufSize = bufSize.ToArray(), - BufOffset = offsets, - ArenaFloats = arenaFloats, - InputValue = inputValue, - OutputValue = outputValue, - Nhwc = opt.Nhwc, - PhysicalIo = opt.Nhwc && opt.PhysicalIo, - }; - } - - /// Structural fusions over the exec node list (single consumer chains). - private static void Fuse(List nodes, List values) - { - // pass 1: Pad -> Conv (fold reflect/constant padding into the conv) - // pass 2: Conv -> Relu (activation epilogue) - // pass 3: Div(x,√2) Erf Add(x,1) Mul(x,0.5) Mul(x0) -> Gelu - // pass 4: Pad(+2) -> 5x Slice -> 10x Slice + 5x Concat -> Concat -> Slice - // (shift-MLP) -> ChannelShift - for (int i = 0; i < nodes.Count; i++) - { - ExecNode n = nodes[i]; - if (n.Op == ExecOp.Conv && n.P is ConvP cp && cp.Pads.All(p => p == 0)) - { - int src = n.In[0]; - ExecNode? pad = nodes.FirstOrDefault(m => m.Out.Length == 1 && m.Out[0] == src && m.Op == ExecOp.Pad); - if (pad != null && SoleUse(nodes, src, i) && pad.P is PadP pp && - PadsOnlySpatial(pp.Pads, values[src].Shape.Length)) - { - cp.Pads = [pp.Pads[2], pp.Pads[3], pp.Pads[2 + values[src].Shape.Length], pp.Pads[3 + values[src].Shape.Length]]; - cp.PadMode = pp.Mode; - cp.PadValue = pp.Value; - n.In[0] = pad.In[0]; - nodes.RemoveAt(nodes.IndexOf(pad)); - i--; - continue; - } - } - } - // Relu/Sigmoid/elementwise-tail fusions - for (int i = 0; i + 1 < nodes.Count; i++) - { - ExecNode a = nodes[i], b = nodes[i + 1]; - if (a.Op == ExecOp.Conv && b.Op == ExecOp.Relu && b.In[0] == a.Out[0] && - SoleUse(nodes, a.Out[0], i + 1)) - { - ((ConvP)a.P!).Act = ActivationKind.Relu; - // rewrite consumers of b.Out to use a.Out - Redirect(nodes, values, b.Out[0], a.Out[0]); - nodes.RemoveAt(i + 1); - i--; - } - else if (a.Op == ExecOp.MaxPool && b.Op == ExecOp.Relu && b.In[0] == a.Out[0] && - SoleUse(nodes, a.Out[0], i + 1)) - { - ((PoolP)a.P!).FuseRelu = true; - Redirect(nodes, values, b.Out[0], a.Out[0]); - nodes.RemoveAt(i + 1); - i--; - } - else if (a.Op == ExecOp.ResizeBilinear && b.Op == ExecOp.Relu && b.In[0] == a.Out[0] && - SoleUse(nodes, a.Out[0], i + 1)) - { - ((ResizeP)a.P!).FuseRelu = true; - Redirect(nodes, values, b.Out[0], a.Out[0]); - nodes.RemoveAt(i + 1); - i--; - } - else if (a.Op == ExecOp.Elementwise && b.Op == ExecOp.Relu && b.In[0] == a.Out[0] && - SoleUse(nodes, a.Out[0], i + 1)) - { - ((EwP)a.P!).Act = ActivationKind.Relu; - Redirect(nodes, values, b.Out[0], a.Out[0]); - nodes.RemoveAt(i + 1); - i--; - } - } - // Conv -> Add residual (skip connections): fold same-shape addend into conv epilogue - for (int i = 0; i + 1 < nodes.Count; i++) - { - ExecNode a = nodes[i], b = nodes[i + 1]; - if (a.Op != ExecOp.Conv || b.Op != ExecOp.Elementwise || - ((EwP)b.P!).Op != EwOp.Add || - ((EwP)b.P!).Act is not (ActivationKind.None or ActivationKind.Relu)) - continue; - var cp = (ConvP)a.P!; - if (cp.Act != ActivationKind.None || cp.Group > 1) continue; // depthwise shader has no residual path - int outV = a.Out[0]; - int other = b.In[0] == outV ? b.In[1] : b.In[1] == outV ? b.In[0] : -1; - if (other < 0 || !SoleUse(nodes, outV, i + 1)) continue; - if (!values[other].Shape.SequenceEqual(values[outV].Shape)) continue; - cp.AddValue = other; - cp.Act = ((EwP)b.P!).Act; // epilogue: +residual then activation (matches conv shader) - Redirect(nodes, values, b.Out[0], outV); - nodes.RemoveAt(i + 1); - i--; - } - FuseGelu(nodes, values); - FuseShift(nodes, values); - CommuteActBeforeTranspose(nodes, values); - FuseLateAct(nodes, values); - } - - // act(T(x)) == T(act(x)) for elementwise unary act: move a sole-consumer - // act node in front of its transpose producer so producer fusions (e.g. - // Gemm->Gelu) can see it. - private static void CommuteActBeforeTranspose(List nodes, List values) - { - for (int g = 0; g < nodes.Count; g++) - { - ExecNode act = nodes[g]; - if (act.Op is not (ExecOp.Relu or ExecOp.Gelu or ExecOp.Sigmoid or ExecOp.Erf) - || act.In.Length != 1) continue; - int t = nodes.FindIndex(m => m.Out.Length == 1 && m.Out[0] == act.In[0]); - if (t < 0 || t >= g || nodes[t].Op != ExecOp.Transpose) continue; - ExecNode tr = nodes[t]; - if (tr.In.Length != 1 || !SoleUse(nodes, tr.Out[0], g)) continue; - // rewire as x --act--> y --T--> z: swap the two Out ids so - // downstream consumers of z read the transposed value as before. - // y now holds act(x) in x's layout — fix its registered shape. - int y = tr.Out[0], z = act.Out[0]; - values[y].Shape = values[tr.In[0]].Shape; - act.In[0] = tr.In[0]; act.Out[0] = y; - tr.In[0] = y; tr.Out[0] = z; - nodes.RemoveAt(g); - nodes.Insert(t, act); - } - } - - // Sole-consumer activation fusion, buffer-aware (consumers reached through - // reshape/squeeze Buf-aliases count as uses of the producer's buffer) and - // adjacency-independent. Retargets the act's output buffer onto the - // producer's so downstream aliases keep their shape metadata. - private static void FuseLateAct(List nodes, List values) - { - for (int i = 0; i < nodes.Count; i++) - { - ExecNode a = nodes[i]; - bool can = a.P switch - { - ConvP c => c.Act == ActivationKind.None, - PoolP p => !p.FuseRelu, - ResizeP r => !r.FuseRelu, - EwP e => e.Act == ActivationKind.None, - GemmP g => g.Act == ActivationKind.None, - _ => false, - }; - if (!can || a.Out.Length != 1) continue; - int ob = values[a.Out[0]].Buf; - if (ob < 0) continue; - - int u = -1, uses = 0; - for (int j = 0; j < nodes.Count && uses <= 1; j++) - if (j != i && nodes[j].In.Any(iv => values[iv].Buf == ob)) { u = j; uses++; } - if (uses != 1) continue; - ExecNode b = nodes[u]; - if (b.In.Length != 1 || values[b.In[0]].Buf != ob) continue; - ActivationKind got = b.Op switch - { - ExecOp.Relu => ActivationKind.Relu, - ExecOp.Gelu => ActivationKind.Gelu, - ExecOp.Sigmoid => ActivationKind.Sigmoid, - _ => ActivationKind.None, - }; - bool ok = a.P switch - { - ConvP => got is ActivationKind.Relu or ActivationKind.Gelu, - PoolP or ResizeP or EwP => got == ActivationKind.Relu, - GemmP => got is ActivationKind.Relu or ActivationKind.Gelu, - _ => false, - }; - if (!ok) continue; - - switch (a.P) - { - case ConvP c: c.Act = got; break; - case PoolP p: p.FuseRelu = true; break; - case ResizeP r: r.FuseRelu = true; break; - case EwP e: e.Act = ActivationKind.Relu; break; - case GemmP g: g.Act = got; break; - } - int ab = values[b.Out[0]].Buf; - if (ab >= 0 && ab != ob) - { - foreach (CValue v in values) - if (v.Buf == ab) v.Buf = ob; - // b.Out is now an unproduced view of a.Out: record it so the - // layout pass can follow the alias chain - values[b.Out[0]].AliasOf = a.Out[0]; - } - else - { - Redirect(nodes, values, b.Out[0], a.Out[0]); - } - nodes.RemoveAt(u); - if (u < i) i--; - } - } - - /// - /// NHWC memory fusions (after layout planning) that keep full-resolution - /// tensors from ever being stored: - /// - /// UpHead skip recompute: when the skip is a cheap direct conv of a smaller - /// input (stem conv: 3/6 → 16/32 channels), UpHead reads the conv's input and - /// recomputes the skip rows, so the skip need not stay live across the network. - /// Conv → MaxPool(2x2/2): the conv output's only reader is the pool, so the - /// conv emits pooled rows directly (). - /// - /// Vulkan plans () only get what its shaders implement: small-ic - /// (stem) convs with staged weights, both for the recompute and for conv+pool. - /// - private static void FuseMemory(List nodes, List values, int outputValue, bool gpu) - { - int Producer(int v) => nodes.FindIndex(m => m.Out.Length == 1 && m.Out[0] == v); - int Uses(int v) - { - int b = values[v].Buf; - if (b < 0 || v == outputValue || values[outputValue].Buf == b) return int.MaxValue; - int n = 0; - foreach (ExecNode m in nodes) - { - foreach (int iv in m.In) if (values[iv].Buf == b) n++; - int a1 = m.P switch { ConvP c => c.AddValue, _ => -1 }; - int a2 = m.P switch { ConvP c => c.BiasValue, GemmP g => g.BiasValue, _ => -1 }; - if (a1 >= 0 && values[a1].Buf == b) n++; - if (a2 >= 0 && values[a2].Buf == b) n++; - } - return n; - } - bool Nhwc4(int v) => values[v].Shape.Length == 4 && values[v].Layout is [0, 2, 3, 1]; - bool SameHw(int a, int b) => values[a].Shape[2] == values[b].Shape[2] && values[a].Shape[3] == values[b].Shape[3]; - - // 1) UpHead skip recompute - foreach (ExecNode u in nodes) - { - if (u.P is not UpHeadP up || up.SkipConv is not null) continue; - int s = u.In[1], ci = Producer(s); - if (ci < 0 || nodes[ci].P is not ConvP cp || cp.Group != 1 || cp.AddValue >= 0 || cp.Pool is not null - || cp.Strides.Any(x => x != 1) || cp.Dilations.Any(x => x != 1)) continue; - int xin = nodes[ci].In[0]; - CValue w = values[nodes[ci].In[1]]; - if (w.Floats is null || w.Shape.Length != 4 || !Nhwc4(s) || !Nhwc4(xin)) continue; - int ic = values[xin].Shape[1], c = values[s].Shape[1]; - // cheap to recompute, much smaller to keep; the conv itself stays scheduled - // for its other reader(s) (and keeps the shared ConvP packed) - if (ic * 2 > c || Kernels.Nhwc.WinogradEligible(cp, w.Shape, ic) || Uses(s) < 2) continue; - if (cp.BiasValue >= 0 && values[cp.BiasValue].Floats is null) continue; - if (gpu && (!GpuSmallConv(cp, w.Shape, ic) || c > 32 || !SameHw(xin, s))) continue; - u.In[1] = xin; - up.SkipConv = cp; - up.SkipW = nodes[ci].In[1]; - up.SkipB = cp.BiasValue; - } - - // 2) Conv -> MaxPool(2x2, stride 2, no pad, exact halving) - for (int pi = 0; pi < nodes.Count; pi++) - { - ExecNode m = nodes[pi]; - if (m.Op != ExecOp.MaxPool || m.P is not PoolP pp || pp.Kernel is not [2, 2] || pp.Strides is not [2, 2] - || pp.Pads.Any(x => x != 0) || m.In.Length != 1) continue; - int v = m.In[0], ci = Producer(v); - if (ci < 0 || ci > pi || nodes[ci].P is not ConvP cp || cp.Group != 1 || cp.AddValue >= 0 || cp.Pool is not null - || Uses(v) != 1 || !Nhwc4(v) || !Nhwc4(m.Out[0])) continue; - int[] cs = values[v].Shape, ps = values[m.Out[0]].Shape; - if (cs[2] != ps[2] * 2 || cs[3] != ps[3] * 2) continue; - if (gpu && (!GpuSmallConv(cp, values[nodes[ci].In[1]].Shape, values[nodes[ci].In[0]].Shape[1]) || !SameHw(nodes[ci].In[0], v))) continue; - cp.Pool = pp; - nodes[ci].Out = [m.Out[0]]; - nodes.RemoveAt(pi); - pi--; - } - } - - /// - /// Upsample fusions (NHWC plans): - /// - /// Resize → (Relu) → Add(·, [Relu](skip)) → 1x1 Conv (few outputs) → (Sigmoid|Relu) - /// becomes one — the network tail, where every - /// intermediate is a full-resolution tensor that would otherwise round-trip memory. - /// Resize → Add(·, [Relu](skip)) becomes a Resize with an addend (In[1]). - /// - /// Fused nodes take the position of the last node they replace, so every input is - /// produced before them. Uses are counted by buffer (aliases and conv/gemm param - /// value ids included), so an intermediate is only dropped when nothing else reads it. - /// - private static void FuseUpsample(List nodes, List values, int outputValue) - { - int Producer(int v) => nodes.FindIndex(m => m.Out.Length == 1 && m.Out[0] == v); - int Uses(int v) - { - int b = values[v].Buf; - if (b < 0 || v == outputValue || values[outputValue].Buf == b) return int.MaxValue; - int n = 0; - foreach (ExecNode m in nodes) - { - foreach (int iv in m.In) if (values[iv].Buf == b) n++; - int a1 = m.P switch { ConvP c => c.AddValue, _ => -1 }; - int a2 = m.P switch { ConvP c => c.BiasValue, GemmP g => g.BiasValue, _ => -1 }; - if (a1 >= 0 && values[a1].Buf == b) n++; - if (a2 >= 0 && values[a2].Buf == b) n++; - } - return n; - } - bool Same(int a, int b) => values[a].Shape.SequenceEqual(values[b].Shape); - - // resolve add(resize(x), other): (resize node, skip value, skip relu node or null) - (ExecNode rs, int skip, ExecNode? relu)? MatchAdd(ExecNode add) - { - if (add.Op != ExecOp.Elementwise || add.In.Length != 2 || add.P is not EwP { Op: EwOp.Add, Act: ActivationKind.None }) - return null; - for (int k = 0; k < 2; k++) - { - int u = add.In[k], o = add.In[1 - k]; - int ri = Producer(u); - if (ri < 0 || nodes[ri].Op != ExecOp.ResizeBilinear || nodes[ri].In.Length != 1) continue; - if (Uses(u) != 1 || !Same(u, add.Out[0]) || !Same(o, add.Out[0]) || values[o].Buf < 0) continue; - int pi = Producer(o); - if (pi >= 0 && nodes[pi].Op == ExecOp.Relu && nodes[pi].In.Length == 1 && Uses(o) == 1 - && values[nodes[pi].In[0]].Buf >= 0 && Same(nodes[pi].In[0], o)) - return (nodes[ri], nodes[pi].In[0], nodes[pi]); - return (nodes[ri], o, null); - } - return null; - } - - // 1) tails: ... -> Add -> Conv1x1 -> (act) - for (int ci = 0; ci < nodes.Count; ci++) - { - ExecNode c = nodes[ci]; - if (c.Op != ExecOp.Conv || c.P is not ConvP cp || cp.Group != 1 || cp.Act != ActivationKind.None - || cp.AddValue >= 0 || cp.Kernel[0] != 1 || cp.Kernel[1] != 1 || cp.Strides.Any(s => s != 1) - || cp.Pads.Any(p => p != 0) || cp.Dilations.Any(d => d != 1)) continue; - CValue w = values[c.In[1]]; - if (w.Floats is null || w.Shape.Length != 4 || w.Shape[0] > 8 || w.Shape[1] > 128 || w.Shape[1] % 4 != 0) continue; // kernel limits (oc lanes, staged weights, vec4 channels) - if (cp.BiasValue >= 0 && values[cp.BiasValue].Floats is null) continue; - int s = c.In[0]; - int ai = Producer(s); - if (ai < 0 || Uses(s) != 1 || values[s].Shape.Length != 4) continue; - ExecNode addNode = nodes[ai]; - if (MatchAdd(addNode) is not { } mt) continue; - var (rs, skip, relu) = mt; - - ExecNode? act = null; - int y = c.Out[0]; - if (Uses(y) == 1) - { - act = nodes.FirstOrDefault(m => m.In.Length == 1 && m.In[0] == y - && m.Op is ExecOp.Sigmoid or ExecOp.Relu && values[m.Out[0]].Buf >= 0); - } - var head = new ExecNode - { - Op = ExecOp.UpHead, - In = cp.BiasValue >= 0 ? [rs.In[0], skip, c.In[1], cp.BiasValue] : [rs.In[0], skip, c.In[1]], - Out = [act?.Out[0] ?? y], - P = new UpHeadP - { - Scales = ((ResizeP)rs.P!).Scales, - ResizeRelu = ((ResizeP)rs.P!).FuseRelu, - SkipRelu = relu is not null, - Act = act is null ? ActivationKind.None : act.Op == ExecOp.Sigmoid ? ActivationKind.Sigmoid : ActivationKind.Relu, - }, - SrcNode = c.SrcNode, - Name = c.Name + ".uphead", - }; - nodes[ci] = head; - nodes.Remove(rs); nodes.Remove(addNode); - if (relu is not null) nodes.Remove(relu); - if (act is not null) nodes.Remove(act); - ci = -1; // indices shifted; rescan (few nodes, cheap) - } - - // 2) Resize -> Add - for (int ai = 0; ai < nodes.Count; ai++) - { - ExecNode add = nodes[ai]; - if (MatchAdd(add) is not { } mt) continue; - var (rs, skip, relu) = mt; - int ri = nodes.IndexOf(rs); - var rp = (ResizeP)rs.P!; - rs.In = [rs.In[0], skip]; - rp.AddRelu = relu is not null; - rs.Out = [add.Out[0]]; - nodes[ai] = rs; // producer precedes consumer: ri < ai - nodes.RemoveAt(ri); - if (relu is not null) nodes.Remove(relu); - ai = -1; - } - } - - private static void Redirect(List nodes, List values, int oldValue, int newValue) - { - foreach (ExecNode m in nodes) - for (int i = 0; i < m.In.Length; i++) - if (m.In[i] == oldValue) m.In[i] = newValue; - if (values[oldValue].Name.Length != 0 && values[newValue].Name.Length == 0) - values[newValue].Name = values[oldValue].Name; - } - - /// True when value `v`'s only consumer is node index `useIdx`, once. - private static bool SoleUse(List nodes, int v, int useIdx) - { - int count = 0; - for (int i = 0; i < nodes.Count; i++) - foreach (int inp in nodes[i].In) - if (inp == v) - { - if (i != useIdx) return false; - if (++count > 1) return false; - } - return count == 1; - } - - private static bool PadsOnlySpatial(int[] pads, int rank) - { - if (pads.Length != 2 * rank) return false; - for (int i = 0; i < 2; i++) if (pads[i] != 0 || pads[i + rank] != 0) return false; - return true; - } - - private static void FuseGelu(List nodes, List values) - { - // Div(X, sqrt2) -> Erf -> Add(+1) -> Mul(*0.5) -> Mul(*X) - for (int i = 0; i + 4 < nodes.Count; i++) - { - ExecNode d = nodes[i], erf = nodes[i + 1], a1 = nodes[i + 2], mh = nodes[i + 3], mx = nodes[i + 4]; - if (d.Op != ExecOp.Elementwise || ((EwP)d.P!).Op != EwOp.Div || - erf.Op != ExecOp.Erf || - a1.Op != ExecOp.Elementwise || ((EwP)a1.P!).Op != EwOp.Add || - mh.Op != ExecOp.Elementwise || ((EwP)mh.P!).Op != EwOp.Mul || - mx.Op != ExecOp.Elementwise || ((EwP)mx.P!).Op != EwOp.Mul) - continue; - if (erf.In[0] != d.Out[0] || a1.In[0] != erf.Out[0] && a1.In[1] != erf.Out[0] || - (mh.In[0] != a1.Out[0] && mh.In[1] != a1.Out[0]) || - (mx.In[0] != mh.Out[0] && mx.In[1] != mh.Out[0])) - continue; - if (!IsScalarConst(values, d.In[1], MathF.Sqrt(2f))) continue; - int a1Other = a1.In[0] == erf.Out[0] ? a1.In[1] : a1.In[0]; - int mhOther = mh.In[0] == a1.Out[0] ? mh.In[1] : mh.In[0]; - int mxOther = mx.In[0] == mh.Out[0] ? mx.In[1] : mx.In[0]; - if (!IsScalarConst(values, a1Other, 1f) || !IsScalarConst(values, mhOther, 0.5f)) continue; - if (mxOther != d.In[0]) continue; - // emit single Gelu - values[mx.Out[0]].Shape = values[d.In[0]].Shape; - nodes[i] = new ExecNode { Op = ExecOp.Gelu, In = [d.In[0]], Out = mx.Out, P = null, SrcNode = d.SrcNode, Name = d.Name }; - nodes.RemoveRange(i + 1, 4); - } - } - - private static bool IsScalarConst(List values, int v, float expected) - { - CValue cv = values[v]; - return cv.Floats is { Length: 1 } f && MathF.Abs(f[0] - expected) < 1e-6f; - } - - /// - /// Recognizes the shift-MLP subgraph: Pad(const ±p on H or W) -> channel - /// Split (5 groups via Slice) -> per-group roll (Slice+Slice+Concat on the - /// spatial axis) -> Concat(axis=1) -> Slice(unpad). Replaced by a single - /// ChannelShift node writing the shifted [1,C,h,w] tensor. - /// - private static void FuseShift(List nodes, List values) - { - for (int i = 0; i < nodes.Count; i++) - { - ExecNode pad = nodes[i]; - if (pad.Op != ExecOp.Pad || pad.P is not PadP pp || pp.Mode != PadMode.Constant) - continue; - int inId = pad.In[0], padOut = pad.Out[0]; - int rank = values[padOut].Shape.Length; - if (rank != 4) continue; - // spatial pad amount (must be uniform ±p on exactly one spatial axis pair? ours: both H,W ±2) - int padH = pp.Pads[2], padW = pp.Pads[3]; - if (pp.Pads[2 + rank] != padH || pp.Pads[3 + rank] != padW) continue; - - // find all consumer slices of padOut (channel split pieces) - var pieces = nodes.Where(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == padOut).ToList(); - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($"[shift?] pad {pad.Name} pieces={pieces.Count}"); - if (pieces.Count != 5) continue; - int[] padShape = values[padOut].Shape; - // SliceP.Starts/Ends are per-Axes-position arrays (len == Axes.Length). - // For a slice emitted from a Split over axis C, every axes entry is - // identity except the one at axis 1, which carries the group range. - bool RangeAt(SliceP s, int ax, int[] inShape, out int st, out int en) - { - st = 0; en = inShape[ax]; - for (int k = 0; k < s.Axes.Length; k++) - if (s.Axes[k] == ax) { st = s.Starts[k]; en = s.Ends[k]; break; } - return s.Axes.Where((a, k) => a != ax).All(a => - s.Starts[Array.IndexOf(s.Axes, a)] == 0 && - s.Ends[Array.IndexOf(s.Axes, a)] >= inShape[a]); - } - var pieceInfo = pieces - .Select(m => (node: m, s: (SliceP)m.P!)) - .Where(t => RangeAt(t.s, 1, padShape, out _, out _)) - .OrderBy(t => { RangeAt(t.s, 1, padShape, out int st, out _); return st; }) - .ToArray(); - if (pieceInfo.Length != 5) - { - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($" pieceInfo={pieceInfo.Length} (OnlyAxis fail)"); - continue; - } - int[] groupEnds = pieceInfo.Select(t => - { - RangeAt(t.s, 1, padShape, out _, out int en); return en; - }).ToArray(); - - // for each piece find its roll: two Slice consumers + one Concat - int[] offsets = new int[5]; - int shiftAxis = -1; - int[] pieceOutIds = pieceInfo.Select(t => t.node.Out[0]).ToArray(); - var smallConcatOuts = new List(5); - bool fail = false; - var toRemove = new List(pieces); - for (int gI = 0; gI < 5; gI++) - { - int pieceId = pieceOutIds[gI]; - var rollSlices = nodes.Where(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == pieceId).ToList(); - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - { - var allCons = nodes.Where(m => m.In.Contains(pieceId)) - .Select(m => m.Op.ToString()); - Console.Error.WriteLine($" g{gI} piece {pieceId} consumers=[{string.Join(",", allCons)}] rollSlices={rollSlices.Count}"); - } - if (rollSlices.Count != 2) { fail = true; break; } - var concat = nodes.FirstOrDefault(m => m.Op == ExecOp.Concat && m.In.Length == 2 && - m.In.Contains(rollSlices[0].Out[0]) && m.In.Contains(rollSlices[1].Out[0])); - if (concat is null) { fail = true; break; } - var sp = (SliceP)rollSlices[0].P!; var sp2 = (SliceP)rollSlices[1].P!; - // both roll slices must slice a single spatial axis - int SliceAxis(SliceP s, int[] pieceShape) - { - int found = -1; - for (int k = 0; k < s.Axes.Length; k++) - { - int d = s.Axes[k]; - bool sliced = s.Starts[k] != 0 || s.Ends[k] < pieceShape[d]; - if (sliced && found != -1) return -1; - if (sliced) found = d; - } - return found; - } - int[] pieceShape = values[pieceId].Shape; - int ax = SliceAxis(sp, pieceShape); - int ax2 = SliceAxis(sp2, pieceShape); - // allow one side to be unsliced (identity roll for the 0-offset group) - if (ax < 0) ax = ax2; - if (ax2 >= 0 && ax2 != ax) ax = -1; - if (ax < 0 || (ax != 2 && ax != 3)) - { - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($" ax={ax} ax2={ax2} s0.Axes=[{string.Join(",", sp.Axes)}] s1.Axes=[{string.Join(",", sp2.Axes)}]"); - fail = true; break; - } - if (shiftAxis == -1) shiftAxis = ax; - else if (shiftAxis != ax) { fail = true; break; } - int L = pieceShape[ax]; - // concat order = [B, A] where A = piece[0..cut] is the "head" - var sA = rollSlices[0].Out[0] == concat.In[0] ? sp2 : sp; - var sB = rollSlices[0].Out[0] == concat.In[0] ? sp : sp2; - RangeAt(sA, ax, pieceShape, out int saSt, out int saEn); - RangeAt(sB, ax, pieceShape, out int sbSt, out int sbEn); - // sanity: A is the head [0..cut], B is the tail [cut..L] - if (saSt != 0 || sbSt != saEn || sbEn < L) { fail = true; break; } - int cut = saEn - saSt; - // effective shift on UNPADDED axis: out[y] = orig[y + s], zero-fill. - // rolled[i] = padded[(i+cut) mod L]; sliced[y] = rolled[y+padAmt] - // = padded[(y+padAmt+cut) mod L]; orig coord = that - padAmt. - int dim = values[pad.In[0]].Shape[shiftAxis]; - int padAmt = shiftAxis == 2 ? padH : padW; - offsets[gI] = ((padAmt + cut) % L) - padAmt; - toRemove.AddRange(rollSlices); - toRemove.Add(concat); - smallConcatOuts.Add(concat.Out[0]); - } - ExecNode? bigConcat = fail ? null : nodes.FirstOrDefault(m => - m.Op == ExecOp.Concat && m.In.Length == 5 && - smallConcatOuts.All(m.In.Contains)); - if (fail || bigConcat is null) - { - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - { - Console.Error.WriteLine($" fail={fail} bigConcat={(bigConcat is null ? "null" : "ok")} (gI loop)"); - foreach (var cc in nodes.Where(m => m.Op == ExecOp.Concat)) - Console.Error.WriteLine($" concat in[{string.Join("+", cc.In.Select(v => v.ToString()))}] outs=[{string.Join("+", cc.Out.Select(v => v.ToString()))}]"); - } - continue; - } - // unpad slice consuming bigConcat output - var unpad = nodes.FirstOrDefault(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == bigConcat.Out[0]); - if (unpad is null) - { - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($" unpad missing for concat out {bigConcat.Out[0]}"); - continue; - } - toRemove.Add(bigConcat); - toRemove.Add(unpad); - toRemove.Add(pad); - - var shift = new ExecNode - { - Op = ExecOp.ChannelShift, - In = [inId], - Out = unpad.Out, - P = new ShiftP { Axis = shiftAxis, GroupEnds = groupEnds, Offsets = offsets }, - SrcNode = pad.SrcNode, - Name = pad.Name + ".shift", - }; - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($"[shift!] {pad.Name} axis={shiftAxis} ends=[{string.Join(",", groupEnds)}] offs=[{string.Join(",", offsets)}]"); - foreach (ExecNode m in toRemove) nodes.Remove(m); - nodes.Insert(Math.Min(i, nodes.Count), shift); - } - } - - /// Lifetime-based arena packing (def-order, free-list reuse). - private static void PlanArena(List nodes, List values, - List bufSize, int inputValue, int outputValue, - out int[] offsets, out int arenaFloats) - { - int bufCount = bufSize.Count; - const int Align = 16; // floats (64B) - int AlignUp(int v) => (v + Align - 1) & ~(Align - 1); - - int[] def = Enumerable.Repeat(int.MaxValue, bufCount).ToArray(); - int[] use = Enumerable.Repeat(-1, bufCount).ToArray(); - for (int ni = 0; ni < nodes.Count; ni++) - { - ExecNode n = nodes[ni]; - foreach (int iv in n.In) - { - int b = values[iv].Buf; - if (b >= 0) { use[b] = Math.Max(use[b], ni); def[b] = Math.Min(def[b], ni); } - } - // value ids carried in params (conv bias/residual, gemm bias) count as uses - int aux = n.P switch - { - ConvP c => Math.Max(c.BiasValue, c.AddValue), - GemmP g => g.BiasValue, - _ => -1, - }; - if (aux >= 0) - { - // note: for ConvP, BiasValue and AddValue may differ — scan both - if (n.P is ConvP cp2) - foreach (int iv in new[] { cp2.BiasValue, cp2.AddValue }) - { - if (iv < 0) continue; - int b = values[iv].Buf; - if (b >= 0) { use[b] = Math.Max(use[b], ni); def[b] = Math.Min(def[b], ni); } - } - else - { - int b = values[aux].Buf; - if (b >= 0) { use[b] = Math.Max(use[b], ni); def[b] = Math.Min(def[b], ni); } - } - } - foreach (int ov in n.Out) - { - int b = values[ov].Buf; - if (b >= 0) def[b] = Math.Min(def[b], ni); - } - } - int inputBuf = values[inputValue].Buf; - int outputBuf = values[outputValue].Buf; - def[inputBuf] = -1; - // keep input + output live for the whole run: callers may read - // Input (e.g. two-stage pipelines reusing it) after Run(). - use[inputBuf] = nodes.Count; - use[outputBuf] = nodes.Count; - - var order = Enumerable.Range(0, bufCount) - .Where(b => def[b] != int.MaxValue) - .OrderBy(b => def[b]).ThenByDescending(b => bufSize[b]).ToList(); - var active = new List(); // placed buf ids still live - var free = new List<(int off, int size)>(); - offsets = new int[bufCount]; - Array.Fill(offsets, -1); - int bump = 0; - foreach (int b in order) - { - // release dead bufs before allocating this one - for (int i = active.Count - 1; i >= 0; i--) - { - int ob = active[i]; - if (use[ob] < def[b]) - { - active.RemoveAt(i); - free.Add((offsets[ob], AlignUp(bufSize[ob]))); - } - } - int need = AlignUp(bufSize[b]); - int bestIdx = -1; - for (int fi = 0; fi < free.Count; fi++) - if (free[fi].size >= need && (bestIdx < 0 || free[fi].size < free[bestIdx].size)) - bestIdx = fi; - if (bestIdx >= 0) - { - var chunk = free[bestIdx]; - offsets[b] = chunk.off; - free.RemoveAt(bestIdx); - if (chunk.size > need) free.Add((chunk.off + need, chunk.size - need)); - } - else - { - offsets[b] = bump; - bump += need; - } - active.Add(b); - } - arenaFloats = bump; - - // greedy by size (largest first, lowest-waste gap among lifetime-overlapping - // placements): usually much tighter than def-order free lists; keep the better one - { - // the input (live for the whole run) goes first, at offset 0: pipelines can - // then stack another plan's arena right after it - var bySize = order.OrderByDescending(b => b == inputBuf).ThenByDescending(b => bufSize[b]).ThenBy(b => def[b]).ToList(); - var placed = new List(bySize.Count); - int[] off2 = new int[bufCount]; - Array.Fill(off2, -1); - int top = 0; - var conflicts = new List<(int off, int end)>(); - foreach (int b in bySize) - { - int need = AlignUp(bufSize[b]); - conflicts.Clear(); - foreach (int o in placed) - if (!(use[o] < def[b] || use[b] < def[o])) - conflicts.Add((off2[o], off2[o] + AlignUp(bufSize[o]))); - conflicts.Sort(); - int best = -1, bestWaste = int.MaxValue, cur = 0; - foreach (var (o, e) in conflicts) - { - if (o - cur >= need && o - cur - need < bestWaste) { best = cur; bestWaste = o - cur - need; } - cur = Math.Max(cur, e); - } - off2[b] = best >= 0 ? best : cur; - top = Math.Max(top, off2[b] + need); - placed.Add(b); - } - if (top <= bump) - { - offsets = off2; - arenaFloats = top; - } - } - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "3") - { - long peak = 0; int at = -1; - for (int ni = -1; ni <= nodes.Count; ni++) - { - long s = 0; - for (int b = 0; b < bufCount; b++) if (def[b] != int.MaxValue && def[b] <= ni && use[b] >= ni) s += AlignUp(bufSize[b]); - if (s > peak) { peak = s; at = ni; } - } - Console.Error.WriteLine($"[arena] {arenaFloats * 4.0 / 1e6:F1} MB (freelist {bump * 4.0 / 1e6:F1}), live-peak {peak * 4.0 / 1e6:F1} MB at #{at}, input {bufSize[inputBuf] * 4.0 / 1e6:F1} MB off {offsets[inputBuf]}"); - for (int b = 0; b < bufCount; b++) - if (def[b] != int.MaxValue && def[b] <= at && use[b] >= at) - Console.Error.WriteLine($" b{b,-4} {bufSize[b] * 4.0 / 1e6,8:F1} MB #{def[b]}..#{use[b]} {(def[b] >= 0 && def[b] < nodes.Count ? nodes[def[b]].Op : "input")}"); - } - } - - private static readonly HashSet IntOps = new(StringComparer.Ordinal) - { "Squeeze", "Unsqueeze", "Div", "Mod", "Add", "Sub", "Mul", "Reshape", - "Concat", "Slice", "Gather", "Cast", "Floor", "Ceil", "Neg", "Abs", - "Min", "Max", "Equal", "Less", "Where", "ConstantOfShape", "Expand", - "Identity", "Tile" }; - - private void EmitNode(OnnxNodeData n) - { - string op = n.OpType; - // Shape → int tensor of input dims - if (op == "Shape") - { - CValue x = V(Input(0, n)); - if (x.Shape is null) throw new InvalidDataException("Shape input has no known shape."); - int start = (int)n.IntAttr("start", 0); - int end = (int)n.IntAttr("end", x.Shape.Length); - int rank = x.Shape.Length; - if (start < 0) start += rank; - if (end < 0) end += rank; - start = Math.Clamp(start, 0, rank); - end = Math.Clamp(end, 0, rank); - long[] dims = x.Shape[start..end].Select(d => (long)d).ToArray(); - AddValue(new CValue { Name = n.Outputs[0], Shape = [dims.Length], DType = 7, Ints = dims }); - return; - } - if (op == "Size") - { - CValue x = V(Input(0, n)); - AddValue(new CValue { Name = n.Outputs[0], Shape = [], DType = 7, Ints = [x.ElementCount] }); - return; - } - if (op == "Constant") - { - OnnxAttributeData? a = n.Attr("value"); - if (a?.Tensor is OnnxTensorData t) - { - if (t.DataType == OnnxTensorData.TypeFloat) - AddValue(new CValue { Name = n.Outputs[0], Shape = t.DimsArray(), DType = 1, Floats = t.AsFloats() }); - else - AddValue(new CValue { Name = n.Outputs[0], Shape = t.DimsArray(), DType = t.DataType, Ints = t.AsInt64s() }); - return; - } - throw new NotSupportedException("Constant without tensor value."); - } - - // int-only evaluation: if every present input is a const int tensor - // and the op is elementwise/shape math, evaluate now. - bool allInt = n.Inputs.All(s => s.Length == 0 || (V(_byName[s]).IsInt)); - if (allInt && IntOps.Contains(op)) - { - EvalInt(n); - return; - } - // Cast on int → still int - if (op == "Cast" && V(Input(0, n)).IsInt) - { - int to = (int)n.IntAttr("to"); - CValue x = V(Input(0, n)); - if (to == 1) - AddValue(new CValue { Name = n.Outputs[0], Shape = x.Shape, DType = 1, Floats = x.Ints!.Select(v => (float)v).ToArray() }); - else - AddValue(new CValue { Name = n.Outputs[0], Shape = x.Shape, DType = 7, Ints = x.Ints }); - return; - } - - // ---- float ops ---- - switch (op) - { - case "Pad": EmitPad(n); break; - case "Conv": EmitConv(n); break; - case "MaxPool": EmitPool(n); break; - case "Relu": EmitUnary(n, ExecOp.Relu); break; - case "Sigmoid": EmitUnary(n, ExecOp.Sigmoid); break; - case "Erf": EmitUnary(n, ExecOp.Erf); break; - case "LayerNormalization": EmitLayerNorm(n); break; - case "Gemm": EmitGemm(n); break; - case "Resize": EmitResize(n); break; - case "Transpose": EmitTranspose(n); break; - case "Slice": EmitSlice(n); break; - case "Concat": EmitConcat(n); break; - case "Split": EmitSplit(n); break; - case "Reshape": EmitReshape(n); break; - case "Squeeze": EmitSqueeze(n); break; - case "Unsqueeze": EmitUnsqueeze(n); break; - case "Add" or "Sub" or "Mul" or "Div": EmitElementwise(n, op); break; - case "Identity": Alias(n); break; - default: - throw new NotSupportedException($"Op '{op}' not supported (node '{n.Name}')."); - } - } - - // ---------- int evaluation ---------- - private void EvalInt(OnnxNodeData n) - { - string op = n.OpType; - CValue In(int i) => V(Input(i, n)); - CValue Out(long[] data, int[] shape) - { - var v = new CValue { Name = n.Outputs[0], Shape = shape, DType = 7, Ints = data }; - AddValue(v); - return v; - } - switch (op) - { - case "Squeeze": - { - CValue x = In(0); - long[]? axes = n.Inputs.Count > 1 && n.Inputs[1].Length > 0 ? In(1).Ints : null; - int[] shape; - if (axes is null) - shape = x.Shape.Where(d => d != 1).ToArray(); - else - { - var dims = x.Shape.ToList(); - foreach (long a in axes.OrderByDescending(v => v)) - { - int ax = (int)(a < 0 ? a + dims.Count : a); - if (dims[ax] != 1) throw new InvalidDataException("Squeeze axis not 1."); - dims.RemoveAt(ax); - } - shape = dims.ToArray(); - } - Out(x.Ints!, shape); - return; - } - case "Unsqueeze": - { - CValue x = In(0); - long[] axes = In(1).Ints!; - var dims = x.Shape.ToList(); - foreach (long a in axes.OrderBy(v => v)) - { - int ax = (int)(a < 0 ? a + dims.Count + 1 : a); - dims.Insert(ax, 1); - } - Out(x.Ints!, dims.ToArray()); - return; - } - case "Reshape": - { - CValue x = In(0); - long[] spec = In(1).Ints!; - bool allowZero = n.IntAttr("allowzero") == 1; - Out(x.Ints!, ResolveReshape(x.Shape, spec, allowZero, x.Ints!.Length)); - return; - } - case "Concat": - { - int axis = (int)n.IntAttr("axis"); - var inputs = Inputs(n).Select(V).ToArray(); - int rank = inputs[0].Shape.Length; - if (axis < 0) axis += rank; - int[] shape = (int[])inputs[0].Shape.Clone(); - shape[axis] = inputs.Sum(v => v.Shape[axis]); - long[] data = IntConcat(inputs.Select(v => v.Ints!).ToArray(), - inputs.Select(v => v.Shape).ToArray(), axis, shape); - Out(data, shape); - return; - } - case "Slice": - { - CValue x = In(0); - long[] starts = In(1).Ints!; - long[] ends = n.Inputs.Count > 2 && n.Inputs[2].Length > 0 ? In(2).Ints! : x.Shape.Select(d => (long)d).ToArray(); - long[] axes = n.Inputs.Count > 3 && n.Inputs[3].Length > 0 ? In(3).Ints! : Enumerable.Range(0, starts.Length).Select(i => (long)i).ToArray(); - long[] steps = n.Inputs.Count > 4 && n.Inputs[4].Length > 0 ? In(4).Ints! : starts.Select(_ => 1L).ToArray(); - SliceBounds(x.Shape, starts, ends, axes, steps, - out int[] outShape, out int[] st, out int[] en, out int[] stepI); - long[] data = IntSlice(x.Ints!, x.Shape, - axes.Select(a => (int)(a < 0 ? a + x.Shape.Length : a)).ToArray(), st, stepI, outShape); - Out(data, outShape); - return; - } - case "Gather": - { - CValue x = In(0); CValue idx = In(1); - int axis = (int)n.IntAttr("axis"); - if (axis < 0) axis += x.Shape.Length; - if (x.Shape.Length != 1 || axis != 0) throw new NotSupportedException("Gather eval supports 1-D axis 0."); - Out(idx.Ints!.Select(i => x.Ints![checked((int)i)]).ToArray(), idx.Shape); - return; - } - case "Identity": - { - CValue x = In(0); - Out(x.Ints!, x.Shape); - return; - } - case "Add" or "Sub" or "Mul" or "Div" or "Mod" or "Min" or "Max": - { - long[] a = In(0).Ints!, b = In(1).Ints!; - long[] r = new long[Math.Max(a.Length, b.Length)]; - for (int i = 0; i < r.Length; i++) - { - long av = a.Length == 1 ? a[0] : a[i], bv = b.Length == 1 ? b[0] : b[i]; - r[i] = op switch - { - "Add" => av + bv, "Sub" => av - bv, "Mul" => av * bv, - "Div" => av / bv, "Mod" => ((av % bv) + bv) % bv, - "Min" => Math.Min(av, bv), _ => Math.Max(av, bv), - }; - } - int[] shape = In(0).Shape.Length >= In(1).Shape.Length ? In(0).Shape : In(1).Shape; - Out(r, shape); - return; - } - case "Neg" or "Abs" or "Floor" or "Ceil": - { - long[] a = In(0).Ints!; - // Floor/Ceil on ints are identity - long[] r = a.Select(v => op == "Neg" ? -v : op == "Abs" ? Math.Abs(v) : v).ToArray(); - Out(r, In(0).Shape); - return; - } - default: - throw new NotSupportedException($"Int eval for '{op}' not implemented (node '{n.Name}')."); - } - } - - // ---------- float ops ---------- - private CValue ActivationValue(string name, int[] shape) - { - int elems = 1; - foreach (int d in shape) elems = checked(elems * d); - return new CValue { Name = name, Shape = shape, DType = 1, Buf = NewBuf(elems) }; - } - - private void Alias(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - AddValue(new CValue { Name = n.Outputs[0], Shape = x.Shape, DType = 1, Buf = x.Buf, Floats = x.Floats, Ints = x.Ints, AliasOf = x.Buf >= 0 ? Input(0, n) : -1 }); - } - - private void EmitPad(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - long[] pads = V(Input(1, n)).Ints!; - float val = n.Inputs.Count > 2 && n.Inputs[2].Length > 0 ? V(Input(2, n)).Floats![0] : 0f; - PadMode mode = n.StringAttr("mode", "constant") switch - { - "reflect" => PadMode.Reflect, - "constant" => PadMode.Constant, - var s => throw new NotSupportedException($"Pad mode '{s}'"), - }; - int[] shape = x.Shape.ToArray(); - for (int i = 0; i < shape.Length; i++) - shape[i] += checked((int)(pads[i] + pads[i + shape.Length])); - Emit(ExecOp.Pad, [Input(0, n)], ActivationValue(n.Outputs[0], shape), - new PadP { Pads = pads.Select(p => (int)p).ToArray(), Mode = mode, Value = val }, n); - } - - private void EmitConv(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - CValue w = V(Input(1, n)); - int[] wd = w.Shape; - int[] pads = n.IntsAttr("pads").Select(v => (int)v).ToArray(); - if (pads.Length == 0) pads = new int[2 * (wd.Length - 2)]; - int[] strides = n.IntsAttr("strides").Select(v => (int)v).ToArray(); - if (strides.Length == 0) strides = Enumerable.Repeat(1, wd.Length - 2).ToArray(); - int[] dil = n.IntsAttr("dilations").Select(v => (int)v).ToArray(); - if (dil.Length == 0) dil = Enumerable.Repeat(1, wd.Length - 2).ToArray(); - int group = (int)n.IntAttr("group", 1); - int[] outShape = new int[wd.Length]; - outShape[0] = x.Shape[0]; - outShape[1] = wd[0]; - for (int d = 0; d < wd.Length - 2; d++) - { - int eff = dil[d] * (wd[2 + d] - 1) + 1; - outShape[2 + d] = (x.Shape[2 + d] + pads[d] + pads[d + wd.Length - 2] - eff) / strides[d] + 1; - } - int biasV = n.Inputs.Count > 2 && n.Inputs[2].Length > 0 ? Input(2, n) : -1; - Emit(ExecOp.Conv, [Input(0, n), Input(1, n)], ActivationValue(n.Outputs[0], outShape), - new ConvP - { - Pads = pads, Strides = strides, Dilations = dil, - Kernel = wd[2..], Group = group, BiasValue = biasV, - }, n); - } - - private void EmitPool(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - int[] kernel = n.IntsAttr("kernel_shape").Select(v => (int)v).ToArray(); - int[] strides = n.IntsAttr("strides").Select(v => (int)v).ToArray(); - if (strides.Length == 0) strides = (int[])kernel.Clone(); - int[] pads = n.IntsAttr("pads").Select(v => (int)v).ToArray(); - if (pads.Length == 0) pads = new int[2 * kernel.Length]; - int[] outShape = (int[])x.Shape.Clone(); - for (int d = 0; d < kernel.Length; d++) - outShape[2 + d] = (x.Shape[2 + d] + pads[d] + pads[d + kernel.Length] - kernel[d]) / strides[d] + 1; - Emit(ExecOp.MaxPool, [Input(0, n)], ActivationValue(n.Outputs[0], outShape), - new PoolP { Kernel = kernel, Strides = strides, Pads = pads }, n); - } - - private void EmitUnary(OnnxNodeData n, ExecOp op) - { - CValue x = V(Input(0, n)); - Emit(op, [Input(0, n)], ActivationValue(n.Outputs[0], x.Shape), null, n); - } - - private void EmitLayerNorm(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - float eps = n.FloatAttr("epsilon", 1e-5f); - int axis = (int)n.IntAttr("axis", -1); - if (axis < 0) axis += x.Shape.Length; - if (axis != x.Shape.Length - 1) - throw new NotSupportedException("LayerNorm only over last dim."); - Emit(ExecOp.LayerNorm, [Input(0, n), Input(1, n), Input(2, n)], - ActivationValue(n.Outputs[0], x.Shape), new LnP { Eps = eps }, n); - } - - private void EmitGemm(OnnxNodeData n) - { - CValue a = V(Input(0, n)); - CValue b = V(Input(1, n)); - bool transB = n.IntAttr("transB") == 1; - float alpha = n.FloatAttr("alpha", 1f); - if (alpha != 1f) throw new NotSupportedException("Gemm alpha != 1."); - int k = transB ? b.Shape[1] : b.Shape[0]; - int ncol = transB ? b.Shape[0] : b.Shape[1]; - if (a.Shape[^1] != k) throw new InvalidDataException("Gemm K mismatch."); - int[] outShape = (int[])a.Shape.Clone(); - outShape[^1] = ncol; - int biasV = n.Inputs.Count > 2 && n.Inputs[2].Length > 0 ? Input(2, n) : -1; - Emit(ExecOp.Gemm, [Input(0, n), Input(1, n)], ActivationValue(n.Outputs[0], outShape), - new GemmP { TransB = transB, BiasValue = biasV }, n); - } - - private void EmitResize(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - float[] scales; - int[]? sizesI = null; - if (n.Inputs.Count > 3 && n.Inputs[3].Length > 0) - { - // sizes input (int tensor): authoritative output shape - long[] sizes = V(Input(3, n)).Ints!; - sizesI = sizes.Select(s => (int)s).ToArray(); - scales = sizes.Select((s, i) => (float)s / x.Shape[i]).ToArray(); - } - else - { - scales = V(Input(2, n)).Floats!; - } - if (n.StringAttr("mode") != "linear") - throw new NotSupportedException("Resize mode != linear."); - int[] outShape = sizesI ?? (int[])x.Shape.Clone(); - if (sizesI == null) - for (int i = 0; i < outShape.Length; i++) - outShape[i] = (int)MathF.Floor(x.Shape[i] * scales[i]); - Emit(ExecOp.ResizeBilinear, [Input(0, n)], ActivationValue(n.Outputs[0], outShape), - new ResizeP { Scales = scales }, n); - } - - private void EmitTranspose(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - int[] perm = n.IntsAttr("perm").Select(v => (int)v).ToArray(); - if (perm.Length == 0) perm = Enumerable.Range(0, x.Shape.Length).Reverse().ToArray(); - int[] outShape = perm.Select(p => x.Shape[p]).ToArray(); - Emit(ExecOp.Transpose, [Input(0, n)], ActivationValue(n.Outputs[0], outShape), - new TransposeP { Perm = perm }, n); - } - - private void EmitSlice(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - long[] starts = V(Input(1, n)).Ints!; - long[] ends = V(Input(2, n)).Ints!; - long[] axes = n.Inputs.Count > 3 && n.Inputs[3].Length > 0 - ? V(Input(3, n)).Ints! : Enumerable.Range(0, starts.Length).Select(i => (long)i).ToArray(); - long[] steps = n.Inputs.Count > 4 && n.Inputs[4].Length > 0 - ? V(Input(4, n)).Ints! : starts.Select(_ => 1L).ToArray(); - SliceBounds(x.Shape, starts, ends, axes, steps, - out int[] outShape, out int[] st, out int[] en, out int[] stepI); - Emit(ExecOp.Slice, [Input(0, n)], ActivationValue(n.Outputs[0], outShape), - new SliceP { Starts = st, Ends = en, - Axes = axes.Select(a => (int)a).ToArray(), Steps = stepI }, n); - } - - private void EmitConcat(OnnxNodeData n) - { - int axis = (int)n.IntAttr("axis"); - int[] ins = Inputs(n); - int[] shape = (int[])V(ins[0]).Shape.Clone(); - int rank = shape.Length; - if (axis < 0) axis += rank; - shape[axis] = ins.Sum(id => V(id).Shape[axis]); - Emit(ExecOp.Concat, ins, ActivationValue(n.Outputs[0], shape), - new ConcatP { Axis = axis }, n); - } - - private void EmitSplit(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - int axis = (int)n.IntAttr("axis"); - if (axis < 0) axis += x.Shape.Length; - int[] sizes; - if (n.Inputs.Count > 1 && n.Inputs[1].Length > 0) - sizes = V(Input(1, n)).Ints!.Select(v => (int)v).ToArray(); - else - { - int outputs = (int)n.IntAttr("num_outputs", n.Outputs.Count); - int each = x.Shape[axis] / outputs; - sizes = Enumerable.Repeat(each, outputs).ToArray(); - sizes[^1] += x.Shape[axis] - each * outputs; - } - // Split is implemented as a per-output Slice at exec time. - int[] starts = new int[x.Shape.Length]; - for (int i = 0; i < sizes.Length; i++) - { - int[] shape = (int[])x.Shape.Clone(); - shape[axis] = sizes[i]; - var slice = new SliceP - { - Starts = (int[])starts.Clone(), - Ends = (int[])x.Shape.Clone(), - Axes = Enumerable.Range(0, x.Shape.Length).ToArray(), - Steps = Enumerable.Repeat(1, x.Shape.Length).ToArray(), - }; - slice.Ends[axis] = starts[axis] + sizes[i]; - _nodes.Add(new ExecNode { Op = ExecOp.Slice, In = [Input(0, n)], Out = [AddValue(ActivationValue(n.Outputs[i], shape))], P = slice, SrcNode = _nodes.Count, Name = n.Name }); - starts[axis] += sizes[i]; - } - } - - private void EmitReshape(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - long[] spec = V(Input(1, n)).Ints!; - bool allowZero = n.IntAttr("allowzero") == 1; - int[] shape = ResolveReshape(x.Shape, spec, allowZero, checked((int)x.ElementCount)); - // zero-copy alias (consts keep their payload so consumers see a materialized tensor) - AddValue(new CValue { Name = n.Outputs[0], Shape = shape, DType = 1, Buf = x.Buf, Floats = x.Floats, AliasOf = x.Buf >= 0 ? Input(0, n) : -1 }); - } - - private void EmitSqueeze(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - long[]? axes = n.Inputs.Count > 1 && n.Inputs[1].Length > 0 ? V(Input(1, n)).Ints : null; - int[] shape; - if (axes is null) shape = x.Shape.Where(d => d != 1).ToArray(); - else - { - var dims = x.Shape.ToList(); - foreach (long a in axes.OrderByDescending(v => v)) dims.RemoveAt((int)(a < 0 ? a + dims.Count : a)); - shape = dims.ToArray(); - } - AddValue(new CValue { Name = n.Outputs[0], Shape = shape, DType = 1, Buf = x.Buf, Floats = x.Floats, AliasOf = x.Buf >= 0 ? Input(0, n) : -1 }); - } - - private void EmitUnsqueeze(OnnxNodeData n) - { - CValue x = V(Input(0, n)); - long[] axes = V(Input(1, n)).Ints!; - var dims = x.Shape.ToList(); - foreach (long a in axes.OrderBy(v => v)) dims.Insert((int)(a < 0 ? a + dims.Count + 1 : a), 1); - AddValue(new CValue { Name = n.Outputs[0], Shape = dims.ToArray(), DType = 1, Buf = x.Buf, Floats = x.Floats, AliasOf = x.Buf >= 0 ? Input(0, n) : -1 }); - } - - private void EmitElementwise(OnnxNodeData n, string op) - { - int a = Input(0, n), b = Input(1, n); - CValue va = V(a), vb = V(b); - int[] shape = BroadcastShape(va.Shape, vb.Shape); - var eop = op switch { "Add" => EwOp.Add, "Sub" => EwOp.Sub, "Mul" => EwOp.Mul, "Div" => EwOp.Div, _ => throw new() }; - Emit(ExecOp.Elementwise, [a, b], ActivationValue(n.Outputs[0], shape), new EwP { Op = eop }, n); - } - - // ---------- helpers ---------- - private static int[] BroadcastShape(int[] a, int[] b) - { - int rank = Math.Max(a.Length, b.Length); - int[] r = new int[rank]; - for (int i = 0; i < rank; i++) - { - int ad = i - (rank - a.Length) >= 0 ? a[i - (rank - a.Length)] : 1; - int bd = i - (rank - b.Length) >= 0 ? b[i - (rank - b.Length)] : 1; - if (ad != bd && ad != 1 && bd != 1) - throw new InvalidDataException($"Broadcast mismatch {ad} vs {bd}."); - r[i] = Math.Max(ad, bd); - } - return r; - } - - internal static int[] ResolveReshape(int[] inShape, long[] spec, bool allowZero, int inElems) - { - int[] shape = new int[spec.Length]; - int neg = -1; long prod = 1; - for (int i = 0; i < spec.Length; i++) - { - long s = spec[i]; - if (s == -1) { neg = i; shape[i] = 1; } - else if (s == 0 && !allowZero) { shape[i] = inShape[i]; prod *= shape[i]; } - else { shape[i] = checked((int)s); prod *= shape[i]; } - } - if (neg >= 0) shape[neg] = checked((int)(inElems / prod)); - return shape; - } - - internal static void SliceBounds(int[] inShape, long[] starts, long[] ends, - long[] axes, long[] steps, out int[] outShape, out int[] stOut, out int[] enOut, out int[] stepOut) - { - int rank = inShape.Length; - stOut = new int[starts.Length]; - enOut = new int[starts.Length]; - stepOut = new int[starts.Length]; - outShape = (int[])inShape.Clone(); - for (int i = 0; i < starts.Length; i++) - { - int ax = (int)(axes[i] < 0 ? axes[i] + rank : axes[i]); - int dim = inShape[ax]; - int step = (int)steps[i]; - long s = starts[i] < 0 ? starts[i] + dim : starts[i]; - long e = ends[i] < 0 ? ends[i] + dim : ends[i]; - if (step > 0) - { - s = Math.Clamp(s, 0, dim); - e = Math.Clamp(e, 0, dim); - } - else - { - s = Math.Clamp(s, -1, dim - 1); - e = Math.Clamp(e, -1, dim - 1); - } - int len = step > 0 ? Math.Max(0, (int)(e - s + step - 1) / step) - : Math.Max(0, (int)(s - e - step - 1) / -step); - stOut[i] = (int)s; - enOut[i] = (int)e; - stepOut[i] = step; - outShape[ax] = len; - } - } - - private static long[] IntConcat(long[][] inputs, int[][] shapes, int axis, int[] outShape) - { - // small tensors only — generic n-d concat - int outElems = outShape.Aggregate(1, (a, b) => a * b); - long[] dst = new long[outElems]; - int[] strides = Strides(outShape); - int[] coord = new int[outShape.Length]; - for (int i = 0; i < inputs.Length; i++) - { - int[] inStrides = Strides(shapes[i]); - int inElems = shapes[i].Aggregate(1, (a, b) => a * b); - for (int e = 0; e < inElems; e++) - { - int rem = e, srcOff = 0; - for (int d = 0; d < shapes[i].Length; d++) - { - coord[d] = rem / inStrides[d]; - srcOff += coord[d] * inStrides[d]; - rem %= inStrides[d]; - } - int dstOff = 0; - for (int d = 0; d < outShape.Length; d++) - dstOff += (coord[d] + (d == axis ? shapes.Take(i).Sum(s => s[axis]) : 0)) * strides[d]; - dst[dstOff] = inputs[i][srcOff]; - } - } - return dst; - } - - private static long[] IntSlice(long[] src, int[] inShape, int[] axes, int[] starts, int[] steps, int[] outShape) - { - int rank = inShape.Length; - int[] fullStart = new int[rank], fullStep = Enumerable.Repeat(1, rank).ToArray(); - for (int i = 0; i < axes.Length; i++) { fullStart[axes[i]] = starts[i]; fullStep[axes[i]] = steps[i]; } - int outElems = outShape.Aggregate(1, (a, b) => a * b); - long[] dst = new long[outElems]; - int[] inStrides = Strides(inShape), outStrides = Strides(outShape); - for (int e = 0; e < outElems; e++) - { - int rem = e, srcOff = 0; - for (int d = 0; d < rank; d++) - { - int cc = rem / outStrides[d]; - rem %= outStrides[d]; - srcOff += (fullStart[d] + cc * fullStep[d]) * inStrides[d]; - } - dst[e] = src[srcOff]; - } - return dst; - } - - internal static int[] Strides(int[] shape) - { - int[] s = new int[shape.Length]; - int acc = 1; - for (int i = shape.Length - 1; i >= 0; i--) { s[i] = acc; acc *= shape[i]; } - return s; - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/Layout.cs b/src/Sdcb.SimdGcdrNet/Engine/Layout.cs deleted file mode 100644 index 755fef0..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/Layout.cs +++ /dev/null @@ -1,111 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// Physical-layout algebra. A layout L lists, for each physical -/// (storage) dim i, the logical dim it holds: storage is row-major over -/// [shape[L[0]], shape[L[1]], ...]. Logical shapes never change — -/// fusion passes and perm reasoning stay in logical space; kernels address by -/// . Size-1 dims carry no stride, so layouts that order the -/// non-1 dims identically are interchangeable (). -/// -internal static class Layout -{ - /// Channels-last for rank-4 [n,c,h,w]: storage [n,h,w,c]. - public static readonly int[] Nhwc = [0, 2, 3, 1]; - - public static int[] Identity(int rank) - { - int[] l = new int[rank]; - for (int i = 0; i < rank; i++) l[i] = i; - return l; - } - - public static bool IsIdentity(int[] l) - { - for (int i = 0; i < l.Length; i++) if (l[i] != i) return false; - return true; - } - - public static int[] Phys(int[] shape, int[] l) - { - int[] p = new int[l.Length]; - for (int i = 0; i < l.Length; i++) p[i] = shape[l[i]]; - return p; - } - - public static int[] Inverse(int[] p) - { - int[] inv = new int[p.Length]; - for (int i = 0; i < p.Length; i++) inv[p[i]] = i; - return inv; - } - - /// - /// Layout of y = Transpose(x, perm) as a view of x's storage. y logical k - /// is x logical perm[k], so x logical d is y logical perm⁻¹[d]: - /// L'[i] = perm⁻¹[L[i]]. Always representable — every Transpose dissolves. - /// - public static int[] Transposed(int[] l, int[] perm) - { - int[] inv = Inverse(perm); - int[] r = new int[l.Length]; - for (int i = 0; i < l.Length; i++) r[i] = inv[l[i]]; - return r; - } - - /// Same storage order for (size-1 dims ignored). - public static bool Equivalent(int[] shape, int[] a, int[] b) => - a.Where(d => shape[d] != 1).SequenceEqual(b.Where(d => shape[d] != 1)); - - /// Physical perm relaying storage from layout to - /// : out phys dim i = in phys dim q[i]. - public static int[] PhysPerm(int[] from, int[] to) => - to.Select(d => Array.IndexOf(from, d)).ToArray(); - - /// - /// Layout of a zero-copy reshape → - /// of storage laid out as , or null when the reshape - /// needs data movement. The logical reshape factors into groups (a run of - /// source dims ↔ a run of target dims with equal product); it is a view iff - /// each group's non-1 source dims sit consecutively, in logical order, in - /// storage. Target dims are then ordered by their group's storage position. - /// - public static int[]? Reshaped(int[] src, int[] l, int[] dst) - { - int[] sd = Enumerable.Range(0, src.Length).Where(d => src[d] != 1).ToArray(); - int[] td = Enumerable.Range(0, dst.Length).Where(d => dst[d] != 1).ToArray(); - // storage rank of each non-1 source dim - int[] rank = new int[src.Length]; - int r = 0; - foreach (int d in l) if (src[d] != 1) rank[d] = r++; - - double[] key = new double[dst.Length]; - int i = 0, j = 0; - while (i < sd.Length || j < td.Length) - { - if (i >= sd.Length || j >= td.Length) return null; // element count mismatch - var gs = new List { sd[i] }; long ps = src[sd[i++]]; - var gt = new List { td[j] }; long pt = dst[td[j++]]; - while (ps != pt) - { - if (ps < pt) { if (i >= sd.Length) return null; gs.Add(sd[i]); ps *= src[sd[i++]]; } - else { if (j >= td.Length) return null; gt.Add(td[j]); pt *= dst[td[j++]]; } - } - for (int k = 1; k < gs.Count; k++) - if (rank[gs[k]] != rank[gs[k - 1]] + 1) return null; - for (int k = 0; k < gt.Count; k++) - key[gt[k]] = rank[gs[0]] + k / (double)(gt.Count + 1); - } - // size-1 target dims ride right behind their logical predecessor (or lead), - // so an identity source yields an identity target - double prev = -1; - for (int e = 0; e < dst.Length; e++) - { - if (dst[e] != 1) prev = key[e]; - else key[e] = prev + 1e-3 * (e + 1) / (dst.Length + 1); - } - return Enumerable.Range(0, dst.Length).OrderBy(e => key[e]).ThenBy(e => e).ToArray(); - } - - public static string Str(int[]? l) => l is null ? "-" : "[" + string.Join(',', l) + "]"; -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/LayoutPlanner.cs b/src/Sdcb.SimdGcdrNet/Engine/LayoutPlanner.cs deleted file mode 100644 index 87c0194..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/LayoutPlanner.cs +++ /dev/null @@ -1,243 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// Physical layout pass (runs after fusion, before arena planning). Every -/// activation value gets a ; logical shapes are -/// untouched, so fusions and golden shapes stay in logical NCHW terms. -/// -/// Rules (global channels-last; no runtime segment decisions — this model is a -/// homogeneous conv + token-MLP graph): -/// -/// Conv / MaxPool / Resize / Pad / ChannelShift consume and produce -/// rank-4 NHWC (). -/// LayerNorm / Gemm need their last logical dim innermost and keep the -/// input layout; elementwise / unary ops follow their full-shape input. -/// Every graph Transpose dissolves into a zero-copy view -/// (); reshape/squeeze aliases stay views when -/// can express them. Layout bookkeeping follows -/// the Buf alias chain (), not just ExecNodes — -/// aliases have no producer node. -/// A consumer whose requirement is not met gets an explicit relayout -/// (a physical Transpose node). On this model the only ones are the I/O -/// boundary relayouts when physicalIo is off. -/// -/// Op params that name axes are rewritten into physical terms (Slice axes, -/// Concat axis, relayout perms), so generic kernels address purely by -/// . -/// -internal sealed class LayoutPlanner -{ - private readonly List _nodes; - private readonly List _values; - private readonly List _bufSize; - private readonly Dictionary _perm = []; // dissolved transpose → logical perm - private readonly Dictionary<(int v, string l), int> _relayout = []; - private int _at; // index of the node being planned - private int _dissolved; - - private LayoutPlanner(List nodes, List values, List bufSize) - { - _nodes = nodes; _values = values; _bufSize = bufSize; - } - - public static void Run(List nodes, List values, List bufSize, - int inputValue, ref int outputValue, bool physicalIo) - { - var lp = new LayoutPlanner(nodes, values, bufSize); - CValue inV = values[inputValue]; - inV.Layout = physicalIo && inV.Shape.Length == 4 ? Layout.Nhwc : Layout.Identity(inV.Shape.Length); - - for (lp._at = 0; lp._at < nodes.Count; lp._at++) - lp.Plan(nodes[lp._at]); - - lp.L(outputValue); - if (!physicalIo) - { - CValue ov = values[outputValue]; - int[] id = Layout.Identity(ov.Shape.Length); - if (!Layout.Equivalent(ov.Shape, ov.Layout!, id)) - { - lp._at = nodes.Count; - outputValue = lp.Relayout(outputValue, id); - } - } - // dead aliases (never consumed): identity layout so dumps never see null. - // Resolving them could insert relayout nodes for values nobody reads. - for (int v = 0; v < values.Count; v++) - if (values[v].Buf >= 0 && values[v].Layout is null) - values[v].Layout = Layout.Identity(values[v].Shape.Length); - - if (Environment.GetEnvironmentVariable("GCDRNET_DUMP") == "1") - Console.Error.WriteLine($"[layout] dissolved {lp._dissolved} transposes, {lp._relayout.Count} relayouts"); - } - - /// Resolves (and caches) the layout of value , following aliases. - private int[] L(int v) - { - CValue cv = _values[v]; - if (cv.Layout is not null) return cv.Layout; - if (cv.Buf < 0) return cv.Layout = Layout.Identity(cv.Shape.Length); // const - if (cv.AliasOf < 0) - throw new InvalidOperationException( - $"layout: value '{cv.Name}' (v{v}) consumed before its producer was planned."); - int s = cv.AliasOf; - int[] ls = L(s); - if (_perm.TryGetValue(v, out int[]? perm)) - { - cv.Buf = _values[s].Buf; - return cv.Layout = Layout.Transposed(ls, perm); - } - int[]? r = Layout.Reshaped(_values[s].Shape, ls, cv.Shape); - if (r is null) - { - // the reshape needs data movement: materialize the source in - // logical order, then the reshape is an identity-layout view of that - int[] id = Layout.Identity(_values[s].Shape.Length); - s = Relayout(s, id); - cv.AliasOf = s; - r = Layout.Reshaped(_values[s].Shape, id, cv.Shape) - ?? throw new InvalidOperationException("identity reshape must be a view"); - } - cv.Buf = _values[s].Buf; - return cv.Layout = r; - } - - /// Inserts (before the current node) a physical copy of in layout . - private int Relayout(int v, int[] target) - { - var key = (v, Layout.Str(target)); - if (_relayout.TryGetValue(key, out int done)) return done; - CValue src = _values[v]; - int[] from = L(v); - int buf = _bufSize.Count; - _bufSize.Add(checked((int)src.ElementCount)); - int id = _values.Count; - _values.Add(new CValue { Name = src.Name + "@relayout", Shape = src.Shape, DType = 1, Buf = buf, Layout = target }); - _nodes.Insert(_at++, new ExecNode - { - Op = ExecOp.Transpose, In = [v], Out = [id], - P = new TransposeP { Perm = Layout.PhysPerm(from, target) }, - Name = $"relayout{Layout.Str(from)}->{Layout.Str(target)}", - }); - _relayout[key] = id; - return id; - } - - private int Need(int v, int[] target) => - Layout.Equivalent(_values[v].Shape, L(v), target) ? v : Relayout(v, target); - - private void Need(ExecNode n, int k, int[] target) => n.In[k] = Need(n.In[k], target); - - private static bool LastInnermost(int[] shape, int[] l) - { - int r = shape.Length - 1; - if (r < 0 || shape[r] == 1) return true; - for (int i = l.Length - 1; i >= 0; i--) - if (shape[l[i]] != 1) return l[i] == r; - return true; - } - - private void SetOut(ExecNode n, int[] l) => _values[n.Out[0]].Layout = l; - - private void Plan(ExecNode n) - { - switch (n.Op) - { - case ExecOp.Transpose: - { - // dissolve: y = T(x) becomes a view of x's storage - int x = n.In[0], o = n.Out[0]; - L(x); - _values[o].AliasOf = x; - _values[o].Layout = null; - _perm[o] = ((TransposeP)n.P!).Perm; - L(o); - _nodes.RemoveAt(_at--); - _dissolved++; - break; - } - case ExecOp.Conv or ExecOp.MaxPool or ExecOp.ResizeBilinear or ExecOp.Pad or ExecOp.ChannelShift: - { - if (_values[n.In[0]].Shape.Length != 4 || _values[n.Out[0]].Shape.Length != 4) - throw new NotSupportedException($"layout: {n.Op} '{n.Name}' is not rank-4."); - if (n.P is PadP pp) - { - int r = 4; - if (pp.Pads[0] != 0 || pp.Pads[1] != 0 || pp.Pads[r] != 0 || pp.Pads[r + 1] != 0) - throw new NotSupportedException($"layout: Pad '{n.Name}' pads N/C."); - } - if (n.P is ConvP cp) - { - if (cp.Dilations.Any(d => d != 1)) - throw new NotSupportedException($"layout: dilated Conv '{n.Name}'."); - if (cp.AddValue >= 0) cp.AddValue = Need(cp.AddValue, Layout.Nhwc); - } - Need(n, 0, Layout.Nhwc); - if (n.Op == ExecOp.ResizeBilinear) - for (int k = 1; k < n.In.Length; k++) Need(n, k, Layout.Nhwc); // fused addend - SetOut(n, Layout.Nhwc); - break; - } - case ExecOp.UpHead: - { - // In = [x, skip, w, bias?]: activations channels-last, weights stay logical - if (_values[n.In[0]].Shape.Length != 4 || _values[n.In[1]].Shape.Length != 4) - throw new NotSupportedException($"layout: UpHead '{n.Name}' is not rank-4."); - Need(n, 0, Layout.Nhwc); - Need(n, 1, Layout.Nhwc); - SetOut(n, Layout.Nhwc); - break; - } - case ExecOp.LayerNorm or ExecOp.Gemm: - { - int x = n.In[0]; - if (!LastInnermost(_values[x].Shape, L(x))) - Need(n, 0, Layout.Identity(_values[x].Shape.Length)); - SetOut(n, L(n.In[0])); - break; - } - case ExecOp.Elementwise: - { - int[] os = _values[n.Out[0]].Shape; - int rank = os.Length; - int ri = Array.FindIndex(n.In, v => _values[v].IsActivation && _values[v].Shape.SequenceEqual(os)); - int[] lo = ri >= 0 ? L(n.In[ri]) : Layout.Identity(rank); - for (int k = 0; k < n.In.Length; k++) - { - CValue cv = _values[n.In[k]]; - if (cv.ElementCount == 1) continue; - // operand storage must match lo restricted to its own dims - int off = rank - cv.Shape.Length; - int[] tv = lo.Where(d => d >= off).Select(d => d - off).ToArray(); - Need(n, k, tv); - } - SetOut(n, lo); - break; - } - case ExecOp.Relu or ExecOp.Sigmoid or ExecOp.Erf or ExecOp.Gelu or ExecOp.Copy: - SetOut(n, L(n.In[0])); - break; - case ExecOp.Slice: - { - int[] lx = L(n.In[0]); - int rank = lx.Length; - var sp = (SliceP)n.P!; - for (int i = 0; i < sp.Axes.Length; i++) - sp.Axes[i] = Array.IndexOf(lx, sp.Axes[i] < 0 ? sp.Axes[i] + rank : sp.Axes[i]); - SetOut(n, lx); - break; - } - case ExecOp.Concat: - { - int[] lo = L(n.In[0]); - for (int k = 1; k < n.In.Length; k++) Need(n, k, lo); - var cp = (ConcatP)n.P!; - cp.Axis = Array.IndexOf(lo, cp.Axis); - SetOut(n, lo); - break; - } - default: - throw new NotSupportedException($"layout: no rule for {n.Op} '{n.Name}'."); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Engine/Session.cs b/src/Sdcb.SimdGcdrNet/Engine/Session.cs deleted file mode 100644 index 2c06d33..0000000 --- a/src/Sdcb.SimdGcdrNet/Engine/Session.cs +++ /dev/null @@ -1,327 +0,0 @@ -using System.Runtime.InteropServices; -using Sdcb.SimdGcdrNet.Kernels; -using Sdcb.SimdGcdrNet.Onnx; - -namespace Sdcb.SimdGcdrNet.Engine; - -/// -/// A reusable inference session over a compiled . -/// Not thread-safe; create one per thread or serialize externally. -/// -public sealed unsafe class InferenceSession : IDisposable -{ - private readonly ExecPlan _plan; - // native so Dispose frees it deterministically: multi-GB LOH arenas of evicted - // sessions otherwise pile up until a (rare, on big-RAM hosts) gen2 GC - private nint _arena; - private readonly long _arenaBytes; - private readonly bool _ownsArena; - private readonly int[] _offsets; - private readonly long[] _nodeTicks; - private readonly int _inputValue, _outputValue; - - /// Creates a session over that owns a fresh native arena. - public InferenceSession(CompiledModel model) : this(model.Plan) { } - - internal InferenceSession(ExecPlan plan) : this(plan, 0) { } - - /// - /// Session over . With = 0 the session owns a - /// fresh zeroed native arena; otherwise it borrows the caller's arena (at least - /// ArenaFloats floats; contents need not be zero — every buffer is written before it is - /// read), which outlives the session and is never freed by it. - /// - internal InferenceSession(ExecPlan plan, nint arena) - { - _plan = plan; - _arenaBytes = (long)plan.ArenaFloats * sizeof(float); - if (arena == 0) - { - // calloc: large blocks come straight from fresh OS pages (already zero, faulted in lazily - // by the parallel kernels) instead of a single-threaded multi-GB memset up front - _arena = (nint)NativeMemory.AllocZeroed((nuint)Math.Max(_arenaBytes, 64)); - _ownsArena = true; - if (_arenaBytes > 0) GC.AddMemoryPressure(_arenaBytes); - } - else _arena = arena; - _offsets = plan.BufOffset; - _inputValue = plan.InputValue; - _outputValue = plan.OutputValue; - _nodeTicks = new long[plan.Nodes.Length]; - CpuKernels.Pack(plan, null); - } - /// Input tensor shape (e.g. [1,3,H,W]). - public int[] InputShape => _plan.Values[_inputValue].Shape; - /// Output tensor shape (e.g. [1,3,H,W]). - public int[] OutputShape => _plan.Values[_outputValue].Shape; - - /// The writable input window inside the arena (NCHW float). - public Span Input => GetSpan(_inputValue); - /// The output tensor after . - public ReadOnlySpan Output => GetSpan(_outputValue); - - private Span GetSpan(int valueId) - { - CValue v = _plan.Values[valueId]; - if (v.Buf >= 0) - return new Span(Arena + _offsets[v.Buf], checked((int)v.ElementCount)); - if (v.Floats != null) return v.Floats; - throw new InvalidOperationException($"Value '{v.Name}' has no float storage."); - } - - private ReadOnlySpan GetIn(int valueId) - { - if (valueId < 0) return ReadOnlySpan.Empty; - return GetSpan(valueId); - } - - /// Runs the whole graph. Input must already be written to . - public void Run() - { - ExecNode[] nodes = _plan.Nodes; - CValue[] values = _plan.Values; - for (int i = 0; i < nodes.Length; i++) - { - ExecNode n = nodes[i]; - long t0 = s_profile ? System.Diagnostics.Stopwatch.GetTimestamp() : 0; - try { Exec(n); } - catch (Exception ex) - { - string In(int v) => v < 0 ? "-" : $"{values[v].Name}:{string.Join('x', values[v].Shape)}"; - throw new InvalidOperationException( - $"node[{i}] {n.Op} '{n.Name}' in=[{string.Join(", ", n.In.Select(In))}] out=[{string.Join(", ", n.Out.Select(In))}]", ex); - } - if (s_profile) _nodeTicks[i] += System.Diagnostics.Stopwatch.GetTimestamp() - t0; - } - } - - private static readonly bool s_profile = Environment.GetEnvironmentVariable("GCDRNET_PROF") == "1"; - - /// Per-op-type profile (ticks), populated when GCDRNET_PROF=1. - internal Dictionary Profile() - { - var d = new Dictionary(); - for (int i = 0; i < _plan.Nodes.Length; i++) - { - ExecOp op = _plan.Nodes[i].Op; - d.TryGetValue(op, out var cur); - d[op] = cur + TimeSpan.FromSeconds((double)_nodeTicks[i] / System.Diagnostics.Stopwatch.Frequency); - } - return d; - } - - /// Per-node profile (node index, op, name, ticks→time), populated when GCDRNET_PROF=1. - internal IEnumerable<(int i, ExecNode n, TimeSpan t)> NodeProfile() => - _plan.Nodes.Select((n, i) => (i, n, TimeSpan.FromSeconds((double)_nodeTicks[i] / System.Diagnostics.Stopwatch.Frequency))); - - /// Clears the GCDRNET_PROF counters (e.g. after a warmup run). - internal void ResetProfile() => Array.Clear(_nodeTicks); - - /// Debug/test: run a single node by index. Arena must be pre-initialized. - internal void ExecNode(int i) => Exec(_plan.Nodes[i]); - - internal int[] ShapeOf(int valueId) => _plan.Values[valueId].Shape; - - /// Debug/test: read an activation value's arena region. - internal ReadOnlySpan ValueSpan(int valueId) => GetSpan(valueId); - - private void Exec(ExecNode n) - { - CValue[] values = _plan.Values; - if (_plan.Nhwc && ExecNhwc(n)) return; - switch (n.Op) - { - case ExecOp.Conv: - { - var p = (ConvP)n.P!; - CValue x = values[n.In[0]], w = values[n.In[1]], o = values[n.Out[0]]; - ReadOnlySpan bias = p.BiasValue >= 0 ? GetIn(p.BiasValue) : ReadOnlySpan.Empty; - ReadOnlySpan add = p.AddValue >= 0 ? GetIn(p.AddValue) : ReadOnlySpan.Empty; - Ops.Conv(GetIn(n.In[0]), x.Shape, GetIn(n.In[1]), w.Shape, bias, - GetSpan(n.Out[0]), o.Shape, p, add); - break; - } - case ExecOp.MaxPool: - { - var p = (PoolP)n.P!; - Ops.MaxPool(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p); - break; - } - case ExecOp.Elementwise: - { - var p = (EwP)n.P!; - Ops.Elementwise(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetIn(n.In[1]), values[n.In[1]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p.Op, p.Act); - break; - } - case ExecOp.Relu: Ops.Relu(GetIn(n.In[0]), GetSpan(n.Out[0])); break; - case ExecOp.Sigmoid: Ops.Sigmoid(GetIn(n.In[0]), GetSpan(n.Out[0])); break; - case ExecOp.Erf: Ops.Erf(GetIn(n.In[0]), GetSpan(n.Out[0])); break; - case ExecOp.Gelu: Ops.Gelu(GetIn(n.In[0]), GetSpan(n.Out[0])); break; - case ExecOp.LayerNorm: - { - var p = (LnP)n.P!; - Ops.LayerNorm(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetIn(n.In[1]), GetIn(n.In[2]), GetSpan(n.Out[0]), p.Eps); - break; - } - case ExecOp.Gemm: - { - var p = (GemmP)n.P!; - ReadOnlySpan bias = p.BiasValue >= 0 ? GetIn(p.BiasValue) : ReadOnlySpan.Empty; - Ops.Gemm(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetIn(n.In[1]), values[n.In[1]].PhysShape, bias, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p); - break; - } - case ExecOp.ResizeBilinear: - { - var p = (ResizeP)n.P!; - Ops.ResizeBilinear(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p.Scales, - p.FuseRelu ? ActivationKind.Relu : ActivationKind.None); - break; - } - case ExecOp.Pad: - { - var p = (PadP)n.P!; - Ops.Pad(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p); - break; - } - case ExecOp.Transpose: - { - var p = (TransposeP)n.P!; - Ops.Transpose(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p.Perm); - break; - } - case ExecOp.Slice: - { - var p = (SliceP)n.P!; - Ops.Slice(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p); - break; - } - case ExecOp.Concat: - { - var p = (ConcatP)n.P!; - var ins = new FBuf[n.In.Length]; - var shapes = new int[n.In.Length][]; - for (int k = 0; k < n.In.Length; k++) - { - CValue cv = values[n.In[k]]; - ins[k] = cv.Buf >= 0 - ? new FBuf(Arena + _offsets[cv.Buf], checked((int)cv.ElementCount)) - : new FBuf(cv.Floats!, 0, checked((int)cv.ElementCount)); - shapes[k] = cv.PhysShape; - } - Ops.Concat(ins, shapes, GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p.Axis); - break; - } - case ExecOp.ChannelShift: - { - var p = (ShiftP)n.P!; - Ops.ChannelShift(GetIn(n.In[0]), values[n.In[0]].PhysShape, - GetSpan(n.Out[0]), values[n.Out[0]].PhysShape, p); - break; - } - case ExecOp.Copy: - GetIn(n.In[0]).CopyTo(GetSpan(n.Out[0])); - break; - default: - throw new NotSupportedException($"ExecOp {n.Op}"); - } - } - - /// Channels-last kernels for the layout-sensitive ops; false = generic (storage-shape) path. - private bool ExecNhwc(ExecNode n) - { - CValue[] values = _plan.Values; - switch (n.Op) - { - case ExecOp.Conv: - { - var p = (ConvP)n.P!; - int[] i = values[n.In[0]].PhysShape, o = values[n.Out[0]].PhysShape; - if (p.Pool is not null) - { - Nhwc.ConvPool(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), o[1], o[2], o[3], - p, (NhwcConvPlan)p.KernelPlan!); - return true; - } - ReadOnlySpan add = p.AddValue >= 0 ? GetIn(p.AddValue) : ReadOnlySpan.Empty; - Nhwc.Conv(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), o[1], o[2], o[3], - p, (NhwcConvPlan)p.KernelPlan!, add); - return true; - } - case ExecOp.MaxPool: - { - int[] i = values[n.In[0]].PhysShape, o = values[n.Out[0]].PhysShape; - Nhwc.MaxPool(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), o[1], o[2], (PoolP)n.P!); - return true; - } - case ExecOp.ResizeBilinear: - { - var p = (ResizeP)n.P!; - int[] i = values[n.In[0]].PhysShape, o = values[n.Out[0]].PhysShape; - Nhwc.ResizeBilinear(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), o[1], o[2], - p.Scales[2], p.Scales[3], p.FuseRelu ? ActivationKind.Relu : ActivationKind.None, - n.In.Length > 1 ? GetIn(n.In[1]) : default, p.AddRelu); - return true; - } - case ExecOp.UpHead: - { - var p = (UpHeadP)n.P!; - int[] i = values[n.In[0]].PhysShape, o = values[n.Out[0]].PhysShape, k = values[n.In[1]].PhysShape; - Nhwc.UpHead(GetIn(n.In[0]), i[1], i[2], i[3], GetIn(n.In[1]), GetSpan(n.Out[0]), o[1], o[2], - p.Scales[2], p.Scales[3], p, (UpHeadPlan)p.KernelPlan!, k[1], k[2], k[3]); - return true; - } - case ExecOp.Pad: - { - int[] i = values[n.In[0]].PhysShape, o = values[n.Out[0]].PhysShape; - Nhwc.Pad(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), o[1], o[2], (PadP)n.P!); - return true; - } - case ExecOp.Gemm when ((GemmP)n.P!).KernelPlan is NhwcConvPlan gk: - { - var p = (GemmP)n.P!; - int[] a = values[n.In[0]].PhysShape; - int k = a[^1], m = (int)(values[n.In[0]].ElementCount / k); - Nhwc.Gemm(GetIn(n.In[0]), m, k, GetSpan(n.Out[0]), values[n.Out[0]].Shape[^1], gk, p.Act); - return true; - } - case ExecOp.ChannelShift: - { - int[] i = values[n.In[0]].PhysShape; - Nhwc.ChannelShift(GetIn(n.In[0]), i[1], i[2], i[3], GetSpan(n.Out[0]), (ShiftP)n.P!); - return true; - } - default: - return false; - } - } - - private float* Arena => _arena != 0 ? (float*)_arena : throw new ObjectDisposedException(nameof(InferenceSession)); - - /// Frees the native arena (if owned by this session). - public void Dispose() - { - Free(); - GC.SuppressFinalize(this); - } - - /// Frees the native arena if was not called. - ~InferenceSession() => Free(); - - private void Free() - { - nint p = Interlocked.Exchange(ref _arena, 0); - if (p == 0 || !_ownsArena) return; - NativeMemory.Free((void*)p); - if (_arenaBytes > 0) GC.RemoveMemoryPressure(_arenaBytes); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/ChannelShiftOp.cs b/src/Sdcb.SimdGcdrNet/Extensions/ChannelShiftOp.cs new file mode 100644 index 0000000..b7a9f64 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/ChannelShiftOp.cs @@ -0,0 +1,189 @@ +using Sdcb.SimdOnnx.Backends.Metal; +using Sdcb.SimdOnnx.Backends.Vulkan; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; + +namespace Sdcb.SimdGcdrNet.Extensions; + +internal sealed class ShiftP +{ + public int Axis; // logical spatial axis index (2 = H, 3 = W) + public required int[] GroupEnds; // cumulative channel-group end indices (len 5) + public required int[] Offsets; // per-group shift (len 5) +} + +/// Fused shift-MLP: per channel group, shift along H or W (zero-fill). In = [x], P = . +internal sealed class ChannelShiftOp : ExtOp +{ + public static readonly ChannelShiftOp Instance = new(); + public override string Name => "ChannelShift"; + + public override void RunCpu(InferenceSession s, ExecNode n) + { + var p = (ShiftP)n.P!; + CValue[] v = s.Plan.Values; + if (s.Plan.Nhwc) + { + int[] i = v[n.In[0]].PhysShape; + GcdrKernels.ChannelShift(s.GetIn(n.In[0]), i[1], i[2], i[3], s.GetSpan(n.Out[0]), p); + } + else + GcdrKernels.ChannelShiftNchw(s.GetIn(n.In[0]), v[n.In[0]].PhysShape, s.GetSpan(n.Out[0]), v[n.Out[0]].PhysShape, p); + } + + private static (uint[] pc, bool v4) Pc(int[] si, ShiftP p, bool vec4) + { + var ge = p.GroupEnds; var of = p.Offsets; + if (ge.Length > 5) throw new NotSupportedException("GPU shift: > 5 groups"); + var pc = new List { (uint)si[3], (uint)si[1], (uint)si[2], (uint)p.Axis }; + for (int i = 0; i < 5; i++) pc.Add((uint)(i < ge.Length ? ge[i] : si[3])); + for (int i = 0; i < 5; i++) pc.Add((uint)(i < of.Length ? of[i] : 0)); + return (pc.ToArray(), vec4 && si[3] % 4 == 0 && ge.All(e => e % 4 == 0)); + } + + public override IEnumerable EmitVulkan(VkGraphModel g, ExecNode n) + { + g.RequireNhwc(n.In[0]); g.RequireNhwc(n.Out[0]); + int[] si = g.Plan.Values[n.In[0]].PhysShape; // [1,h,w,c] + var (pc, v4) = Pc(si, (ShiftP)n.P!, g.Device.Vec4Eltwise); + uint count = (uint)g.Plan.Values[n.Out[0]].ElementCount / (v4 ? 4u : 1u); + yield return g.Dispatch(n, v4 ? "shift4" : "shift", [g.Bind(n.In[0]), g.Bind(n.Out[0])], + VkGraphModel.PcOf(pc), VkGraphModel.G(count), owner: GcdrExtensions.Owner); + } + + public override IEnumerable EmitMetal(MtlGraphModel g, ExecNode n) + { + g.RequireNhwc(n.In[0]); g.RequireNhwc(n.Out[0]); + int[] si = g.Plan.Values[n.In[0]].PhysShape; + var (pc, v4) = Pc(si, (ShiftP)n.P!, true); + uint count = (uint)g.Plan.Values[n.Out[0]].ElementCount / (v4 ? 4u : 1u); + yield return g.Dispatch(n, v4 ? "shift4" : "shift", [g.Bind(n.In[0]), g.Bind(n.Out[0])], + MtlGraphModel.PcOf(pc), MtlGraphModel.G(count), owner: GcdrExtensions.Owner); + } +} + +/// +/// Recognizes the shift-MLP subgraph: Pad(const ±p on H or W) -> channel Split (5 groups via +/// Slice) -> per-group roll (Slice+Slice+Concat on the spatial axis) -> Concat(axis=1) -> +/// Slice(unpad). Replaced by a single node writing the shifted [1,C,h,w]. +/// +internal sealed class FuseShiftPass : GraphPass +{ + public override PassStage Stage => PassStage.Logical; + + public override void Run(PassContext ctx) + { + List nodes = ctx.Nodes; + List values = ctx.Values; + for (int i = 0; i < nodes.Count; i++) + { + ExecNode pad = nodes[i]; + if (pad.Op != ExecOp.Pad || pad.P is not PadP pp || pp.Mode != PadMode.Constant) + continue; + int inId = pad.In[0], padOut = pad.Out[0]; + int rank = values[padOut].Shape.Length; + if (rank != 4) continue; + int padH = pp.Pads[2], padW = pp.Pads[3]; + if (pp.Pads[2 + rank] != padH || pp.Pads[3 + rank] != padW) continue; + + // channel split pieces of the padded tensor + var pieces = nodes.Where(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == padOut).ToList(); + if (pieces.Count != 5) continue; + int[] padShape = values[padOut].Shape; + // SliceP.Starts/Ends are per-Axes-position arrays. For a slice emitted from a Split + // over axis C, every axes entry is identity except axis 1 (the group range). + bool RangeAt(SliceP s, int ax, int[] inShape, out int st, out int en) + { + st = 0; en = inShape[ax]; + for (int k = 0; k < s.Axes.Length; k++) + if (s.Axes[k] == ax) { st = s.Starts[k]; en = s.Ends[k]; break; } + return s.Axes.Where(a => a != ax).All(a => + s.Starts[Array.IndexOf(s.Axes, a)] == 0 && + s.Ends[Array.IndexOf(s.Axes, a)] >= inShape[a]); + } + var pieceInfo = pieces + .Select(m => (node: m, s: (SliceP)m.P!)) + .Where(t => RangeAt(t.s, 1, padShape, out _, out _)) + .OrderBy(t => { RangeAt(t.s, 1, padShape, out int st, out _); return st; }) + .ToArray(); + if (pieceInfo.Length != 5) continue; + int[] groupEnds = pieceInfo.Select(t => { RangeAt(t.s, 1, padShape, out _, out int en); return en; }).ToArray(); + + // each piece's roll: two Slice consumers + one Concat + int[] offsets = new int[5]; + int shiftAxis = -1; + int[] pieceOutIds = pieceInfo.Select(t => t.node.Out[0]).ToArray(); + var smallConcatOuts = new List(5); + bool fail = false; + var toRemove = new List(pieces); + for (int gI = 0; gI < 5; gI++) + { + int pieceId = pieceOutIds[gI]; + var rollSlices = nodes.Where(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == pieceId).ToList(); + if (rollSlices.Count != 2) { fail = true; break; } + var concat = nodes.FirstOrDefault(m => m.Op == ExecOp.Concat && m.In.Length == 2 && + m.In.Contains(rollSlices[0].Out[0]) && m.In.Contains(rollSlices[1].Out[0])); + if (concat is null) { fail = true; break; } + var sp = (SliceP)rollSlices[0].P!; var sp2 = (SliceP)rollSlices[1].P!; + // both roll slices must slice a single spatial axis + int SliceAxis(SliceP s, int[] pieceShape) + { + int found = -1; + for (int k = 0; k < s.Axes.Length; k++) + { + int d = s.Axes[k]; + bool sliced = s.Starts[k] != 0 || s.Ends[k] < pieceShape[d]; + if (sliced && found != -1) return -1; + if (sliced) found = d; + } + return found; + } + int[] pieceShape = values[pieceId].Shape; + int ax = SliceAxis(sp, pieceShape); + int ax2 = SliceAxis(sp2, pieceShape); + // one side may be unsliced (identity roll for the 0-offset group) + if (ax < 0) ax = ax2; + if (ax2 >= 0 && ax2 != ax) ax = -1; + if (ax < 0 || (ax != 2 && ax != 3)) { fail = true; break; } + if (shiftAxis == -1) shiftAxis = ax; + else if (shiftAxis != ax) { fail = true; break; } + int L = pieceShape[ax]; + // concat order = [B, A] where A = piece[0..cut] is the "head" + var sA = rollSlices[0].Out[0] == concat.In[0] ? sp2 : sp; + var sB = rollSlices[0].Out[0] == concat.In[0] ? sp : sp2; + RangeAt(sA, ax, pieceShape, out int saSt, out int saEn); + RangeAt(sB, ax, pieceShape, out int sbSt, out int sbEn); + if (saSt != 0 || sbSt != saEn || sbEn < L) { fail = true; break; } + int cut = saEn - saSt; + // effective shift on the unpadded axis: out[y] = orig[y + s], zero-fill. + // rolled[i] = padded[(i+cut) mod L]; sliced[y] = rolled[y+padAmt] + int padAmt = shiftAxis == 2 ? padH : padW; + offsets[gI] = ((padAmt + cut) % L) - padAmt; + toRemove.AddRange(rollSlices); + toRemove.Add(concat); + smallConcatOuts.Add(concat.Out[0]); + } + ExecNode? bigConcat = fail ? null : nodes.FirstOrDefault(m => + m.Op == ExecOp.Concat && m.In.Length == 5 && smallConcatOuts.All(m.In.Contains)); + if (fail || bigConcat is null) continue; + var unpad = nodes.FirstOrDefault(m => m.Op == ExecOp.Slice && m.In.Length == 1 && m.In[0] == bigConcat.Out[0]); + if (unpad is null) continue; + toRemove.Add(bigConcat); + toRemove.Add(unpad); + toRemove.Add(pad); + + var shift = new ExecNode + { + Op = ExecOp.Ext, + Ext = ChannelShiftOp.Instance, + In = [inId], + Out = unpad.Out, + P = new ShiftP { Axis = shiftAxis, GroupEnds = groupEnds, Offsets = offsets }, + SrcNode = pad.SrcNode, + Name = pad.Name + ".shift", + }; + foreach (ExecNode m in toRemove) nodes.Remove(m); + nodes.Insert(Math.Min(i, nodes.Count), shift); + } + } +} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/GcdrExtensions.cs b/src/Sdcb.SimdGcdrNet/Extensions/GcdrExtensions.cs new file mode 100644 index 0000000..bc7ce33 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/GcdrExtensions.cs @@ -0,0 +1,16 @@ +using System.Reflection; +using Sdcb.SimdOnnx.Engine; + +namespace Sdcb.SimdGcdrNet.Extensions; + +/// The GCDRNet-specific fusions (shift-MLP, upsample tail) registered with the engine. +internal static class GcdrExtensions +{ + /// Assembly embedding the extension shaders. + public static Assembly Owner { get; } = typeof(GcdrExtensions).Assembly; + + public static EngineExtensions Instance { get; } = new EngineExtensions() + .Add(new FuseShiftPass()) + .Add(new FuseUpHeadPass()) + .Add(new UpHeadSkipRecomputePass()); +} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Avx512.cs b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Avx512.cs new file mode 100644 index 0000000..df81c46 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Avx512.cs @@ -0,0 +1,88 @@ +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; +using static Sdcb.SimdOnnx.Kernels.Nhwc; + +using F32x8 = System.Runtime.Intrinsics.Vector256; +using F32x16 = System.Runtime.Intrinsics.Vector512; +using V128 = System.Runtime.Intrinsics.Vector128; +using V4 = System.Runtime.Intrinsics.Vector128; +using V256 = System.Runtime.Intrinsics.Vector256; + +namespace Sdcb.SimdGcdrNet.Extensions; + +internal static unsafe partial class GcdrKernels +{ + /// + /// for the AVX-512 tier: 16-lane resize, and the c-long oc dot runs as four + /// interleaved per-pixel FMA chains (one chain per pixel was FMA-latency bound). Each pixel keeps + /// its bias-initialized, channel-ascending chain, so outputs match the AVX2 path bit for bit. + /// + internal static void UpHead512(float* sp, int iw, int c, float* kpp, bool skipConv, DenseArgs ca, float* dp, int oh, int ow, + int[] y0, int[] y1, int[] x0, int[] x1, float[] wy, float[] wx, bool relu, bool skipRelu, + float* wp, float* bp, int oc, ActivationKind act) + { + bool v512 = c % 16 == 0; + For(oh, (long)oh * ow * c * (oc + 4), y => + { + float* s = stackalloc float[4 * c + 8]; + float* acc = stackalloc float[4 * 8]; + float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c; + var vY = new System.Numerics.Vector(wy[y]); + var vY16 = Vector512.Create(wy[y]); + float wY = wy[y]; + float* orow = dp + (long)y * ow * oc; + float* krow = kpp + (long)y * ow * c; + float[]? band = skipConv ? BandBuf((long)ow * c) : null; + fixed (float* sb = band) + { + if (skipConv) + { + ConvRow(ca, y, sb, null); // the skip row, recomputed by the same conv kernel + krow = sb; + } + for (int x = 0; x < ow; x += 4) + { + int n = Math.Min(4, ow - x); + for (int i = 0; i < n; i++) + { + int xx = x + i; + float* a = r0 + (long)x0[xx] * c, b = r0 + (long)x1[xx] * c, e = r1 + (long)x0[xx] * c, f = r1 + (long)x1[xx] * c; + if (v512) ResizePixel512(a, b, e, f, Vector512.Create(wx[xx]), vY16, c, relu, krow + (long)xx * c, skipRelu, s + i * c); + else ResizePixel(a, b, e, f, wx[xx], wY, new System.Numerics.Vector(wx[xx]), vY, c, relu, krow + (long)xx * c, skipRelu, s + i * c); + } + UpHeadDot4(s, c, wp, bp, acc, n); + for (int i = 0; i < n; i++) + { + float* o = orow + (long)(x + i) * oc; + for (int l = 0; l < oc; l++) o[l] = acc[i * 8 + l]; + } + } + if (act != ActivationKind.None) ActRow(orow, ow * oc, act, true); + } + }); + } + + /// acc[i*8 + l] = bias[l] + Σ_ch s_i[ch]·w[ch][l] for 4 pixels (s_i = s + i*c), chains interleaved. + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + internal static void UpHeadDot4(float* s, int c, float* wp, float* bp, float* acc, int n) + { + float* s0 = s, s1 = n > 1 ? s + c : s, s2 = n > 2 ? s + 2 * c : s, s3 = n > 3 ? s + 3 * c : s; + F32x8 a0 = Avx.LoadVector256(bp), a1 = a0, a2 = a0, a3 = a0; + float* w = wp; + for (int ch = 0; ch < c; ch++, w += 8) + { + F32x8 wv = Avx.LoadVector256(w); + a0 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s0 + ch), wv, a0); + a1 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s1 + ch), wv, a1); + a2 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s2 + ch), wv, a2); + a3 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s3 + ch), wv, a3); + } + Avx.Store(acc, a0); Avx.Store(acc + 8, a1); Avx.Store(acc + 16, a2); Avx.Store(acc + 24, a3); + } +} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Neon.cs b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Neon.cs new file mode 100644 index 0000000..9e9c556 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.Neon.cs @@ -0,0 +1,59 @@ +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; +using static Sdcb.SimdOnnx.Kernels.Nhwc; + +using F32x8 = System.Runtime.Intrinsics.Vector256; +using F32x16 = System.Runtime.Intrinsics.Vector512; +using V128 = System.Runtime.Intrinsics.Vector128; +using V4 = System.Runtime.Intrinsics.Vector128; +using V256 = System.Runtime.Intrinsics.Vector256; + +namespace Sdcb.SimdGcdrNet.Extensions; + +internal static unsafe partial class GcdrKernels +{ + /// + /// One output row for oc <= 4, c % 4 == 0: per 4 channels the bilinear + /// lerps (fma), [relu], + [relu](skip), then fmla by element into one 4-lane accumulator against + /// the packed [c][8] weights — the resized pixel never leaves registers. + /// + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + internal static void UpHeadRowNeon(float* r0, float* r1, int* x0, int* x1, float* wx, float wy, float* krow, + int c, int ow, float* wp, float* bp, float* orow, int oc, bool relu, bool skipRelu, ActivationKind act) + { + V4 vY = Vector128.Create(wy), bias = Ld4(bp); + for (int x = 0; x < ow; x++) + { + float* a = r0 + (long)x0[x] * c, b = r0 + (long)x1[x] * c, e = r1 + (long)x0[x] * c, f = r1 + (long)x1[x] * c; + float* k = krow + (long)x * c; + V4 vX = Vector128.Create(wx[x]), acc = bias; + float* w = wp; + for (int ch = 0; ch < c; ch += 4, w += 32) + { + V4 va = Ld4(a + ch), ve = Ld4(e + ch); + V4 top = Fma4(va, AdvSimd.Subtract(Ld4(b + ch), va), vX); + V4 bot = Fma4(ve, AdvSimd.Subtract(Ld4(f + ch), ve), vX); + V4 v = Fma4(top, AdvSimd.Subtract(bot, top), vY); + if (relu) v = AdvSimd.Max(v, V4.Zero); + V4 s = Ld4(k + ch); + v = AdvSimd.Add(v, skipRelu ? AdvSimd.Max(s, V4.Zero) : s); + acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w), v, 0); + acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 8), v, 1); + acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 16), v, 2); + acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 24), v, 3); + } + acc = Act4(acc, act); + float* o = orow + (long)x * oc; + o[0] = acc.ToScalar(); + if (oc > 1) o[1] = acc.GetElement(1); + if (oc > 2) o[2] = acc.GetElement(2); + if (oc > 3) o[3] = acc.GetElement(3); + } + } +} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.PackedSimd.cs b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.PackedSimd.cs new file mode 100644 index 0000000..35f5daf --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.PackedSimd.cs @@ -0,0 +1,59 @@ +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; +using static Sdcb.SimdOnnx.Kernels.Nhwc; + +using F32x8 = System.Runtime.Intrinsics.Vector256; +using F32x16 = System.Runtime.Intrinsics.Vector512; +using V128 = System.Runtime.Intrinsics.Vector128; +using V4 = System.Runtime.Intrinsics.Vector128; +using V256 = System.Runtime.Intrinsics.Vector256; + +namespace Sdcb.SimdGcdrNet.Extensions; + +internal static unsafe partial class GcdrKernels +{ + /// + /// One output row for oc <= 4, c % 4 == 0 (see ): + /// per 4 channels the bilinear lerps, [relu], + [relu](skip), then each channel is splatted back + /// from a 16-byte spill (WASM SIMD has no by-element multiply) into one 4-lane accumulator. + /// Same per-element ops as the unfused Vector<float> path. + /// + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + internal static void UpHeadRowPk(float* r0, float* r1, int* x0, int* x1, float* wx, float wy, float* krow, + int c, int ow, float* wp, float* bp, float* orow, int oc, bool relu, bool skipRelu, ActivationKind act) + { + V4 vY = PackedSimd.Splat(wy), bias = LdP(bp); + float* s = stackalloc float[4]; + for (int x = 0; x < ow; x++) + { + float* a = r0 + (long)x0[x] * c, b = r0 + (long)x1[x] * c, e = r1 + (long)x0[x] * c, f = r1 + (long)x1[x] * c; + float* k = krow + (long)x * c; + V4 vX = PackedSimd.Splat(wx[x]), acc = bias; + float* w = wp; + for (int ch = 0; ch < c; ch += 4, w += 32) + { + V4 va = LdP(a + ch), ve = LdP(e + ch); + V4 top = MadP(va, PackedSimd.Subtract(LdP(b + ch), va), vX); + V4 bot = MadP(ve, PackedSimd.Subtract(LdP(f + ch), ve), vX); + V4 v = MadP(top, PackedSimd.Subtract(bot, top), vY); + if (relu) v = PackedSimd.PseudoMax(V4.Zero, v); + V4 sk = LdP(k + ch); + v = PackedSimd.Add(v, skipRelu ? PackedSimd.PseudoMax(V4.Zero, sk) : sk); + PackedSimd.Store(s, v); + acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s), LdP(w)); + acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 1), LdP(w + 8)); + acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 2), LdP(w + 16)); + acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 3), LdP(w + 24)); + } + PackedSimd.Store(s, ActP(acc, act)); + float* o = orow + (long)x * oc; + for (int l = 0; l < oc; l++) o[l] = s[l]; + } + } +} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.cs b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.cs new file mode 100644 index 0000000..eb01525 --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/GcdrKernels.cs @@ -0,0 +1,312 @@ +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; +using static Sdcb.SimdOnnx.Kernels.Nhwc; + +using F32x8 = System.Runtime.Intrinsics.Vector256; +using F32x16 = System.Runtime.Intrinsics.Vector512; +using V128 = System.Runtime.Intrinsics.Vector128; +using V4 = System.Runtime.Intrinsics.Vector128; +using V256 = System.Runtime.Intrinsics.Vector256; + +namespace Sdcb.SimdGcdrNet.Extensions; + +/// Packed UpHead weights: [c][8] oc lanes (zero-padded) + bias[8]. +internal sealed class UpHeadPlan +{ + public required float[] W; + public required float[] Bias; + public int Oc, C; +} + + +/// +/// CPU kernels of the GCDR extension ops (, ), +/// built on the engine's NHWC helpers (). +/// +internal static unsafe partial class GcdrKernels +{ + private static readonly ParallelOptions s_po = new() { MaxDegreeOfParallelism = Avx2Kernels.Threads }; + + /// Packs UpHead weights [oc, c, 1, 1] as [c][8] (oc lanes, zero-padded) + bias[8]. + public static UpHeadPlan PackUpHead(ReadOnlySpan w, int oc, int c, ReadOnlySpan bias) + { + if (oc > 8) throw new NotSupportedException($"UpHead supports oc <= 8 (got {oc})."); + float[] pw = new float[c * 8], pb = new float[8]; + for (int o = 0; o < oc; o++) + for (int ch = 0; ch < c; ch++) pw[ch * 8 + o] = w[o * c + ch]; + if (!bias.IsEmpty) bias.CopyTo(pb); + return new UpHeadPlan { W = pw, Bias = pb, Oc = oc, C = c }; + } + + /// + /// out[y,x,o] = act(bias[o] + Σ_c w[o,c]·s[c]), s = [relu](resize(src)) + [relu](skip): the + /// full-resolution resize/add tensors live only in a per-pixel stack buffer. Per-element + /// math follows the unfused nodes (bias-initialized, channel-ascending accumulation). + /// + public static void UpHead(ReadOnlySpan src, int ih, int iw, int c, ReadOnlySpan skip, + Span dst, int oh, int ow, float sy, float sx, UpHeadP p, UpHeadPlan kp, + int skipH = 0, int skipW = 0, int skipC = 0) + { + // SkipConv: `skip` is the conv's input [skipH, skipW, skipC]; each task recomputes + // its skip row (same direct-conv kernel → same values) instead of reading a stored one + ConvP? sc = p.SkipConv; + var skp = sc?.KernelPlan as NhwcConvPlan; + if (kp.C != c || dst.Length != (long)oh * ow * kp.Oc + || (sc is null ? skip.Length != (long)oh * ow * c + : skp is null || skp.Depthwise || skp.WinoU is not null || sc.AddValue >= 0 || skip.Length != (long)skipH * skipW * skipC)) + throw new NotSupportedException("UpHead shape mismatch."); + int[] y0 = new int[oh], y1 = new int[oh], x0 = new int[ow], x1 = new int[ow]; + float[] wy = new float[oh], wx = new float[ow]; + ResizeAxis(oh, ih, sy, y0, y1, wy); + ResizeAxis(ow, iw, sx, x0, x1, wx); + bool relu = p.ResizeRelu, skipRelu = p.SkipRelu, avx = UseAvx; + int oc = kp.Oc; + bool neonUp = UseNeon && c % 4 == 0 && oc <= 4; + bool pkUp = UsePacked && c % 4 == 0 && oc <= 4; + ActivationKind act = p.Act; + int[] rm = [], cm = []; + float[] padVec = []; + if (sc is not null) + { + rm = Map(oh, sc.Kernel[0], sc.Strides[0], sc.Pads[0], skipH, sc.PadMode); + cm = Map(ow, sc.Kernel[1], sc.Strides[1], sc.Pads[1], skipW, sc.PadMode); + padVec = new float[skipC * sc.Kernel[1] + Ob]; // one constant-pad kernel row (Dense512Row) + Array.Fill(padVec, sc.PadValue); + } + fixed (float* sp0 = src, dp0 = dst, kp0 = skip, wp0 = kp.W, bp0 = kp.Bias) + fixed (float* cw = skp?.W, cb = skp?.Bias, pv = padVec) + fixed (int* rmp = rm, cmp = cm) + { + float* sp = sp0, dp = dp0, kpp = kp0, wp = wp0, bp = bp0; + var ca = sc is null ? default + : new DenseArgs(kp0, cw, cb, pv, rmp, cmp, skipW, skipC, ow, c, sc.Kernel[0], sc.Kernel[1], skp!.Blocks, sc.Act, avx); + if (UseAvx512) + { + UpHead512(sp, iw, c, kpp, sc is not null, ca, dp, oh, ow, y0, y1, x0, x1, wy, wx, relu, skipRelu, wp, bp, oc, act); + return; + } + For(oh, (long)oh * ow * c * (oc + 4), y => + { + float* s = stackalloc float[c + 8]; + float* acc = stackalloc float[8]; + float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c; + var vY = new Vector(wy[y]); + float wY = wy[y]; + float* orow = dp + (long)y * ow * oc; + float[]? band = sc is null ? null : BandBuf((long)ow * c); + fixed (float* sb = band) + { + float* krow = kpp + (long)y * ow * c; + if (sc is not null) + { + ConvRow(ca, y, sb, null); + krow = sb; + } + if (neonUp) + { + fixed (int* xa = x0, xb = x1) + fixed (float* wxp = wx) + UpHeadRowNeon(r0, r1, xa, xb, wxp, wY, krow, c, ow, wp, bp, orow, oc, relu, skipRelu, act); + return; + } + if (pkUp) + { + fixed (int* xa = x0, xb = x1) + fixed (float* wxp = wx) + UpHeadRowPk(r0, r1, xa, xb, wxp, wY, krow, c, ow, wp, bp, orow, oc, relu, skipRelu, act); + return; + } + for (int x = 0; x < ow; x++) + { + ResizePixel(r0 + (long)x0[x] * c, r0 + (long)x1[x] * c, r1 + (long)x0[x] * c, r1 + (long)x1[x] * c, + wx[x], wY, new Vector(wx[x]), vY, c, relu, krow + (long)x * c, skipRelu, s); + float* o = orow + (long)x * oc; + if (avx) + { + F32x8 a = Avx.LoadVector256(bp); + float* w = wp; + for (int ch = 0; ch < c; ch++, w += 8) + a = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s + ch), Avx.LoadVector256(w), a); + Avx.Store(acc, a); + } + else if (UseNeon) + { + V128 a = AdvSimd.LoadVector128(bp), b = AdvSimd.LoadVector128(bp + 4); + float* w = wp; + for (int ch = 0; ch < c; ch++, w += 8) + { + V128 v = AdvSimd.LoadAndReplicateToVector128(s + ch); + a = AdvSimd.FusedMultiplyAdd(a, AdvSimd.LoadVector128(w), v); + b = AdvSimd.FusedMultiplyAdd(b, AdvSimd.LoadVector128(w + 4), v); + } + AdvSimd.Store(acc, a); AdvSimd.Store(acc + 4, b); + } + else + { + for (int l = 0; l < 8; l++) acc[l] = bp[l]; + float* w = wp; + for (int ch = 0; ch < c; ch++, w += 8) + { + float v = s[ch]; + for (int l = 0; l < oc; l++) acc[l] += v * w[l]; + } + } + for (int l = 0; l < oc; l++) o[l] = acc[l]; + } + if (act != ActivationKind.None) ActRow(orow, ow * oc, act, avx); + } + }); + } + } + + + // out[y,x,c] = in[y+off_g, x, c] (logical axis 2) or in[y, x+off_g, c] (axis 3), zero-fill + public static void ChannelShift(ReadOnlySpan src, int h, int w, int c, + Span dst, ShiftP p) + { + int[] ends = p.GroupEnds, offs = p.Offsets; + bool alongH = p.Axis == 2; + fixed (float* sp0 = src, dp0 = dst) + { + float* sp = sp0, dp = dp0; + For(h, (long)h * w * c, y => + { + for (int x = 0; x < w; x++) + { + float* o = dp + ((long)y * w + x) * c; + int g0 = 0; + for (int g = 0; g < offs.Length; g++) + { + int g1 = ends[g]; + int sy = alongH ? y + offs[g] : y, sx = alongH ? x : x + offs[g]; + if ((uint)sy < (uint)h && (uint)sx < (uint)w) + Copy(sp + ((long)sy * w + sx) * c + g0, o + g0, g1 - g0); + else + Fill(o + g0, g1 - g0, 0f); + g0 = g1; + } + if (g0 < c) Copy(sp + ((long)y * w + x) * c + g0, o + g0, c - g0); + } + }); + } + } + // out[c,y,x] = in[c, y+dy_g] (axis=2) or in[c,y,x+dx_g] (axis=3), zero-fill + public static void ChannelShiftNchw(ReadOnlySpan src, int[] inShape, Span dst, + int[] outShape, ShiftP p) + { + int c = inShape[1], h = inShape[2], w = inShape[3]; + if (Avx2.IsSupported && Fma.IsSupported && c * h * w >= 8192) + { + ChannelShiftNchwAvx2(src, c, h, w, dst, p.Axis, p.Offsets, p.GroupEnds); + return; + } + int g0 = 0; + for (int g = 0; g < p.Offsets.Length; g++) + { + int g1 = p.GroupEnds[g]; + int off = p.Offsets[g]; + for (int ch = g0; ch < g1; ch++) + { + ReadOnlySpan s = src.Slice(ch * h * w, h * w); + Span d = dst.Slice(ch * h * w, h * w); + if (p.Axis == 2) + { + for (int y = 0; y < h; y++) + { + int sy = y + off; + if ((uint)sy < (uint)h) s.Slice(sy * w, w).CopyTo(d.Slice(y * w, w)); + else d.Slice(y * w, w).Clear(); + } + } + else if (Math.Abs(off) >= w) + { + d.Clear(); // shifted fully out of range -> all zeros + } + else + { + // out[c,y,x] = in[c,y,x+off], zero-fill — same convention as axis 2 + if (off >= 0) + for (int y = 0; y < h; y++) + { + ReadOnlySpan sr = s.Slice(y * w, w); + Span dr = d.Slice(y * w, w); + sr.Slice(off).CopyTo(dr.Slice(0, w - off)); + dr.Slice(w - off).Clear(); + } + else + for (int y = 0; y < h; y++) + { + ReadOnlySpan sr = s.Slice(y * w, w); + Span dr = d.Slice(y * w, w); + sr.Slice(0, w + off).CopyTo(dr.Slice(-off)); + dr.Slice(0, -off).Clear(); + } + } + } + g0 = g1; + } + } + private static void ChannelShiftNchwAvx2(ReadOnlySpan src, int c, int h, int w, + Span dst, int axis, int[] offsets, int[] groupEnds) + { + fixed (float* sp0 = src, dp0 = dst) + fixed (int* offp = offsets, endp = groupEnds) + { + float* sp = sp0, dp = dp0; + int* offa = offp, enda = endp; + int groups = offsets.Length; + Parallel.For(0, c * h, s_po, t => + { + int ch = t / h, y = t % h; + int off = 0; + for (int g = 0, g0 = 0; g < groups; g++) + { + int g1 = enda[g]; + if (ch >= g0 && ch < g1) { off = offa[g]; break; } + g0 = g1; + } + float* sr = sp + (ch * h + y) * w; + float* dr = dp + (ch * h + y) * w; + if (axis == 2) + { + int sy = y + off; + if ((uint)sy < (uint)h) + Buffer.MemoryCopy(sp + (ch * h + sy) * w, dr, w * 4L, w * 4L); + else + { + F32x8 z = F32x8.Zero; + int x = 0; + for (; x + 8 <= w; x += 8) Avx.Store(dr + x, z); + for (; x < w; x++) dr[x] = 0f; + } + } + else if (Math.Abs(off) >= w) + { + int x = 0; + for (; x + 8 <= w; x += 8) Avx.Store(dr + x, F32x8.Zero); + for (; x < w; x++) dr[x] = 0f; + } + else if (off >= 0) + { + // dr[x] = sr[x+off]; tail zero + Buffer.MemoryCopy(sr + off, dr, (w - off) * 4L, (w - off) * 4L); + int x = w - off; + for (; x < w; x++) dr[x] = 0f; + } + else + { + // dr[x] = sr[x+off] with off<0: head zero, then copy + int x = 0; + for (; x + 8 <= -off; x += 8) Avx.Store(dr + x, F32x8.Zero); + for (; x < -off; x++) dr[x] = 0f; + Buffer.MemoryCopy(sr, dr - off, (w + off) * 4L, (w + off) * 4L); + } + }); + } + }} diff --git a/src/Sdcb.SimdGcdrNet/Extensions/UpHeadOp.cs b/src/Sdcb.SimdGcdrNet/Extensions/UpHeadOp.cs new file mode 100644 index 0000000..d23536a --- /dev/null +++ b/src/Sdcb.SimdGcdrNet/Extensions/UpHeadOp.cs @@ -0,0 +1,279 @@ +using Sdcb.SimdOnnx.Backends.Metal; +using Sdcb.SimdOnnx.Backends.Vulkan; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Kernels; + +namespace Sdcb.SimdGcdrNet.Extensions; + +/// +/// Fused NHWC network tail (In = [x, skip, w, bias?]): +/// out[o] = act(bias[o] + Σ_c w[o,c]·s[c]) with s = [relu](resize(x)) + [relu](skip). +/// Replaces Resize → (Relu) → Add → 1x1 Conv → (act) so the full-resolution +/// intermediates are never stored. +/// +internal sealed class UpHeadP +{ + public required float[] Scales; + public bool ResizeRelu; + public bool SkipRelu; + public ActivationKind Act; + public object? KernelPlan; // backend-private packed weights + // In[1] is the input of this (still scheduled) conv and the skip is recomputed, + // so the full-resolution skip tensor need not stay live + public ConvP? SkipConv; + public int SkipW = -1, SkipB = -1; // the skip conv's weight / bias value ids + + public override string ToString() => $"rrelu={ResizeRelu} srelu={SkipRelu} act={Act}" + + (SkipConv is { } sc ? $" skip=conv{sc.Kernel[0]}x{sc.Kernel[1]}(In[1]) sact={sc.Act}" : ""); +} + +internal sealed class UpHeadOp : ExtOp +{ + public static readonly UpHeadOp Instance = new(); + public override string Name => "UpHead"; + + public override IEnumerable ParamRefs(ExecNode n) + { + var p = (UpHeadP)n.P!; + if (p.SkipW >= 0) yield return p.SkipW; + if (p.SkipB >= 0) yield return p.SkipB; + } + + public override void PlanLayout(LayoutPlanner lp, ExecNode n) + { + // activations channels-last, weights stay logical + if (lp.Values[n.In[0]].Shape.Length != 4 || lp.Values[n.In[1]].Shape.Length != 4) + throw new NotSupportedException($"layout: UpHead '{n.Name}' is not rank-4."); + lp.Need(n, 0, Layout.Nhwc); + lp.Need(n, 1, Layout.Nhwc); + lp.Values[n.Out[0]].Layout = Layout.Nhwc; + } + + public override void PackCpu(ExecPlan plan, ExecNode n, CpuKernels.Cache? cache) + { + var up = (UpHeadP)n.P!; + if (up.KernelPlan is not null) return; + CValue w = plan.Values[n.In[2]]; + float[]? b = n.In.Length > 3 ? plan.Values[n.In[3]].Floats : null; + object kp = CpuKernels.Get(cache, n, w.Floats!, b, () => GcdrKernels.PackUpHead(w.Floats!, w.Shape[0], w.Shape[1], b)); + Interlocked.CompareExchange(ref up.KernelPlan, kp, null); + } + + public override void RunCpu(InferenceSession s, ExecNode n) + { + var p = (UpHeadP)n.P!; + CValue[] v = s.Plan.Values; + int[] i = v[n.In[0]].PhysShape, o = v[n.Out[0]].PhysShape, k = v[n.In[1]].PhysShape; + GcdrKernels.UpHead(s.GetIn(n.In[0]), i[1], i[2], i[3], s.GetIn(n.In[1]), s.GetSpan(n.Out[0]), o[1], o[2], + p.Scales[2], p.Scales[3], p, (UpHeadPlan)p.KernelPlan!, k[1], k[2], k[3]); + } + + public override IEnumerable<(int value, ConstKind kind)> GpuConsts(ExecNode n) + { + var up = (UpHeadP)n.P!; + yield return (n.In[2], ConstKind.Raw); + if (n.In.Length > 3) yield return (n.In[3], ConstKind.F32); + if (up.SkipConv is not null) + { + yield return (up.SkipW, ConstKind.ConvW); + if (up.SkipB >= 0) yield return (up.SkipB, ConstKind.F32); + } + } + + private static (uint c, uint oc, uint flags) Common(CValue[] V, ExecNode n, string backend) + { + var p = (UpHeadP)n.P!; + int[] si = V[n.In[0]].PhysShape, so = V[n.Out[0]].PhysShape; + uint c = (uint)si[3], oc = (uint)so[3]; + if (oc > 8 || c > 128 || c % 4 != 0) throw new NotSupportedException($"{backend} UpHead: c={c} oc={oc}"); + uint act = p.Act switch + { + ActivationKind.Relu => 1u, ActivationKind.Sigmoid => 2u, ActivationKind.None => 0u, + _ => throw new NotSupportedException($"{backend} UpHead: {p.Act}"), + }; + uint flags = (p.ResizeRelu ? 1u : 0u) | (p.SkipRelu ? 2u : 0u) | (n.In.Length > 3 ? 4u : 0u) | (act << 3); + return (c, oc, flags); + } + + private static byte[] SkipPc(CValue[] V, ExecNode n, uint c, string backend) + { + var p = (UpHeadP)n.P!; + var sc = p.SkipConv!; + // skip recomputed from the stem conv's input (GraphCompiler.GpuSmallConv, c <= 32) + int[] xi = V[n.In[1]].PhysShape, cw = V[p.SkipW].Shape, so = V[n.Out[0]].PhysShape; + if (!GraphCompiler.GpuSmallConv(sc, cw, xi[3]) || cw[0] != c || c > 32 || xi[1] != so[1] || xi[2] != so[2]) + throw new NotSupportedException($"{backend} UpHead: unsupported skip conv"); + uint cf = (sc.BiasValue >= 0 ? 1u : 0u) | (sc.PadMode == PadMode.Reflect ? 2u : 0u) + | (sc.Act == ActivationKind.Relu ? 4u : 0u); + return VkGraphModel.PcOfF(VkGraphModel.U(xi[3], sc.Kernel[0], sc.Kernel[1], sc.Pads[0], sc.Pads[1], (int)cf), + sc.PadMode == PadMode.Constant ? sc.PadValue : 0f); + } + + public override IEnumerable EmitVulkan(VkGraphModel g, ExecNode n) + { + var p = (UpHeadP)n.P!; + CValue[] V = g.Plan.Values; + g.RequireNhwc(n.In[0]); g.RequireNhwc(n.In[1]); g.RequireNhwc(n.Out[0]); + int[] si = V[n.In[0]].PhysShape, so = V[n.Out[0]].PhysShape; + var (c, oc, flags) = Common(V, n, "Vulkan"); + byte[] head = VkGraphModel.PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], p.Scales[2], p.Scales[3]); + VkBind bias = n.In.Length > 3 ? g.Bind(n.In[3], ConstKind.F32) : g.BindOrDummy(-1, ConstKind.F32); + if (p.SkipConv is not null) + { + uint np = g.Device.ConvDot && g.F16 ? 2u : 1u; + yield return g.Dispatch(n, np > 1 ? $"uphead_sc{np}" : "uphead_sc", + [g.Bind(n.In[0]), g.Bind(n.In[1]), g.Bind(n.In[2]), bias, g.Bind(n.Out[0]), + g.Bind(p.SkipW, ConstKind.ConvW), g.BindOrDummy(p.SkipB, ConstKind.F32)], + [.. head, .. SkipPc(V, n, c, "Vulkan")], + VkGraphModel.G((uint)so[1] * (((uint)so[2] + np - 1) / np)), owner: GcdrExtensions.Owner); + yield break; + } + yield return g.Dispatch(n, "uphead", + [g.Bind(n.In[0]), g.Bind(n.In[1]), g.Bind(n.In[2]), bias, g.Bind(n.Out[0])], + head, VkGraphModel.G((uint)so[1] * (uint)so[2]), owner: GcdrExtensions.Owner); + } + + public override IEnumerable EmitMetal(MtlGraphModel g, ExecNode n) + { + var p = (UpHeadP)n.P!; + CValue[] V = g.Plan.Values; + g.RequireNhwc(n.In[0]); g.RequireNhwc(n.In[1]); g.RequireNhwc(n.Out[0]); + int[] si = V[n.In[0]].PhysShape, so = V[n.Out[0]].PhysShape; + var (c, oc, flags) = Common(V, n, "Metal"); + byte[] head = MtlGraphModel.PcOfF([c, (uint)si[1], (uint)si[2], (uint)so[1], (uint)so[2], oc, flags], p.Scales[2], p.Scales[3]); + MtlBind bias = n.In.Length > 3 ? g.Bind(n.In[3], ConstKind.F32) : g.BindOrDummy(-1, ConstKind.F32); + if (p.SkipConv is not null) + { + yield return g.Dispatch(n, "uphead_sc", + [g.Bind(n.In[0]), g.Bind(n.In[1]), g.Bind(n.In[2]), bias, g.Bind(n.Out[0]), + g.Bind(p.SkipW, ConstKind.ConvW), g.BindOrDummy(p.SkipB, ConstKind.F32)], + [.. head, .. SkipPc(V, n, c, "Metal")], + MtlGraphModel.G((uint)so[1] * (uint)so[2]), owner: GcdrExtensions.Owner); + yield break; + } + yield return g.Dispatch(n, "uphead", + [g.Bind(n.In[0]), g.Bind(n.In[1]), g.Bind(n.In[2]), bias, g.Bind(n.Out[0])], + head, MtlGraphModel.G((uint)so[1] * (uint)so[2]), owner: GcdrExtensions.Owner); + } +} + +/// +/// Resize → (Relu) → Add(·, [Relu](skip)) → 1x1 Conv (few outputs) → (Sigmoid|Relu) becomes one +/// node in NHWC plans — the network tail, where every intermediate is a +/// full-resolution tensor that would otherwise round-trip memory. The fused node takes the +/// position of the last node it replaces, so every input is produced before it. +/// +internal sealed class FuseUpHeadPass : GraphPass +{ + public override PassStage Stage => PassStage.Logical; + + public override void Run(PassContext ctx) + { + if (!ctx.Options.Nhwc || Env.Flag("NOFUSE_UP") == "1") return; + List nodes = ctx.Nodes; + List values = ctx.Values; + + // resolve add(resize(x), other): (resize node, skip value, skip relu node or null) + (ExecNode rs, int skip, ExecNode? relu)? MatchAdd(ExecNode add) + { + if (add.Op != ExecOp.Elementwise || add.In.Length != 2 || add.P is not EwP { Op: EwOp.Add, Act: ActivationKind.None }) + return null; + for (int k = 0; k < 2; k++) + { + int u = add.In[k], o = add.In[1 - k]; + int ri = ctx.Producer(u); + if (ri < 0 || nodes[ri].Op != ExecOp.ResizeBilinear || nodes[ri].In.Length != 1) continue; + if (ctx.Uses(u) != 1 || !ctx.SameShape(u, add.Out[0]) || !ctx.SameShape(o, add.Out[0]) || values[o].Buf < 0) continue; + int pi = ctx.Producer(o); + if (pi >= 0 && nodes[pi].Op == ExecOp.Relu && nodes[pi].In.Length == 1 && ctx.Uses(o) == 1 + && values[nodes[pi].In[0]].Buf >= 0 && ctx.SameShape(nodes[pi].In[0], o)) + return (nodes[ri], nodes[pi].In[0], nodes[pi]); + return (nodes[ri], o, null); + } + return null; + } + + for (int ci = 0; ci < nodes.Count; ci++) + { + ExecNode c = nodes[ci]; + if (c.Op != ExecOp.Conv || c.P is not ConvP cp || cp.Group != 1 || cp.Act != ActivationKind.None + || cp.AddValue >= 0 || cp.Kernel[0] != 1 || cp.Kernel[1] != 1 || cp.Strides.Any(s => s != 1) + || cp.Pads.Any(p => p != 0) || cp.Dilations.Any(d => d != 1)) continue; + CValue w = values[c.In[1]]; + // kernel limits (oc lanes, staged weights, vec4 channels) + if (w.Floats is null || w.Shape.Length != 4 || w.Shape[0] > 8 || w.Shape[1] > 128 || w.Shape[1] % 4 != 0) continue; + if (cp.BiasValue >= 0 && values[cp.BiasValue].Floats is null) continue; + int s = c.In[0]; + int ai = ctx.Producer(s); + if (ai < 0 || ctx.Uses(s) != 1 || values[s].Shape.Length != 4) continue; + ExecNode addNode = nodes[ai]; + if (MatchAdd(addNode) is not { } mt) continue; + var (rs, skip, relu) = mt; + + ExecNode? act = null; + int y = c.Out[0]; + if (ctx.Uses(y) == 1) + act = nodes.FirstOrDefault(m => m.In.Length == 1 && m.In[0] == y + && m.Op is ExecOp.Sigmoid or ExecOp.Relu && values[m.Out[0]].Buf >= 0); + var head = new ExecNode + { + Op = ExecOp.Ext, + Ext = UpHeadOp.Instance, + In = cp.BiasValue >= 0 ? [rs.In[0], skip, c.In[1], cp.BiasValue] : [rs.In[0], skip, c.In[1]], + Out = [act?.Out[0] ?? y], + P = new UpHeadP + { + Scales = ((ResizeP)rs.P!).Scales, + ResizeRelu = ((ResizeP)rs.P!).FuseRelu, + SkipRelu = relu is not null, + Act = act is null ? ActivationKind.None : act.Op == ExecOp.Sigmoid ? ActivationKind.Sigmoid : ActivationKind.Relu, + }, + SrcNode = c.SrcNode, + Name = c.Name + ".uphead", + }; + nodes[ci] = head; + nodes.Remove(rs); nodes.Remove(addNode); + if (relu is not null) nodes.Remove(relu); + if (act is not null) nodes.Remove(act); + ci = -1; // indices shifted; rescan (few nodes, cheap) + } + } +} + +/// +/// UpHead skip recompute (after layout planning): when the skip is a cheap direct conv of a +/// smaller input (stem conv: 3/6 → 16/32 channels), UpHead reads the conv's input and recomputes +/// the skip rows, so the skip need not stay live across the network. GPU plans only get it for the +/// small-ic convs their shaders stage (). +/// +internal sealed class UpHeadSkipRecomputePass : GraphPass +{ + public override PassStage Stage => PassStage.Physical; + + public override void Run(PassContext ctx) + { + List nodes = ctx.Nodes; + List values = ctx.Values; + foreach (ExecNode u in nodes) + { + if (u.P is not UpHeadP up || up.SkipConv is not null) continue; + int s = u.In[1], ci = ctx.Producer(s); + if (ci < 0 || nodes[ci].P is not ConvP cp || cp.Group != 1 || cp.AddValue >= 0 || cp.Pool is not null + || cp.Strides.Any(x => x != 1) || cp.Dilations.Any(x => x != 1)) continue; + int xin = nodes[ci].In[0]; + CValue w = values[nodes[ci].In[1]]; + if (w.Floats is null || w.Shape.Length != 4 || !ctx.IsNhwc4(s) || !ctx.IsNhwc4(xin)) continue; + int ic = values[xin].Shape[1], c = values[s].Shape[1]; + // cheap to recompute, much smaller to keep; the conv itself stays scheduled + // for its other reader(s) (and keeps the shared ConvP packed) + if (ic * 2 > c || Nhwc.WinogradEligible(cp, w.Shape, ic) || ctx.Uses(s) < 2) continue; + if (cp.BiasValue >= 0 && values[cp.BiasValue].Floats is null) continue; + bool sameHw = values[xin].Shape[2] == values[s].Shape[2] && values[xin].Shape[3] == values[s].Shape[3]; + if (ctx.Options.Gpu && (!GraphCompiler.GpuSmallConv(cp, w.Shape, ic) || c > 32 || !sameHw)) continue; + u.In[1] = xin; + up.SkipConv = cp; + up.SkipW = nodes[ci].In[1]; + up.SkipB = cp.BiasValue; + } + } +} diff --git a/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs b/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs index a4c12ae..a83518f 100644 --- a/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs +++ b/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs @@ -1,8 +1,11 @@ using System.Collections.Concurrent; using System.Runtime.InteropServices; -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdGcdrNet.Extensions; using Sdcb.SimdGcdrNet.Kernels; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdOnnx.Onnx; +using VkBackend = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend; +using MetalBackend = Sdcb.SimdOnnx.Backends.Metal.MetalBackend; namespace Sdcb.SimdGcdrNet; @@ -51,7 +54,7 @@ public sealed class GcdrEnhancer : IDisposable // change): one pool serves every size; idle workers beyond the first are kept // only while their device memory fits the budget // (3 GB; integrated GPUs: at most a quarter of the shared device-local heap) - private static long VkIdleBudget => (long)Backends.Vulkan.VkGcdrPipeline.BudgetFor(Backends.Vulkan.VkBackend.Default.Dev); + private static long VkIdleBudget => (long)Backends.Vulkan.VkGcdrPipeline.BudgetFor(VkBackend.Default.Dev); private SizePool? _vkPool; private SizePool? _mtlPool; private long _poolsCreated, _workersCreated, _poolsEvicted; @@ -95,7 +98,7 @@ public static GcdrEnhancer FromStreams(Stream gcnetOnnx, Stream drnetOnnx, // pipeline sessions keep I/O in plan layout: the glue kernels below read/write // channels-last directly, so the plan has no boundary relayouts - private static readonly CompileOptions CpuPlan = new(CompileOptions.DefaultNhwc, PhysicalIo: true); + private static readonly CompileOptions CpuPlan = new(CompileOptions.DefaultNhwc, PhysicalIo: true, Extensions: GcdrExtensions.Instance); private static bool CpuNhwc => CpuPlan.Nhwc; private static long Key(int h, int w) => (long)h * 65536 + w; @@ -143,7 +146,7 @@ private SizePool CpuPool() { if (_vkPool is { } p) return p; p = new SizePool(MaxPerSize, - () => { Interlocked.Increment(ref _workersCreated); return Backends.Vulkan.VkBackend.Default.NewWorker(_gcModel, _drModel); }, + () => { Interlocked.Increment(ref _workersCreated); return new Backends.Vulkan.VkGcdrPipeline(VkBackend.Default, _gcModel, _drModel); }, w => w.DeviceBytes, VkIdleBudget); if (Interlocked.CompareExchange(ref _vkPool, p, null) is { } won) { p.Dispose(); return won; } Interlocked.Increment(ref _poolsCreated); @@ -154,7 +157,7 @@ private SizePool CpuPool() { if (_mtlPool is { } p) return p; p = new SizePool(MaxPerSize, - () => { Interlocked.Increment(ref _workersCreated); return Backends.Metal.MetalBackend.Default.NewWorker(_gcModel, _drModel); }, + () => { Interlocked.Increment(ref _workersCreated); return new Backends.Metal.MtlGcdrPipeline(MetalBackend.Default, _gcModel, _drModel); }, w => w.DeviceBytes, (long)Backends.Metal.MtlGcdrPipeline.ArenaBudget); if (Interlocked.CompareExchange(ref _mtlPool, p, null) is { } won) { p.Dispose(); return won; } Interlocked.Increment(ref _poolsCreated); @@ -275,7 +278,7 @@ internal static (long gcOff, long total) StackArenas(ExecPlan gc, ExecPlan dr, l private static bool VulkanOk() { - try { _ = Sdcb.SimdGcdrNet.Backends.Vulkan.VkBackend.Default.Dev.DeviceName; return true; } + try { _ = VkBackend.Default.Dev.DeviceName; return true; } catch { return false; } } @@ -283,11 +286,11 @@ private static bool MetalOk() { try { - var dev = Backends.Metal.MetalBackend.Default.Dev; + var dev = MetalBackend.Default.Dev; // Apple7+ (M1+) for simdgroup matrix ops; Pipe() forces the embedded // MSL library compile now rather than mid-Enhance on macOS 13. Intel Macs can // claim Apple7 yet build the simdgroup-MMA conv with 0 threads/group (Pipe throws). - return dev.SupportsApple7 && dev.Pipe("bgr2nhwc") is not null && dev.Pipe("conv_mm_32_0") is not null; + return dev.SupportsApple7 && dev.Pipe("bgr2nhwc", GcdrExtensions.Owner) is not null && dev.Pipe("conv_mm_32_0") is not null; } catch { return false; } } diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Avx2Kernels.cs b/src/Sdcb.SimdGcdrNet/Kernels/Avx2Kernels.cs deleted file mode 100644 index c8615d5..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Avx2Kernels.cs +++ /dev/null @@ -1,970 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; - -using F32x8 = System.Runtime.Intrinsics.Vector256; -using V256 = System.Runtime.Intrinsics.Vector256; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// AVX2+FMA kernels: multi-threaded, register-tiled NCHW conv (OC4 x YB2), -/// deinterleaved stride-2 conv, depthwise, 1x1, maxpool, GEMM, LayerNorm. -/// Inputs are pinned for the duration of each call so Parallel.For lambdas -/// can capture raw pointers (spans are not capturable). Edge pixels go -/// through a scalar spill path so every pad mode stays exact. -/// -internal static unsafe partial class Avx2Kernels -{ - internal static readonly int Threads = InitThreads(); - private static int InitThreads() - { - string? s = Environment.GetEnvironmentVariable("GCDRNET_THREADS"); - return int.TryParse(s, out int t) && t > 0 ? t : Environment.ProcessorCount; - } - private static readonly ParallelOptions PO = new() { MaxDegreeOfParallelism = Threads }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 Fma(F32x8 a, F32x8 b, F32x8 c) => V256.FusedMultiplyAdd(a, b, c); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 Ld(float* p) => Avx.LoadVector256(p); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void St(float* p, F32x8 v) => Avx.Store(p, v); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 Bc(float* p) => Avx2.BroadcastScalarToVector256(p); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int Reflect(int i, int dim) - { - if (dim <= 1) return 0; - while (i < 0 || i >= dim) i = i < 0 ? -i : 2 * dim - 2 - i; - return i; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void ApplyAct(ref F32x8 v, ActivationKind act) - { - if (act == ActivationKind.Relu) v = V256.Max(v, F32x8.Zero); - else if (act == ActivationKind.Sigmoid) v = SimdMath.SigmoidCore(v); - else if (act == ActivationKind.Gelu) v = SimdMath.GeluCore(v); - } - - // lane-wise fetch with pad resolution (edge chunks only) - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 Spill8(float* sp, int ih, int iw, int sy, int sx, - PadMode mode, float padVal, float* spill) - { - for (int l = 0; l < 8; l++) - { - int px = sx + l; - spill[l] = (uint)px < (uint)iw ? sp[sy * iw + px] - : mode == PadMode.Reflect ? sp[sy * iw + Reflect(px, iw)] - : padVal; - } - return Ld(spill); - } - - private static void StoreRow(float* dp, int oStride, - int xw, ActivationKind act, F32x8 a0, F32x8 a1, F32x8 a2, F32x8 a3) - { - ApplyAct(ref a0, act); ApplyAct(ref a1, act); - ApplyAct(ref a2, act); ApplyAct(ref a3, act); - if (xw == 8) - { - St(dp, a0); - St(dp + oStride, a1); - St(dp + 2 * oStride, a2); - St(dp + 3 * oStride, a3); - return; - } - float* tmp = stackalloc float[32]; - St(tmp, a0); St(tmp + 8, a1); St(tmp + 16, a2); St(tmp + 24, a3); - for (int l = 0; l < xw; l++) - for (int j = 0; j < 4; j++) - dp[j * oStride + l] = tmp[j * 8 + l]; - } - - // ======================== dense conv 3x3 s1 ======================== - public static void Conv3x3S1(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, Span dst, - int oc, int oh, int ow, int pt, int pl, - PadMode mode, float padVal, ActivationKind act) - { - int yb = (oh + 1) / 2, ob = oc / 4; - bool hasBias = !bias.IsEmpty; - fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst) - { - float* sp = sp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, ob * yb, PO, tile => - { - Conv3x3Tile(sp, ic, ih, iw, wp, hasBias ? bp : null, dp, oc, oh, ow, - pt, pl, mode, padVal, act, tile / yb * 4, tile % yb * 2); - }); - } - } - - private static void Conv3x3Tile(float* sp, int ic, int ih, int iw, - float* wp, float* bp, float* dp, - int oc, int oh, int ow, int pt, int pl, - PadMode mode, float padVal, ActivationKind act, int o0, int y0) - { - bool two = y0 + 1 < oh; - int y1 = two ? y0 + 1 : y0; - int wStride = ic * 9; - int ostride = oh * ow; - float* spill = stackalloc float[8]; - - for (int x0 = 0; x0 < ow; x0 += 8) - { - int xw = Math.Min(8, ow - x0); - F32x8 a0, a1, a2, a3, b0, b1, b2, b3; - if (bp is null) - { - a0 = a1 = a2 = a3 = b0 = b1 = b2 = b3 = F32x8.Zero; - } - else - { - a0 = b0 = Bc(bp + o0); - a1 = b1 = Bc(bp + o0 + 1); - a2 = b2 = Bc(bp + o0 + 2); - a3 = b3 = Bc(bp + o0 + 3); - } - for (int c = 0; c < ic; c++) - { - float* sbase = sp + c * ih * iw; - float* wb = wp + (o0 * ic + c) * 9; - for (int ky = 0; ky < 3; ky++) - { - int syA = y0 + ky - pt, syB = y1 + ky - pt; - int ryA = (uint)syA < (uint)ih ? syA - : mode == PadMode.Reflect ? Reflect(syA, ih) : -1; - int ryB = !two ? -1 - : (uint)syB < (uint)ih ? syB - : mode == PadMode.Reflect ? Reflect(syB, ih) : -1; - for (int kx = 0; kx < 3; kx++) - { - int sx = x0 + kx - pl; - int wi = ky * 3 + kx; - F32x8 wv0 = Bc(wb + wi); - F32x8 wv1 = Bc(wb + wi + wStride); - F32x8 wv2 = Bc(wb + wi + 2 * wStride); - F32x8 wv3 = Bc(wb + wi + 3 * wStride); - bool interior = (uint)sx < (uint)iw && sx + 8 <= iw; - if (ryA >= 0) - { - F32x8 v = interior - ? Ld(sbase + ryA * iw + sx) - : Spill8(sbase, ih, iw, ryA, sx, mode, padVal, spill); - a0 = Fma(v, wv0, a0); a1 = Fma(v, wv1, a1); a2 = Fma(v, wv2, a2); a3 = Fma(v, wv3, a3); - } - else if (padVal != 0f) - { - F32x8 pv = V256.Create(padVal); - a0 = Fma(pv, wv0, a0); a1 = Fma(pv, wv1, a1); - a2 = Fma(pv, wv2, a2); a3 = Fma(pv, wv3, a3); - } - if (ryB >= 0) - { - F32x8 v = interior - ? Ld(sbase + ryB * iw + sx) - : Spill8(sbase, ih, iw, ryB, sx, mode, padVal, spill); - b0 = Fma(v, wv0, b0); b1 = Fma(v, wv1, b1); b2 = Fma(v, wv2, b2); b3 = Fma(v, wv3, b3); - } - else if (ryB < 0 && padVal != 0f) - { - F32x8 pv = V256.Create(padVal); - b0 = Fma(pv, wv0, b0); b1 = Fma(pv, wv1, b1); - b2 = Fma(pv, wv2, b2); b3 = Fma(pv, wv3, b3); - } - } - } - } - float* db = dp + (o0 * oh + y0) * ow + x0; - StoreRow(db, ostride, xw, act, a0, a1, a2, a3); - if (two) StoreRow(db + ow, ostride, xw, act, b0, b1, b2, b3); - } - } - - // ======================== conv 1x1 ======================== - public static void Conv1x1(ReadOnlySpan src, int ic, int hw, - ReadOnlySpan w, ReadOnlySpan bias, Span dst, - int oc, ActivationKind act) - { - int stripes = Math.Clamp(hw / 32768, 1, 8); - int stride = hw / stripes; - bool hasBias = !bias.IsEmpty; - fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst) - { - float* sp = sp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, ((oc + 3) / 4) * stripes, PO, tile => - { - int o0 = tile / stripes * 4; - int hw0 = tile % stripes * stride; - int hw1 = tile % stripes == stripes - 1 ? hw : hw0 + stride; - Conv1x1Strip(sp, ic, wp, hasBias ? bp : null, dp, - o0, Math.Min(4, oc - o0), hw0, hw1, hw, act); - }); - } - } - - private static void Conv1x1Strip(float* sp, int ic, - float* wp, float* bp, float* dp, - int o0, int ocLim, int hw0, int hw1, int hw, ActivationKind act) - { - int i = hw0; - for (; i + 8 <= hw1; i += 8) - { - F32x8 a0 = V256.Create(bp is null ? 0f : bp[o0]); - F32x8 a1 = V256.Create(ocLim > 1 && bp is not null ? bp[o0 + 1] : 0f); - F32x8 a2 = V256.Create(ocLim > 2 && bp is not null ? bp[o0 + 2] : 0f); - F32x8 a3 = V256.Create(ocLim > 3 && bp is not null ? bp[o0 + 3] : 0f); - for (int c = 0; c < ic; c++) - { - F32x8 v = Ld(sp + c * hw + i); - a0 = Fma(v, Bc(wp + o0 * ic + c), a0); - if (ocLim > 1) a1 = Fma(v, Bc(wp + (o0 + 1) * ic + c), a1); - if (ocLim > 2) a2 = Fma(v, Bc(wp + (o0 + 2) * ic + c), a2); - if (ocLim > 3) a3 = Fma(v, Bc(wp + (o0 + 3) * ic + c), a3); - } - float* db = dp + o0 * hw + i; - ApplyAct(ref a0, act); - St(db, a0); - if (ocLim > 1) { ApplyAct(ref a1, act); St(db + hw, a1); } - if (ocLim > 2) { ApplyAct(ref a2, act); St(db + 2 * hw, a2); } - if (ocLim > 3) { ApplyAct(ref a3, act); St(db + 3 * hw, a3); } - } - for (; i < hw1; i++) - for (int j = 0; j < ocLim; j++) - { - float acc = bp is null ? 0f : bp[o0 + j]; - for (int c = 0; c < ic; c++) - acc += sp[c * hw + i] * wp[(o0 + j) * ic + c]; - float r = act == ActivationKind.Relu ? MathF.Max(acc, 0f) - : act == ActivationKind.Sigmoid ? 1f / (1f + MathF.Exp(-acc)) : acc; - dp[(o0 + j) * hw + i] = r; - } - } - - // ======================== conv 3x3 s2 (patch embed) ======================== - private static readonly Vector256 Evens = Vector256.Create(0, 2, 4, 6, 0, 0, 0, 0); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 LoadStride2(float* p) - { - F32x8 a = Ld(p); - F32x8 b = Ld(p + 8); - var ea = Avx2.PermuteVar8x32(a.AsInt32(), Evens); - var eb = Avx2.PermuteVar8x32(b.AsInt32(), Evens); - return V256.Create(ea.GetLower().AsSingle(), eb.GetLower().AsSingle()); - } - - public static void Conv3x3S2(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, Span dst, - int oc, int oh, int ow, int pt, int pl, - PadMode mode, float padVal, ActivationKind act) - { - int yb = (oh + 1) / 2, ob = oc / 4; - bool hasBias = !bias.IsEmpty; - fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst) - { - float* sp = sp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, ob * yb, PO, tile => - { - ConvS2Tile(sp, ic, ih, iw, wp, hasBias ? bp : null, dp, oc, oh, ow, - pt, pl, mode, padVal, act, tile / yb * 4, tile % yb * 2); - }); - } - } - - private static void ConvS2Tile(float* sp, int ic, int ih, int iw, - float* wp, float* bp, float* dp, - int oc, int oh, int ow, int pt, int pl, - PadMode mode, float padVal, ActivationKind act, int o0, int y0) - { - bool two = y0 + 1 < oh; - int y1 = two ? y0 + 1 : y0; - int wStride = ic * 9; - int ostride = oh * ow; - float* spill = stackalloc float[16]; - - for (int x0 = 0; x0 < ow; x0 += 8) - { - int xw = Math.Min(8, ow - x0); - F32x8 a0, a1, a2, a3, b0, b1, b2, b3; - if (bp is null) - { - a0 = a1 = a2 = a3 = b0 = b1 = b2 = b3 = F32x8.Zero; - } - else - { - a0 = b0 = Bc(bp + o0); - a1 = b1 = Bc(bp + o0 + 1); - a2 = b2 = Bc(bp + o0 + 2); - a3 = b3 = Bc(bp + o0 + 3); - } - for (int c = 0; c < ic; c++) - { - float* sbase = sp + c * ih * iw; - float* wb = wp + (o0 * ic + c) * 9; - for (int ky = 0; ky < 3; ky++) - { - int syA = y0 * 2 + ky - pt, syB = y1 * 2 + ky - pt; - int ryA = (uint)syA < (uint)ih ? syA - : mode == PadMode.Reflect ? Reflect(syA, ih) : -1; - int ryB = !two ? -1 - : (uint)syB < (uint)ih ? syB - : mode == PadMode.Reflect ? Reflect(syB, ih) : -1; - for (int kx = 0; kx < 3; kx++) - { - int sx = x0 * 2 + kx - pl; - int wi = ky * 3 + kx; - F32x8 wv0 = Bc(wb + wi); - F32x8 wv1 = Bc(wb + wi + wStride); - F32x8 wv2 = Bc(wb + wi + 2 * wStride); - F32x8 wv3 = Bc(wb + wi + 3 * wStride); - bool interior = sx >= 0 && sx + 15 < iw; - if (ryA >= 0) - { - F32x8 v; - if (interior) - { - v = LoadStride2(sbase + ryA * iw + sx); - } - else - { - for (int l = 0; l < 8; l++) - { - int px = sx + 2 * l; - spill[l] = x0 + l < ow && (uint)px < (uint)iw - ? sbase[ryA * iw + px] - : x0 + l < ow && mode == PadMode.Reflect - ? sbase[ryA * iw + Reflect(px, iw)] - : padVal; - } - v = Ld(spill); - } - a0 = Fma(v, wv0, a0); a1 = Fma(v, wv1, a1); a2 = Fma(v, wv2, a2); a3 = Fma(v, wv3, a3); - } - else if (padVal != 0f) - { - F32x8 pv = V256.Create(padVal); - a0 = Fma(pv, wv0, a0); a1 = Fma(pv, wv1, a1); - a2 = Fma(pv, wv2, a2); a3 = Fma(pv, wv3, a3); - } - if (ryB >= 0) - { - F32x8 v; - if (interior) - { - v = LoadStride2(sbase + ryB * iw + sx); - } - else - { - for (int l = 0; l < 8; l++) - { - int px = sx + 2 * l; - spill[l] = x0 + l < ow && (uint)px < (uint)iw - ? sbase[ryB * iw + px] - : x0 + l < ow && mode == PadMode.Reflect - ? sbase[ryB * iw + Reflect(px, iw)] - : padVal; - } - v = Ld(spill); - } - b0 = Fma(v, wv0, b0); b1 = Fma(v, wv1, b1); b2 = Fma(v, wv2, b2); b3 = Fma(v, wv3, b3); - } - else if (ryB < 0 && padVal != 0f) - { - F32x8 pv = V256.Create(padVal); - b0 = Fma(pv, wv0, b0); b1 = Fma(pv, wv1, b1); - b2 = Fma(pv, wv2, b2); b3 = Fma(pv, wv3, b3); - } - } - } - } - float* db = dp + (o0 * oh + y0) * ow + x0; - StoreRow(db, ostride, xw, act, a0, a1, a2, a3); - if (two) StoreRow(db + ow, ostride, xw, act, b0, b1, b2, b3); - } - } - - // ======================== depthwise conv ======================== - public static void Depthwise(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, Span dst, - int oh, int ow, int kh, int kw, int pt, int pl, - PadMode mode, float padVal, ActivationKind act) - { - bool hasBias = !bias.IsEmpty; - fixed (float* sp0 = src, wp0 = w, bp0 = bias, dp0 = dst) - { - float* sp = sp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, ic * oh, PO, t => - { - int c = t / oh, y = t % oh; - float* sbase = sp + c * ih * iw; - float* drow = dp + (c * oh + y) * ow; - float* wk = wp + c * kh * kw; - float b = bp is null ? 0f : bp[c]; - float* spill = stackalloc float[16]; - for (int x0 = 0; x0 < ow; x0 += 8) - { - int xw = Math.Min(8, ow - x0); - F32x8 acc = V256.Create(b); - for (int ky = 0; ky < kh; ky++) - { - int sy = y + ky - pt; - int ry = (uint)sy < (uint)ih ? sy - : mode == PadMode.Reflect ? Reflect(sy, ih) : -1; - for (int kx = 0; kx < kw; kx++) - { - int sx = x0 + kx - pl; - float wv = wk[ky * kw + kx]; - F32x8 v; - if (ry >= 0 && sx >= 0 && sx + 8 <= iw) - { - v = Ld(sbase + ry * iw + sx); - } - else - { - for (int l = 0; l < 8; l++) - { - int px = sx + l; - spill[l] = x0 + l < ow && ry >= 0 && (uint)px < (uint)iw - ? sbase[ry * iw + px] - : x0 + l < ow && ry >= 0 && mode == PadMode.Reflect - ? sbase[ry * iw + Reflect(px, iw)] - : padVal; - } - v = Ld(spill); - } - acc = Fma(v, V256.Create(wv), acc); - } - } - ApplyAct(ref acc, act); - if (xw == 8) - { - St(drow + x0, acc); - } - else - { - St(spill, acc); - for (int l = 0; l < xw; l++) drow[x0 + l] = spill[l]; - } - } - }); - } - } - - // ======================== maxpool 2x2 s2 ======================== - public static void MaxPool2x2(ReadOnlySpan src, int c, int ih, int iw, - Span dst, int oh, int ow, bool fuseRelu) - { - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - Parallel.For(0, c * oh, PO, t => - { - int ch = t / oh, y = t % oh; - float* sbase = sp + ch * ih * iw; - float* drow = dp + (ch * oh + y) * ow; - int r0 = Math.Min(y * 2, ih - 1) * iw, r1 = Math.Min(y * 2 + 1, ih - 1) * iw; - float* spill = stackalloc float[16]; - for (int x0 = 0; x0 < ow; x0 += 8) - { - int xw = Math.Min(8, ow - x0); - int sx = x0 * 2; - F32x8 m; - if (sx + 15 < iw) - { - m = V256.Max(LoadStride2(sbase + r0 + sx), LoadStride2(sbase + r0 + sx + 1)); - m = V256.Max(m, V256.Max(LoadStride2(sbase + r1 + sx), LoadStride2(sbase + r1 + sx + 1))); - } - else - { - for (int l = 0; l < 8; l++) - { - if (x0 + l >= ow) { spill[l] = float.NegativeInfinity; continue; } - int xx = Math.Min(sx + 2 * l, iw - 2); - float mx = MathF.Max(sbase[r0 + xx], sbase[r0 + xx + 1]); - spill[l] = MathF.Max(mx, MathF.Max(sbase[r1 + xx], sbase[r1 + xx + 1])); - } - m = Ld(spill); - } - if (fuseRelu) m = V256.Max(m, F32x8.Zero); - if (xw == 8) - { - St(drow + x0, m); - } - else - { - St(spill, m); - for (int l = 0; l < xw; l++) drow[x0 + l] = spill[l]; - } - } - }); - } - } - - // ======================== GEMM (B row-major [K,N]) ======================== - public static void GemmKN(ReadOnlySpan a, int M, int K, - ReadOnlySpan b, int N, ReadOnlySpan bias, Span dst) - { - bool hasBias = !bias.IsEmpty; - fixed (float* ap0 = a, bp0 = b, cp0 = bias, dp0 = dst) - { - float* ap = ap0, bp = bp0, cp = cp0, dp = dp0; - Parallel.For(0, (M + 3) / 4, PO, t => - { - int m0 = t * 4; - GemmTile(ap, m0, Math.Min(4, M - m0), K, bp, N, hasBias ? cp : null, dp); - }); - } - } - - private static void GemmTile(float* ap, int m0, int mc, int K, - float* bp, int N, float* cp, float* dp) - { - int nv = N - N % 16; - for (int n0 = 0; n0 < nv; n0 += 16) - { - F32x8 c00, c01; - if (cp is null) - { - c00 = c01 = F32x8.Zero; - } - else - { - c00 = Ld(cp + n0); - c01 = Ld(cp + n0 + 8); - } - F32x8 c10 = c00, c11 = c01, c20 = c00, c21 = c01, c30 = c00, c31 = c01; - for (int k = 0; k < K; k++) - { - F32x8 bv0 = Ld(bp + k * N + n0); - F32x8 bv1 = Ld(bp + k * N + n0 + 8); - F32x8 av = Bc(ap + m0 * K + k); - c00 = Fma(av, bv0, c00); c01 = Fma(av, bv1, c01); - if (mc > 1) { av = Bc(ap + (m0 + 1) * K + k); c10 = Fma(av, bv0, c10); c11 = Fma(av, bv1, c11); } - if (mc > 2) { av = Bc(ap + (m0 + 2) * K + k); c20 = Fma(av, bv0, c20); c21 = Fma(av, bv1, c21); } - if (mc > 3) { av = Bc(ap + (m0 + 3) * K + k); c30 = Fma(av, bv0, c30); c31 = Fma(av, bv1, c31); } - } - float* db = dp + m0 * N + n0; - St(db, c00); St(db + 8, c01); - if (mc > 1) { St(db + N, c10); St(db + N + 8, c11); } - if (mc > 2) { St(db + 2 * N, c20); St(db + 2 * N + 8, c21); } - if (mc > 3) { St(db + 3 * N, c30); St(db + 3 * N + 8, c31); } - } - for (int n0 = nv; n0 < N; n0++) - for (int m = 0; m < mc; m++) - { - float s = cp is null ? 0f : cp[n0]; - for (int k = 0; k < K; k++) - s += ap[(m0 + m) * K + k] * bp[k * N + n0]; - dp[(m0 + m) * N + n0] = s; - } - } - - // ======================== LayerNorm ======================== - public static void LayerNorm(ReadOnlySpan x, int rows, int c, - ReadOnlySpan w, ReadOnlySpan b, Span dst, float eps) - { - fixed (float* xp0 = x, wp0 = w, bp0 = b, dp0 = dst) - { - float* xp = xp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, rows, PO, r => - { - float* xr = xp + r * c; - float* dr = dp + r * c; - F32x8 sum = F32x8.Zero, sq = F32x8.Zero; - int i = 0; - for (; i + 8 <= c; i += 8) - { - F32x8 v = Ld(xr + i); - sum = V256.Add(sum, v); - sq = Fma(v, v, sq); - } - float s = 0, ss = 0; - for (int l = 0; l < 8; l++) { s += sum[l]; ss += sq[l]; } - for (; i < c; i++) { float v = xr[i]; s += v; ss += v * v; } - float mean = s / c; - float inv = 1f / MathF.Sqrt(MathF.Max(ss / c - mean * mean, 0f) + eps); - F32x8 vm = V256.Create(mean), vi = V256.Create(inv); - for (i = 0; i + 8 <= c; i += 8) - { - F32x8 v = Ld(xr + i); - F32x8 wv = Ld(wp + i); - F32x8 bv = Ld(bp + i); - F32x8 t = V256.Subtract(v, vm); - St(dr + i, Fma(t, V256.Multiply(vi, wv), bv)); - } - for (; i < c; i++) dr[i] = (xr[i] - mean) * inv * wp[i] + bp[i]; - }); - } - } - - // ======================== parallel unary ======================== - // op: 0 relu, 1 sigmoid, 2 gelu, 3 erf - public static void Unary(ReadOnlySpan src, Span dst, int n, int op) - { - int chunks = Math.Clamp(n / 8192, 1, Threads * 4); - int stride = (n + chunks - 1) / chunks; - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - Parallel.For(0, chunks, PO, t => - { - int i0 = t * stride, i1 = Math.Min(n, i0 + stride); - int i = i0; - for (; i + 8 <= i1; i += 8) - { - F32x8 v = Ld(sp + i); - ApplyUnary(ref v, op); - St(dp + i, v); - } - for (; i < i1; i++) dp[i] = UnaryScalar(sp[i], op); - }); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void ApplyUnary(ref F32x8 v, int op) - { - if (op == 0) v = V256.Max(v, F32x8.Zero); - else if (op == 1) v = SimdMath.SigmoidCore(v); - else if (op == 2) v = SimdMath.GeluCore(v); - else v = SimdMath.ErfCore(v); - } - - private static float UnaryScalar(float x, int op) => op switch - { - 0 => MathF.Max(x, 0f), - 1 => 1f / (1f + MathF.Exp(-x)), - 2 => 0.5f * x * (1f + SimdMath.ErfScalar(x * 0.70710678118654752f)), - _ => SimdMath.ErfScalar(x), - }; - - // ======================== parallel binary elementwise ======================== - // op: 0 add, 1 sub, 2 mul, 3 div - public static void Ew2(ReadOnlySpan a, ReadOnlySpan b, - Span dst, int n, int op, ActivationKind act = ActivationKind.None) - { - int chunks = Math.Clamp(n / 8192, 1, Threads * 4); - int stride = (n + chunks - 1) / chunks; - fixed (float* ap0 = a, bp0 = b, dp0 = dst) - { - float* ap = ap0, bp = bp0, dp = dp0; - Parallel.For(0, chunks, PO, t => - { - int i0 = t * stride, i1 = Math.Min(n, i0 + stride); - int i = i0; - for (; i + 8 <= i1; i += 8) - { - F32x8 v = EwCore(Ld(ap + i), Ld(bp + i), op); - ApplyAct(ref v, act); - St(dp + i, v); - } - for (; i < i1; i++) - { - float v = EwScalar(ap[i], bp[i], op); - dp[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - }); - } - } - - // scalar broadcast: dst[i] = src[i] op s (rev: dst[i] = s op src[i]) - public static void Ew1(ReadOnlySpan src, Span dst, - int n, float s, int op, bool rev = false, ActivationKind act = ActivationKind.None) - { - int chunks = Math.Clamp(n / 8192, 1, Threads * 4); - int stride = (n + chunks - 1) / chunks; - fixed (float* ap0 = src, dp0 = dst) - { - float* ap = ap0, dp = dp0; - F32x8 vs = V256.Create(s); - Parallel.For(0, chunks, PO, t => - { - int i0 = t * stride, i1 = Math.Min(n, i0 + stride); - int i = i0; - for (; i + 8 <= i1; i += 8) - { - F32x8 v = Ld(ap + i); - v = rev ? EwCore(vs, v, op) : EwCore(v, vs, op); - ApplyAct(ref v, act); - St(dp + i, v); - } - for (; i < i1; i++) - { - float v = rev ? EwScalar(s, ap[i], op) : EwScalar(ap[i], s, op); - dp[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - }); - } - } - - // broadcast over a contiguous inner block: b[bo] constant per outer row - public static void EwInner(ReadOnlySpan a, ReadOnlySpan b, - int[] bStr, int[] outShape, int[] outStr, int rank, int inner, - Span dst, int n, int op, ActivationKind act = ActivationKind.None) - { - int outer = n / inner; - fixed (float* ap0 = a, bp0 = b, dp0 = dst) - { - float* ap = ap0, bp = bp0, dp = dp0; - Parallel.For(0, outer, PO, e => - { - int rem = e, bo = 0; - for (int d = 0; d < rank - 1; d++) - { - int cc = rem / (outStr[d] / inner); - rem %= outStr[d] / inner; - bo += cc * bStr[d]; - } - float s = bp[bo]; - F32x8 vs = V256.Create(s); - float* ar = ap + e * inner, dr = dp + e * inner; - int i = 0; - for (; i + 8 <= inner; i += 8) - { - F32x8 v = EwCore(Ld(ar + i), vs, op); - ApplyAct(ref v, act); - St(dr + i, v); - } - for (; i < inner; i++) - { - float v = EwScalar(ar[i], s, op); - dr[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - }); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 EwCore(F32x8 a, F32x8 b, int op) => op switch - { - 0 => V256.Add(a, b), - 1 => V256.Subtract(a, b), - 2 => V256.Multiply(a, b), - _ => V256.Divide(a, b), - }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static float EwScalar(float a, float b, int op) => op switch - { - 0 => a + b, 1 => a - b, 2 => a * b, _ => a / b, - }; - - // ======================== bilinear resize (half-pixel) ======================== - public static void Bilinear(ReadOnlySpan src, int c, int ih, int iw, - Span dst, int oh, int ow, - int[] y0, int[] y1, float[] wy, int[] x0, int[] x1, float[] wx, - ActivationKind act = ActivationKind.None) - { - fixed (float* sp0 = src, dp0 = dst) - fixed (int* y0p = y0, y1p = y1, x0p = x0, x1p = x1) - fixed (float* wyp = wy, wxp = wx) - { - float* sp = sp0, dp = dp0; - int* y0a = y0p, y1a = y1p, x0a = x0p, x1a = x1p; - float* wya = wyp, wxa = wxp; - Parallel.For(0, c * oh, PO, t => - { - int ch = t / oh, y = t % oh; - float* r0 = sp + (ch * ih + y0a[y]) * iw; - float* r1 = sp + (ch * ih + y1a[y]) * iw; - float* dr = dp + (ch * oh + y) * ow; - F32x8 wyv = V256.Create(wya[y]); - int x = 0; - for (; x + 8 <= ow; x += 8) - { - var i0 = Avx.LoadVector256(x0a + x); - var i1 = Avx.LoadVector256(x1a + x); - F32x8 wxv = Ld(wxa + x); - F32x8 t0 = Avx2.GatherVector256(r0, i0, 4); - F32x8 t1 = Avx2.GatherVector256(r0, i1, 4); - F32x8 b0 = Avx2.GatherVector256(r1, i0, 4); - F32x8 b1 = Avx2.GatherVector256(r1, i1, 4); - F32x8 top = Fma(V256.Subtract(t1, t0), wxv, t0); - F32x8 bot = Fma(V256.Subtract(b1, b0), wxv, b0); - F32x8 v = Fma(V256.Subtract(bot, top), wyv, top); - ApplyAct(ref v, act); - St(dr + x, v); - } - for (; x < ow; x++) - { - float t0 = r0[x0a[x]], t1v = r0[x1a[x]]; - float b0 = r1[x0a[x]], b1v = r1[x1a[x]]; - float top = t0 + (t1v - t0) * wxa[x]; - float bot = b0 + (b1v - b0) * wxa[x]; - float v = top + (bot - top) * wya[y]; - dr[x] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - }); - } - } - - // ======================== tiled transpose [a,b] -> [b,a] ======================== - public static void Transpose2D(ReadOnlySpan src, int a, int b, - Span dst, int copies = 1, int srcStride = 0, int dstStride = 0) - { - if (srcStride == 0) srcStride = a * b; - if (dstStride == 0) dstStride = a * b; - int ya = (a + 7) / 8, xb = (b + 7) / 8; - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - Parallel.For(0, copies * ya, PO, t => - { - int o = t / ya, ty = t % ya; - float* sb = sp + o * srcStride, db = dp + o * dstStride; - for (int tx = 0; tx < xb; tx++) - { - int y0 = ty * 8, x0 = tx * 8; - if (y0 + 8 <= a && x0 + 8 <= b) - { - T8(sb + y0 * b + x0, b, db + x0 * a + y0, a); - } - else - { - int ye = Math.Min(y0 + 8, a), xe = Math.Min(x0 + 8, b); - for (int y = y0; y < ye; y++) - for (int x = x0; x < xe; x++) - db[x * a + y] = sb[y * b + x]; - } - } - }); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void T8(float* s, int ls, float* d, int ld) - { - F32x8 r0 = Ld(s), r1 = Ld(s + ls), r2 = Ld(s + 2 * ls), r3 = Ld(s + 3 * ls); - F32x8 r4 = Ld(s + 4 * ls), r5 = Ld(s + 5 * ls), r6 = Ld(s + 6 * ls), r7 = Ld(s + 7 * ls); - F32x8 t0 = Avx.UnpackLow(r0, r1), t1 = Avx.UnpackHigh(r0, r1); - F32x8 t2 = Avx.UnpackLow(r2, r3), t3 = Avx.UnpackHigh(r2, r3); - F32x8 t4 = Avx.UnpackLow(r4, r5), t5 = Avx.UnpackHigh(r4, r5); - F32x8 t6 = Avx.UnpackLow(r6, r7), t7 = Avx.UnpackHigh(r6, r7); - const byte LO = 0x44, HI = 0xEE; - F32x8 u0 = Avx.Shuffle(t0, t2, LO), u1 = Avx.Shuffle(t0, t2, HI); - F32x8 u2 = Avx.Shuffle(t1, t3, LO), u3 = Avx.Shuffle(t1, t3, HI); - F32x8 u4 = Avx.Shuffle(t4, t6, LO), u5 = Avx.Shuffle(t4, t6, HI); - F32x8 u6 = Avx.Shuffle(t5, t7, LO), u7 = Avx.Shuffle(t5, t7, HI); - const byte L = 0x20, H = 0x31; - St(d, Avx.Permute2x128(u0, u4, L)); - St(d + ld, Avx.Permute2x128(u1, u5, L)); - St(d + 2 * ld, Avx.Permute2x128(u2, u6, L)); - St(d + 3 * ld, Avx.Permute2x128(u3, u7, L)); - St(d + 4 * ld, Avx.Permute2x128(u0, u4, H)); - St(d + 5 * ld, Avx.Permute2x128(u1, u5, H)); - St(d + 6 * ld, Avx.Permute2x128(u2, u6, H)); - St(d + 7 * ld, Avx.Permute2x128(u3, u7, H)); - } - - // ======================== pad ======================== - public static void Pad4(ReadOnlySpan src, int nc, int ih, int iw, - Span dst, int oh, int ow, int[] mapY, int[] mapX, float padVal) - { - // contiguous interior run of mapX (identity-mapped region) - int xl = 0; while (xl < ow && mapX[xl] < 0) xl++; - int xr = ow; while (xr > xl && mapX[xr - 1] < 0) xr--; - bool interior = true; - for (int x = xl; x < xr; x++) if (mapX[x] != x - xl + mapX[xl]) { interior = false; break; } - int srcX0 = xl < xr ? mapX[xl] : 0; - fixed (float* sp0 = src, dp0 = dst) - fixed (int* myp = mapY, mxp = mapX) - { - float* sp = sp0, dp = dp0; - int* mya = myp, mxa = mxp; - Parallel.For(0, nc * oh, PO, t => - { - int ch = t / oh, y = t % oh; - float* dr = dp + (ch * oh + y) * ow; - int my = mya[y]; - if (my < 0) - { - F32x8 pv = V256.Create(padVal); - int x = 0; - for (; x + 8 <= ow; x += 8) St(dr + x, pv); - for (; x < ow; x++) dr[x] = padVal; - return; - } - float* sr = sp + (ch * ih + my) * iw; - for (int x = 0; x < xl; x++) dr[x] = mxa[x] < 0 ? padVal : sr[mxa[x]]; - for (int x = xr; x < ow; x++) dr[x] = mxa[x] < 0 ? padVal : sr[mxa[x]]; - if (interior) - { - float* s = sr + srcX0; - int i = 0, len = xr - xl; - for (; i + 8 <= len; i += 8) St(dr + xl + i, Ld(s + i)); - for (; i < len; i++) dr[xl + i] = s[i]; - } - else - { - for (int x = xl; x < xr; x++) dr[x] = sr[mxa[x]]; - } - }); - } - } - - // ======================== channel shift ======================== - public static void ChannelShift(ReadOnlySpan src, int c, int h, int w, - Span dst, int axis, int[] offsets, int[] groupEnds) - { - fixed (float* sp0 = src, dp0 = dst) - fixed (int* offp = offsets, endp = groupEnds) - { - float* sp = sp0, dp = dp0; - int* offa = offp, enda = endp; - int groups = offsets.Length; - Parallel.For(0, c * h, PO, t => - { - int ch = t / h, y = t % h; - int off = 0; - for (int g = 0, g0 = 0; g < groups; g++) - { - int g1 = enda[g]; - if (ch >= g0 && ch < g1) { off = offa[g]; break; } - g0 = g1; - } - float* sr = sp + (ch * h + y) * w; - float* dr = dp + (ch * h + y) * w; - if (axis == 2) - { - int sy = y + off; - if ((uint)sy < (uint)h) - Buffer.MemoryCopy(sp + (ch * h + sy) * w, dr, w * 4L, w * 4L); - else - { - F32x8 z = F32x8.Zero; - int x = 0; - for (; x + 8 <= w; x += 8) St(dr + x, z); - for (; x < w; x++) dr[x] = 0f; - } - } - else if (Math.Abs(off) >= w) - { - int x = 0; - for (; x + 8 <= w; x += 8) St(dr + x, F32x8.Zero); - for (; x < w; x++) dr[x] = 0f; - } - else if (off >= 0) - { - // dr[x] = sr[x+off]; tail zero - Buffer.MemoryCopy(sr + off, dr, (w - off) * 4L, (w - off) * 4L); - int x = w - off; - for (; x < w; x++) dr[x] = 0f; - } - else - { - // dr[x] = sr[x+off] with off<0: head zero, then copy - int x = 0; - for (; x + 8 <= -off; x += 8) St(dr + x, F32x8.Zero); - for (; x < -off; x++) dr[x] = 0f; - Buffer.MemoryCopy(sr, dr - off, (w + off) * 4L, (w + off) * 4L); - } - }); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/NeonKernels.cs b/src/Sdcb.SimdGcdrNet/Kernels/NeonKernels.cs deleted file mode 100644 index c578885..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/NeonKernels.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; - -using V4 = System.Runtime.Intrinsics.Vector128; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// AArch64 AdvSimd (NEON) versions of the non-conv ops that have AVX2 kernels. -internal static unsafe class NeonKernels -{ - private static readonly ParallelOptions PO = new() { MaxDegreeOfParallelism = Avx2Kernels.Threads }; - - /// Row-wise LayerNorm over the last axis (c); rows split across threads in groups. - public static void LayerNorm(ReadOnlySpan x, int rows, int c, - ReadOnlySpan w, ReadOnlySpan b, Span dst, float eps) - { - int per = Math.Max(1, 16384 / c), tasks = (rows + per - 1) / per; - fixed (float* xp0 = x, wp0 = w, bp0 = b, dp0 = dst) - { - float* xp = xp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, tasks, PO, t => - { - for (int r = t * per, re = Math.Min(rows, r + per); r < re; r++) - LayerNormRow(xp + (long)r * c, dp + (long)r * c, c, wp, bp, eps); - }); - } - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void LayerNormRow(float* xr, float* dr, int c, float* wp, float* bp, float eps) - { - V4 sum = V4.Zero, sq = V4.Zero; - int i = 0; - for (; i + 4 <= c; i += 4) - { - V4 v = AdvSimd.LoadVector128(xr + i); - sum = AdvSimd.Add(sum, v); - sq = AdvSimd.FusedMultiplyAdd(sq, v, v); - } - float s = Vector128.Sum(sum), ss = Vector128.Sum(sq); - for (; i < c; i++) { float v = xr[i]; s += v; ss += v * v; } - float mean = s / c; - float inv = 1f / MathF.Sqrt(MathF.Max(ss / c - mean * mean, 0f) + eps); - V4 vm = Vector128.Create(mean), vi = Vector128.Create(inv); - for (i = 0; i + 4 <= c; i += 4) - { - V4 t = AdvSimd.Subtract(AdvSimd.LoadVector128(xr + i), vm); - AdvSimd.Store(dr + i, AdvSimd.FusedMultiplyAdd(AdvSimd.LoadVector128(bp + i), t, - AdvSimd.Multiply(vi, AdvSimd.LoadVector128(wp + i)))); - } - for (; i < c; i++) dr[i] = (xr[i] - mean) * inv * wp[i] + bp[i]; - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Avx512.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Avx512.cs deleted file mode 100644 index c379e04..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Avx512.cs +++ /dev/null @@ -1,392 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; - -using F32x8 = System.Runtime.Intrinsics.Vector256; -using F32x16 = System.Runtime.Intrinsics.Vector512; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// AVX-512 tier of the NHWC conv / Gemm micro-kernel: pixels × 32 oc -/// (two packed 16-oc blocks, one zmm each) per pass, so every broadcast feeds two FMAs and -/// the 32 zmm registers hold the whole accumulator tile. Per output element the reduction -/// is the AVX2 one (bias-init, taps then channels ascending, one FMA each) and the -/// epilogue reuses 's math, so results are bit-identical to AVX2. -/// The k loops store to a scratch tile and never call out (calls make RyuJIT spill the -/// accumulators); the residual/activation epilogue runs afterwards. -/// -internal static unsafe partial class Nhwc -{ - private const int Tile5 = 10; - - /// AVX-512 tier switch: Avx512F.IsSupported, off with GCDRNET_AVX512=0 or the - /// .NET 8/9-style DOTNET_EnableAVX512F=0 (the .NET 10 JIT only honours DOTNET_EnableAVX512). - /// Mutable for in-process A/B tests. - internal static bool UseAvx512 = Avx512F.IsSupported - && Environment.GetEnvironmentVariable("GCDRNET_AVX512") != "0" - && Environment.GetEnvironmentVariable("DOTNET_EnableAVX512F") != "0"; - - /// Tile5 rows × 32 oc: dst[r*ds + 0..15] ← block wb, dst[r*ds + 16..31] ← block wb + wStep. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void Tile512x32(float** ptrs, int taps, int ic, float* wb, long wStep, float* bb, float* dst, int ds) - { - F32x16 a0 = Avx512F.LoadVector512(bb), b0 = Avx512F.LoadVector512(bb + Ob); - F32x16 a1 = a0, a2 = a0, a3 = a0, a4 = a0, a5 = a0, a6 = a0, a7 = a0, a8 = a0, a9 = a0; - F32x16 b1 = b0, b2 = b0, b3 = b0, b4 = b0, b5 = b0, b6 = b0, b7 = b0, b8 = b0, b9 = b0; - for (int tap = 0; tap < taps; tap++) - { - float** pp = ptrs + tap * Tile5; - float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3], p4 = pp[4], p5 = pp[5], p6 = pp[6], p7 = pp[7], p8 = pp[8], p9 = pp[9]; - float* w = wb + (long)tap * ic * Ob; - float* x = w + wStep; - for (int c = 0; c < ic; c++, w += Ob, x += Ob) - { - F32x16 wa = Avx512F.LoadVector512(w), wx = Avx512F.LoadVector512(x); - F32x16 v = Vector512.Create(p0[c]); - a0 = Avx512F.FusedMultiplyAdd(v, wa, a0); b0 = Avx512F.FusedMultiplyAdd(v, wx, b0); - v = Vector512.Create(p1[c]); - a1 = Avx512F.FusedMultiplyAdd(v, wa, a1); b1 = Avx512F.FusedMultiplyAdd(v, wx, b1); - v = Vector512.Create(p2[c]); - a2 = Avx512F.FusedMultiplyAdd(v, wa, a2); b2 = Avx512F.FusedMultiplyAdd(v, wx, b2); - v = Vector512.Create(p3[c]); - a3 = Avx512F.FusedMultiplyAdd(v, wa, a3); b3 = Avx512F.FusedMultiplyAdd(v, wx, b3); - v = Vector512.Create(p4[c]); - a4 = Avx512F.FusedMultiplyAdd(v, wa, a4); b4 = Avx512F.FusedMultiplyAdd(v, wx, b4); - v = Vector512.Create(p5[c]); - a5 = Avx512F.FusedMultiplyAdd(v, wa, a5); b5 = Avx512F.FusedMultiplyAdd(v, wx, b5); - v = Vector512.Create(p6[c]); - a6 = Avx512F.FusedMultiplyAdd(v, wa, a6); b6 = Avx512F.FusedMultiplyAdd(v, wx, b6); - v = Vector512.Create(p7[c]); - a7 = Avx512F.FusedMultiplyAdd(v, wa, a7); b7 = Avx512F.FusedMultiplyAdd(v, wx, b7); - v = Vector512.Create(p8[c]); - a8 = Avx512F.FusedMultiplyAdd(v, wa, a8); b8 = Avx512F.FusedMultiplyAdd(v, wx, b8); - v = Vector512.Create(p9[c]); - a9 = Avx512F.FusedMultiplyAdd(v, wa, a9); b9 = Avx512F.FusedMultiplyAdd(v, wx, b9); - } - } - Avx512F.Store(dst, a0); Avx512F.Store(dst + Ob, b0); dst += ds; - Avx512F.Store(dst, a1); Avx512F.Store(dst + Ob, b1); dst += ds; - Avx512F.Store(dst, a2); Avx512F.Store(dst + Ob, b2); dst += ds; - Avx512F.Store(dst, a3); Avx512F.Store(dst + Ob, b3); dst += ds; - Avx512F.Store(dst, a4); Avx512F.Store(dst + Ob, b4); dst += ds; - Avx512F.Store(dst, a5); Avx512F.Store(dst + Ob, b5); dst += ds; - Avx512F.Store(dst, a6); Avx512F.Store(dst + Ob, b6); dst += ds; - Avx512F.Store(dst, a7); Avx512F.Store(dst + Ob, b7); dst += ds; - Avx512F.Store(dst, a8); Avx512F.Store(dst + Ob, b8); dst += ds; - Avx512F.Store(dst, a9); Avx512F.Store(dst + Ob, b9); - } - - /// Tile5 rows × 16 oc (odd block count tail): dst[r*ds + 0..15]. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void Tile512x16(float** ptrs, int taps, int ic, float* wb, float* bb, float* dst, int ds) - { - F32x16 a0 = Avx512F.LoadVector512(bb); - F32x16 a1 = a0, a2 = a0, a3 = a0, a4 = a0, a5 = a0, a6 = a0, a7 = a0, a8 = a0, a9 = a0; - for (int tap = 0; tap < taps; tap++) - { - float** pp = ptrs + tap * Tile5; - float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3], p4 = pp[4], p5 = pp[5], p6 = pp[6], p7 = pp[7], p8 = pp[8], p9 = pp[9]; - float* w = wb + (long)tap * ic * Ob; - for (int c = 0; c < ic; c++, w += Ob) - { - F32x16 wa = Avx512F.LoadVector512(w); - a0 = Avx512F.FusedMultiplyAdd(Vector512.Create(p0[c]), wa, a0); - a1 = Avx512F.FusedMultiplyAdd(Vector512.Create(p1[c]), wa, a1); - a2 = Avx512F.FusedMultiplyAdd(Vector512.Create(p2[c]), wa, a2); - a3 = Avx512F.FusedMultiplyAdd(Vector512.Create(p3[c]), wa, a3); - a4 = Avx512F.FusedMultiplyAdd(Vector512.Create(p4[c]), wa, a4); - a5 = Avx512F.FusedMultiplyAdd(Vector512.Create(p5[c]), wa, a5); - a6 = Avx512F.FusedMultiplyAdd(Vector512.Create(p6[c]), wa, a6); - a7 = Avx512F.FusedMultiplyAdd(Vector512.Create(p7[c]), wa, a7); - a8 = Avx512F.FusedMultiplyAdd(Vector512.Create(p8[c]), wa, a8); - a9 = Avx512F.FusedMultiplyAdd(Vector512.Create(p9[c]), wa, a9); - } - } - Avx512F.Store(dst, a0); dst += ds; - Avx512F.Store(dst, a1); dst += ds; - Avx512F.Store(dst, a2); dst += ds; - Avx512F.Store(dst, a3); dst += ds; - Avx512F.Store(dst, a4); dst += ds; - Avx512F.Store(dst, a5); dst += ds; - Avx512F.Store(dst, a6); dst += ds; - Avx512F.Store(dst, a7); dst += ds; - Avx512F.Store(dst, a8); dst += ds; - Avx512F.Store(dst, a9); - } - - /// Runs the 32- or 16-wide kernel for one tile into ([Tile5][32]). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Tile512(float** ptrs, int nblk, int taps, int ic, float* wb, long wStep, float* bb, float* part) - { - if (nblk == 2) Tile512x32(ptrs, taps, ic, wb, wStep, bb, part, 2 * Ob); - else Tile512x16(ptrs, taps, ic, wb, bb, part, 2 * Ob); - } - - /// Epilogue of one tile: part ([Tile5][32]) → rows × (lanes0 + lanes1) outputs, + residual, act. - [MethodImpl(MethodImplOptions.NoInlining)] - private static void Epilogue512(float* part, int rows, int nblk, float* outp, float* addp, int oc, - int lanes0, int lanes1, ActivationKind act) - { - bool simple = act is ActivationKind.None or ActivationKind.Relu; - for (int r = 0; r < rows; r++, part += 2 * Ob) - { - float* o = outp + (long)r * oc; - float* ad = addp == null ? null : addp + (long)r * oc; - for (int h = 0; h < nblk; h++) - { - int lanes = h == 0 ? lanes0 : lanes1; - float* s = part + h * Ob, oh = o + h * Ob, ah = ad == null ? null : ad + h * Ob; - if (lanes == Ob && simple) - { - F32x16 v = Avx512F.LoadVector512(s); - if (ah != null) v = Avx512F.Add(v, Avx512F.LoadVector512(ah)); - if (act == ActivationKind.Relu) v = Avx512F.Max(v, F32x16.Zero); - Avx512F.Store(oh, v); - } - else - Store16(oh, ah, Avx.LoadVector256(s), Avx.LoadVector256(s + 8), lanes, act); - } - } - } - - private static void Dense512(ReadOnlySpan src, int iw, int ic, Span dst, - int oh, int ow, int oc, int kh, int kw, int[] rm, int[] cm, float[] padVec, - NhwcConvPlan kp, ReadOnlySpan add, ActivationKind act) - { - int taps = kh * kw, nb = kp.Blocks, np = (nb + 1) / 2; - bool hasAdd = !add.IsEmpty; - fixed (float* sp0 = src, dp0 = dst, ap0 = add, wp0 = kp.W, bp0 = kp.Bias, pv0 = padVec) - fixed (int* rm0 = rm, cm0 = cm) - { - var a = new DenseArgs(sp0, wp0, bp0, pv0, rm0, cm0, iw, ic, ow, oc, kh, kw, nb, act, true); - float* dp = dp0, ap = ap0; - For(oh * np, (long)oh * ow * oc * taps * ic / 64, t => - { - int y = t / np, blk = t % np * 2; - long o = (long)y * ow * oc; - Dense512Row(a, y, blk, dp + o, hasAdd ? ap + o : null); - }); - } - } - - /// All oc blocks of direct-conv output row (the row-level entry of - /// the fused paths: conv+pool bands, UpHead skip recompute). AVX-512 tier: block pairs through - /// (needs a padVec of ic*kw + 16 floats); else . - private static void ConvRow(in DenseArgs a, int y, float* orow, float* arow) - { - if (UseAvx512) - for (int blk = 0; blk < a.Nb; blk += 2) Dense512Row(a, y, blk, orow, arow); - else - for (int blk = 0; blk < a.Nb; blk++) DenseRow(a, y, blk, orow, arow); - } - - /// Output row , oc blocks blk and blk+1 (if present) of a direct conv - /// into (the row's first pixel; addend row likewise). - private static void Dense512Row(in DenseArgs a, int y, int blk, float* orow, float* arow) - { - int kh = a.Kh, kw = a.Kw, taps = kh * kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, nblk = Math.Min(2, a.Nb - blk); - long wStep = (long)taps * ic * Ob; - float** ptrs = stackalloc float*[Tile5 * taps]; - float* part = stackalloc float[Tile5 * 2 * Ob]; - float* wb = a.W + blk * wStep; - float* bb = a.Bias + blk * Ob; - int lanes0 = Math.Min(Ob, oc - blk * Ob), lanes1 = nblk == 2 ? Math.Min(Ob, oc - (blk + 1) * Ob) : 0; - int* rm = a.Rm, cm = a.Cm; - for (int x0 = 0; x0 < ow; x0 += Tile5) - { - int rows = Math.Min(Tile5, ow - x0); - // every column of the tile reads kw adjacent input pixels (no column pad/reflect): one - // kernel row is then a single kw*ic float run in the order of the packed [tap][ic] - // weights, so the tile walks kh "taps" of kw*ic channels (fewer pointers, longer k loops) - bool merged = kw > 1; - for (int i = 0; i < rows && merged; i++) - { - int* cx = cm + (x0 + i) * kw; - merged = cx[0] >= 0; - for (int kx = 1; kx < kw; kx++) merged &= cx[kx] == cx[0] + kx; - } - if (merged) - { - for (int ky = 0; ky < kh; ky++) - { - int r = rm[y * kh + ky]; - for (int i = 0; i < Tile5; i++) - { - int c = cm[(x0 + Math.Min(i, rows - 1)) * kw]; - ptrs[ky * Tile5 + i] = r < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic; - } - } - Tile512(ptrs, nblk, kh, kw * ic, wb, wStep, bb, part); - } - else - { - for (int tap = 0; tap < taps; tap++) - { - int r = rm[y * kh + tap / kw]; - for (int i = 0; i < Tile5; i++) - { - int c = cm[(x0 + Math.Min(i, rows - 1)) * kw + tap % kw]; - ptrs[tap * Tile5 + i] = r < 0 || c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic; - } - } - Tile512(ptrs, nblk, taps, ic, wb, wStep, bb, part); - } - long o = (long)x0 * oc + blk * Ob; - Epilogue512(part, rows, nblk, orow + o, arow != null ? arow + o : null, oc, lanes0, lanes1, a.Act); - } - } - - private static void Gemm512(ReadOnlySpan a, int m, int k, Span dst, int n, - NhwcConvPlan kp, ActivationKind act) - { - const int G = 8; // row tiles per task sharing one weight panel walk - int nb = kp.Blocks, np = (nb + 1) / 2, tiles = (m + Tile5 - 1) / Tile5, groups = (tiles + G - 1) / G; - long wStep = (long)k * Ob; - fixed (float* ap0 = a, dp0 = dst, wp0 = kp.W, bp0 = kp.Bias) - { - float* ap = ap0, dp = dp0, wp = wp0, bp = bp0; - For(groups * np, (long)m * n * k / 64, t => - { - int g = t / np, blk = t % np * 2, nblk = Math.Min(2, nb - blk); - float** ptrs = stackalloc float*[Tile5]; - float* part = stackalloc float[Tile5 * 2 * Ob]; - float* wb = wp + blk * wStep, bb = bp + blk * Ob; - int lanes0 = Math.Min(Ob, n - blk * Ob), lanes1 = nblk == 2 ? Math.Min(Ob, n - (blk + 1) * Ob) : 0; - for (int tile = g * G, te = Math.Min(tiles, tile + G); tile < te; tile++) - { - int m0 = tile * Tile5, rows = Math.Min(Tile5, m - m0); - for (int i = 0; i < Tile5; i++) ptrs[i] = ap + (long)(m0 + Math.Min(i, rows - 1)) * k; - Tile512(ptrs, nblk, 1, k, wb, wStep, bb, part); - Epilogue512(part, rows, nblk, dp + (long)m0 * n + blk * Ob, null, n, lanes0, lanes1, act); - } - }); - } - } - - // ======================== memory-bound NHWC ops (c % 16 == 0) ======================== - - /// with 16-lane vectors: same per-lane ops (and IEEE Max) → same bits. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void ResizePixel512(float* a, float* b, float* e, float* f, F32x16 vX, F32x16 vY, - int c, bool relu, float* add, bool addRelu, float* o) - { - for (int ch = 0; ch < c; ch += 16) - { - var va = Vector512.Load(a + ch); var ve = Vector512.Load(e + ch); - var top = va + (Vector512.Load(b + ch) - va) * vX; - var bot = ve + (Vector512.Load(f + ch) - ve) * vX; - var v = top + (bot - top) * vY; - if (relu) v = Vector512.Max(v, F32x16.Zero); - if (add != null) - { - var sk = Vector512.Load(add + ch); - v += addRelu ? Vector512.Max(sk, F32x16.Zero) : sk; - } - v.Store(o + ch); - } - } - - /// MaxPool, one output pixel per step: max over the valid taps in registers, one store - /// (the generic path copies the first tap and read-modify-writes the output per tap). - private static void MaxPool512(float* sp, int iw, int c, float* dp, int oh, int ow, int kh, int kw, - int[] rm, int[] cm, bool relu) - { - For(oh, (long)oh * ow * c * kh * kw, y => - { - float** taps = stackalloc float*[kh * kw]; - for (int x = 0; x < ow; x++) - { - int n = 0; - for (int ky = 0; ky < kh; ky++) - { - int r = rm[y * kh + ky]; - if (r < 0) continue; - for (int kx = 0; kx < kw; kx++) - { - int cc = cm[x * kw + kx]; - if (cc >= 0) taps[n++] = sp + ((long)r * iw + cc) * c; - } - } - float* o = dp + ((long)y * ow + x) * c; - if (n == 0) { Fill(o, c, 0f); continue; } - for (int ch = 0; ch < c; ch += 16) - { - var v = Vector512.Load(taps[0] + ch); - for (int k = 1; k < n; k++) v = Vector512.Max(v, Vector512.Load(taps[k] + ch)); - if (relu) v = Vector512.Max(v, F32x16.Zero); - v.Store(o + ch); - } - } - }); - } - - /// - /// for the AVX-512 tier: 16-lane resize, and the c-long oc dot runs as four - /// interleaved per-pixel FMA chains (one chain per pixel was FMA-latency bound). Each pixel keeps - /// its bias-initialized, channel-ascending chain, so outputs match the AVX2 path bit for bit. - /// - private static void UpHead512(float* sp, int iw, int c, float* kpp, bool skipConv, DenseArgs ca, float* dp, int oh, int ow, - int[] y0, int[] y1, int[] x0, int[] x1, float[] wy, float[] wx, bool relu, bool skipRelu, - float* wp, float* bp, int oc, ActivationKind act) - { - bool v512 = c % 16 == 0; - For(oh, (long)oh * ow * c * (oc + 4), y => - { - float* s = stackalloc float[4 * c + 8]; - float* acc = stackalloc float[4 * 8]; - float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c; - var vY = new System.Numerics.Vector(wy[y]); - var vY16 = Vector512.Create(wy[y]); - float wY = wy[y]; - float* orow = dp + (long)y * ow * oc; - float* krow = kpp + (long)y * ow * c; - float[]? band = skipConv ? BandBuf((long)ow * c) : null; - fixed (float* sb = band) - { - if (skipConv) - { - ConvRow(ca, y, sb, null); // the skip row, recomputed by the same conv kernel - krow = sb; - } - for (int x = 0; x < ow; x += 4) - { - int n = Math.Min(4, ow - x); - for (int i = 0; i < n; i++) - { - int xx = x + i; - float* a = r0 + (long)x0[xx] * c, b = r0 + (long)x1[xx] * c, e = r1 + (long)x0[xx] * c, f = r1 + (long)x1[xx] * c; - if (v512) ResizePixel512(a, b, e, f, Vector512.Create(wx[xx]), vY16, c, relu, krow + (long)xx * c, skipRelu, s + i * c); - else ResizePixel(a, b, e, f, wx[xx], wY, new System.Numerics.Vector(wx[xx]), vY, c, relu, krow + (long)xx * c, skipRelu, s + i * c); - } - UpHeadDot4(s, c, wp, bp, acc, n); - for (int i = 0; i < n; i++) - { - float* o = orow + (long)(x + i) * oc; - for (int l = 0; l < oc; l++) o[l] = acc[i * 8 + l]; - } - } - if (act != ActivationKind.None) ActRow(orow, ow * oc, act, true); - } - }); - } - - /// acc[i*8 + l] = bias[l] + Σ_ch s_i[ch]·w[ch][l] for 4 pixels (s_i = s + i*c), chains interleaved. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void UpHeadDot4(float* s, int c, float* wp, float* bp, float* acc, int n) - { - float* s0 = s, s1 = n > 1 ? s + c : s, s2 = n > 2 ? s + 2 * c : s, s3 = n > 3 ? s + 3 * c : s; - F32x8 a0 = Avx.LoadVector256(bp), a1 = a0, a2 = a0, a3 = a0; - float* w = wp; - for (int ch = 0; ch < c; ch++, w += 8) - { - F32x8 wv = Avx.LoadVector256(w); - a0 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s0 + ch), wv, a0); - a1 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s1 + ch), wv, a1); - a2 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s2 + ch), wv, a2); - a3 = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s3 + ch), wv, a3); - } - Avx.Store(acc, a0); Avx.Store(acc + 8, a1); Avx.Store(acc + 16, a2); Avx.Store(acc + 24, a3); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Neon.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Neon.cs deleted file mode 100644 index 277f774..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.Neon.cs +++ /dev/null @@ -1,373 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using Sdcb.SimdGcdrNet.Engine; - -using V4 = System.Runtime.Intrinsics.Vector128; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// AArch64 AdvSimd (NEON) tier: the 6 × 16 conv / Gemm micro-kernel as fmla-by-element on -/// 4-channel input loads, 24 register accumulators (of 32 vector registers), used wherever the -/// x86 tiers take ; direct convs feed it kernel-row runs (kw·ic contiguous -/// NHWC floats per tap row) so the 3- / 6-channel stems run the same loop. -/// -internal static unsafe partial class Nhwc -{ - /// NEON tier switch: AdvSimd.Arm64.IsSupported (DOTNET_EnableAdvSimd=0 drops to the - /// scalar kernels). Settable only for the in-process tier comparison tests. - internal static bool UseNeon = AdvSimd.Arm64.IsSupported; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 Ld4(float* p) => AdvSimd.LoadVector128(p); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 Fma4(V4 acc, V4 a, V4 b) => AdvSimd.FusedMultiplyAdd(acc, a, b); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 Act4(V4 v, ActivationKind act) => act switch - { - ActivationKind.Relu => AdvSimd.Max(v, V4.Zero), - ActivationKind.Sigmoid => SimdMath.SigmoidCore(v), - ActivationKind.Gelu => SimdMath.GeluCore(v), - _ => v, - }; - - /// 6 × 16 tile: into a scratch tile, then the residual / - /// activation epilogue per row. - private static void TileNeon(float** ptrs, int rows, int taps, int ic, float* wb, float* bb, - float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - float* part = stackalloc float[Tile * Ob]; - TileNeonK(ptrs, taps, ic, wb, bb, part, Ob); - TileNeonStore(part, rows, outp, addp, oc, lanes, act); - } - - /// Residual / activation epilogue of a finished 6 × 16 scratch tile. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void TileNeonStore(float* part, int rows, float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - for (int r = 0; r < rows; r++, part += Ob) - { - float* o = outp + (long)r * oc; - float* ad = addp == null ? null : addp + (long)r * oc; - if (lanes == Ob) - { - for (int l = 0; l < Ob; l += 4) - { - V4 v = Ld4(part + l); - if (ad != null) v = AdvSimd.Add(v, Ld4(ad + l)); - AdvSimd.Store(o + l, Act4(v, act)); - } - continue; - } - for (int l = 0; l < lanes; l++) - o[l] = ActS(ad != null ? part[l] + ad[l] : part[l], act); - } - } - - /// 6 × 16 k loop over all ic channels of every tap (weights [tap][ic][16]), bias-initialized. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void TileNeonK(float** ptrs, int taps, int ic, float* wb, float* bb, float* dst, int ds) => - TileNeonK(ptrs, taps, 0, ic, (long)ic * Ob, wb, bb, dst, ds); - - /// - /// The 6 × 16 k loop: dst[r*ds + 0..15] = init + Σ x·w for all rows, where - /// ptrs[tap*Tile + r] + coff = row r's ic inputs of tap and wb + tap·wTap its [ic][16] weights; - /// init = bias , or (bb == null) the partial sums already in dst. Four - /// channels per step (one 128-bit load per pixel, fmla by element; the next weight vector is - /// loaded one group ahead for the in-order A53), ic % 4 leftovers by broadcast. Never calls - /// out, so the 24 accumulators stay in registers. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void TileNeonK(float** ptrs, int taps, int coff, int ic, long wTap, float* wb, float* bb, float* dst, int ds, - bool relu = false) - { - V4 a0, a1, a2, a3, a4, a5, b0, b1, b2, b3, b4, b5, c0, c1, c2, c3, c4, c5, d0, d1, d2, d3, d4, d5; - if (bb != null) - { - a0 = Ld4(bb); b0 = Ld4(bb + 4); c0 = Ld4(bb + 8); d0 = Ld4(bb + 12); - a1 = a2 = a3 = a4 = a5 = a0; - b1 = b2 = b3 = b4 = b5 = b0; - c1 = c2 = c3 = c4 = c5 = c0; - d1 = d2 = d3 = d4 = d5 = d0; - } - else - { - float* s = dst; - a0 = Ld4(s); b0 = Ld4(s + 4); c0 = Ld4(s + 8); d0 = Ld4(s + 12); s += ds; - a1 = Ld4(s); b1 = Ld4(s + 4); c1 = Ld4(s + 8); d1 = Ld4(s + 12); s += ds; - a2 = Ld4(s); b2 = Ld4(s + 4); c2 = Ld4(s + 8); d2 = Ld4(s + 12); s += ds; - a3 = Ld4(s); b3 = Ld4(s + 4); c3 = Ld4(s + 8); d3 = Ld4(s + 12); s += ds; - a4 = Ld4(s); b4 = Ld4(s + 4); c4 = Ld4(s + 8); d4 = Ld4(s + 12); s += ds; - a5 = Ld4(s); b5 = Ld4(s + 4); c5 = Ld4(s + 8); d5 = Ld4(s + 12); - } - int ic4 = ic & ~3; - for (int tap = 0; tap < taps; tap++) - { - float** pp = ptrs + tap * Tile; - float* p0 = pp[0] + coff, p1 = pp[1] + coff, p2 = pp[2] + coff, p3 = pp[3] + coff, p4 = pp[4] + coff, p5 = pp[5] + coff; - float* w = wb + tap * wTap; - int c = 0; - for (; c < ic4; c += 4, w += 4 * Ob) - { - V4 x0 = Ld4(p0 + c), x1 = Ld4(p1 + c), x2 = Ld4(p2 + c); - V4 x3 = Ld4(p3 + c), x4 = Ld4(p4 + c), x5 = Ld4(p5 + c); - V4 wv0 = Ld4(w), wv1; - wv1 = Ld4(w + 4); - a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 0); - a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 0); - a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 0); - a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 0); - a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 0); - a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 0); - wv0 = Ld4(w + 8); - b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 0); - b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 0); - b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 0); - b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 0); - b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 0); - b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 0); - wv1 = Ld4(w + 12); - c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 0); - c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 0); - c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 0); - c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 0); - c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 0); - c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 0); - wv0 = Ld4(w + 16); - d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 0); - d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 0); - d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 0); - d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 0); - d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 0); - d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 0); - wv1 = Ld4(w + 20); - a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 1); - a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 1); - a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 1); - a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 1); - a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 1); - a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 1); - wv0 = Ld4(w + 24); - b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 1); - b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 1); - b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 1); - b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 1); - b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 1); - b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 1); - wv1 = Ld4(w + 28); - c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 1); - c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 1); - c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 1); - c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 1); - c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 1); - c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 1); - wv0 = Ld4(w + 32); - d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 1); - d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 1); - d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 1); - d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 1); - d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 1); - d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 1); - wv1 = Ld4(w + 36); - a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 2); - a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 2); - a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 2); - a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 2); - a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 2); - a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 2); - wv0 = Ld4(w + 40); - b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 2); - b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 2); - b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 2); - b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 2); - b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 2); - b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 2); - wv1 = Ld4(w + 44); - c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 2); - c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 2); - c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 2); - c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 2); - c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 2); - c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 2); - wv0 = Ld4(w + 48); - d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 2); - d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 2); - d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 2); - d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 2); - d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 2); - d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 2); - wv1 = Ld4(w + 52); - a0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a0, wv0, x0, 3); - a1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a1, wv0, x1, 3); - a2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a2, wv0, x2, 3); - a3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a3, wv0, x3, 3); - a4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a4, wv0, x4, 3); - a5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(a5, wv0, x5, 3); - wv0 = Ld4(w + 56); - b0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b0, wv1, x0, 3); - b1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b1, wv1, x1, 3); - b2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b2, wv1, x2, 3); - b3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b3, wv1, x3, 3); - b4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b4, wv1, x4, 3); - b5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(b5, wv1, x5, 3); - wv1 = Ld4(w + 60); - c0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c0, wv0, x0, 3); - c1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c1, wv0, x1, 3); - c2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c2, wv0, x2, 3); - c3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c3, wv0, x3, 3); - c4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c4, wv0, x4, 3); - c5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(c5, wv0, x5, 3); - d0 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d0, wv1, x0, 3); - d1 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d1, wv1, x1, 3); - d2 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d2, wv1, x2, 3); - d3 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d3, wv1, x3, 3); - d4 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d4, wv1, x4, 3); - d5 = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(d5, wv1, x5, 3); - } - for (; c < ic; c++, w += Ob) - { - V4 w0 = Ld4(w), w1 = Ld4(w + 4), w2 = Ld4(w + 8), w3 = Ld4(w + 12); - V4 x = AdvSimd.LoadAndReplicateToVector128(p0 + c); - a0 = Fma4(a0, w0, x); b0 = Fma4(b0, w1, x); c0 = Fma4(c0, w2, x); d0 = Fma4(d0, w3, x); - x = AdvSimd.LoadAndReplicateToVector128(p1 + c); - a1 = Fma4(a1, w0, x); b1 = Fma4(b1, w1, x); c1 = Fma4(c1, w2, x); d1 = Fma4(d1, w3, x); - x = AdvSimd.LoadAndReplicateToVector128(p2 + c); - a2 = Fma4(a2, w0, x); b2 = Fma4(b2, w1, x); c2 = Fma4(c2, w2, x); d2 = Fma4(d2, w3, x); - x = AdvSimd.LoadAndReplicateToVector128(p3 + c); - a3 = Fma4(a3, w0, x); b3 = Fma4(b3, w1, x); c3 = Fma4(c3, w2, x); d3 = Fma4(d3, w3, x); - x = AdvSimd.LoadAndReplicateToVector128(p4 + c); - a4 = Fma4(a4, w0, x); b4 = Fma4(b4, w1, x); c4 = Fma4(c4, w2, x); d4 = Fma4(d4, w3, x); - x = AdvSimd.LoadAndReplicateToVector128(p5 + c); - a5 = Fma4(a5, w0, x); b5 = Fma4(b5, w1, x); c5 = Fma4(c5, w2, x); d5 = Fma4(d5, w3, x); - } - } - if (relu) - { - V4 z = V4.Zero; - a0 = AdvSimd.Max(a0, z); b0 = AdvSimd.Max(b0, z); c0 = AdvSimd.Max(c0, z); d0 = AdvSimd.Max(d0, z); - a1 = AdvSimd.Max(a1, z); b1 = AdvSimd.Max(b1, z); c1 = AdvSimd.Max(c1, z); d1 = AdvSimd.Max(d1, z); - a2 = AdvSimd.Max(a2, z); b2 = AdvSimd.Max(b2, z); c2 = AdvSimd.Max(c2, z); d2 = AdvSimd.Max(d2, z); - a3 = AdvSimd.Max(a3, z); b3 = AdvSimd.Max(b3, z); c3 = AdvSimd.Max(c3, z); d3 = AdvSimd.Max(d3, z); - a4 = AdvSimd.Max(a4, z); b4 = AdvSimd.Max(b4, z); c4 = AdvSimd.Max(c4, z); d4 = AdvSimd.Max(d4, z); - a5 = AdvSimd.Max(a5, z); b5 = AdvSimd.Max(b5, z); c5 = AdvSimd.Max(c5, z); d5 = AdvSimd.Max(d5, z); - } - StoreRow(dst, a0, b0, c0, d0); dst += ds; - StoreRow(dst, a1, b1, c1, d1); dst += ds; - StoreRow(dst, a2, b2, c2, d2); dst += ds; - StoreRow(dst, a3, b3, c3, d3); dst += ds; - StoreRow(dst, a4, b4, c4, d4); dst += ds; - StoreRow(dst, a5, b5, c5, d5); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void StoreRow(float* d, V4 a, V4 b, V4 c, V4 e) - { - AdvSimd.Store(d, a); AdvSimd.Store(d + 4, b); AdvSimd.Store(d + 8, c); AdvSimd.Store(d + 12, e); - } - - [ThreadStatic] private static float[]? t_runs; - - /// - /// One output row for oc <= 4, c % 4 == 0: per 4 channels the bilinear - /// lerps (fma), [relu], + [relu](skip), then fmla by element into one 4-lane accumulator against - /// the packed [c][8] weights — the resized pixel never leaves registers. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void UpHeadRowNeon(float* r0, float* r1, int* x0, int* x1, float* wx, float wy, float* krow, - int c, int ow, float* wp, float* bp, float* orow, int oc, bool relu, bool skipRelu, ActivationKind act) - { - V4 vY = Vector128.Create(wy), bias = Ld4(bp); - for (int x = 0; x < ow; x++) - { - float* a = r0 + (long)x0[x] * c, b = r0 + (long)x1[x] * c, e = r1 + (long)x0[x] * c, f = r1 + (long)x1[x] * c; - float* k = krow + (long)x * c; - V4 vX = Vector128.Create(wx[x]), acc = bias; - float* w = wp; - for (int ch = 0; ch < c; ch += 4, w += 32) - { - V4 va = Ld4(a + ch), ve = Ld4(e + ch); - V4 top = Fma4(va, AdvSimd.Subtract(Ld4(b + ch), va), vX); - V4 bot = Fma4(ve, AdvSimd.Subtract(Ld4(f + ch), ve), vX); - V4 v = Fma4(top, AdvSimd.Subtract(bot, top), vY); - if (relu) v = AdvSimd.Max(v, V4.Zero); - V4 s = Ld4(k + ch); - v = AdvSimd.Add(v, skipRelu ? AdvSimd.Max(s, V4.Zero) : s); - acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w), v, 0); - acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 8), v, 1); - acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 16), v, 2); - acc = AdvSimd.Arm64.FusedMultiplyAddBySelectedScalar(acc, Ld4(w + 24), v, 3); - } - acc = Act4(acc, act); - float* o = orow + (long)x * oc; - o[0] = acc.ToScalar(); - if (oc > 1) o[1] = acc.GetElement(1); - if (oc > 2) o[2] = acc.GetElement(2); - if (oc > 3) o[3] = acc.GetElement(3); - } - } - - /// Thread-local pinned scratch (the pointer outlives any fixed scope). - private static T* Pinned(ref T[]? slot, int n) where T : unmanaged - { - if (slot is not { } b || b.Length < n) slot = b = GC.AllocateUninitializedArray(n, pinned: true); - return (T*)Unsafe.AsPointer(ref System.Runtime.InteropServices.MemoryMarshal.GetArrayDataReference(b)); - } - - /// - /// on NEON: a k×k conv as kh taps over kernel-row runs. For each tap row - /// the kw·ic inputs of a pixel are contiguous in NHWC (weights [ky][kx][ic][16] are already - /// [ky][kx·ic][16]), so interior pixels point straight into the source; pixels whose run - /// crosses a padded / reflected border are gathered into a thread-local buffer, constant-pad - /// rows read . - /// - private static void DenseRowNeon(in DenseArgs a, int y, int blk, float* orow, float* arow) - { - int kh = a.Kh, kw = a.Kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, run = kw * ic; - float** ptrs = stackalloc float*[Tile * kh]; - float* part = stackalloc float[Tile * Ob]; - float* wb = a.W + (long)blk * kh * run * Ob; - float* bb = a.Bias + blk * Ob; - int lanes = Math.Min(Ob, oc - blk * Ob); - bool direct = lanes == Ob && arow == null && a.Act is ActivationKind.None or ActivationKind.Relu; - float* gather = null; - for (int x0 = 0; x0 < ow; x0 += Tile) - { - int rows = Math.Min(Tile, ow - x0); - for (int ky = 0; ky < kh; ky++) - { - int r = a.Rm[y * kh + ky]; - for (int i = 0; i < Tile; i++) - { - int* cmx = a.Cm + (x0 + Math.Min(i, rows - 1)) * kw; - int cx = cmx[0]; - float* p; - if (r < 0) p = a.PadVec; - else if (cx >= 0 && cmx[kw - 1] == cx + kw - 1) p = a.Src + ((long)r * a.Iw + cx) * ic; - else - { - if (gather == null) gather = Pinned(ref t_runs, Tile * kh * run); - p = gather + (ky * Tile + i) * run; - for (int kx = 0; kx < kw; kx++) - { - int c = cmx[kx]; - Buffer.MemoryCopy(c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic, p + kx * ic, ic * 4L, ic * 4L); - } - } - ptrs[ky * Tile + i] = p; - } - } - long o = (long)x0 * oc + blk * Ob; - if (direct && rows == Tile) - { - // full tile, no residual / transcendental epilogue: store straight from the accumulators - TileNeonK(ptrs, kh, 0, run, (long)run * Ob, wb, bb, orow + o, oc, a.Act == ActivationKind.Relu); - continue; - } - TileNeonK(ptrs, kh, run, wb, bb, part, Ob); - TileNeonStore(part, rows, orow + o, arow != null ? arow + o : null, oc, lanes, a.Act); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.PackedSimd.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.PackedSimd.cs deleted file mode 100644 index ba2568a..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.PackedSimd.cs +++ /dev/null @@ -1,251 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Wasm; -using Sdcb.SimdGcdrNet.Engine; - -using V4 = System.Runtime.Intrinsics.Vector128; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// WebAssembly SIMD128 (PackedSimd) tier, after SimdPaddleOCR's Nhwc.Conv.PackedSimd. WASM SIMD has -/// no FMA (relaxed SIMD is off) and V8 maps v128 values onto at most 15 allocatable XMM registers -/// on x64, so the conv / Gemm micro-kernel is pixels × 8 oc (8 accumulators + -/// 2 weights + 1 splat), run twice per 16-oc packed block; each input float is broadcast straight -/// from memory (v128.load32_splat). Every reduction keeps the Vector<float> tier's per-lane -/// order (bias-init, taps then channels ascending, separate multiply then add), so conv sums are -/// bit-identical to it; direct convs read NHWC kernel-row runs like the NEON tier (same k order). -/// -internal static unsafe partial class Nhwc -{ - /// PackedSimd tier switch: PackedSimd.IsSupported (browser-wasm with SIMD). - /// Settable only for in-process tier comparisons (WasmBench --tiers / --vector). - internal static bool UsePacked = PackedSimd.IsSupported; - - private const int PkTile = 4; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 LdP(float* p) => PackedSimd.LoadVector128(p); - - /// acc + a·b, rounded twice (no FMA): the Vector<float> tier's acc += a * b. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 MadP(V4 acc, V4 a, V4 b) => PackedSimd.Add(acc, PackedSimd.Multiply(a, b)); - - // pmax(0, v): one maxps in V8 (f32x4.max needs a NaN / -0 fix-up); equals Max(v, 0) for finite v - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static V4 ActP(V4 v, ActivationKind act) => act switch - { - ActivationKind.Relu => PackedSimd.PseudoMax(V4.Zero, v), - ActivationKind.Sigmoid => SimdMath.SigmoidPk(v), - ActivationKind.Gelu => SimdMath.GeluPk(v), - _ => v, - }; - - /// - /// rows × 16 oc: dst[r*ds + 0..15] = bias + Σ x·w, where ptrs[tap*ps + r] are - /// row r's ic inputs of tap and wb + tap·wTap its [ic][16] weights. Two 8-oc passes; the k loop - /// never calls out, so the accumulators stay in registers. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void TilePkK(float** ptrs, int ps, int taps, int ic, long wTap, float* wb, float* bb, float* dst, int ds) - { - for (int h = 0; h < Ob; h += 8) - { - V4 a0 = LdP(bb + h), b0 = LdP(bb + h + 4); - V4 a1 = a0, a2 = a0, a3 = a0, b1 = b0, b2 = b0, b3 = b0; - for (int tap = 0; tap < taps; tap++) - { - float** pp = ptrs + tap * ps; - float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3]; - float* w = wb + tap * wTap + h; - for (int c = 0; c < ic; c++, w += Ob) - { - V4 w0 = LdP(w), w1 = LdP(w + 4); - V4 v = PackedSimd.LoadScalarAndSplatVector128(p0 + c); - a0 = MadP(a0, v, w0); b0 = MadP(b0, v, w1); - v = PackedSimd.LoadScalarAndSplatVector128(p1 + c); - a1 = MadP(a1, v, w0); b1 = MadP(b1, v, w1); - v = PackedSimd.LoadScalarAndSplatVector128(p2 + c); - a2 = MadP(a2, v, w0); b2 = MadP(b2, v, w1); - v = PackedSimd.LoadScalarAndSplatVector128(p3 + c); - a3 = MadP(a3, v, w0); b3 = MadP(b3, v, w1); - } - } - float* d = dst + h; - PackedSimd.Store(d, a0); PackedSimd.Store(d + 4, b0); d += ds; - PackedSimd.Store(d, a1); PackedSimd.Store(d + 4, b1); d += ds; - PackedSimd.Store(d, a2); PackedSimd.Store(d + 4, b2); d += ds; - PackedSimd.Store(d, a3); PackedSimd.Store(d + 4, b3); - } - } - - /// Residual / activation epilogue of a finished [rows][16] scratch tile. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void StorePk(float* part, int rows, float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - for (int r = 0; r < rows; r++, part += Ob) - { - float* o = outp + (long)r * oc; - float* ad = addp == null ? null : addp + (long)r * oc; - if (lanes == Ob) - { - for (int l = 0; l < Ob; l += 4) - { - V4 v = LdP(part + l); - if (ad != null) v = PackedSimd.Add(v, LdP(ad + l)); - PackedSimd.Store(o + l, ActP(v, act)); - } - continue; - } - for (int l = 0; l < lanes; l++) - o[l] = ActS(ad != null ? part[l] + ad[l] : part[l], act); - } - } - - /// on PackedSimd: kh taps over NHWC kernel-row runs (kw·ic contiguous - /// floats, see ); border-crossing runs are gathered, constant-pad rows read PadVec. - private static void DenseRowPk(in DenseArgs a, int y, int blk, float* orow, float* arow) - { - int kh = a.Kh, kw = a.Kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, run = kw * ic; - float** ptrs = stackalloc float*[PkTile * kh]; - float* part = stackalloc float[PkTile * Ob]; - float* wb = a.W + (long)blk * kh * run * Ob; - float* bb = a.Bias + blk * Ob; - int lanes = Math.Min(Ob, oc - blk * Ob); - float* gather = null; - for (int x0 = 0; x0 < ow; x0 += PkTile) - { - int rows = Math.Min(PkTile, ow - x0); - for (int ky = 0; ky < kh; ky++) - { - int r = a.Rm[y * kh + ky]; - for (int i = 0; i < PkTile; i++) - { - int* cmx = a.Cm + (x0 + Math.Min(i, rows - 1)) * kw; - int cx = cmx[0]; - float* p; - if (r < 0) p = a.PadVec; - else if (cx >= 0 && cmx[kw - 1] == cx + kw - 1) p = a.Src + ((long)r * a.Iw + cx) * ic; - else - { - if (gather == null) gather = Pinned(ref t_runs, PkTile * kh * run); - p = gather + (ky * PkTile + i) * run; - for (int kx = 0; kx < kw; kx++) - { - int c = cmx[kx]; - Buffer.MemoryCopy(c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic, p + kx * ic, ic * 4L, ic * 4L); - } - } - ptrs[ky * PkTile + i] = p; - } - } - long o = (long)x0 * oc + blk * Ob; - TilePkK(ptrs, PkTile, kh, run, (long)run * Ob, wb, bb, part, Ob); - StorePk(part, rows, orow + o, arow != null ? arow + o : null, oc, lanes, a.Act); - } - } - - /// on PackedSimd: -row tiles, 12 per task sharing one weight panel walk. - private static void GemmPk(ReadOnlySpan a, int m, int k, Span dst, int n, NhwcConvPlan kp, ActivationKind act) - { - const int G = 12; - int nb = kp.Blocks, tiles = (m + PkTile - 1) / PkTile, groups = (tiles + G - 1) / G; - fixed (float* ap0 = a, dp0 = dst, wp0 = kp.W, bp0 = kp.Bias) - { - float* ap = ap0, dp = dp0, wp = wp0, bp = bp0; - For(groups * nb, (long)m * n * k / 64, t => - { - int g = t / nb, blk = t % nb; - float** ptrs = stackalloc float*[PkTile]; - float* part = stackalloc float[PkTile * Ob]; - float* wb = wp + (long)blk * k * Ob, bb = bp + blk * Ob; - int lanes = Math.Min(Ob, n - blk * Ob); - for (int tile = g * G, te = Math.Min(tiles, tile + G); tile < te; tile++) - { - int m0 = tile * PkTile, rows = Math.Min(PkTile, m - m0); - for (int i = 0; i < PkTile; i++) ptrs[i] = ap + (long)(m0 + Math.Min(i, rows - 1)) * k; - TilePkK(ptrs, PkTile, 1, k, (long)k * Ob, wb, bb, part, Ob); - StorePk(part, rows, dp + (long)m0 * n + blk * Ob, null, n, lanes, act); - } - }); - } - } - - /// - /// One depthwise output pixel on PackedSimd: 16 then 4 channels per step, bias-initialized, taps - /// ascending, + residual, activation. Returns the first channel left for the scalar tail. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static int DepthwisePixelPk(float** ptrs, int taps, int c, float* wp, float* bp, float* ad, float* o, ActivationKind act) - { - int ch = 0; - for (; ch + 16 <= c; ch += 16) - { - V4 a0 = LdP(bp + ch), a1 = LdP(bp + ch + 4), a2 = LdP(bp + ch + 8), a3 = LdP(bp + ch + 12); - float* w = wp + ch; - for (int tap = 0; tap < taps; tap++, w += c) - { - float* s = ptrs[tap] + ch; - a0 = MadP(a0, LdP(s), LdP(w)); - a1 = MadP(a1, LdP(s + 4), LdP(w + 4)); - a2 = MadP(a2, LdP(s + 8), LdP(w + 8)); - a3 = MadP(a3, LdP(s + 12), LdP(w + 12)); - } - if (ad != null) - { - a0 = PackedSimd.Add(a0, LdP(ad + ch)); a1 = PackedSimd.Add(a1, LdP(ad + ch + 4)); - a2 = PackedSimd.Add(a2, LdP(ad + ch + 8)); a3 = PackedSimd.Add(a3, LdP(ad + ch + 12)); - } - PackedSimd.Store(o + ch, ActP(a0, act)); PackedSimd.Store(o + ch + 4, ActP(a1, act)); - PackedSimd.Store(o + ch + 8, ActP(a2, act)); PackedSimd.Store(o + ch + 12, ActP(a3, act)); - } - for (; ch + 4 <= c; ch += 4) - { - V4 acc = LdP(bp + ch); - for (int tap = 0; tap < taps; tap++) - acc = MadP(acc, LdP(ptrs[tap] + ch), LdP(wp + tap * c + ch)); - if (ad != null) acc = PackedSimd.Add(acc, LdP(ad + ch)); - PackedSimd.Store(o + ch, ActP(acc, act)); - } - return ch; - } - - /// - /// One output row for oc <= 4, c % 4 == 0 (see ): - /// per 4 channels the bilinear lerps, [relu], + [relu](skip), then each channel is splatted back - /// from a 16-byte spill (WASM SIMD has no by-element multiply) into one 4-lane accumulator. - /// Same per-element ops as the unfused Vector<float> path. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void UpHeadRowPk(float* r0, float* r1, int* x0, int* x1, float* wx, float wy, float* krow, - int c, int ow, float* wp, float* bp, float* orow, int oc, bool relu, bool skipRelu, ActivationKind act) - { - V4 vY = PackedSimd.Splat(wy), bias = LdP(bp); - float* s = stackalloc float[4]; - for (int x = 0; x < ow; x++) - { - float* a = r0 + (long)x0[x] * c, b = r0 + (long)x1[x] * c, e = r1 + (long)x0[x] * c, f = r1 + (long)x1[x] * c; - float* k = krow + (long)x * c; - V4 vX = PackedSimd.Splat(wx[x]), acc = bias; - float* w = wp; - for (int ch = 0; ch < c; ch += 4, w += 32) - { - V4 va = LdP(a + ch), ve = LdP(e + ch); - V4 top = MadP(va, PackedSimd.Subtract(LdP(b + ch), va), vX); - V4 bot = MadP(ve, PackedSimd.Subtract(LdP(f + ch), ve), vX); - V4 v = MadP(top, PackedSimd.Subtract(bot, top), vY); - if (relu) v = PackedSimd.PseudoMax(V4.Zero, v); - V4 sk = LdP(k + ch); - v = PackedSimd.Add(v, skipRelu ? PackedSimd.PseudoMax(V4.Zero, sk) : sk); - PackedSimd.Store(s, v); - acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s), LdP(w)); - acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 1), LdP(w + 8)); - acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 2), LdP(w + 16)); - acc = MadP(acc, PackedSimd.LoadScalarAndSplatVector128(s + 3), LdP(w + 24)); - } - PackedSimd.Store(s, ActP(acc, act)); - float* o = orow + (long)x * oc; - for (int l = 0; l < oc; l++) o[l] = s[l]; - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.cs b/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.cs deleted file mode 100644 index e3122af..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Nhwc.cs +++ /dev/null @@ -1,904 +0,0 @@ -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.Wasm; -using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; - -using F32x8 = System.Runtime.Intrinsics.Vector256; -using V128 = System.Runtime.Intrinsics.Vector128; -using V256 = System.Runtime.Intrinsics.Vector256; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// Backend-private packed conv weights for the NHWC CPU kernels (lives in ). -internal sealed class NhwcConvPlan -{ - public required float[] W; // dense: [oc/16][tap][ic][16] (oc zero-padded); depthwise: [tap][c] - public required float[] Bias; // dense: padded to blocks*16; depthwise: [c] - public bool Depthwise; - public int Blocks; - public float[]? WinoU; // Winograd F(4x4,3x3) weights [36][oc/16][ic][16], null = direct conv -} - -/// Packed weights: [c][8] oc lanes (zero-padded) + bias[8]. -internal sealed class UpHeadPlan -{ - public required float[] W; - public required float[] Bias; - public int Oc, C; -} - -/// -/// Channels-last (NHWC, N==1) kernels. Shapes are passed in storage order -/// ([h,w,c]). Every pad mode goes through precomputed source row/col maps -/// (-1 = constant pad → a pad-value pixel vector), so the inner loops never -/// branch on borders. Dense/pointwise conv: register-tiled 6 pixels × 16 oc -/// AVX2 micro-kernel (after SimdPaddleOCR's Nhwc.Conv) with bias-init and a -/// fused residual + activation epilogue; AVX-512 widens it to 10 × 32 (Nhwc.Avx512.cs), -/// NEON keeps 6 × 16 (Nhwc.Neon.cs), WASM SIMD128 narrows it to 4 × 8 (Nhwc.PackedSimd.cs); -/// other tiers use Vector<float>. -/// -internal static unsafe partial class Nhwc -{ - private const int Ob = 16, Tile = 6; - private static readonly ParallelOptions PO = new() { MaxDegreeOfParallelism = Avx2Kernels.Threads }; - private static bool UseAvx => Avx2.IsSupported && Fma.IsSupported; - - private static void For(int n, long work, Action body) - { - if (work < 32768 || n == 1) { for (int i = 0; i < n; i++) body(i); return; } - Parallel.For(0, n, PO, body); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int Reflect(int i, int dim) - { - if (dim <= 1) return 0; - while (i < 0 || i >= dim) i = i < 0 ? -i : 2 * dim - 2 - i; - return i; - } - - /// map[o*k+kk] = source index of output o, tap kk, or -1 for constant pad. - private static int[] Map(int outLen, int k, int stride, int pad, int inLen, PadMode mode) - { - int[] m = new int[outLen * k]; - for (int o = 0; o < outLen; o++) - for (int kk = 0; kk < k; kk++) - { - int s = o * stride + kk - pad; - m[o * k + kk] = (uint)s < (uint)inLen ? s : mode == PadMode.Reflect ? Reflect(s, inLen) : -1; - } - return m; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static float ActS(float v, ActivationKind act) => - act == ActivationKind.Relu ? MathF.Max(v, 0f) : - act == ActivationKind.Sigmoid ? 1f / (1f + MathF.Exp(-v)) : - act == ActivationKind.Gelu ? 0.5f * v * (1f + SimdMath.ErfScalar(v * 0.70710678f)) : v; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static F32x8 Act8(F32x8 v, ActivationKind act) => act switch - { - ActivationKind.Relu => V256.Max(v, F32x8.Zero), - ActivationKind.Sigmoid => SimdMath.SigmoidCore(v), - ActivationKind.Gelu => SimdMath.GeluCore(v), - _ => v, - }; - - // ======================== weight packing ======================== - public static NhwcConvPlan Pack(ReadOnlySpan w, int[] wShape, ReadOnlySpan bias, int group, int ic) - { - int oc = wShape[0], icg = wShape[1], taps = wShape[2] * wShape[3]; - if (group > 1) - { - if (group != ic || icg != 1 || oc != ic) - throw new NotSupportedException($"NHWC grouped conv supports depthwise only (g={group}, ic={ic}, oc={oc})."); - float[] pw = new float[taps * oc]; - for (int c = 0; c < oc; c++) - for (int t = 0; t < taps; t++) pw[t * oc + c] = w[c * taps + t]; - float[] pb = new float[oc]; - if (!bias.IsEmpty) bias.CopyTo(pb); - return new NhwcConvPlan { W = pw, Bias = pb, Depthwise = true }; - } - int nb = (oc + Ob - 1) / Ob; - float[] p = new float[checked(nb * taps * ic * Ob)]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < taps; t++) - p[(((o / Ob) * taps + t) * ic + c) * Ob + o % Ob] = w[(o * ic + c) * taps + t]; - float[] b = new float[nb * Ob]; - if (!bias.IsEmpty) bias.CopyTo(b); - return new NhwcConvPlan { W = p, Bias = b, Blocks = nb }; - } - - /// Packs Gemm B ([K,N], or [N,K] when transB) as [N/16][K][16] — a 1x1 conv panel. - public static NhwcConvPlan PackGemm(ReadOnlySpan b, int k, int n, bool transB, ReadOnlySpan bias) - { - int nb = (n + Ob - 1) / Ob; - float[] p = new float[checked(nb * k * Ob)]; - for (int o = 0; o < n; o++) - for (int c = 0; c < k; c++) - p[((o / Ob) * k + c) * Ob + o % Ob] = transB ? b[o * k + c] : b[c * n + o]; - float[] pb = new float[nb * Ob]; - if (!bias.IsEmpty) bias.CopyTo(pb); - return new NhwcConvPlan { W = p, Bias = pb, Blocks = nb }; - } - - /// dst[M,N] = act(a[M,K]·B + bias) through the 6x16 conv micro-kernel. - public static void Gemm(ReadOnlySpan a, int m, int k, Span dst, int n, - NhwcConvPlan kp, ActivationKind act) - { - if (UseAvx512) { Gemm512(a, m, k, dst, n, kp, act); return; } - if (UsePacked) { GemmPk(a, m, k, dst, n, kp, act); return; } - const int G = 8; // row tiles per task sharing one weight panel walk - int nb = kp.Blocks, tiles = (m + Tile - 1) / Tile, groups = (tiles + G - 1) / G; - bool avx = UseAvx; - fixed (float* ap0 = a, dp0 = dst, wp0 = kp.W, bp0 = kp.Bias) - { - float* ap = ap0, dp = dp0, wp = wp0, bp = bp0; - For(groups * nb, (long)m * n * k / 64, t => - { - int g = t / nb, blk = t % nb; - float** ptrs = stackalloc float*[Tile]; - float* wb = wp + (long)blk * k * Ob, bb = bp + blk * Ob; - int lanes = Math.Min(Ob, n - blk * Ob); - for (int tile = g * G, te = Math.Min(tiles, tile + G); tile < te; tile++) - { - int m0 = tile * Tile, rows = Math.Min(Tile, m - m0); - for (int i = 0; i < Tile; i++) ptrs[i] = ap + (long)(m0 + Math.Min(i, rows - 1)) * k; - float* outp = dp + (long)m0 * n + blk * Ob; - if (avx) TileAvx(ptrs, rows, 1, k, wb, bb, outp, null, n, lanes, act); - else TileVec(ptrs, rows, 1, k, wb, bb, outp, null, n, lanes, act); - } - }); - } - } - - // ======================== conv ======================== - public static void Conv(ReadOnlySpan src, int ih, int iw, int ic, - Span dst, int oh, int ow, int oc, ConvP p, NhwcConvPlan kp, ReadOnlySpan add) - { - if (!add.IsEmpty && add.Length != dst.Length) - throw new NotSupportedException($"conv residual addend has {add.Length} elems, expected {dst.Length}."); - if (kp.WinoU is not null) - { - WinogradConv(src, ih, iw, ic, dst, oh, ow, oc, p, kp, add); - return; - } - int kh = p.Kernel[0], kw = p.Kernel[1]; - int[] rm = Map(oh, kh, p.Strides[0], p.Pads[0], ih, p.PadMode); - int[] cm = Map(ow, kw, p.Strides[1], p.Pads[1], iw, p.PadMode); - float[] padVec = new float[ic * kw + Ob]; // one constant-pad kernel row (Dense512 row runs) - Array.Fill(padVec, p.PadValue); - if (kp.Depthwise) - Depthwise(src, iw, ic, dst, oh, ow, kh, kw, rm, cm, padVec, kp, add, p.Act); - else if (UseAvx512) - Dense512(src, iw, ic, dst, oh, ow, oc, kh, kw, rm, cm, padVec, kp, add, p.Act); - else - Dense(src, iw, ic, dst, oh, ow, oc, kh, kw, rm, cm, padVec, kp, add, p.Act); - } - - private static void Dense(ReadOnlySpan src, int iw, int ic, Span dst, - int oh, int ow, int oc, int kh, int kw, int[] rm, int[] cm, float[] padVec, - NhwcConvPlan kp, ReadOnlySpan add, ActivationKind act) - { - int taps = kh * kw, nb = kp.Blocks; - bool hasAdd = !add.IsEmpty, avx = UseAvx; - fixed (float* sp0 = src, dp0 = dst, ap0 = add, wp0 = kp.W, bp0 = kp.Bias, pv0 = padVec) - fixed (int* rm0 = rm, cm0 = cm) - { - var a = new DenseArgs(sp0, wp0, bp0, pv0, rm0, cm0, iw, ic, ow, oc, kh, kw, nb, act, avx); - float* dp = dp0, ap = ap0; - For(oh * nb, (long)oh * ow * oc * taps * ic / 64, t => - { - int y = t / nb, blk = t % nb; - long o = (long)y * ow * oc; - DenseRow(a, y, blk, dp + o, hasAdd ? ap + o : null); - }); - } - } - - /// Per-call direct-conv constants (pointers into pinned/fixed storage). - private readonly unsafe struct DenseArgs(float* src, float* w, float* bias, float* padVec, int* rm, int* cm, - int iw, int ic, int ow, int oc, int kh, int kw, int nb, ActivationKind act, bool avx) - { - public readonly float* Src = src, W = w, Bias = bias, PadVec = padVec; - public readonly int* Rm = rm, Cm = cm; - public readonly int Iw = iw, Ic = ic, Ow = ow, Oc = oc, Kh = kh, Kw = kw, Nb = nb; - public readonly ActivationKind Act = act; - public readonly bool Avx = avx; - } - - /// Output row , oc block of a direct conv - /// into (the row's first pixel; addend row likewise). - private static void DenseRow(in DenseArgs a, int y, int blk, float* orow, float* arow) - { - if (UseNeon) - { - DenseRowNeon(a, y, blk, orow, arow); - return; - } - if (UsePacked) - { - DenseRowPk(a, y, blk, orow, arow); - return; - } - int taps = a.Kh * a.Kw, ic = a.Ic, ow = a.Ow, oc = a.Oc, kw = a.Kw; - float** ptrs = stackalloc float*[Tile * taps]; - float* wb = a.W + (long)blk * taps * ic * Ob; - float* bb = a.Bias + blk * Ob; - int lanes = Math.Min(Ob, oc - blk * Ob); - for (int x0 = 0; x0 < ow; x0 += Tile) - { - int rows = Math.Min(Tile, ow - x0); - for (int tap = 0; tap < taps; tap++) - { - int r = a.Rm[y * a.Kh + tap / kw]; - for (int i = 0; i < Tile; i++) - { - int c = a.Cm[(x0 + Math.Min(i, rows - 1)) * kw + tap % kw]; - ptrs[tap * Tile + i] = r < 0 || c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic; - } - } - long o = (long)x0 * oc + blk * Ob; - float* outp = orow + o; - float* addp = arow != null ? arow + o : null; - if (a.Avx) TileAvx(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, a.Act); - else TileVec(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, a.Act); - } - } - - /// 6 × 16 tile: into a scratch tile, then the - /// residual / activation epilogue () per row. - private static void TileAvx(float** ptrs, int rows, int taps, int ic, float* wb, float* bb, - float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - float* part = stackalloc float[Tile * Ob]; - TileAvxK(ptrs, taps, ic, wb, bb, part, Ob); - for (int r = 0; r < rows; r++, part += Ob) - Store16(outp + (long)r * oc, addp == null ? null : addp + (long)r * oc, - Avx.LoadVector256(part), Avx.LoadVector256(part + 8), lanes, act); - } - - /// - /// The 6 × 16 k loop: dst[r*ds + 0..15] = bias + Σ x·w for all rows. It never - /// calls out — passing the accumulators to a helper (by reference under the x64 ABIs) makes - /// RyuJIT keep all twelve on the stack, a load + store around every FMA. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void TileAvxK(float** ptrs, int taps, int ic, float* wb, float* bb, float* dst, int ds) - { - F32x8 a0 = Avx.LoadVector256(bb), b0 = Avx.LoadVector256(bb + 8); - F32x8 a1 = a0, a2 = a0, a3 = a0, a4 = a0, a5 = a0; - F32x8 b1 = b0, b2 = b0, b3 = b0, b4 = b0, b5 = b0; - for (int tap = 0; tap < taps; tap++) - { - float** pp = ptrs + tap * Tile; - float* p0 = pp[0], p1 = pp[1], p2 = pp[2], p3 = pp[3], p4 = pp[4], p5 = pp[5]; - float* w = wb + (long)tap * ic * Ob; - for (int c = 0; c < ic; c++, w += Ob) - { - F32x8 w0 = Avx.LoadVector256(w), w1 = Avx.LoadVector256(w + 8); - F32x8 v0 = Avx.BroadcastScalarToVector256(p0 + c), v1 = Avx.BroadcastScalarToVector256(p1 + c); - a0 = Fma.MultiplyAdd(v0, w0, a0); b0 = Fma.MultiplyAdd(v0, w1, b0); - a1 = Fma.MultiplyAdd(v1, w0, a1); b1 = Fma.MultiplyAdd(v1, w1, b1); - F32x8 v2 = Avx.BroadcastScalarToVector256(p2 + c), v3 = Avx.BroadcastScalarToVector256(p3 + c); - a2 = Fma.MultiplyAdd(v2, w0, a2); b2 = Fma.MultiplyAdd(v2, w1, b2); - a3 = Fma.MultiplyAdd(v3, w0, a3); b3 = Fma.MultiplyAdd(v3, w1, b3); - F32x8 v4 = Avx.BroadcastScalarToVector256(p4 + c), v5 = Avx.BroadcastScalarToVector256(p5 + c); - a4 = Fma.MultiplyAdd(v4, w0, a4); b4 = Fma.MultiplyAdd(v4, w1, b4); - a5 = Fma.MultiplyAdd(v5, w0, a5); b5 = Fma.MultiplyAdd(v5, w1, b5); - } - } - Avx.Store(dst, a0); Avx.Store(dst + 8, b0); dst += ds; - Avx.Store(dst, a1); Avx.Store(dst + 8, b1); dst += ds; - Avx.Store(dst, a2); Avx.Store(dst + 8, b2); dst += ds; - Avx.Store(dst, a3); Avx.Store(dst + 8, b3); dst += ds; - Avx.Store(dst, a4); Avx.Store(dst + 8, b4); dst += ds; - Avx.Store(dst, a5); Avx.Store(dst + 8, b5); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Store16(float* o, float* add, F32x8 a, F32x8 b, int lanes, ActivationKind act) - { - if (lanes == Ob) - { - if (add != null) { a = Avx.Add(a, Avx.LoadVector256(add)); b = Avx.Add(b, Avx.LoadVector256(add + 8)); } - Avx.Store(o, Act8(a, act)); Avx.Store(o + 8, Act8(b, act)); - return; - } - // channel tail (e.g. the final oc=3 conv): only the valid lanes touch memory - float* t = stackalloc float[Ob]; - Avx.Store(t, a); Avx.Store(t + 8, b); - for (int l = 0; l < lanes; l++) - o[l] = ActS(add != null ? t[l] + add[l] : t[l], act); - } - - private static void TileVec(float** ptrs, int rows, int taps, int ic, float* wb, float* bb, - float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - if (UseNeon) - { - TileNeon(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, act); - return; - } - int VL = Vector.Count; - // software Vector (DOTNET_EnableHWIntrinsic=0) is far slower than plain scalar - bool vec = Vector.IsHardwareAccelerated && Ob % VL == 0; - if (vec && VL == 4) - { - TileVec4(ptrs, rows, taps, ic, wb, bb, outp, addp, oc, lanes, act); - return; - } - float* acc = stackalloc float[Ob]; - for (int i = 0; i < rows; i++) - { - for (int l = 0; l < Ob; l++) acc[l] = bb[l]; - for (int tap = 0; tap < taps; tap++) - { - float* p = ptrs[tap * Tile + i]; - float* w = wb + (long)tap * ic * Ob; - for (int c = 0; c < ic; c++, w += Ob) - { - float v = p[c]; - if (vec) - { - var vv = new Vector(v); - for (int l = 0; l < Ob; l += VL) - Simd.VectorStore(acc + l, Simd.VectorLoad(acc + l) + Simd.VectorLoad(w + l) * vv); - } - else - for (int l = 0; l < Ob; l++) acc[l] += v * w[l]; - } - } - float* o = outp + (long)i * oc; - float* ad = addp == null ? null : addp + (long)i * oc; - for (int l = 0; l < lanes; l++) - o[l] = ActS(ad != null ? acc[l] + ad[l] : acc[l], act); - } - } - - /// - /// 128-bit Vector<float> tile (AVX2 disabled): two pixel rows × 16 oc in eight register - /// accumulators (+4 weight vectors), instead of a stack accumulator round-tripped through - /// memory on every multiply-add. Same per-lane op order (acc + w·x) as the generic loop. - /// - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void TileVec4(float** ptrs, int rows, int taps, int ic, float* wb, float* bb, - float* outp, float* addp, int oc, int lanes, ActivationKind act) - { - float* tmp = stackalloc float[2 * Ob]; - for (int i = 0; i < rows; i += 2) - { - Vector a0 = Simd.VectorLoad(bb), a1 = Simd.VectorLoad(bb + 4), - a2 = Simd.VectorLoad(bb + 8), a3 = Simd.VectorLoad(bb + 12); - Vector c0 = a0, c1 = a1, c2 = a2, c3 = a3; - for (int tap = 0; tap < taps; tap++) - { - // ptrs holds Tile entries per tap (short tiles repeat their last pixel) - float* p = ptrs[tap * Tile + i], q = ptrs[tap * Tile + i + 1]; - float* w = wb + (long)tap * ic * Ob; - for (int c = 0; c < ic; c++, w += Ob) - { - Vector w0 = Simd.VectorLoad(w), w1 = Simd.VectorLoad(w + 4), - w2 = Simd.VectorLoad(w + 8), w3 = Simd.VectorLoad(w + 12); - var vp = new Vector(p[c]); - a0 += w0 * vp; a1 += w1 * vp; a2 += w2 * vp; a3 += w3 * vp; - var vq = new Vector(q[c]); - c0 += w0 * vq; c1 += w1 * vq; c2 += w2 * vq; c3 += w3 * vq; - } - } - Simd.VectorStore(tmp, a0); Simd.VectorStore(tmp + 4, a1); - Simd.VectorStore(tmp + 8, a2); Simd.VectorStore(tmp + 12, a3); - Simd.VectorStore(tmp + 16, c0); Simd.VectorStore(tmp + 20, c1); - Simd.VectorStore(tmp + 24, c2); Simd.VectorStore(tmp + 28, c3); - for (int r = 0; r < 2 && i + r < rows; r++) - { - float* o = outp + (long)(i + r) * oc; - float* ad = addp == null ? null : addp + (long)(i + r) * oc; - float* t = tmp + r * Ob; - for (int l = 0; l < lanes; l++) - o[l] = ActS(ad != null ? t[l] + ad[l] : t[l], act); - } - } - } - - private static void Depthwise(ReadOnlySpan src, int iw, int c, Span dst, - int oh, int ow, int kh, int kw, int[] rm, int[] cm, float[] padVec, - NhwcConvPlan kp, ReadOnlySpan add, ActivationKind act) - { - int taps = kh * kw; - bool hasAdd = !add.IsEmpty, avx = UseAvx; - int VL = Vector.Count; - fixed (float* sp0 = src, dp0 = dst, ap0 = add, wp0 = kp.W, bp0 = kp.Bias, pv0 = padVec) - fixed (int* rm0 = rm, cm0 = cm) - { - float* sp = sp0, dp = dp0, ap = ap0, wp = wp0, bp = bp0, pv = pv0; - int* rmp = rm0, cmp = cm0; - For(oh, (long)oh * ow * c * taps, y => - { - float** ptrs = stackalloc float*[taps]; - for (int x = 0; x < ow; x++) - { - for (int tap = 0; tap < taps; tap++) - { - int r = rmp[y * kh + tap / kw], cc = cmp[x * kw + tap % kw]; - ptrs[tap] = r < 0 || cc < 0 ? pv : sp + ((long)r * iw + cc) * c; - } - long ob = ((long)y * ow + x) * c; - float* o = dp + ob; - float* ad = hasAdd ? ap + ob : null; - int ch = 0; - if (UseNeon) - { - for (; ch + 4 <= c; ch += 4) - { - V128 acc = AdvSimd.LoadVector128(bp + ch); - for (int tap = 0; tap < taps; tap++) - acc = AdvSimd.FusedMultiplyAdd(acc, AdvSimd.LoadVector128(ptrs[tap] + ch), AdvSimd.LoadVector128(wp + tap * c + ch)); - if (ad != null) acc = AdvSimd.Add(acc, AdvSimd.LoadVector128(ad + ch)); - AdvSimd.Store(o + ch, Act4(acc, act)); - } - } - else if (avx) - { - for (; ch + 8 <= c; ch += 8) - { - F32x8 acc = Avx.LoadVector256(bp + ch); - for (int tap = 0; tap < taps; tap++) - acc = Fma.MultiplyAdd(Avx.LoadVector256(ptrs[tap] + ch), Avx.LoadVector256(wp + tap * c + ch), acc); - if (ad != null) acc = Avx.Add(acc, Avx.LoadVector256(ad + ch)); - Avx.Store(o + ch, Act8(acc, act)); - } - } - else if (UsePacked) - ch = DepthwisePixelPk(ptrs, taps, c, wp, bp, ad, o, act); - else if (Vector.IsHardwareAccelerated && act is ActivationKind.None or ActivationKind.Relu) - { - for (; ch + VL <= c; ch += VL) - { - var acc = Simd.VectorLoad(bp + ch); - for (int tap = 0; tap < taps; tap++) - acc += Simd.VectorLoad(ptrs[tap] + ch) * Simd.VectorLoad(wp + tap * c + ch); - if (ad != null) acc += Simd.VectorLoad(ad + ch); - if (act == ActivationKind.Relu) acc = Vector.Max(acc, Vector.Zero); - Simd.VectorStore(o + ch, acc); - } - } - for (; ch < c; ch++) - { - float acc = bp[ch]; - for (int tap = 0; tap < taps; tap++) acc += ptrs[tap][ch] * wp[tap * c + ch]; - if (ad != null) acc += ad[ch]; - o[ch] = ActS(acc, act); - } - } - }); - } - } - - // ======================== pixel-vector helpers ======================== - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Copy(float* s, float* d, int n) => - Buffer.MemoryCopy(s, d, (long)n * 4, (long)n * 4); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Fill(float* d, int n, float v) => new Span(d, n).Fill(v); - - // ======================== conv + 2x2 max pool ======================== - [ThreadStatic] private static float[]? t_band; - - private static float[] BandBuf(long floats) => - t_band is { } b && b.Length >= floats ? b : (t_band = new float[checked((int)floats)]); - - /// - /// Dense / Winograd conv with a fused 2x2 stride-2 max pool (): - /// each task produces a band of conv rows (2 direct, 4 = one Winograd tile row) in a - /// thread-local buffer and pools it straight into ([ph, pw, oc]), - /// so the full-resolution conv output is never stored. Per-element math is the same as - /// the separate Conv → MaxPool nodes. - /// - public static void ConvPool(ReadOnlySpan src, int ih, int iw, int ic, - Span dst, int ph, int pw, int oc, ConvP p, NhwcConvPlan kp) - { - int oh = ph * 2, ow = pw * 2; - if (kp.Depthwise || p.AddValue >= 0 || dst.Length != (long)ph * pw * oc) - throw new NotSupportedException("ConvPool shape/kind mismatch."); - bool relu = p.Pool!.FuseRelu, wino = kp.WinoU is not null; - int band = wino ? 4 : 2, tasks = (oh + band - 1) / band; - int kh = p.Kernel[0], kw = p.Kernel[1]; - float[] padVec = new float[ic * kw + Ob]; // one constant-pad kernel row (Dense512Row) - Array.Fill(padVec, p.PadValue); - int[] rm = Map(oh, kh, p.Strides[0], p.Pads[0], ih, p.PadMode); - int[] cm = Map(ow, kw, p.Strides[1], p.Pads[1], iw, p.PadMode); - fixed (float* sp = src, dp0 = dst, wp = kp.W, up = kp.WinoU, bp = kp.Bias, pv = padVec) - fixed (int* rmp = rm, cmp = cm) - { - var da = new DenseArgs(sp, wp, bp, pv, rmp, cmp, iw, ic, ow, oc, kh, kw, kp.Blocks, p.Act, UseAvx); - var wa = wino ? new WinoArgs(sp, up, bp, pv, ih, iw, ic, oh, ow, oc, kp.Blocks, p) : default; - float* dp = dp0; - For(tasks, (long)oh * ow * oc * kh * kw * ic / 64, t => - { - int y0 = t * band, rows = Math.Min(band, oh - y0); - float[] buf = BandBuf((long)band * ow * oc); - fixed (float* sb = buf) - { - if (wino) - for (int t0 = y0 / 4 * wa.Tw, te = t0 + wa.Tw; t0 < te; t0 += wa.T) - WinoTiles(wa, t0, Math.Min(wa.T, te - t0), sb, null, y0); - else - for (int r = 0; r < rows; r++) - ConvRow(da, y0 + r, sb + (long)r * ow * oc, null); - PoolBand(sb, rows / 2, ow, oc, dp + (long)(y0 / 2) * pw * oc, pw, relu); - } - }); - } - } - - /// 2x2/2 max over conv row pairs of a band; same op order as . - private static void PoolBand(float* sb, int prows, int ow, int c, float* drow, int pw, bool relu) - { - int VL = Vector.Count; - for (int r = 0; r < prows; r++) - { - float* top = sb + (long)(2 * r) * ow * c, bot = top + (long)ow * c; - float* orow = drow + (long)r * pw * c; - for (int x = 0; x < pw; x++) - { - float* o = orow + (long)x * c; - float* s0 = top + (long)(2 * x) * c, s1 = s0 + c, s2 = bot + (long)(2 * x) * c, s3 = s2 + c; - int ch = 0; - for (; ch + VL <= c; ch += VL) - { - var v = Vector.Max(Simd.VectorLoad(s0 + ch), Simd.VectorLoad(s1 + ch)); - v = Vector.Max(v, Simd.VectorLoad(s2 + ch)); - Simd.VectorStore(o + ch, Vector.Max(v, Simd.VectorLoad(s3 + ch))); - } - for (; ch < c; ch++) o[ch] = MathF.Max(MathF.Max(MathF.Max(s0[ch], s1[ch]), s2[ch]), s3[ch]); - if (relu) - for (ch = 0; ch < c; ch++) o[ch] = MathF.Max(o[ch], 0f); - } - } - } - - // ======================== maxpool ======================== - public static void MaxPool(ReadOnlySpan src, int ih, int iw, int c, - Span dst, int oh, int ow, PoolP p) - { - int kh = p.Kernel[0], kw = p.Kernel[1]; - int[] rm = Map(oh, kh, p.Strides[0], p.Pads[0], ih, PadMode.Constant); - int[] cm = Map(ow, kw, p.Strides[1], p.Pads[1], iw, PadMode.Constant); - bool relu = p.FuseRelu; - int VL = Vector.Count; - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - if (UseAvx512 && c % 16 == 0) { MaxPool512(sp, iw, c, dp, oh, ow, kh, kw, rm, cm, relu); return; } - For(oh, (long)oh * ow * c * kh * kw, y => - { - for (int x = 0; x < ow; x++) - { - float* o = dp + ((long)y * ow + x) * c; - bool any = false; - for (int ky = 0; ky < kh; ky++) - { - int r = rm[y * kh + ky]; - if (r < 0) continue; - for (int kx = 0; kx < kw; kx++) - { - int cc = cm[x * kw + kx]; - if (cc < 0) continue; - float* s = sp + ((long)r * iw + cc) * c; - if (!any) { Copy(s, o, c); any = true; continue; } - int ch = 0; - for (; ch + VL <= c; ch += VL) - Simd.VectorStore(o + ch, Vector.Max(Simd.VectorLoad(o + ch), Simd.VectorLoad(s + ch))); - for (; ch < c; ch++) o[ch] = MathF.Max(o[ch], s[ch]); - } - } - if (!any) Fill(o, c, 0f); - else if (relu) - for (int ch = 0; ch < c; ch++) o[ch] = MathF.Max(o[ch], 0f); - } - }); - } - } - - // ======================== bilinear resize (half-pixel) ======================== - private static void ResizeAxis(int outLen, int inLen, float scale, int[] i0, int[] i1, float[] wt) - { - for (int o = 0; o < outLen; o++) - { - float f = (o + 0.5f) / scale - 0.5f; - float fl = MathF.Floor(f); - i0[o] = Math.Clamp((int)fl, 0, inLen - 1); - i1[o] = Math.Clamp((int)fl + 1, 0, inLen - 1); - wt[o] = f - fl; - } - } - - /// One output pixel of the bilinear resize into (c channels): - /// v = [relu](lerp) + [relu](add) — the same per-element ops as separate Resize/Relu/Add nodes. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void ResizePixel(float* a, float* b, float* e, float* f, float wX, float wY, - Vector vX, Vector vY, int c, bool relu, float* add, bool addRelu, float* o) - { - int VL = Vector.Count, ch = 0; - for (; ch + VL <= c; ch += VL) - { - var va = Simd.VectorLoad(a + ch); var ve = Simd.VectorLoad(e + ch); - var top = va + (Simd.VectorLoad(b + ch) - va) * vX; - var bot = ve + (Simd.VectorLoad(f + ch) - ve) * vX; - var v = top + (bot - top) * vY; - if (relu) v = Vector.Max(v, Vector.Zero); - if (add != null) - { - var s = Simd.VectorLoad(add + ch); - v += addRelu ? Vector.Max(s, Vector.Zero) : s; - } - Simd.VectorStore(o + ch, v); - } - for (; ch < c; ch++) - { - float top = a[ch] + (b[ch] - a[ch]) * wX; - float bot = e[ch] + (f[ch] - e[ch]) * wX; - float v = top + (bot - top) * wY; - if (relu) v = MathF.Max(v, 0f); - if (add != null) v += addRelu ? MathF.Max(add[ch], 0f) : add[ch]; - o[ch] = v; - } - } - - /// NHWC bilinear resize; optional same-shape addend (out = [relu](resize) + [relu](add)). - public static void ResizeBilinear(ReadOnlySpan src, int ih, int iw, int c, - Span dst, int oh, int ow, float sy, float sx, ActivationKind act, - ReadOnlySpan add = default, bool addRelu = false) - { - if (!add.IsEmpty && add.Length != dst.Length) - throw new NotSupportedException($"resize addend has {add.Length} elems, expected {dst.Length}."); - int[] y0 = new int[oh], y1 = new int[oh], x0 = new int[ow], x1 = new int[ow]; - float[] wy = new float[oh], wx = new float[ow]; - ResizeAxis(oh, ih, sy, y0, y1, wy); - ResizeAxis(ow, iw, sx, x0, x1, wx); - bool relu = act == ActivationKind.Relu, hasAdd = !add.IsEmpty, v512 = UseAvx512 && c % 16 == 0; - fixed (float* sp0 = src, dp0 = dst, ap0 = add) - { - float* sp = sp0, dp = dp0, ap = ap0; - For(oh, (long)oh * ow * c, y => - { - float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c; - var vY = new Vector(wy[y]); - float wY = wy[y]; - if (v512) - { - var vY16 = Vector512.Create(wY); - for (int x = 0; x < ow; x++) - { - long ob = ((long)y * ow + x) * c; - ResizePixel512(r0 + (long)x0[x] * c, r0 + (long)x1[x] * c, r1 + (long)x0[x] * c, r1 + (long)x1[x] * c, - Vector512.Create(wx[x]), vY16, c, relu, hasAdd ? ap + ob : null, addRelu, dp + ob); - } - return; - } - for (int x = 0; x < ow; x++) - { - long ob = ((long)y * ow + x) * c; - ResizePixel(r0 + (long)x0[x] * c, r0 + (long)x1[x] * c, r1 + (long)x0[x] * c, r1 + (long)x1[x] * c, - wx[x], wY, new Vector(wx[x]), vY, c, relu, hasAdd ? ap + ob : null, addRelu, dp + ob); - } - }); - } - } - - // ======================== fused upsample tail ======================== - /// Packs UpHead weights [oc, c, 1, 1] as [c][8] (oc lanes, zero-padded) + bias[8]. - public static UpHeadPlan PackUpHead(ReadOnlySpan w, int oc, int c, ReadOnlySpan bias) - { - if (oc > 8) throw new NotSupportedException($"UpHead supports oc <= 8 (got {oc})."); - float[] pw = new float[c * 8], pb = new float[8]; - for (int o = 0; o < oc; o++) - for (int ch = 0; ch < c; ch++) pw[ch * 8 + o] = w[o * c + ch]; - if (!bias.IsEmpty) bias.CopyTo(pb); - return new UpHeadPlan { W = pw, Bias = pb, Oc = oc, C = c }; - } - - /// - /// out[y,x,o] = act(bias[o] + Σ_c w[o,c]·s[c]), s = [relu](resize(src)) + [relu](skip): the - /// full-resolution resize/add tensors live only in a per-pixel stack buffer. Per-element - /// math follows the unfused nodes (bias-initialized, channel-ascending accumulation). - /// - public static void UpHead(ReadOnlySpan src, int ih, int iw, int c, ReadOnlySpan skip, - Span dst, int oh, int ow, float sy, float sx, UpHeadP p, UpHeadPlan kp, - int skipH = 0, int skipW = 0, int skipC = 0) - { - // SkipConv: `skip` is the conv's input [skipH, skipW, skipC]; each task recomputes - // its skip row (same direct-conv kernel → same values) instead of reading a stored one - ConvP? sc = p.SkipConv; - var skp = sc?.KernelPlan as NhwcConvPlan; - if (kp.C != c || dst.Length != (long)oh * ow * kp.Oc - || (sc is null ? skip.Length != (long)oh * ow * c - : skp is null || skp.Depthwise || skp.WinoU is not null || sc.AddValue >= 0 || skip.Length != (long)skipH * skipW * skipC)) - throw new NotSupportedException("UpHead shape mismatch."); - int[] y0 = new int[oh], y1 = new int[oh], x0 = new int[ow], x1 = new int[ow]; - float[] wy = new float[oh], wx = new float[ow]; - ResizeAxis(oh, ih, sy, y0, y1, wy); - ResizeAxis(ow, iw, sx, x0, x1, wx); - bool relu = p.ResizeRelu, skipRelu = p.SkipRelu, avx = UseAvx; - int oc = kp.Oc; - bool neonUp = UseNeon && c % 4 == 0 && oc <= 4; - bool pkUp = UsePacked && c % 4 == 0 && oc <= 4; - ActivationKind act = p.Act; - int[] rm = [], cm = []; - float[] padVec = []; - if (sc is not null) - { - rm = Map(oh, sc.Kernel[0], sc.Strides[0], sc.Pads[0], skipH, sc.PadMode); - cm = Map(ow, sc.Kernel[1], sc.Strides[1], sc.Pads[1], skipW, sc.PadMode); - padVec = new float[skipC * sc.Kernel[1] + Ob]; // one constant-pad kernel row (Dense512Row) - Array.Fill(padVec, sc.PadValue); - } - fixed (float* sp0 = src, dp0 = dst, kp0 = skip, wp0 = kp.W, bp0 = kp.Bias) - fixed (float* cw = skp?.W, cb = skp?.Bias, pv = padVec) - fixed (int* rmp = rm, cmp = cm) - { - float* sp = sp0, dp = dp0, kpp = kp0, wp = wp0, bp = bp0; - var ca = sc is null ? default - : new DenseArgs(kp0, cw, cb, pv, rmp, cmp, skipW, skipC, ow, c, sc.Kernel[0], sc.Kernel[1], skp!.Blocks, sc.Act, avx); - if (UseAvx512) - { - UpHead512(sp, iw, c, kpp, sc is not null, ca, dp, oh, ow, y0, y1, x0, x1, wy, wx, relu, skipRelu, wp, bp, oc, act); - return; - } - For(oh, (long)oh * ow * c * (oc + 4), y => - { - float* s = stackalloc float[c + 8]; - float* acc = stackalloc float[8]; - float* r0 = sp + (long)y0[y] * iw * c, r1 = sp + (long)y1[y] * iw * c; - var vY = new Vector(wy[y]); - float wY = wy[y]; - float* orow = dp + (long)y * ow * oc; - float[]? band = sc is null ? null : BandBuf((long)ow * c); - fixed (float* sb = band) - { - float* krow = kpp + (long)y * ow * c; - if (sc is not null) - { - ConvRow(ca, y, sb, null); - krow = sb; - } - if (neonUp) - { - fixed (int* xa = x0, xb = x1) - fixed (float* wxp = wx) - UpHeadRowNeon(r0, r1, xa, xb, wxp, wY, krow, c, ow, wp, bp, orow, oc, relu, skipRelu, act); - return; - } - if (pkUp) - { - fixed (int* xa = x0, xb = x1) - fixed (float* wxp = wx) - UpHeadRowPk(r0, r1, xa, xb, wxp, wY, krow, c, ow, wp, bp, orow, oc, relu, skipRelu, act); - return; - } - for (int x = 0; x < ow; x++) - { - ResizePixel(r0 + (long)x0[x] * c, r0 + (long)x1[x] * c, r1 + (long)x0[x] * c, r1 + (long)x1[x] * c, - wx[x], wY, new Vector(wx[x]), vY, c, relu, krow + (long)x * c, skipRelu, s); - float* o = orow + (long)x * oc; - if (avx) - { - F32x8 a = Avx.LoadVector256(bp); - float* w = wp; - for (int ch = 0; ch < c; ch++, w += 8) - a = Fma.MultiplyAdd(Avx.BroadcastScalarToVector256(s + ch), Avx.LoadVector256(w), a); - Avx.Store(acc, a); - } - else if (UseNeon) - { - V128 a = AdvSimd.LoadVector128(bp), b = AdvSimd.LoadVector128(bp + 4); - float* w = wp; - for (int ch = 0; ch < c; ch++, w += 8) - { - V128 v = AdvSimd.LoadAndReplicateToVector128(s + ch); - a = AdvSimd.FusedMultiplyAdd(a, AdvSimd.LoadVector128(w), v); - b = AdvSimd.FusedMultiplyAdd(b, AdvSimd.LoadVector128(w + 4), v); - } - AdvSimd.Store(acc, a); AdvSimd.Store(acc + 4, b); - } - else - { - for (int l = 0; l < 8; l++) acc[l] = bp[l]; - float* w = wp; - for (int ch = 0; ch < c; ch++, w += 8) - { - float v = s[ch]; - for (int l = 0; l < oc; l++) acc[l] += v * w[l]; - } - } - for (int l = 0; l < oc; l++) o[l] = acc[l]; - } - if (act != ActivationKind.None) ActRow(orow, ow * oc, act, avx); - } - }); - } - } - - private static void ActRow(float* p, int n, ActivationKind act, bool avx) - { - int i = 0; - if (avx) - for (; i + 8 <= n; i += 8) Avx.Store(p + i, Act8(Avx.LoadVector256(p + i), act)); - else if (UseNeon) - for (; i + 4 <= n; i += 4) AdvSimd.Store(p + i, Act4(AdvSimd.LoadVector128(p + i), act)); - else if (UsePacked) - for (; i + 4 <= n; i += 4) PackedSimd.Store(p + i, ActP(LdP(p + i), act)); - else if (act == ActivationKind.Sigmoid) - { - var row = new Span(p + i, n - i); - SimdMath.Sigmoid(row, row); - return; - } - for (; i < n; i++) p[i] = ActS(p[i], act); - } - - // ======================== pad (spatial only) ======================== - public static void Pad(ReadOnlySpan src, int ih, int iw, int c, - Span dst, int oh, int ow, PadP p) - { - int rank = p.Pads.Length / 2; - if (rank != 4) throw new NotSupportedException("NHWC Pad supports rank-4 only."); - int[] rm = Map(oh, 1, 1, p.Pads[2], ih, p.Mode); - int[] cm = Map(ow, 1, 1, p.Pads[3], iw, p.Mode); - float val = p.Value; - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - For(oh, (long)oh * ow * c, y => - { - int r = rm[y]; - for (int x = 0; x < ow; x++) - { - float* o = dp + ((long)y * ow + x) * c; - int cc = cm[x]; - if (r < 0 || cc < 0) Fill(o, c, val); - else Copy(sp + ((long)r * iw + cc) * c, o, c); - } - }); - } - } - - // ======================== channel shift ======================== - // out[y,x,c] = in[y+off_g, x, c] (logical axis 2) or in[y, x+off_g, c] (axis 3), zero-fill - public static void ChannelShift(ReadOnlySpan src, int h, int w, int c, - Span dst, ShiftP p) - { - int[] ends = p.GroupEnds, offs = p.Offsets; - bool alongH = p.Axis == 2; - fixed (float* sp0 = src, dp0 = dst) - { - float* sp = sp0, dp = dp0; - For(h, (long)h * w * c, y => - { - for (int x = 0; x < w; x++) - { - float* o = dp + ((long)y * w + x) * c; - int g0 = 0; - for (int g = 0; g < offs.Length; g++) - { - int g1 = ends[g]; - int sy = alongH ? y + offs[g] : y, sx = alongH ? x : x + offs[g]; - if ((uint)sy < (uint)h && (uint)sx < (uint)w) - Copy(sp + ((long)sy * w + sx) * c + g0, o + g0, g1 - g0); - else - Fill(o + g0, g1 - g0, 0f); - g0 = g1; - } - if (g0 < c) Copy(sp + ((long)y * w + x) * c + g0, o + g0, c - g0); - } - }); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.Neon.cs b/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.Neon.cs deleted file mode 100644 index d70409a..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.Neon.cs +++ /dev/null @@ -1,75 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// NEON Winograd schedule for layers whose transformed weights U (36·oc·ic floats) outgrow the L2 -/// caches (the deep, low-resolution layers): the per-task tile blocks of stream -/// every U panel once per block of tiles — DRAM-bound on the RK3399 (512 KB A53 / 1 MB A72 L2). -/// Instead tiles are taken in large groups and each group runs three parallel phases: input -/// transforms into V[36][Tg][ic], one GEMM task per (xi, oc block) that loads its U panel once for -/// the whole group, then the output transforms with the usual epilogue. Same per-element math as -/// (same transforms, same k loop), so the outputs are bit-identical. -/// -internal static unsafe partial class Nhwc -{ - /// U sizes (floats) above this take on NEON (512 KB). - private const long NeonWinoUFloats = 128 * 1024; - - [ThreadStatic] private static float[]? t_wgV, t_wgM; - - private static void WinogradGrouped(in WinoArgs a, float* dp, float* ap) - { - int ic = a.Ic, nb = a.Nb, tw = a.Tw, oh = a.Oh, ow = a.Ow, oc = a.Oc; - int nt = (oh + 3) / 4 * tw; - // V group ~2 MB (spread over the L2s, re-read nb times), whole Tile-row GEMM tiles - int tg = Math.Clamp(524288 / (36 * ic) / Tile * Tile, Tile, (nt + Tile - 1) / Tile * Tile); - float* V = Pinned(ref t_wgV, 36 * tg * ic), M = Pinned(ref t_wgM, 36 * nb * tg * Ob); - WinoArgs ca = a; - for (int g0 = 0; g0 < nt; g0 += tg) - { - int tn = Math.Min(tg, nt - g0); - Parallel.For(0, (tn + 7) / 8, PO, k => - { - float** dp6 = stackalloc float*[36]; - for (int t = k * 8, te = Math.Min(tn, t + 8); t < te; t++) - { - int ty = (g0 + t) / tw, tx = (g0 + t) % tw; - int iy0 = ty * 4 - ca.PadT, ix0 = tx * 4 - ca.PadL; - for (int i = 0; i < 6; i++) - { - int r = SrcIdx(iy0 + i, ca.Ih, ca.Reflect); - for (int j = 0; j < 6; j++) - { - int c = SrcIdx(ix0 + j, ca.Iw, ca.Reflect); - dp6[i * 6 + j] = r < 0 || c < 0 ? ca.PadVec : ca.Src + ((long)r * ca.Iw + c) * ic; - } - } - InputTransform(dp6, ic, V + (long)t * ic, (long)tg * ic); - } - }); - Parallel.For(0, 36 * nb, PO, j => - { - int xi = j / nb, blk = j % nb; - float** ptrs = stackalloc float*[Tile]; - float* zero = stackalloc float[Ob]; - new Span(zero, Ob).Clear(); - float* wb = ca.U + ((long)xi * nb + blk) * ic * Ob; - float* vx = V + (long)xi * tg * ic, mx = M + ((long)xi * nb + blk) * tg * Ob; - // all Tile rows land in M (tg is a multiple of Tile); rows past tn are never read - for (int r0 = 0; r0 < tn; r0 += Tile) - { - int rows = Math.Min(Tile, tn - r0); - for (int i = 0; i < Tile; i++) ptrs[i] = vx + (long)(r0 + Math.Min(i, rows - 1)) * ic; - TileNeonK(ptrs, 1, ic, wb, zero, mx + (long)r0 * Ob, Ob); - } - }); - int chunks = (tn + 15) / 16; - Parallel.For(0, chunks * nb, PO, j => - { - int blk = j % nb, t0 = j / nb * 16; - float* tmp = stackalloc float[16 * Ob]; - WinoOutput(M + ((long)blk * tg + t0) * Ob, (long)nb * tg * Ob, Ob, Math.Min(16, tn - t0), g0 + t0, - tw, oh, ow, oc, blk * Ob, ca.Bias + blk * Ob, dp, ap, 0, ca.Act, tmp, false, false); - }); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.cs b/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.cs deleted file mode 100644 index aea5029..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/NhwcWinograd.cs +++ /dev/null @@ -1,399 +0,0 @@ -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.Wasm; -using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; -using F32x8 = System.Runtime.Intrinsics.Vector256; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// Winograd F(4x4, 3x3) for dense 3x3 stride-1 NHWC convs: 36 multiplies per -/// 4x4 output tile and (ic, oc) pair instead of 144. Tiles are processed in -/// cache-sized blocks per task — input transform into a block-local -/// V[36][T][ic], then per 16-oc block 36 small GEMMs through the 6x16 conv -/// micro-kernel into M[36][T][16], then the output transform with the usual -/// bias / residual / activation epilogue — so the 2.25x-expanded transform -/// tensors never leave L2. -/// -internal static unsafe partial class Nhwc -{ - private static readonly bool s_winograd = Environment.GetEnvironmentVariable("GCDRNET_WINOGRAD") != "0"; - - [ThreadStatic] private static float[]? t_wino; - - /// Dense 3x3 s1 convs with enough channels for the transforms to amortize. - internal static bool WinogradEligible(ConvP p, int[] wShape, int ic) => - s_winograd && Vector.IsHardwareAccelerated && p.Group == 1 - && wShape[2] == 3 && wShape[3] == 3 && p.Strides[0] == 1 && p.Strides[1] == 1 - && p.Dilations[0] == 1 && p.Dilations[1] == 1 - && ic >= 16 && ic % Vector.Count == 0 && ic % 8 == 0; - - // G (6x3) of F(4,3): U = G g G^T - private static readonly double[,] s_g = - { - { 1.0 / 4, 0, 0 }, - { -1.0 / 6, -1.0 / 6, -1.0 / 6 }, - { -1.0 / 6, 1.0 / 6, -1.0 / 6 }, - { 1.0 / 24, 1.0 / 12, 1.0 / 6 }, - { 1.0 / 24, -1.0 / 12, 1.0 / 6 }, - { 0, 0, 1 }, - }; - - /// Transformed weights [36][oc/16][ic][16] (oc zero-padded), from ONNX [oc][ic][3][3]. - internal static float[] PackWinograd(ReadOnlySpan w, int oc, int ic) - { - int nb = (oc + Ob - 1) / Ob; - float[] u = new float[checked(36 * nb * ic * Ob)]; - double* gg = stackalloc double[18]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - { - ReadOnlySpan k = w.Slice((o * ic + c) * 9, 9); - // gg = G·k (6x3) - for (int i = 0; i < 6; i++) - for (int j = 0; j < 3; j++) - gg[i * 3 + j] = s_g[i, 0] * k[j] + s_g[i, 1] * k[3 + j] + s_g[i, 2] * k[6 + j]; - // U = gg·G^T (6x6) - for (int i = 0; i < 6; i++) - for (int j = 0; j < 6; j++) - { - double v = gg[i * 3] * s_g[j, 0] + gg[i * 3 + 1] * s_g[j, 1] + gg[i * 3 + 2] * s_g[j, 2]; - u[(((i * 6 + j) * nb + o / Ob) * ic + c) * Ob + o % Ob] = (float)v; - } - } - return u; - } - - /// Per-call Winograd constants shared by every tile block. - private readonly unsafe struct WinoArgs - { - public readonly float* Src, U, Bias, PadVec; - public readonly int Ih, Iw, Ic, Oh, Ow, Oc, Tw, T, Nb, PadT, PadL; - public readonly bool Reflect, Avx, Avx512, Pk; - public readonly ActivationKind Act; - - public WinoArgs(float* src, float* u, float* bias, float* padVec, int ih, int iw, int ic, - int oh, int ow, int oc, int nb, ConvP p) - { - Src = src; U = u; Bias = bias; PadVec = padVec; - Ih = ih; Iw = iw; Ic = ic; Oh = oh; Ow = ow; Oc = oc; Nb = nb; - Tw = (ow + 3) / 4; - Avx512 = UseAvx512; - Avx = UseAvx; - Pk = UsePacked; - // V block ~256 KB (AVX2); the AVX-512 tier (Tile5-row x 32-oc GEMM tiles) measured - // best at ~64 KB on Zen 5, NEON at ~16 KB (one Tile row for ic >= 32: four RK3399 A53s share 512 KB L2) - int tr = Avx512 ? Tile5 : Pk ? PkTile : Tile, vBudget = Avx512 ? 16384 : UseNeon ? 4096 : 65536; - T = Math.Clamp(vBudget / (36 * ic) / tr * tr, tr, 60 / tr * tr); - PadT = p.Pads[0]; PadL = p.Pads[1]; - Reflect = p.PadMode == PadMode.Reflect; - Act = p.Act; - } - } - - private static void WinogradConv(ReadOnlySpan src, int ih, int iw, int ic, - Span dst, int oh, int ow, int oc, ConvP p, NhwcConvPlan kp, ReadOnlySpan add) - { - float[] padVec = new float[ic]; - Array.Fill(padVec, p.PadValue); - bool hasAdd = !add.IsEmpty; - fixed (float* sp = src, dp0 = dst, ap0 = add, up = kp.WinoU, bp = kp.Bias, pv = padVec) - { - var a = new WinoArgs(sp, up, bp, pv, ih, iw, ic, oh, ow, oc, kp.Blocks, p); - int nt = (oh + 3) / 4 * a.Tw, chunks = (nt + a.T - 1) / a.T; - float* dp = dp0, ap = hasAdd ? ap0 : null; - if (UseNeon && 36L * a.Nb * ic * Ob > NeonWinoUFloats) - { - WinogradGrouped(a, dp, ap); - return; - } - For(chunks, (long)oh * ow * oc * ic, chunk => - { - int t0 = chunk * a.T; - WinoTiles(a, t0, Math.Min(a.T, nt - t0), dp, ap, 0); - }); - } - } - - /// - /// Tiles [t0, t0+tn) (raster order, tn <= T) of one Winograd conv, written to - /// whose row 0 is output row (rows past - /// Oh are skipped). Every tile's math is independent of how tiles are grouped. - /// - private static void WinoTiles(in WinoArgs a, int t0, int tn, float* dp, float* ap, int rowBase) - { - int ic = a.Ic, T = a.T, tw = a.Tw, nb = a.Nb, oc = a.Oc, ow = a.Ow, oh = a.Oh; - bool avx = a.Avx, avx512 = a.Avx512, pk = a.Pk; - // AVX-512: Tile5-row x 32-oc GEMM tiles over block pairs; M rows hold both blocks - int tr = avx512 ? Tile5 : pk ? PkTile : Tile, step = avx512 ? 2 : 1, mRow = step * Ob; - int vSize = 36 * T * ic, mSize = 36 * T * mRow; - int need = vSize + mSize + 16 * Ob + 2 * Ob; - float[] buf = t_wino is { } tb && tb.Length >= need ? tb : (t_wino = new float[need]); - fixed (float* b0 = buf) - { - float* V = b0, M = b0 + vSize, tmp = M + mSize, zero = tmp + 16 * Ob; - new Span(zero, 2 * Ob).Clear(); - float** dp6 = stackalloc float*[36]; - for (int t = 0; t < tn; t++) - { - int ty = (t0 + t) / tw, tx = (t0 + t) % tw; - int iy0 = ty * 4 - a.PadT, ix0 = tx * 4 - a.PadL; - for (int i = 0; i < 6; i++) - { - int r = SrcIdx(iy0 + i, a.Ih, a.Reflect); - for (int j = 0; j < 6; j++) - { - int c = SrcIdx(ix0 + j, a.Iw, a.Reflect); - dp6[i * 6 + j] = r < 0 || c < 0 ? a.PadVec : a.Src + ((long)r * a.Iw + c) * ic; - } - } - if (avx512 && ic % 16 == 0) InputTransform512(dp6, ic, V + (long)t * ic, (long)T * ic); - else InputTransform(dp6, ic, V + (long)t * ic, (long)T * ic); - } - - float** ptrs = stackalloc float*[Math.Max(Tile, Tile5)]; - for (int blk0 = 0; blk0 < nb; blk0 += step) - { - int nblk = Math.Min(step, nb - blk0); - for (int xi = 0; xi < 36; xi++) - { - float* wb = a.U + ((long)xi * nb + blk0) * ic * Ob; - float* vx = V + (long)xi * T * ic, mx = M + (long)xi * T * mRow; - for (int r0 = 0; r0 < tn; r0 += tr) - { - int rows = Math.Min(tr, tn - r0); - for (int i = 0; i < tr; i++) ptrs[i] = vx + (long)(r0 + Math.Min(i, rows - 1)) * ic; - // AVX2 / NEON / PackedSimd write all tr rows into M (T is a multiple of tr); rows past tn are never read - if (avx512) Tile512(ptrs, nblk, 1, ic, wb, (long)ic * Ob, zero, mx + (long)r0 * mRow); - else if (avx) TileAvxK(ptrs, 1, ic, wb, zero, mx + (long)r0 * Ob, Ob); - else if (UseNeon) TileNeonK(ptrs, 1, ic, wb, zero, mx + (long)r0 * Ob, Ob); - else if (pk) TilePkK(ptrs, PkTile, 1, ic, (long)ic * Ob, wb, zero, mx + (long)r0 * Ob, Ob); - else TileVec(ptrs, rows, 1, ic, wb, zero, mx + (long)r0 * Ob, null, Ob, Ob, ActivationKind.None); - } - } - for (int h = 0; h < nblk; h++) - { - int blk = blk0 + h; - WinoOutput(M + h * Ob, (long)T * mRow, mRow, tn, t0, tw, oh, ow, oc, blk * Ob, a.Bias + blk * Ob, - dp, ap, rowBase, a.Act, tmp, avx, avx512); - } - } - } - } - - /// Output transform + bias/residual/activation epilogue of one 16-oc block for tn tiles; - /// tile t's 36 products are m[xi*mStride + t*mRow + 0..15]. - private static void WinoOutput(float* m, long mStride, int mRow, int tn, int t0, int tw, int oh, int ow, int oc, - int ocOff, float* bb, float* dp, float* ap, int rowBase, ActivationKind act, float* tmp, bool avx, bool avx512) - { - int lanes = Math.Min(Ob, oc - ocOff); - bool fast512 = avx512 && lanes == Ob && act is ActivationKind.None or ActivationKind.Relu; - bool fastNeon = UseNeon && lanes == Ob; - bool fastPk = UsePacked && lanes == Ob; - for (int t = 0; t < tn; t++) - { - int ty = (t0 + t) / tw, tx = (t0 + t) % tw; - if (avx512) OutputTransform512(m + (long)t * mRow, mStride, tmp); - else OutputTransform(m + (long)t * mRow, mStride, tmp); - for (int py = 0; py < 4; py++) - { - int oy = ty * 4 + py; - if (oy >= oh) break; - for (int px = 0; px < 4; px++) - { - int ox = tx * 4 + px; - if (ox >= ow) break; - long o = ((long)(oy - rowBase) * ow + ox) * oc + ocOff; - float* tp = tmp + (py * 4 + px) * Ob; - if (fast512) - { - var v = Avx512F.Add(Avx512F.LoadVector512(tp), Avx512F.LoadVector512(bb)); - if (ap != null) v = Avx512F.Add(v, Avx512F.LoadVector512(ap + o)); - if (act == ActivationKind.Relu) v = Avx512F.Max(v, Vector512.Zero); - Avx512F.Store(dp + o, v); - } - else if (fastNeon) - for (int l = 0; l < Ob; l += 4) - { - var v = AdvSimd.Add(AdvSimd.LoadVector128(tp + l), AdvSimd.LoadVector128(bb + l)); - if (ap != null) v = AdvSimd.Add(v, AdvSimd.LoadVector128(ap + o + l)); - AdvSimd.Store(dp + o + l, Act4(v, act)); - } - else if (fastPk) - for (int l = 0; l < Ob; l += 4) - { - var v = PackedSimd.Add(LdP(tp + l), LdP(bb + l)); - if (ap != null) v = PackedSimd.Add(v, LdP(ap + o + l)); - PackedSimd.Store(dp + o + l, ActP(v, act)); - } - else if (avx) - Store16(dp + o, ap != null ? ap + o : null, - Avx.Add(Avx.LoadVector256(tp), Avx.LoadVector256(bb)), - Avx.Add(Avx.LoadVector256(tp + 8), Avx.LoadVector256(bb + 8)), lanes, act); - else - for (int l = 0; l < lanes; l++) - { - float v = tp[l] + bb[l]; - if (ap != null) v += ap[o + l]; - dp[o + l] = ActS(v, act); - } - } - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int SrcIdx(int i, int n, bool reflect) => - (uint)i < (uint)n ? i : reflect ? Reflect(i, n) : -1; - - /// V[xi] = (B^T d B)[xi] for one tile, all channels; V row stride per xi = vStride. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void InputTransform(float** d, int ic, float* v, long vStride) - { - int VL = Vector.Count; - var t = stackalloc Vector[36]; - var four = new Vector(4f); - var five = new Vector(5f); - var two = new Vector(2f); - for (int c = 0; c < ic; c += VL) - { - // columns: t = B^T d - for (int j = 0; j < 6; j++) - { - var d0 = Simd.VectorLoad(d[j] + c); var d1 = Simd.VectorLoad(d[6 + j] + c); - var d2 = Simd.VectorLoad(d[12 + j] + c); var d3 = Simd.VectorLoad(d[18 + j] + c); - var d4 = Simd.VectorLoad(d[24 + j] + c); var d5 = Simd.VectorLoad(d[30 + j] + c); - t[j] = four * d0 - five * d2 + d4; - t[6 + j] = d3 + d4 - four * (d1 + d2); - t[12 + j] = d4 - d3 + four * (d1 - d2); - t[18 + j] = d4 - d2 + two * (d3 - d1); - t[24 + j] = d4 - d2 - two * (d3 - d1); - t[30 + j] = four * d1 - five * d3 + d5; - } - // rows: V = t B - for (int i = 0; i < 6; i++) - { - var r0 = t[i * 6]; var r1 = t[i * 6 + 1]; var r2 = t[i * 6 + 2]; - var r3 = t[i * 6 + 3]; var r4 = t[i * 6 + 4]; var r5 = t[i * 6 + 5]; - float* o = v + c; - long b = (long)(i * 6) * vStride; - Simd.VectorStore(o + b, four * r0 - five * r2 + r4); - Simd.VectorStore(o + b + vStride, r3 + r4 - four * (r1 + r2)); - Simd.VectorStore(o + b + 2 * vStride, r4 - r3 + four * (r1 - r2)); - Simd.VectorStore(o + b + 3 * vStride, r4 - r2 + two * (r3 - r1)); - Simd.VectorStore(o + b + 4 * vStride, r4 - r2 - two * (r3 - r1)); - Simd.VectorStore(o + b + 5 * vStride, four * r1 - five * r3 + r5); - } - } - } - - /// tmp[(py*4+px)*16 + l] = (A^T m A)[py,px] for one tile's 16 oc lanes; m row stride per xi = mStride. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void OutputTransform(float* m, long mStride, float* tmp) - { - int VL = Vector.Count; - var s = stackalloc Vector[24]; - var two = new Vector(2f); - var four = new Vector(4f); - var eight = new Vector(8f); - for (int l = 0; l < Ob; l += VL) - { - // columns: s = A^T m (4x6) - for (int j = 0; j < 6; j++) - { - var m0 = Simd.VectorLoad(m + j * mStride + l); var m1 = Simd.VectorLoad(m + (6 + j) * mStride + l); - var m2 = Simd.VectorLoad(m + (12 + j) * mStride + l); var m3 = Simd.VectorLoad(m + (18 + j) * mStride + l); - var m4 = Simd.VectorLoad(m + (24 + j) * mStride + l); var m5 = Simd.VectorLoad(m + (30 + j) * mStride + l); - var a = m1 + m2; var b = m1 - m2; var c = m3 + m4; var e = m3 - m4; - s[j] = m0 + a + c; - s[6 + j] = b + two * e; - s[12 + j] = a + four * c; - s[18 + j] = b + eight * e + m5; - } - // rows: y = s A (4x4) - for (int i = 0; i < 4; i++) - { - var r0 = s[i * 6]; var r1 = s[i * 6 + 1]; var r2 = s[i * 6 + 2]; - var r3 = s[i * 6 + 3]; var r4 = s[i * 6 + 4]; var r5 = s[i * 6 + 5]; - var a = r1 + r2; var b = r1 - r2; var c = r3 + r4; var e = r3 - r4; - float* o = tmp + i * 4 * Ob + l; - Simd.VectorStore(o, r0 + a + c); - Simd.VectorStore(o + Ob, b + two * e); - Simd.VectorStore(o + 2 * Ob, a + four * c); - Simd.VectorStore(o + 3 * Ob, b + eight * e + r5); - } - } - } - - /// with 16 lanes (AVX-512; ic % 16 == 0). Same per-lane ops → same bits. - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void InputTransform512(float** d, int ic, float* v, long vStride) - { - var t = stackalloc Vector512[36]; - var four = Vector512.Create(4f); - var five = Vector512.Create(5f); - var two = Vector512.Create(2f); - for (int c = 0; c < ic; c += 16) - { - for (int j = 0; j < 6; j++) - { - var d0 = Vector512.Load(d[j] + c); var d1 = Vector512.Load(d[6 + j] + c); - var d2 = Vector512.Load(d[12 + j] + c); var d3 = Vector512.Load(d[18 + j] + c); - var d4 = Vector512.Load(d[24 + j] + c); var d5 = Vector512.Load(d[30 + j] + c); - t[j] = four * d0 - five * d2 + d4; - t[6 + j] = d3 + d4 - four * (d1 + d2); - t[12 + j] = d4 - d3 + four * (d1 - d2); - t[18 + j] = d4 - d2 + two * (d3 - d1); - t[24 + j] = d4 - d2 - two * (d3 - d1); - t[30 + j] = four * d1 - five * d3 + d5; - } - for (int i = 0; i < 6; i++) - { - var r0 = t[i * 6]; var r1 = t[i * 6 + 1]; var r2 = t[i * 6 + 2]; - var r3 = t[i * 6 + 3]; var r4 = t[i * 6 + 4]; var r5 = t[i * 6 + 5]; - float* o = v + c + (long)(i * 6) * vStride; - (four * r0 - five * r2 + r4).Store(o); - (r3 + r4 - four * (r1 + r2)).Store(o + vStride); - (r4 - r3 + four * (r1 - r2)).Store(o + 2 * vStride); - (r4 - r2 + two * (r3 - r1)).Store(o + 3 * vStride); - (r4 - r2 - two * (r3 - r1)).Store(o + 4 * vStride); - (four * r1 - five * r3 + r5).Store(o + 5 * vStride); - } - } - } - - /// with one 16-lane vector per element (AVX-512). - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void OutputTransform512(float* m, long mStride, float* tmp) - { - var s = stackalloc Vector512[24]; - var two = Vector512.Create(2f); - var four = Vector512.Create(4f); - var eight = Vector512.Create(8f); - for (int j = 0; j < 6; j++) - { - var m0 = Vector512.Load(m + j * mStride); var m1 = Vector512.Load(m + (6 + j) * mStride); - var m2 = Vector512.Load(m + (12 + j) * mStride); var m3 = Vector512.Load(m + (18 + j) * mStride); - var m4 = Vector512.Load(m + (24 + j) * mStride); var m5 = Vector512.Load(m + (30 + j) * mStride); - var a = m1 + m2; var b = m1 - m2; var c = m3 + m4; var e = m3 - m4; - s[j] = m0 + a + c; - s[6 + j] = b + two * e; - s[12 + j] = a + four * c; - s[18 + j] = b + eight * e + m5; - } - for (int i = 0; i < 4; i++) - { - var r0 = s[i * 6]; var r1 = s[i * 6 + 1]; var r2 = s[i * 6 + 2]; - var r3 = s[i * 6 + 3]; var r4 = s[i * 6 + 4]; var r5 = s[i * 6 + 5]; - var a = r1 + r2; var b = r1 - r2; var c = r3 + r4; var e = r3 - r4; - float* o = tmp + i * 4 * Ob; - (r0 + a + c).Store(o); - (b + two * e).Store(o + Ob); - (a + four * c).Store(o + 2 * Ob); - (b + eight * e + r5).Store(o + 3 * Ob); - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Ops.cs b/src/Sdcb.SimdGcdrNet/Kernels/Ops.cs deleted file mode 100644 index 3f1eabe..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Ops.cs +++ /dev/null @@ -1,1005 +0,0 @@ -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// Flat-float NCHW kernel library. Dispatch is a plain IsSupported chain: -/// AVX2+FMA hot paths, then the portable Vector<float> tier, else scalar -/// (the reference). Runtime knobs: DOTNET_EnableAVX2 / DOTNET_EnableAVX / -/// DOTNET_EnableHWIntrinsic. -/// -internal static unsafe class Ops -{ - // ======================== Conv ======================== - // src [1,IC,H,W]; w [OC,IC/g,kh,kw]; dst [1,OC,OH,OW]. N==1 assumed. - // `add` is an optional same-shape residual addend, applied before the activation. - public static void Conv(ReadOnlySpan src, int[] inShape, - ReadOnlySpan w, int[] wShape, ReadOnlySpan bias, - Span dst, int[] outShape, ConvP p, ReadOnlySpan add = default) - { - if (!add.IsEmpty && add.Length != dst.Length) - throw new NotSupportedException( - $"conv residual addend has {add.Length} elems, expected {dst.Length}."); - if (!add.IsEmpty) - { - // activation must run after the residual add: conv without act, then fused add+act - var q = new ConvP - { - Pads = p.Pads, Strides = p.Strides, Dilations = p.Dilations, Kernel = p.Kernel, - Group = p.Group, Act = ActivationKind.None, PadMode = p.PadMode, - PadValue = p.PadValue, BiasValue = p.BiasValue, KernelPlan = p.KernelPlan, - }; - Conv(src, inShape, w, wShape, bias, dst, outShape, q); - AddInPlace(dst, add, p.Act); - return; - } - int ic = inShape[1], ih = inShape[2], iw = inShape[3]; - int oc = wShape[0], kh = wShape[2], kw = wShape[3]; - int oh = outShape[2], ow = outShape[3]; - int sh = p.Strides[0], sw = p.Strides[1]; - int pt = p.Pads[0], pl = p.Pads[1], pb = p.Pads[2], pr = p.Pads[3]; - int group = p.Group; - - bool symmetricPad = pt == pb && pl == pr; - if (Avx2.IsSupported && Fma.IsSupported) - { - if (group > 1 && group == ic) - { - Avx2Kernels.Depthwise(src, ic, ih, iw, w, bias, dst, oh, ow, - kh, kw, pt, pl, p.PadMode, p.PadValue, p.Act); - return; - } - if (group == 1 && kh == 3 && kw == 3 && oc % 4 == 0 && symmetricPad) - { - if (sh == 1 && sw == 1) - { - Avx2Kernels.Conv3x3S1(src, ic, ih, iw, w, bias, dst, oc, oh, ow, - pt, pl, p.PadMode, p.PadValue, p.Act); - return; - } - if (sh == 2 && sw == 2) - { - Avx2Kernels.Conv3x3S2(src, ic, ih, iw, w, bias, dst, oc, oh, ow, - pt, pl, p.PadMode, p.PadValue, p.Act); - return; - } - } - if (group == 1 && kh == 1 && kw == 1 && sh == 1 && sw == 1 && pt == 0 && pl == 0 && pb == 0 && pr == 0) - { - Avx2Kernels.Conv1x1(src, ic, ih * iw, w, bias, dst, oc, p.Act); - return; - } - // unspecialized shape: fall through to the Vector tier - } - else if (!Vector.IsHardwareAccelerated) - { - ConvScalar(src, ic, ih, iw, w, bias, dst, oc, oh, ow, kh, kw, sh, sw, - pt, pl, group, p.PadMode, p.PadValue, p.Act); - return; - } - - // Vector tier - if (group > 1) - { - if (group != ic) // grouped (non-depthwise) conv: scalar reference handles groups correctly - { - ConvScalar(src, ic, ih, iw, w, bias, dst, oc, oh, ow, kh, kw, sh, sw, - pt, pl, group, p.PadMode, p.PadValue, p.Act); - return; - } - ConvDepthwiseVec(src, ic, ih, iw, w, bias, dst, oh, ow, kh, kw, - pt, pl, p.PadMode, p.PadValue, p.Act); - return; - } - if (kh == 1 && kw == 1 && sh == 1 && sw == 1 && pt == 0 && pl == 0) - { - Conv1x1Vec(src, ic, ih, iw, w, bias, dst, oc, oh, ow, p.Act); - return; - } - ConvVecW(src, ic, ih, iw, w, bias, dst, oc, oh, ow, kh, kw, sh, sw, - pt, pl, group, p.PadMode, p.PadValue, p.Act); - } - - /// dst = act(dst + add) — residual epilogue for fused Conv+Add. - private static void AddInPlace(Span dst, ReadOnlySpan add, ActivationKind act) - { - int n = dst.Length, i = 0, VL = Vector.Count; - var zero = Vector.Zero; - for (; i + VL <= n; i += VL) - { - var v = Simd.VectorLoad(dst, i) + Simd.VectorLoad(add, i); - if (act == ActivationKind.Relu) v = Vector.Max(v, zero); - Simd.VectorStore(dst, i, v); - } - for (; i < n; i++) - { - float v = dst[i] + add[i]; - dst[i] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - if (act != ActivationKind.None && act != ActivationKind.Relu) - ApplyActVec(dst, act); - } - - // ---- scalar reference conv (N==1, dense or grouped) ---- - private static void ConvScalar(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, - Span dst, int oc, int oh, int ow, int kh, int kw, - int sh, int sw, int pt, int pl, int group, - PadMode padMode, float padValue, ActivationKind act) - { - int icg = ic / group, ocg = oc / group; - for (int g = 0; g < group; g++) - for (int o = g * ocg; o < (g + 1) * ocg; o++) - for (int y = 0; y < oh; y++) - for (int x = 0; x < ow; x++) - { - float acc = bias.IsEmpty ? 0f : bias[o]; - for (int c = g * icg; c < (g + 1) * icg; c++) - for (int ky = 0; ky < kh; ky++) - { - int sy = y * sh + ky - pt; - if ((uint)sy >= (uint)ih) - { - if (padMode == PadMode.Reflect) sy = Reflect(sy, ih); - else continue; - } - for (int kx = 0; kx < kw; kx++) - { - int sx = x * sw + kx - pl; - if ((uint)sx >= (uint)iw) - { - if (padMode == PadMode.Reflect) sx = Reflect(sx, iw); - else continue; - } - acc += src[(c * ih + sy) * iw + sx] * w[((o * icg + c - g * icg) * kh + ky) * kw + kx]; - } - } - dst[(o * oh + y) * ow + x] = ApplyAct(acc, act); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int Reflect(int i, int dim) - { - // ONNX reflect (half-sample symmetric, no edge repeat): -1→1, dim→dim-2 - if (dim <= 1) return 0; // reflect is undefined on a singleton axis; don't spin - while (i < 0 || i >= dim) i = i < 0 ? -i : 2 * dim - 2 - i; - return i; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static float ApplyAct(float v, ActivationKind act) => - act == ActivationKind.Relu ? MathF.Max(v, 0f) : - act == ActivationKind.Sigmoid ? 1f / (1f + MathF.Exp(-v)) : - act == ActivationKind.Gelu ? 0.5f * v * (1f + SimdMath.ErfScalar(v * 0.70710678f)) : v; - - // ---- vector-over-output-W conv: one output row vector per (oc,y) ---- - private static void ConvVecW(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, - Span dst, int oc, int oh, int ow, int kh, int kw, - int sh, int sw, int pt, int pl, int group, - PadMode padMode, float padValue, ActivationKind act) - { - int icg = ic / group; - int VL = Vector.Count; - // interior x-range where all taps in-bounds (no pad needed) - for (int o = 0; o < oc; o++) - { - int g = o / (oc / group); // output channel -> group index - for (int y = 0; y < oh; y++) - { - Span drow = dst.Slice((o * oh + y) * ow, ow); - if (!bias.IsEmpty) drow.Fill(bias[o]); else drow.Clear(); - for (int c = g * icg; c < (g + 1) * icg; c++) - { - ReadOnlySpan sbase = src.Slice(c * ih * iw, ih * iw); - ReadOnlySpan wk = w.Slice((o * icg + (c - g * icg)) * kh * kw, kh * kw); - for (int ky = 0; ky < kh; ky++) - { - int sy = y * sh + ky - pt; - bool yOk = (uint)sy < (uint)ih; - int syR = yOk ? sy : (padMode == PadMode.Reflect ? Reflect(sy, ih) : -1); - for (int kx = 0; kx < kw; kx++) - { - float wv = wk[ky * kw + kx]; - if (wv == 0f) continue; - if (sw == 1) - { - // contiguous reads: dst[x] += wv * src[sy][x + kx - pl] - int xBegin = Math.Max(0, pl - kx); - int xEnd = Math.Min(ow, iw + pl - kx); - if (yOk || padMode == PadMode.Reflect) - { - VecAccMul(drow.Slice(xBegin, xEnd - xBegin), - sbase.Slice(syR * iw + xBegin + kx - pl, xEnd - xBegin), wv); - } - else if (padValue != 0f) - { - for (int x = xBegin; x < xEnd; x++) drow[x] += wv * padValue; - } - // pad regions (constant → add wv*padValue; reflect → gather) - for (int x = 0; x < xBegin; x++) drow[x] += wv * PadFetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue, syR, sy); - for (int x = xEnd; x < ow; x++) drow[x] += wv * PadFetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue, syR, sy); - } - else - { - for (int x = 0; x < ow; x++) - drow[x] += wv * PadFetch(sbase, sy, x * sw + kx - pl, ih, iw, padMode, padValue, syR, sy); - } - } - } - } - ApplyActVec(drow, act); - } - } - - static float PadFetch(ReadOnlySpan sbase, int sy, int sx, int ih, int iw, - PadMode mode, float padValue, int syR, int syOrig) - { - int yy = syOrig, xx = sx; - if ((uint)yy >= (uint)ih || (uint)xx >= (uint)iw) - { - if (mode == PadMode.Constant) return padValue; - yy = Reflect(yy, ih); xx = Reflect(xx, iw); - } - return sbase[yy * iw + xx]; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void VecAccMul(Span dst, ReadOnlySpan src, float w) - { - int i = 0, VL = Vector.Count; - var vw = new Vector(w); - for (; i + VL <= dst.Length; i += VL) - Simd.VectorStore(dst, i, Simd.VectorAddMul(Simd.VectorLoad(dst, i), Simd.VectorLoad(src, i), vw)); - for (; i < dst.Length; i++) dst[i] += src[i] * w; - } - - private static void ApplyActVec(Span row, ActivationKind act) - { - if (act == ActivationKind.None) return; - if (act == ActivationKind.Relu) { Relu(row, row); return; } - if (act == ActivationKind.Gelu) { SimdMath.Gelu(row, row); return; } - if (act == ActivationKind.Sigmoid) { SimdMath.Sigmoid(row, row); return; } - for (int i = 0; i < row.Length; i++) row[i] = ApplyAct(row[i], act); - } - - // ---- 1x1 conv = GEMM over pixels: dst[oc, HW] = W[oc, ic] * src[ic, HW] ---- - private static void Conv1x1Vec(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, - Span dst, int oc, int oh, int ow, ActivationKind act) - { - int hw = ih * iw; - int VL = Vector.Count; - for (int o = 0; o < oc; o++) - { - Span drow = dst.Slice(o * hw, hw); - float b = bias.IsEmpty ? 0f : bias[o]; - for (int i = 0; i < hw; i++) drow[i] = b; - ReadOnlySpan wo = w.Slice(o * ic, ic); - for (int c = 0; c < ic; c++) - { - float wv = wo[c]; - if (wv == 0f) continue; - VecAccMul(drow, src.Slice(c * hw, hw), wv); - } - ApplyActVec(drow, act); - } - } - - // ---- depthwise conv (group == C): dst[c,y,x] ---- - private static void ConvDepthwiseVec(ReadOnlySpan src, int ic, int ih, int iw, - ReadOnlySpan w, ReadOnlySpan bias, - Span dst, int oh, int ow, int kh, int kw, - int pt, int pl, PadMode padMode, float padValue, ActivationKind act) - { - int VL = Vector.Count; - for (int c = 0; c < ic; c++) - { - ReadOnlySpan sbase = src.Slice(c * ih * iw, ih * iw); - ReadOnlySpan wk = w.Slice(c * kh * kw, kh * kw); - for (int y = 0; y < oh; y++) - { - Span drow = dst.Slice((c * oh + y) * ow, ow); - float b = bias.IsEmpty ? 0f : bias[c]; - for (int i = 0; i < ow; i++) drow[i] = b; - for (int ky = 0; ky < kh; ky++) - { - int sy = y + ky - pt; - for (int kx = 0; kx < kw; kx++) - { - float wv = wk[ky * kw + kx]; - if (wv == 0f) continue; - int xBegin = Math.Max(0, pl - kx); - int xEnd = Math.Min(ow, iw + pl - kx); - if ((uint)sy < (uint)ih) - { - VecAccMul(drow.Slice(xBegin, xEnd - xBegin), - sbase.Slice(sy * iw + xBegin + kx - pl, xEnd - xBegin), wv); - } - for (int x = 0; x < xBegin; x++) drow[x] += wv * Fetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue); - for (int x = xEnd; x < ow; x++) drow[x] += wv * Fetch(sbase, sy, x + kx - pl, ih, iw, padMode, padValue); - if ((uint)sy >= (uint)ih && padMode == PadMode.Constant && padValue != 0f) - for (int x = xBegin; x < xEnd; x++) drow[x] += wv * padValue; - else if ((uint)sy >= (uint)ih && padMode == PadMode.Reflect) - for (int x = xBegin; x < xEnd; x++) drow[x] += wv * sbase[Reflect(sy, ih) * iw + x + kx - pl]; - } - } - ApplyActVec(drow, act); - } - } - - static float Fetch(ReadOnlySpan s, int y, int x, int ih, int iw, PadMode m, float pv) - { - if ((uint)y >= (uint)ih || (uint)x >= (uint)iw) - { - if (m == PadMode.Constant) return pv; - y = Reflect(y, ih); x = Reflect(x, iw); - } - return s[y * iw + x]; - } - } - - // ======================== MaxPool ======================== - public static void MaxPool(ReadOnlySpan src, int[] inShape, Span dst, int[] outShape, - PoolP p) - { - int c = inShape[1], ih = inShape[2], iw = inShape[3]; - int oh = outShape[2], ow = outShape[3]; - int kh = p.Kernel[0], kw = p.Kernel[1], sh = p.Strides[0], sw = p.Strides[1]; - int pt = p.Pads[0], pl = p.Pads[1]; - if (Avx2.IsSupported && Fma.IsSupported && - kh == 2 && kw == 2 && sh == 2 && sw == 2 && pt == 0 && pl == 0) - { - Avx2Kernels.MaxPool2x2(src, c, ih, iw, dst, oh, ow, p.FuseRelu); - return; - } - for (int ch = 0; ch < c; ch++) - { - ReadOnlySpan s = src.Slice(ch * ih * iw, ih * iw); - Span d = dst.Slice(ch * oh * ow, oh * ow); - for (int y = 0; y < oh; y++) - for (int x = 0; x < ow; x++) - { - float m = float.NegativeInfinity; - for (int ky = 0; ky < kh; ky++) - { - int sy = y * sh + ky - pt; - if ((uint)sy >= (uint)ih) continue; - for (int kx = 0; kx < kw; kx++) - { - int sx = x * sw + kx - pl; - if ((uint)sx >= (uint)iw) continue; - float v = s[sy * iw + sx]; - if (v > m) m = v; - } - } - if (m == float.NegativeInfinity) m = 0f; - d[y * ow + x] = p.FuseRelu && m < 0f ? 0f : m; - } - } - } - - // ======================== elementwise ======================== - public static void Elementwise(ReadOnlySpan a, int[] aShape, - ReadOnlySpan b, int[] bShape, Span dst, int[] outShape, EwOp op, - ActivationKind act = ActivationKind.None) - { - int n = outShape.Aggregate(1, (x, y) => x * y); - int oi = (int)op; - bool avx = Avx2.IsSupported && Fma.IsSupported && n >= 8192; - // fast path: same shape, contiguous - if (aShape.SequenceEqual(outShape) && bShape.SequenceEqual(outShape)) - { - if (avx) { Avx2Kernels.Ew2(a, b, dst, n, oi, act); return; } - VecBin(a, b, dst, n, oi); goto Done; - } - // scalar broadcast - if (bShape.Aggregate(1, (x, y) => x * y) == 1) - { - if (avx) { Avx2Kernels.Ew1(a, dst, n, b[0], oi, act: act); return; } - VecScalar(a, dst, n, b[0], oi); goto Done; - } - if (aShape.Aggregate(1, (x, y) => x * y) == 1) - { - if (avx) - { - Avx2Kernels.Ew1(b, dst, n, a[0], oi, - rev: op is EwOp.Sub or EwOp.Div, act: act); - return; - } - if (op is EwOp.Sub or EwOp.Div) VecScalarR(b, dst, n, a[0], oi); - else VecScalar(b, dst, n, a[0], oi); - goto Done; - } - // inner-block broadcast: a full-shape, b constant over trailing dims - if (avx && aShape.SequenceEqual(outShape)) - { - int[] bStr = BroadcastStrides(bShape, outShape.Length); - int[] outStr = StridesOf(outShape); - int inner = 1, rank = outShape.Length; - while (rank > 1 && bStr[rank - 1] == 0) - { - inner *= outShape[rank - 1]; - rank--; - } - if (inner > 8 && n % inner == 0) - { - Avx2Kernels.EwInner(a, b, bStr, outShape, outStr, - outShape.Length, inner, dst, n, oi, act); - return; - } - } - // general broadcast (strided) - BroadcastLoop(a, aShape, b, bShape, dst, outShape, op); - Done: - if (act == ActivationKind.Relu) Relu(dst, dst); - } - - private static void VecBin(ReadOnlySpan a, ReadOnlySpan b, Span d, int n, int op) - { - int i = 0, VL = Vector.Count; - for (; i + VL <= n; i += VL) - { - var va = Simd.VectorLoad(a, i); var vb = Simd.VectorLoad(b, i); - var r = op switch { 0 => va + vb, 1 => va - vb, 2 => va * vb, _ => va / vb }; - Simd.VectorStore(d, i, r); - } - for (; i < n; i++) - d[i] = op switch { 0 => a[i] + b[i], 1 => a[i] - b[i], 2 => a[i] * b[i], _ => a[i] / b[i] }; - } - - private static void VecScalar(ReadOnlySpan a, Span d, int n, float s, int op) - { - int i = 0, VL = Vector.Count; - var vs = new Vector(s); - for (; i + VL <= n; i += VL) - { - var va = Simd.VectorLoad(a, i); - var r = op switch { 0 => va + vs, 1 => va - vs, 2 => va * vs, _ => va / vs }; - Simd.VectorStore(d, i, r); - } - for (; i < n; i++) - d[i] = op switch { 0 => a[i] + s, 1 => a[i] - s, 2 => a[i] * s, _ => a[i] / s }; - } - - private static void VecScalarR(ReadOnlySpan b, Span d, int n, float s, int op) - { - for (int i = 0; i < n; i++) - d[i] = op == 1 ? s - b[i] : s / b[i]; - } - - private static void BroadcastLoop(ReadOnlySpan a, int[] aShape, - ReadOnlySpan b, int[] bShape, Span dst, int[] outShape, EwOp op) - { - int rank = outShape.Length; - int[] aStr = BroadcastStrides(aShape, rank), bStr = BroadcastStrides(bShape, rank); - int[] outStr = StridesOf(outShape); - int n = outShape.Aggregate(1, (x, y) => x * y); - for (int e = 0; e < n; e++) - { - int rem = e, ao = 0, bo = 0; - for (int d = 0; d < rank; d++) - { - int c = rem / outStr[d]; - rem %= outStr[d]; - ao += c * aStr[d]; bo += c * bStr[d]; - } - dst[e] = op switch - { - EwOp.Add => a[ao] + b[bo], EwOp.Sub => a[ao] - b[bo], - EwOp.Mul => a[ao] * b[bo], _ => a[ao] / b[bo], - }; - } - } - - private static int[] BroadcastStrides(int[] shape, int rank) - { - int[] s = new int[rank]; - int acc = 1; - for (int i = shape.Length - 1; i >= 0; i--) { s[i + rank - shape.Length] = acc; acc *= shape[i]; } - for (int i = 0; i < rank; i++) - { - int si = i - (rank - shape.Length); - if (si < 0 || shape[si] == 1) s[i] = 0; - } - return s; - } - - private static int[] StridesOf(int[] shape) - { - int[] s = new int[shape.Length]; - int acc = 1; - for (int i = shape.Length - 1; i >= 0; i--) { s[i] = acc; acc *= shape[i]; } - return s; - } - - // ======================== unary ======================== - public static void Relu(ReadOnlySpan src, Span dst) - { - int n = src.Length; - if (Avx2.IsSupported && Fma.IsSupported && n >= 8192) { Avx2Kernels.Unary(src, dst, n, 0); return; } - int i = 0, VL = Vector.Count; - for (; i + VL <= n; i += VL) - Simd.VectorStore(dst, i, Vector.Max(Simd.VectorLoad(src, i), Vector.Zero)); - for (; i < n; i++) dst[i] = MathF.Max(src[i], 0f); - } - - public static void Sigmoid(ReadOnlySpan src, Span dst) - { - if (Avx2.IsSupported && Fma.IsSupported && src.Length >= 8192) { Avx2Kernels.Unary(src, dst, src.Length, 1); return; } - SimdMath.Sigmoid(src, dst); - } - - public static void Erf(ReadOnlySpan src, Span dst) - { - if (Avx2.IsSupported && Fma.IsSupported && src.Length >= 8192) { Avx2Kernels.Unary(src, dst, src.Length, 3); return; } - SimdMath.Erf(src, dst); - } - - public static void Gelu(ReadOnlySpan src, Span dst) - { - if (Avx2.IsSupported && Fma.IsSupported && src.Length >= 8192) { Avx2Kernels.Unary(src, dst, src.Length, 2); return; } - SimdMath.Gelu(src, dst); - } - - // ======================== LayerNorm ======================== - // x [.., rows, C] normalized over last dim; w[C], b[C] - public static void LayerNorm(ReadOnlySpan x, int[] shape, - ReadOnlySpan w, ReadOnlySpan b, Span dst, float eps) - { - int c = shape[^1]; - int rows = (int)(x.Length / c); - if (Avx2.IsSupported && Fma.IsSupported) - { - Avx2Kernels.LayerNorm(x, rows, c, w, b, dst, eps); - return; - } - if (Nhwc.UseNeon) - { - NeonKernels.LayerNorm(x, rows, c, w, b, dst, eps); - return; - } - if (Nhwc.UsePacked) - { - PackedKernels.LayerNorm(x, rows, c, w, b, dst, eps); - return; - } - for (int r = 0; r < rows; r++) - { - ReadOnlySpan xr = x.Slice(r * c, c); - Span dr = dst.Slice(r * c, c); - float s = 0f, s2 = 0f; - for (int i = 0; i < c; i++) { float v = xr[i]; s += v; s2 += v * v; } - float mean = s / c; - float inv = 1f / MathF.Sqrt(s2 / c - mean * mean + eps); - for (int i = 0; i < c; i++) dr[i] = (xr[i] - mean) * inv * w[i] + b[i]; - } - } - - // ======================== Gemm ======================== - // A [M,K] (last-2-dims of A), B [K,N] or [N,K] if transB, bias [N]; dst [M,N] - public static void Gemm(ReadOnlySpan a, int[] aShape, ReadOnlySpan b, - int[] bShape, ReadOnlySpan bias, Span dst, int[] outShape, GemmP p) - { - int m = 1; - for (int i = 0; i < aShape.Length - 1; i++) m *= aShape[i]; - int k = aShape[^1]; - int n = outShape[^1]; - if (!p.TransB && Avx2.IsSupported && Fma.IsSupported) - Avx2Kernels.GemmKN(a, m, k, b, n, bias, dst); - else if (p.TransB) - GemmBT(a, m, k, b, n, bias, dst); - else - GemmKN(a, m, k, b, n, bias, dst); - switch (p.Act) - { - case ActivationKind.Relu: Relu(dst, dst); break; - case ActivationKind.Gelu: SimdMath.Gelu(dst, dst); break; - case ActivationKind.Sigmoid: SimdMath.Sigmoid(dst, dst); break; - } - } - - // B row-major [K,N]: dst[m,n] = Σ_k a[m,k]*b[k,n] — vector over N - private static void GemmKN(ReadOnlySpan a, int M, int K, - ReadOnlySpan b, int N, ReadOnlySpan bias, Span dst) - { - int VL = Vector.Count; - // tile M by 4 for B-row reuse - int m0 = 0; - for (; m0 + 4 <= M; m0 += 4) - { - ReadOnlySpan a0 = a.Slice(m0 * K, K), a1 = a.Slice((m0 + 1) * K, K), - a2 = a.Slice((m0 + 2) * K, K), a3 = a.Slice((m0 + 3) * K, K); - for (int n0 = 0; n0 < N; n0 += VL) - { - int nw = Math.Min(VL, N - n0); - if (nw < VL) break; // tail handled scalar below - var acc0 = bias.IsEmpty ? Vector.Zero : Simd.VectorLoad(bias, n0); - var acc1 = acc0; var acc2 = acc0; var acc3 = acc0; - for (int k = 0; k < K; k++) - { - var bv = Simd.VectorLoad(b, k * N + n0); - acc0 = Simd.VectorAddMul(acc0, bv, a0[k]); - acc1 = Simd.VectorAddMul(acc1, bv, a1[k]); - acc2 = Simd.VectorAddMul(acc2, bv, a2[k]); - acc3 = Simd.VectorAddMul(acc3, bv, a3[k]); - } - Simd.VectorStore(dst, m0 * N + n0, acc0); - Simd.VectorStore(dst, (m0 + 1) * N + n0, acc1); - Simd.VectorStore(dst, (m0 + 2) * N + n0, acc2); - Simd.VectorStore(dst, (m0 + 3) * N + n0, acc3); - } - // N tail - for (int n0 = N - N % VL; n0 < N; n0++) - { - float s0 = bias.IsEmpty ? 0 : bias[n0], s1 = s0, s2 = s0, s3 = s0; - for (int k = 0; k < K; k++) - { - float bv = b[k * N + n0]; - s0 += a0[k] * bv; s1 += a1[k] * bv; s2 += a2[k] * bv; s3 += a3[k] * bv; - } - dst[m0 * N + n0] = s0; dst[(m0 + 1) * N + n0] = s1; - dst[(m0 + 2) * N + n0] = s2; dst[(m0 + 3) * N + n0] = s3; - } - } - for (; m0 < M; m0++) - { - ReadOnlySpan ar = a.Slice(m0 * K, K); - for (int n0 = 0; n0 + VL <= N; n0 += VL) - { - var acc = bias.IsEmpty ? Vector.Zero : Simd.VectorLoad(bias, n0); - for (int k = 0; k < K; k++) - acc = Simd.VectorAddMul(acc, Simd.VectorLoad(b, k * N + n0), ar[k]); - Simd.VectorStore(dst, m0 * N + n0, acc); - } - for (int n0 = N - N % VL; n0 < N; n0++) - { - float s = bias.IsEmpty ? 0 : bias[n0]; - for (int k = 0; k < K; k++) s += ar[k] * b[k * N + n0]; - dst[m0 * N + n0] = s; - } - } - } - - // B row-major [N,K] (transB): dst[m,n] = Σ_k a[m,k]*b[n,k] — dot products - private static void GemmBT(ReadOnlySpan a, int M, int K, - ReadOnlySpan b, int N, ReadOnlySpan bias, Span dst) - { - for (int m = 0; m < M; m++) - { - ReadOnlySpan ar = a.Slice(m * K, K); - for (int n = 0; n < N; n++) - { - float s = bias.IsEmpty ? 0 : bias[n]; - ReadOnlySpan br = b.Slice(n * K, K); - int k = 0, VL = Vector.Count; - var acc = Vector.Zero; - for (; k + VL <= K; k += VL) - acc += Simd.VectorLoad(ar, k) * Simd.VectorLoad(br, k); - for (int i = 0; i < VL; i++) s += acc[i]; - for (; k < K; k++) s += ar[k] * br[k]; - dst[m * N + n] = s; - } - } - } - - // ======================== Resize ======================== - // bilinear, half_pixel, per-axis scale (NCHW; only spatial dims scale != 1) - public static void ResizeBilinear(ReadOnlySpan src, int[] inShape, - Span dst, int[] outShape, float[] scales, - ActivationKind act = ActivationKind.None) - { - int c = inShape[1], ih = inShape[2], iw = inShape[3]; - int oh = outShape[2], ow = outShape[3]; - float sy = scales[2], sx = scales[3]; - // precompute row/col src indices + lerp weights - int[] y0 = new int[oh], y1 = new int[oh]; - float[] wy = new float[oh]; - for (int y = 0; y < oh; y++) - { - float fy = (y + 0.5f) / sy - 0.5f; - float fl = MathF.Floor(fy); - y0[y] = Math.Clamp((int)fl, 0, ih - 1); - y1[y] = Math.Clamp((int)fl + 1, 0, ih - 1); - wy[y] = fy - fl; - } - int[] x0 = new int[ow], x1 = new int[ow]; - float[] wx = new float[ow]; - for (int x = 0; x < ow; x++) - { - float fx = (x + 0.5f) / sx - 0.5f; - float fl = MathF.Floor(fx); - x0[x] = Math.Clamp((int)fl, 0, iw - 1); - x1[x] = Math.Clamp((int)fl + 1, 0, iw - 1); - wx[x] = fx - fl; - } - if (Avx2.IsSupported && Fma.IsSupported && oh * ow >= 8192) - { - Avx2Kernels.Bilinear(src, c, ih, iw, dst, oh, ow, y0, y1, wy, x0, x1, wx, act); - return; - } - for (int ch = 0; ch < c; ch++) - { - ReadOnlySpan s = src.Slice(ch * ih * iw, ih * iw); - Span d = dst.Slice(ch * oh * ow, oh * ow); - for (int y = 0; y < oh; y++) - { - ReadOnlySpan r0 = s.Slice(y0[y] * iw, iw); - ReadOnlySpan r1 = s.Slice(y1[y] * iw, iw); - float wY = wy[y]; - for (int x = 0; x < ow; x++) - { - float wX = wx[x]; - float top = r0[x0[x]] + (r0[x1[x]] - r0[x0[x]]) * wX; - float bot = r1[x0[x]] + (r1[x1[x]] - r1[x0[x]]) * wX; - float v = top + (bot - top) * wY; - d[y * ow + x] = act == ActivationKind.Relu ? MathF.Max(v, 0f) : v; - } - } - } - } - - // ======================== Pad ======================== - public static void Pad(ReadOnlySpan src, int[] inShape, Span dst, - int[] outShape, PadP p) - { - int rank = inShape.Length; - int n = 0, c = 0, ih = 0, iw = 0, oh = 0, ow = 0; - if (rank == 4) { n = inShape[0]; c = inShape[1]; ih = inShape[2]; iw = inShape[3]; oh = outShape[2]; ow = outShape[3]; } - else throw new NotSupportedException("Pad supports rank-4 only."); - int pt = p.Pads[2], pl = p.Pads[3]; - // index maps - int[] mapY = new int[oh], mapX = new int[ow]; - for (int y = 0; y < oh; y++) - { - int sy = y - pt; - if ((uint)sy < (uint)ih) mapY[y] = sy; - else if (p.Mode == PadMode.Reflect) mapY[y] = Reflect(sy, ih); - else mapY[y] = -1; - } - for (int x = 0; x < ow; x++) - { - int sx = x - pl; - if ((uint)sx < (uint)iw) mapX[x] = sx; - else if (p.Mode == PadMode.Reflect) mapX[x] = Reflect(sx, iw); - else mapX[x] = -1; - } - if (Avx2.IsSupported && Fma.IsSupported && oh * ow >= 4096) - { - Avx2Kernels.Pad4(src, n * c, ih, iw, dst, oh, ow, mapY, mapX, p.Value); - return; - } - for (int nc = 0; nc < n * c; nc++) - { - ReadOnlySpan s = src.Slice(nc * ih * iw, ih * iw); - Span d = dst.Slice(nc * oh * ow, oh * ow); - for (int y = 0; y < oh; y++) - { - int my = mapY[y]; - Span drow = d.Slice(y * ow, ow); - if (my < 0) { drow.Fill(p.Value); continue; } - ReadOnlySpan srow = s.Slice(my * iw, iw); - // x regions: [0,xl) pad, [xl,xr) copy, [xr,ow) pad - int xl = 0; while (xl < ow && mapX[xl] < 0) xl++; - int xr = ow; while (xr > xl && mapX[xr - 1] < 0) xr--; - if (p.Mode == PadMode.Constant) - { - for (int x = 0; x < xl; x++) drow[x] = p.Value; - for (int x = xr; x < ow; x++) drow[x] = p.Value; - } - else - { - for (int x = 0; x < xl; x++) drow[x] = srow[mapX[x]]; - for (int x = xr; x < ow; x++) drow[x] = srow[mapX[x]]; - } - for (int x = xl; x < xr; x++) drow[x] = srow[mapX[x]]; - } - } - } - - // ======================== Transpose ======================== - public static void Transpose(ReadOnlySpan src, int[] inShape, Span dst, - int[] outShape, int[] perm) - { - int rank = inShape.Length; - // fast path: last-two-dim swap on any rank (most common here); leading dims must be identity - bool isSwapLast2 = perm.Length == rank && perm[^1] == rank - 2 && perm[^2] == rank - 1; - for (int i = 0; isSwapLast2 && i < rank - 2; i++) isSwapLast2 = perm[i] == i; - if (isSwapLast2) - { - int a = inShape[rank - 2], b = inShape[rank - 1]; - int outer = 1; - for (int i = 0; i < rank - 2; i++) outer *= inShape[i]; - if (Avx2.IsSupported && Fma.IsSupported && a * b >= 8192) - { - Avx2Kernels.Transpose2D(src, a, b, dst, copies: outer); - return; - } - for (int o = 0; o < outer; o++) - Transpose2D(src.Slice(o * a * b, a * b), dst.Slice(o * a * b, a * b), a, b); - return; - } - if (rank == 4 && perm[0] == 0 && perm[1] == 3 && perm[2] == 1 && perm[3] == 2) - { - // [1,h,w,c] -> [1,c,h,w] == 2D transpose of [hw,c]->[c,hw] - int hw = inShape[1] * inShape[2], cc = inShape[3]; - if (Avx2.IsSupported && Fma.IsSupported && hw * cc >= 8192) - { - Avx2Kernels.Transpose2D(src, hw, cc, dst); - return; - } - Transpose2D(src, dst, hw, cc); - return; - } - if (rank == 4 && perm[0] == 0 && perm[1] == 2 && perm[2] == 3 && perm[3] == 1) - { - // [1,c,h,w] -> [1,h,w,c] == 2D transpose of [c,hw]->[hw,c] - int cc = inShape[1], hw = inShape[2] * inShape[3]; - if (Avx2.IsSupported && Fma.IsSupported && hw * cc >= 8192) - { - Avx2Kernels.Transpose2D(src, cc, hw, dst); - return; - } - Transpose2D(src, dst, cc, hw); - return; - } - // generic permute - int[] inStr = StridesOf(inShape), outStr = StridesOf(outShape); - int[] permutedInStr = new int[rank]; - for (int i = 0; i < rank; i++) permutedInStr[i] = inStr[perm[i]]; - int n = outShape.Aggregate(1, (x, y) => x * y); - for (int e = 0; e < n; e++) - { - int rem = e, so = 0; - for (int d = 0; d < rank; d++) - { - int c = rem / outStr[d]; - rem %= outStr[d]; - so += c * permutedInStr[d]; - } - dst[e] = src[so]; - } - } - - // tiled [a,b] -> [b,a] - private static void Transpose2D(ReadOnlySpan src, Span dst, int a, int b) - { - const int T = 16; - for (int y0 = 0; y0 < a; y0 += T) - for (int x0 = 0; x0 < b; x0 += T) - { - int ye = Math.Min(y0 + T, a), xe = Math.Min(x0 + T, b); - for (int y = y0; y < ye; y++) - for (int x = x0; x < xe; x++) - dst[x * a + y] = src[y * b + x]; - } - } - - // ======================== Slice ======================== - public static void Slice(ReadOnlySpan src, int[] inShape, Span dst, - int[] outShape, SliceP p) - { - int rank = inShape.Length; - int[] inStr = StridesOf(inShape), outStr = StridesOf(outShape); - int[] start = new int[rank], step = Enumerable.Repeat(1, rank).ToArray(); - for (int i = 0; i < p.Axes.Length; i++) { start[p.Axes[i]] = p.Starts[i]; step[p.Axes[i]] = p.Steps[i]; } - int total = outShape.Aggregate(1, (a, b) => a * b); - if (total == 0) return; - // fast path: contiguous tail (last sliced axis is innermost and steps 1) - int innerAxis = rank - 1; - if (step[innerAxis] == 1) - { - int innerLen = outShape[innerAxis]; - int innerOff = start[innerAxis]; - int outer = total / innerLen; - for (int e = 0; e < outer; e++) - { - int rem = e, so = 0; - for (int d = 0; d < rank - 1; d++) - { - int cc = rem / (outStr[d] / innerLen); - rem %= outStr[d] / innerLen; - so += (start[d] + cc * step[d]) * inStr[d]; - } - src.Slice(so + innerOff, innerLen).CopyTo(dst.Slice(e * innerLen, innerLen)); - } - return; - } - int n = outShape.Aggregate(1, (a, b) => a * b); - for (int e = 0; e < n; e++) - { - int rem = e, so = 0; - for (int d = 0; d < rank; d++) - { - int cc = rem / outStr[d]; - rem %= outStr[d]; - so += (start[d] + cc * step[d]) * inStr[d]; - } - dst[e] = src[so]; - } - } - - // ======================== Concat ======================== - public static void Concat(FBuf[] ins, int[][] inShapes, Span dst, - int[] outShape, int axis) - { - int inner = 1; - for (int d = axis + 1; d < outShape.Length; d++) inner *= outShape[d]; - int outer = 1; - for (int d = 0; d < axis; d++) outer *= outShape[d]; - int dstOff = 0; - for (int o = 0; o < outer; o++) - { - for (int i = 0; i < ins.Length; i++) - { - int count = inShapes[i][axis] * inner; - if (count == 0) continue; - ins[i].Span.Slice(o * count, count).CopyTo(dst.Slice(dstOff, count)); - dstOff += count; - } - } - } - - // ======================== ChannelShift ======================== - // out[c,y,x] = in[c, y+dy_g] (axis=2) or in[c,y,x+dx_g] (axis=3), zero-fill - public static void ChannelShift(ReadOnlySpan src, int[] inShape, Span dst, - int[] outShape, ShiftP p) - { - int c = inShape[1], h = inShape[2], w = inShape[3]; - if (Avx2.IsSupported && Fma.IsSupported && c * h * w >= 8192) - { - Avx2Kernels.ChannelShift(src, c, h, w, dst, p.Axis, p.Offsets, p.GroupEnds); - return; - } - int g0 = 0; - for (int g = 0; g < p.Offsets.Length; g++) - { - int g1 = p.GroupEnds[g]; - int off = p.Offsets[g]; - for (int ch = g0; ch < g1; ch++) - { - ReadOnlySpan s = src.Slice(ch * h * w, h * w); - Span d = dst.Slice(ch * h * w, h * w); - if (p.Axis == 2) - { - for (int y = 0; y < h; y++) - { - int sy = y + off; - if ((uint)sy < (uint)h) s.Slice(sy * w, w).CopyTo(d.Slice(y * w, w)); - else d.Slice(y * w, w).Clear(); - } - } - else if (Math.Abs(off) >= w) - { - d.Clear(); // shifted fully out of range -> all zeros - } - else - { - // out[c,y,x] = in[c,y,x+off], zero-fill — same convention as axis 2 - if (off >= 0) - for (int y = 0; y < h; y++) - { - ReadOnlySpan sr = s.Slice(y * w, w); - Span dr = d.Slice(y * w, w); - sr.Slice(off).CopyTo(dr.Slice(0, w - off)); - dr.Slice(w - off).Clear(); - } - else - for (int y = 0; y < h; y++) - { - ReadOnlySpan sr = s.Slice(y * w, w); - Span dr = d.Slice(y * w, w); - sr.Slice(0, w + off).CopyTo(dr.Slice(-off)); - dr.Slice(0, -off).Clear(); - } - } - } - g0 = g1; - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/PackedKernels.cs b/src/Sdcb.SimdGcdrNet/Kernels/PackedKernels.cs deleted file mode 100644 index 069d467..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/PackedKernels.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Wasm; - -using V4 = System.Runtime.Intrinsics.Vector128; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// WebAssembly SIMD128 (PackedSimd) versions of the non-conv ops that have AVX2 / NEON kernels. -internal static unsafe class PackedKernels -{ - private static readonly ParallelOptions PO = new() { MaxDegreeOfParallelism = Avx2Kernels.Threads }; - - /// Row-wise LayerNorm over the last axis (c); rows split across threads in groups - /// ( with separate multiply + add). - public static void LayerNorm(ReadOnlySpan x, int rows, int c, - ReadOnlySpan w, ReadOnlySpan b, Span dst, float eps) - { - int per = Math.Max(1, 16384 / c), tasks = (rows + per - 1) / per; - fixed (float* xp0 = x, wp0 = w, bp0 = b, dp0 = dst) - { - float* xp = xp0, wp = wp0, bp = bp0, dp = dp0; - Parallel.For(0, tasks, PO, t => - { - for (int r = t * per, re = Math.Min(rows, r + per); r < re; r++) - LayerNormRow(xp + (long)r * c, dp + (long)r * c, c, wp, bp, eps); - }); - } - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static void LayerNormRow(float* xr, float* dr, int c, float* wp, float* bp, float eps) - { - V4 sum = V4.Zero, sq = V4.Zero; - int i = 0; - for (; i + 4 <= c; i += 4) - { - V4 v = PackedSimd.LoadVector128(xr + i); - sum = PackedSimd.Add(sum, v); - sq = PackedSimd.Add(sq, PackedSimd.Multiply(v, v)); - } - float s = Vector128.Sum(sum), ss = Vector128.Sum(sq); - for (; i < c; i++) { float v = xr[i]; s += v; ss += v * v; } - float mean = s / c; - float inv = 1f / MathF.Sqrt(MathF.Max(ss / c - mean * mean, 0f) + eps); - V4 vm = PackedSimd.Splat(mean), vi = PackedSimd.Splat(inv); - for (i = 0; i + 4 <= c; i += 4) - { - V4 t = PackedSimd.Subtract(PackedSimd.LoadVector128(xr + i), vm); - V4 g = PackedSimd.Multiply(vi, PackedSimd.LoadVector128(wp + i)); - PackedSimd.Store(dr + i, PackedSimd.Add(PackedSimd.LoadVector128(bp + i), PackedSimd.Multiply(t, g))); - } - for (; i < c; i++) dr[i] = (xr[i] - mean) * inv * wp[i] + bp[i]; - } -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/PixelOps.cs b/src/Sdcb.SimdGcdrNet/Kernels/PixelOps.cs index 4e33b58..8ff88dd 100644 --- a/src/Sdcb.SimdGcdrNet/Kernels/PixelOps.cs +++ b/src/Sdcb.SimdGcdrNet/Kernels/PixelOps.cs @@ -1,5 +1,6 @@ using System.Numerics; using System.Runtime.CompilerServices; +using Sdcb.SimdOnnx.Kernels; namespace Sdcb.SimdGcdrNet.Kernels; diff --git a/src/Sdcb.SimdGcdrNet/Kernels/Simd.cs b/src/Sdcb.SimdGcdrNet/Kernels/Simd.cs deleted file mode 100644 index 562903b..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/Simd.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// Shared Vector<float> helpers (width-agnostic SIMD). -internal static class Simd -{ - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector VectorLoad(ReadOnlySpan source, int offset) => - Vector.LoadUnsafe(ref Unsafe.Add(ref MemoryMarshal.GetReference(source), offset)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void VectorStore(Span destination, int offset, Vector value) => - value.StoreUnsafe(ref Unsafe.Add(ref MemoryMarshal.GetReference(destination), offset)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe Vector VectorLoad(float* source) => - Vector.LoadUnsafe(ref Unsafe.AsRef(source)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void VectorStore(float* destination, Vector value) => - value.StoreUnsafe(ref Unsafe.AsRef(destination)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector VectorAddMul(Vector acc, Vector value, float weight) => - acc + value * new Vector(weight); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector VectorAddMul(Vector acc, Vector value, Vector weight) => - acc + value * weight; -} diff --git a/src/Sdcb.SimdGcdrNet/Kernels/SimdMath.cs b/src/Sdcb.SimdGcdrNet/Kernels/SimdMath.cs deleted file mode 100644 index de100ef..0000000 --- a/src/Sdcb.SimdGcdrNet/Kernels/SimdMath.cs +++ /dev/null @@ -1,270 +0,0 @@ -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Wasm; -using System.Runtime.Intrinsics.X86; - -namespace Sdcb.SimdGcdrNet.Kernels; - -/// -/// Vectorized transcendental math: exp / sigmoid / erf / gelu. -/// Precision targets ~1e-6 absolute so fused activations stay well inside the -/// 2e-3 validation budget. -/// -internal static class SimdMath -{ - // ---------- exp(x): cephes-style 2^n * poly(r) ---------- - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector Exp(Vector x) - { - int VL = Vector.Count; - // clamp to avoid overflow in scale - x = Vector.Max(x, new Vector(-87.3f)); - x = Vector.Min(x, new Vector(88.7f)); - float invLn2 = 1.4426950408889634f; - var fx = x * new Vector(invLn2) + new Vector(0.5f); - // floor(fx) - var n = Vector.ConvertToInt32(fx); - var nf = Vector.ConvertToSingle(n); - // nf > fx (i.e. fx negative half case) fix - var mask = Vector.GreaterThan(nf, fx); - nf -= Vector.BitwiseAnd(Vector.One, Vector.AsVectorSingle(mask)); - // r = x - n*ln2 (hi/lo parts for precision) - const float ln2hi = 0.693359375f, ln2lo = -0.00021219444005469058f; - var r = x - nf * new Vector(ln2hi) - nf * new Vector(ln2lo); - // exp(r) poly, degree 5 (cephes) - var p = new Vector(1.9875691500e-4f); - p = p * r + new Vector(1.3981999507e-3f); - p = p * r + new Vector(8.3334519073e-3f); - p = p * r + new Vector(4.1665795894e-2f); - p = p * r + new Vector(1.6666665459e-1f); - p = p * r + new Vector(5.0000001201e-1f); - var y = p * (r * r) + r + Vector.One; - // scale by 2^n - var ni = Vector.ConvertToInt32(nf) + new Vector(127); - var pow2n = Vector.AsVectorSingle(ni << 23); - return y * pow2n; - } - - public static void Exp(ReadOnlySpan src, Span dst) - { - int n = src.Length, i = 0, VL = Vector.Count; - for (; i + VL <= n; i += VL) - Simd.VectorStore(dst, i, Exp(Simd.VectorLoad(src, i))); - for (; i < n; i++) dst[i] = MathF.Exp(src[i]); - } - - // ---------- sigmoid = 1/(1+e^-x) ---------- - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector SigmoidV(Vector x) => - Vector.One / (Vector.One + Exp(-x)); - - public static void Sigmoid(ReadOnlySpan src, Span dst) - { - int n = src.Length, i = 0, VL = Vector.Count; - for (; i + VL <= n; i += VL) - Simd.VectorStore(dst, i, SigmoidV(Simd.VectorLoad(src, i))); - for (; i < n; i++) dst[i] = 1f / (1f + MathF.Exp(-src[i])); - } - - // ---------- erf(x): Abramowitz–Stegun 7.1.26 (max ~1.5e-7) ---------- - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector ErfV(Vector x) - { - var sign = Vector.ConditionalSelect(Vector.LessThan(x, Vector.Zero), - -Vector.One, Vector.One); - var ax = Vector.Abs(x); - var t = Vector.One / (Vector.One + new Vector(0.3275911f) * ax); - var p = new Vector(1.061405429f); - p = p * t + new Vector(-1.453152027f); - p = p * t + new Vector(1.421413741f); - p = p * t + new Vector(-0.284496736f); - p = p * t + new Vector(0.254829592f); - var y = Vector.One - p * t * Exp(-ax * ax); - // apply sign (sign(0)=0 → y*0 = 0 correct) - return y * sign; - } - - public static void Erf(ReadOnlySpan src, Span dst) - { - int n = src.Length, i = 0, VL = Vector.Count; - for (; i + VL <= n; i += VL) - Simd.VectorStore(dst, i, ErfV(Simd.VectorLoad(src, i))); - for (; i < n; i++) dst[i] = ErfScalar(src[i]); - } - - // ---------- gelu(x) = 0.5*x*(1+erf(x/√2)) ---------- - public static void Gelu(ReadOnlySpan src, Span dst) - { - int n = src.Length, i = 0, VL = Vector.Count; - var invSqrt2 = new Vector(0.70710678118654752f); - var half = new Vector(0.5f); - for (; i + VL <= n; i += VL) - { - var x = Simd.VectorLoad(src, i); - var g = half * x * (Vector.One + ErfV(x * invSqrt2)); - Simd.VectorStore(dst, i, g); - } - for (; i < n; i++) dst[i] = 0.5f * src[i] * (1f + ErfScalar(src[i] * 0.70710678118654752f)); - } - - // ---------- AVX2 cores (used by Avx2Kernels fused epilogues) ---------- - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector256 ExpCore(Vector256 x) - { - x = Vector256.Max(x, Vector256.Create(-87.3f)); - x = Vector256.Min(x, Vector256.Create(88.7f)); - var fx = Vector256.FusedMultiplyAdd(x, Vector256.Create(1.4426950408889634f), Vector256.Create(0.5f)); - var nf = Vector256.Floor(fx); - var r = Vector256.FusedMultiplyAdd(nf, Vector256.Create(-0.693359375f), x); - r = Vector256.FusedMultiplyAdd(nf, Vector256.Create(0.00021219444005469058f), r); - var p = Vector256.Create(1.9875691500e-4f); - p = Vector256.FusedMultiplyAdd(p, r, Vector256.Create(1.3981999507e-3f)); - p = Vector256.FusedMultiplyAdd(p, r, Vector256.Create(8.3334519073e-3f)); - p = Vector256.FusedMultiplyAdd(p, r, Vector256.Create(4.1665795894e-2f)); - p = Vector256.FusedMultiplyAdd(p, r, Vector256.Create(1.6666665459e-1f)); - p = Vector256.FusedMultiplyAdd(p, r, Vector256.Create(5.0000001201e-1f)); - var y = Vector256.Add(Vector256.FusedMultiplyAdd(p, Vector256.Multiply(r, r), r), Vector256.One); - var ni = Avx2.Add(Avx2.ConvertToVector256Int32(nf), Vector256.Create(127)); - var pow2n = Avx2.ShiftLeftLogical(ni, 23).AsSingle(); - return Vector256.Multiply(y, pow2n); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector256 SigmoidCore(Vector256 x) => - Vector256.Divide(Vector256.One, - Vector256.Add(Vector256.One, ExpCore(Vector256.Negate(x)))); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector256 ErfCore(Vector256 x) - { - var sign = Avx.Or(Avx.And(x, Vector256.Create(-0f)), - Vector256.One); - var ax = Avx.AndNot(Vector256.Create(-0f), x); - var t = Vector256.Divide(Vector256.One, - Vector256.Add(Vector256.One, Vector256.Multiply(Vector256.Create(0.3275911f), ax))); - var p = Vector256.Create(1.061405429f); - p = Vector256.FusedMultiplyAdd(p, t, Vector256.Create(-1.453152027f)); - p = Vector256.FusedMultiplyAdd(p, t, Vector256.Create(1.421413741f)); - p = Vector256.FusedMultiplyAdd(p, t, Vector256.Create(-0.284496736f)); - p = Vector256.FusedMultiplyAdd(p, t, Vector256.Create(0.254829592f)); - var y = Vector256.Subtract(Vector256.One, - Vector256.Multiply(Vector256.Multiply(p, t), ExpCore(Vector256.Negate(Vector256.Multiply(ax, ax))))); - return Vector256.Multiply(y, sign); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector256 GeluCore(Vector256 x) => - Vector256.Multiply(Vector256.Multiply(Vector256.Create(0.5f), x), - Vector256.Add(Vector256.One, - ErfCore(Vector256.Multiply(x, Vector256.Create(0.70710678118654752f))))); - - // ---------- 128-bit cores (NEON tier fused epilogues; same formulas as the AVX2 cores) ---------- - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 ExpCore(Vector128 x) - { - x = Vector128.Max(x, Vector128.Create(-87.3f)); - x = Vector128.Min(x, Vector128.Create(88.7f)); - var fx = Vector128.FusedMultiplyAdd(x, Vector128.Create(1.4426950408889634f), Vector128.Create(0.5f)); - var nf = Vector128.Floor(fx); - var r = Vector128.FusedMultiplyAdd(nf, Vector128.Create(-0.693359375f), x); - r = Vector128.FusedMultiplyAdd(nf, Vector128.Create(0.00021219444005469058f), r); - var p = Vector128.Create(1.9875691500e-4f); - p = Vector128.FusedMultiplyAdd(p, r, Vector128.Create(1.3981999507e-3f)); - p = Vector128.FusedMultiplyAdd(p, r, Vector128.Create(8.3334519073e-3f)); - p = Vector128.FusedMultiplyAdd(p, r, Vector128.Create(4.1665795894e-2f)); - p = Vector128.FusedMultiplyAdd(p, r, Vector128.Create(1.6666665459e-1f)); - p = Vector128.FusedMultiplyAdd(p, r, Vector128.Create(5.0000001201e-1f)); - var y = Vector128.FusedMultiplyAdd(p, r * r, r) + Vector128.One; - var ni = Vector128.ConvertToInt32(nf) + Vector128.Create(127); - return y * Vector128.ShiftLeft(ni, 23).AsSingle(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 SigmoidCore(Vector128 x) => - Vector128.One / (Vector128.One + ExpCore(-x)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 ErfCore(Vector128 x) - { - var signBit = Vector128.Create(-0f); - var sign = (x & signBit) | Vector128.One; - var ax = Vector128.AndNot(x, signBit); - var t = Vector128.One / (Vector128.One + Vector128.Create(0.3275911f) * ax); - var p = Vector128.Create(1.061405429f); - p = Vector128.FusedMultiplyAdd(p, t, Vector128.Create(-1.453152027f)); - p = Vector128.FusedMultiplyAdd(p, t, Vector128.Create(1.421413741f)); - p = Vector128.FusedMultiplyAdd(p, t, Vector128.Create(-0.284496736f)); - p = Vector128.FusedMultiplyAdd(p, t, Vector128.Create(0.254829592f)); - var y = Vector128.One - p * t * ExpCore(-(ax * ax)); - return y * sign; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 GeluCore(Vector128 x) => - Vector128.Create(0.5f) * x * (Vector128.One + ErfCore(x * Vector128.Create(0.70710678118654752f))); - - // ---------- WASM SIMD128 (PackedSimd) cores: Exp / SigmoidV / ErfV / Gelu above, op for op ---------- - // WASM SIMD has no FMA (relaxed SIMD is off), and Vector128.FusedMultiplyAdd would fall back to - // scalar fmaf, so these keep the Vector tier's separate multiply + add (same bits for finite x). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 MulAddPk(Vector128 a, Vector128 b, Vector128 c) => - PackedSimd.Add(PackedSimd.Multiply(a, b), c); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 ExpPk(Vector128 x) - { - // pmax/pmin: one maxps/minps in V8 (f32x4.max/min need a NaN fix-up sequence) - x = PackedSimd.PseudoMax(Vector128.Create(-87.3f), x); - x = PackedSimd.PseudoMin(Vector128.Create(88.7f), x); - var fx = MulAddPk(x, Vector128.Create(1.4426950408889634f), Vector128.Create(0.5f)); - var nf = PackedSimd.ConvertToSingle(PackedSimd.ConvertToInt32Saturate(fx)); - nf = PackedSimd.Subtract(nf, PackedSimd.And(Vector128.One, PackedSimd.CompareGreaterThan(nf, fx))); - var r = PackedSimd.Subtract(PackedSimd.Subtract(x, PackedSimd.Multiply(nf, Vector128.Create(0.693359375f))), - PackedSimd.Multiply(nf, Vector128.Create(-0.00021219444005469058f))); - var p = Vector128.Create(1.9875691500e-4f); - p = MulAddPk(p, r, Vector128.Create(1.3981999507e-3f)); - p = MulAddPk(p, r, Vector128.Create(8.3334519073e-3f)); - p = MulAddPk(p, r, Vector128.Create(4.1665795894e-2f)); - p = MulAddPk(p, r, Vector128.Create(1.6666665459e-1f)); - p = MulAddPk(p, r, Vector128.Create(5.0000001201e-1f)); - var y = PackedSimd.Add(PackedSimd.Add(PackedSimd.Multiply(p, PackedSimd.Multiply(r, r)), r), Vector128.One); - var ni = PackedSimd.Add(PackedSimd.ConvertToInt32Saturate(nf), Vector128.Create(127)); - return PackedSimd.Multiply(y, PackedSimd.ShiftLeft(ni, 23).AsSingle()); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 SigmoidPk(Vector128 x) => - PackedSimd.Divide(Vector128.One, PackedSimd.Add(Vector128.One, ExpPk(PackedSimd.Negate(x)))); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 ErfPk(Vector128 x) - { - var one = Vector128.One; - var sign = Vector128.ConditionalSelect(PackedSimd.CompareLessThan(x, Vector128.Zero), Vector128.Create(-1f), one); - var ax = PackedSimd.Abs(x); - var t = PackedSimd.Divide(one, PackedSimd.Add(one, PackedSimd.Multiply(Vector128.Create(0.3275911f), ax))); - var p = Vector128.Create(1.061405429f); - p = MulAddPk(p, t, Vector128.Create(-1.453152027f)); - p = MulAddPk(p, t, Vector128.Create(1.421413741f)); - p = MulAddPk(p, t, Vector128.Create(-0.284496736f)); - p = MulAddPk(p, t, Vector128.Create(0.254829592f)); - var y = PackedSimd.Subtract(one, PackedSimd.Multiply(PackedSimd.Multiply(p, t), - ExpPk(PackedSimd.Multiply(PackedSimd.Negate(ax), ax)))); - return PackedSimd.Multiply(y, sign); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector128 GeluPk(Vector128 x) => - PackedSimd.Multiply(PackedSimd.Multiply(Vector128.Create(0.5f), x), - PackedSimd.Add(Vector128.One, ErfPk(PackedSimd.Multiply(x, Vector128.Create(0.70710678118654752f))))); - - internal static float ErfScalar(float x) - { - float ax = MathF.Abs(x); - float t = 1f / (1f + 0.3275911f * ax); - float p = ((((1.061405429f * t - 1.453152027f) * t + 1.421413741f) * t - 0.284496736f) * t + 0.254829592f) * t; - float y = 1f - p * MathF.Exp(-ax * ax); - return x < 0 ? -y : y; - } -} diff --git a/src/Sdcb.SimdGcdrNet/Onnx/OnnxModel.cs b/src/Sdcb.SimdGcdrNet/Onnx/OnnxModel.cs deleted file mode 100644 index fa30861..0000000 --- a/src/Sdcb.SimdGcdrNet/Onnx/OnnxModel.cs +++ /dev/null @@ -1,49 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Onnx; - -/// Validated ONNX model: graph nodes, initializers, IO metadata. -public sealed class OnnxModel -{ - private OnnxModel(int opset, OnnxGraphData graph, Dictionary initializers) - { - Opset = opset; - Graph = graph; - Initializers = initializers; - } - - /// The model's default-domain (ai.onnx) opset version. - public int Opset { get; } - internal OnnxGraphData Graph { get; } - internal Dictionary Initializers { get; } - - /// Loads an ONNX model from the file at . - public static OnnxModel Load(string path) - { - using FileStream fs = File.OpenRead(path); - return Load(fs); - } - - /// Loads an ONNX model by reading to its end. - public static OnnxModel Load(Stream source) => Build(OnnxProtoReader.Parse(source)); - - /// Parses the model straight from (pinned for the duration - /// of the call, no intermediate copy); weights are decoded into model-owned arrays, so the - /// memory may be released afterwards. - public static unsafe OnnxModel Load(ReadOnlySpan source) - { - fixed (byte* p = source) - { - using var ums = new UnmanagedMemoryStream(p, source.Length); - return Load(ums); - } - } - - private static OnnxModel Build(OnnxModelData parsed) - { - OnnxGraphData g = parsed.Graph!; - int opset = checked((int)(parsed.Opsets.FirstOrDefault( - static x => x.Domain.Length == 0 || x.Domain == "ai.onnx")?.Version ?? 0)); - Dictionary init = new(g.Initializers.Count); - foreach (OnnxTensorData t in g.Initializers) init[t.Name] = t; - return new OnnxModel(opset, g, init); - } -} diff --git a/src/Sdcb.SimdGcdrNet/Onnx/OnnxModelData.cs b/src/Sdcb.SimdGcdrNet/Onnx/OnnxModelData.cs deleted file mode 100644 index 92063f4..0000000 --- a/src/Sdcb.SimdGcdrNet/Onnx/OnnxModelData.cs +++ /dev/null @@ -1,195 +0,0 @@ -namespace Sdcb.SimdGcdrNet.Onnx; - -/// Raw parsed ONNX structures (protobuf level, pre-validation). -internal sealed class OnnxModelData -{ - public long IrVersion; - public List Opsets { get; } = []; - public OnnxGraphData? Graph; -} - -internal sealed class OnnxOpsetData -{ - public string Domain = ""; - public long Version; -} - -internal sealed class OnnxGraphData -{ - public List Nodes { get; } = []; - public List Initializers { get; } = []; - public List Inputs { get; } = []; - public List Outputs { get; } = []; - public List ValueInfo { get; } = []; -} - -internal sealed class OnnxNodeData -{ - public List Inputs { get; } = []; - public List Outputs { get; } = []; - public string Name = ""; - public string OpType = ""; - public string Domain = ""; - public List Attributes { get; } = []; - - public OnnxAttributeData? Attr(string name) - { - foreach (OnnxAttributeData a in Attributes) - if (a.Name == name) return a; - return null; - } - - public long IntAttr(string name, long fallback = 0) => Attr(name)?.Int ?? fallback; - public float FloatAttr(string name, float fallback = 0f) => Attr(name)?.Float ?? fallback; - public string StringAttr(string name, string fallback = "") => - Attr(name)?.String is byte[] b ? System.Text.Encoding.UTF8.GetString(b) : fallback; - public long[] IntsAttr(string name) => Attr(name)?.Ints.ToArray() ?? []; -} - -internal sealed class OnnxAttributeData -{ - public string Name = ""; - public int Type; - public float Float; - public long Int; - public byte[]? String; - public OnnxTensorData? Tensor; - public List Floats { get; } = []; - public List Ints { get; } = []; - public List Strings { get; } = []; -} - -internal sealed class OnnxTensorData -{ - public List Dims { get; } = []; - public int DataType; - public List FloatData { get; } = []; - public List Int32Data { get; } = []; - public List StringData { get; } = []; - public List Int64Data { get; } = []; - public byte[]? RawData; - public string Name = ""; - public List DoubleData { get; } = []; - public List UInt64Data { get; } = []; - - public const int TypeFloat = 1, TypeUInt8 = 2, TypeInt8 = 3, TypeUInt16 = 4, TypeInt16 = 5, - TypeInt32 = 6, TypeInt64 = 7, TypeString = 8, TypeBool = 9, TypeFloat16 = 10, - TypeDouble = 11, TypeUInt32 = 12, TypeUInt64 = 13; - - public int[] DimsArray() => Dims.Select(d => checked((int)d)).ToArray(); - - /// Decoded element count (product of dims; scalar = 1). - public long ElementCount() - { - long n = 1; - foreach (long d in Dims) n *= d; - return n; - } - - private float[]? _floats; - - /// - /// This tensor as floats (float/double/int sources), decoded once and shared: every - /// per-size plan of a model references the same read-only array. Raw float bytes are - /// dropped after decoding so the weights live in memory only once. - /// - public float[] AsFloats() - { - lock (this) - { - if (_floats is { } cached) return cached; - _floats = DecodeFloats(); - if (DataType == TypeFloat) RawData = null; - return _floats; - } - } - - private float[] DecodeFloats() - { - int count = checked((int)ElementCount()); - if (RawData is byte[] raw) - { - switch (DataType) - { - case TypeFloat: - { - float[] f = new float[checked(raw.Length / 4)]; - Buffer.BlockCopy(raw, 0, f, 0, raw.Length); - return f; - } - case TypeDouble: - { - double[] d = new double[checked(raw.Length / 8)]; - Buffer.BlockCopy(raw, 0, d, 0, raw.Length); - return d.Select(v => (float)v).ToArray(); - } - case TypeInt32: - { - int[] v = new int[checked(raw.Length / 4)]; - Buffer.BlockCopy(raw, 0, v, 0, raw.Length); - return v.Select(x => (float)x).ToArray(); - } - case TypeInt64: - { - long[] v = new long[checked(raw.Length / 8)]; - Buffer.BlockCopy(raw, 0, v, 0, raw.Length); - return v.Select(x => (float)x).ToArray(); - } - default: - throw new NotSupportedException($"Tensor '{Name}': raw dtype {DataType} cannot be read as float."); - } - } - if (FloatData.Count > 0) return FloatData.ToArray(); - if (Int32Data.Count > 0) return Int32Data.Select(v => (float)v).ToArray(); - if (Int64Data.Count > 0) return Int64Data.Select(v => (float)v).ToArray(); - if (DoubleData.Count > 0) return DoubleData.Select(v => (float)v).ToArray(); - if (count == 0) return []; - throw new InvalidDataException($"Tensor '{Name}' has no float-readable data."); - } - - /// Decodes this tensor to an int64 array. - public long[] AsInt64s() - { - byte[]? rawData; float[]? fl; - lock (this) { rawData = RawData; fl = _floats; } - if (DataType == TypeFloat && fl is not null) return fl.Select(x => (long)x).ToArray(); - if (rawData is byte[] raw) - { - switch (DataType) - { - case TypeInt64: - { - long[] v = new long[checked(raw.Length / 8)]; - Buffer.BlockCopy(raw, 0, v, 0, raw.Length); - return v; - } - case TypeInt32: - { - int[] v = new int[checked(raw.Length / 4)]; - Buffer.BlockCopy(raw, 0, v, 0, raw.Length); - return v.Select(x => (long)x).ToArray(); - } - case TypeFloat: - { - float[] f = new float[checked(raw.Length / 4)]; - Buffer.BlockCopy(raw, 0, f, 0, raw.Length); - return f.Select(x => (long)x).ToArray(); - } - default: - throw new NotSupportedException($"Tensor '{Name}': raw dtype {DataType} cannot be read as int64."); - } - } - if (Int64Data.Count > 0) return Int64Data.ToArray(); - if (Int32Data.Count > 0) return Int32Data.Select(v => (long)v).ToArray(); - if (FloatData.Count > 0) return FloatData.Select(v => (long)v).ToArray(); - if (ElementCount() == 0) return []; - throw new InvalidDataException($"Tensor '{Name}' has no int-readable data."); - } -} - -internal sealed class OnnxValueInfoData -{ - public string Name = ""; - public int ElementType; - public int[]? Shape; // -1 marks a dynamic dimension -} diff --git a/src/Sdcb.SimdGcdrNet/Onnx/OnnxProtoReader.cs b/src/Sdcb.SimdGcdrNet/Onnx/OnnxProtoReader.cs deleted file mode 100644 index d255248..0000000 --- a/src/Sdcb.SimdGcdrNet/Onnx/OnnxProtoReader.cs +++ /dev/null @@ -1,514 +0,0 @@ -using System.Buffers.Binary; -using System.Text; - -namespace Sdcb.SimdGcdrNet.Onnx; - -// A deliberately small protobuf decoder for the ONNX protobuf schema. It is -// kept independent of Google.Protobuf so the core remains entirely managed -// and has no generated-code or native runtime dependency. -// -// The reader consumes the input stream in place. Length-delimited protobuf -// messages are represented by bounded child readers rather than materialized -// as byte arrays. Only values that the model needs to retain (for example -// tensor raw_data and strings) are copied into managed storage. -internal static class OnnxProtoReader -{ - public static OnnxModelData Parse(Stream source) - { - if (source is null) throw new ArgumentNullException(nameof(source)); - ProtoReader reader = new(source, -1); - OnnxModelData model = new(); - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 0: model.IrVersion = checked((long)reader.ReadVarint()); break; - case 8 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxOpsetData opset = new(); - ParseOpset(ref nested, opset); - nested.EnsureFullyConsumed(); - model.Opsets.Add(opset); - break; - } - case 7 when wire == 2: - { - var nested = reader.EnterMessage(); - model.Graph = ParseGraph(ref nested); - nested.EnsureFullyConsumed(); - break; - } - default: reader.Skip(wire); break; - } - } - if (model.Graph is null) throw new InvalidDataException("ONNX ModelProto has no graph."); - return model; - } - - private static void ParseOpset(ref ProtoReader reader, OnnxOpsetData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: result.Domain = reader.ReadString(); break; - case 2 when wire == 0: result.Version = checked((long)reader.ReadVarint()); break; - default: reader.Skip(wire); break; - } - } - } - - private static OnnxGraphData ParseGraph(ref ProtoReader reader) - { - OnnxGraphData result = new(); - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxNodeData node = new(); - ParseNode(ref nested, node); - nested.EnsureFullyConsumed(); - result.Nodes.Add(node); - break; - } - case 5 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxTensorData tensor = new(); - ParseTensor(ref nested, tensor); - nested.EnsureFullyConsumed(); - result.Initializers.Add(tensor); - break; - } - case 11 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxValueInfoData value = new(); - ParseValueInfo(ref nested, value); - nested.EnsureFullyConsumed(); - result.Inputs.Add(value); - break; - } - case 12 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxValueInfoData value = new(); - ParseValueInfo(ref nested, value); - nested.EnsureFullyConsumed(); - result.Outputs.Add(value); - break; - } - case 13 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxValueInfoData value = new(); - ParseValueInfo(ref nested, value); - nested.EnsureFullyConsumed(); - result.ValueInfo.Add(value); - break; - } - default: reader.Skip(wire); break; - } - } - return result; - } - - private static void ParseNode(ref ProtoReader reader, OnnxNodeData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: result.Inputs.Add(reader.ReadString()); break; - case 2 when wire == 2: result.Outputs.Add(reader.ReadString()); break; - case 3 when wire == 2: result.Name = reader.ReadString(); break; - case 4 when wire == 2: result.OpType = reader.ReadString(); break; - case 5 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxAttributeData attribute = new(); - ParseAttribute(ref nested, attribute); - nested.EnsureFullyConsumed(); - result.Attributes.Add(attribute); - break; - } - case 7 when wire == 2: result.Domain = reader.ReadString(); break; - default: reader.Skip(wire); break; - } - } - if (result.OpType.Length == 0) - throw new InvalidDataException( - $"ONNX NodeProto has no op_type (inputs={result.Inputs.Count}, " + - $"outputs={result.Outputs.Count}, name='{result.Name}')."); - } - - private static void ParseAttribute(ref ProtoReader reader, OnnxAttributeData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: result.Name = reader.ReadString(); break; - case 20 when wire == 0: result.Type = checked((int)reader.ReadVarint()); break; - case 2 when wire == 5: result.Float = BitConverter.Int32BitsToSingle(unchecked((int)reader.ReadFixed32())); break; - case 3 when wire == 0: result.Int = unchecked((long)reader.ReadVarint()); break; - case 4 when wire == 2: result.String = reader.ReadBytes(); break; - case 5 when wire == 2: - { - var nested = reader.EnterMessage(); - OnnxTensorData tensor = new(); - ParseTensor(ref nested, tensor); - nested.EnsureFullyConsumed(); - result.Tensor = tensor; - break; - } - case 7 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedFloat(ref nested, result.Floats); - nested.EnsureFullyConsumed(); - break; - } - case 7 when wire == 5: result.Floats.Add(BitConverter.Int32BitsToSingle(unchecked((int)reader.ReadFixed32()))); break; - case 8 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedInt64(ref nested, result.Ints); - nested.EnsureFullyConsumed(); - break; - } - case 8 when wire == 0: result.Ints.Add(unchecked((long)reader.ReadVarint())); break; - case 9 when wire == 2: result.Strings.Add(reader.ReadBytes()); break; - default: reader.Skip(wire); break; - } - } - } - - private static void ParseTensor(ref ProtoReader reader, OnnxTensorData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedInt64(ref nested, result.Dims); - nested.EnsureFullyConsumed(); - break; - } - case 1 when wire == 0: result.Dims.Add(unchecked((long)reader.ReadVarint())); break; - case 2 when wire == 0: result.DataType = checked((int)reader.ReadVarint()); break; - case 4 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedFloat(ref nested, result.FloatData); - nested.EnsureFullyConsumed(); - break; - } - case 4 when wire == 5: result.FloatData.Add(BitConverter.Int32BitsToSingle(unchecked((int)reader.ReadFixed32()))); break; - case 5 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedInt32(ref nested, result.Int32Data); - nested.EnsureFullyConsumed(); - break; - } - case 5 when wire == 0: result.Int32Data.Add(unchecked((int)reader.ReadVarint())); break; - case 6 when wire == 2: result.StringData.Add(reader.ReadBytes()); break; - case 7 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedInt64(ref nested, result.Int64Data); - nested.EnsureFullyConsumed(); - break; - } - case 7 when wire == 0: result.Int64Data.Add(unchecked((long)reader.ReadVarint())); break; - case 8 when wire == 2: result.Name = reader.ReadString(); break; - case 9 when wire == 2: result.RawData = reader.ReadBytes(); break; - case 10 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedDouble(ref nested, result.DoubleData); - nested.EnsureFullyConsumed(); - break; - } - case 10 when wire == 1: result.DoubleData.Add(BitConverter.Int64BitsToDouble(unchecked((long)reader.ReadFixed64()))); break; - case 11 when wire == 2: - { - var nested = reader.EnterMessage(); - ReadPackedUInt64(ref nested, result.UInt64Data); - nested.EnsureFullyConsumed(); - break; - } - case 11 when wire == 1: result.UInt64Data.Add(reader.ReadFixed64()); break; - default: reader.Skip(wire); break; - } - } - if (result.Name.Length == 0) throw new InvalidDataException("ONNX TensorProto has no name."); - } - - private static void ParseValueInfo(ref ProtoReader reader, OnnxValueInfoData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 2: result.Name = reader.ReadString(); break; - case 2 when wire == 2: - { - var nested = reader.EnterMessage(); - ParseType(ref nested, result); - nested.EnsureFullyConsumed(); - break; - } - default: reader.Skip(wire); break; - } - } - } - - private static void ParseType(ref ProtoReader reader, OnnxValueInfoData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - if (field == 1 && wire == 2) - { - var nested = reader.EnterMessage(); - ParseTensorType(ref nested, result); - nested.EnsureFullyConsumed(); - } - else reader.Skip(wire); - } - } - - private static void ParseTensorType(ref ProtoReader reader, OnnxValueInfoData result) - { - while (reader.TryReadTag(out int field, out int wire)) - { - switch (field) - { - case 1 when wire == 0: result.ElementType = checked((int)reader.ReadVarint()); break; - case 2 when wire == 2: - { - var nested = reader.EnterMessage(); - result.Shape = ParseShape(ref nested); - nested.EnsureFullyConsumed(); - break; - } - default: reader.Skip(wire); break; - } - } - } - - private static int[] ParseShape(ref ProtoReader reader) - { - List dims = []; - while (reader.TryReadTag(out int field, out int wire)) - { - if (field == 1 && wire == 2) - { - var dimension = reader.EnterMessage(); - long value = -1; - while (dimension.TryReadTag(out int df, out int dw)) - { - if (df == 1 && dw == 0) value = unchecked((long)dimension.ReadVarint()); - else dimension.Skip(dw); - } - dimension.EnsureFullyConsumed(); - dims.Add(value > 0 && value <= int.MaxValue ? (int)value : -1); - } - else reader.Skip(wire); - } - return dims.ToArray(); - } - - private static void ReadPackedFloat(ref ProtoReader reader, List destination) - { - if ((reader.Remaining & 3) != 0) throw new InvalidDataException("Packed float field is truncated."); - while (reader.Remaining > 0) - destination.Add(BitConverter.Int32BitsToSingle(unchecked((int)reader.ReadFixed32()))); - } - - private static void ReadPackedDouble(ref ProtoReader reader, List destination) - { - if ((reader.Remaining & 7) != 0) throw new InvalidDataException("Packed double field is truncated."); - while (reader.Remaining > 0) - destination.Add(BitConverter.Int64BitsToDouble(unchecked((long)reader.ReadFixed64()))); - } - - private static void ReadPackedInt32(ref ProtoReader reader, List destination) - { - while (reader.Remaining > 0) destination.Add(unchecked((int)reader.ReadVarint())); - } - - private static void ReadPackedInt64(ref ProtoReader reader, List destination) - { - while (reader.Remaining > 0) destination.Add(unchecked((long)reader.ReadVarint())); - } - - private static void ReadPackedUInt64(ref ProtoReader reader, List destination) - { - while (reader.Remaining > 0) destination.Add(reader.ReadVarint()); - } - - private ref struct ProtoReader - { - private readonly Stream _source; - // -1 denotes the unbounded root reader. Nested readers have an exact - // number of bytes left and can never consume their parent message. - private long _remaining; - - public ProtoReader(Stream source, long remaining) - { - _source = source; - _remaining = remaining; - } - - public long Remaining => _remaining; - - public bool TryReadTag(out int field, out int wire) - { - int first = ReadByte(allowEof: true); - if (first < 0) { field = wire = 0; return false; } - ulong tag = ReadVarint(first); - field = checked((int)(tag >> 3)); - wire = checked((int)(tag & 7)); - if (field <= 0) throw new InvalidDataException("Invalid protobuf field number."); - return true; - } - - public ulong ReadVarint() - { - int first = ReadByte(allowEof: false); - return ReadVarint(first); - } - - private ulong ReadVarint(int first) - { - ulong value = 0; - for (int shift = 0; shift < 64; shift += 7) - { - int next = shift == 0 ? first : ReadByte(allowEof: false); - byte b = checked((byte)next); - if (shift == 63 && b > 1) throw new InvalidDataException("Protobuf varint overflows UInt64."); - value |= (ulong)(b & 0x7f) << shift; - if ((b & 0x80) == 0) return value; - } - throw new InvalidDataException("Protobuf varint is too long."); - } - - public uint ReadFixed32() - { - Span bytes = stackalloc byte[4]; - ReadExact(bytes); - return BinaryPrimitives.ReadUInt32LittleEndian(bytes); - } - - public ulong ReadFixed64() - { - Span bytes = stackalloc byte[8]; - ReadExact(bytes); - return BinaryPrimitives.ReadUInt64LittleEndian(bytes); - } - - public byte[] ReadBytes() - { - ulong length = ReadVarint(); - if (length > int.MaxValue) throw new InvalidDataException("Protobuf length-delimited field is too large."); - int count = checked((int)length); - byte[] value = new byte[count]; - ReadExact(value); - return value; - } - - public string ReadString() - { - try { return new UTF8Encoding(false, true).GetString(ReadBytes()); } - catch (DecoderFallbackException ex) { throw new InvalidDataException("ONNX protobuf string is not valid UTF-8.", ex); } - } - - public ProtoReader EnterMessage() - { - ulong length = ReadVarint(); - if (length > long.MaxValue || (_remaining >= 0 && length > (ulong)_remaining)) - throw new InvalidDataException("Protobuf length-delimited field is out of bounds."); - if (_remaining >= 0) _remaining -= checked((long)length); - return new ProtoReader(_source, checked((long)length)); - } - - public void Skip(int wire) - { - switch (wire) - { - case 0: ReadVarint(); break; - case 1: _ = ReadFixed64(); break; - case 2: SkipLengthDelimited(); break; - case 3: - while (TryReadTag(out _, out int nestedWire)) - { - if (nestedWire == 4) return; - Skip(nestedWire); - } - throw new InvalidDataException("Unterminated protobuf group."); - case 4: throw new InvalidDataException("Unexpected protobuf end-group."); - case 5: _ = ReadFixed32(); break; - default: throw new NotSupportedException($"Unsupported protobuf wire type {wire}."); - } - } - - public readonly void EnsureFullyConsumed() - { - if (_remaining != 0) throw new InvalidDataException("Truncated protobuf nested message."); - } - - private void SkipLengthDelimited() - { - ulong length = ReadVarint(); - if (length > long.MaxValue || (_remaining >= 0 && length > (ulong)_remaining)) - throw new InvalidDataException("Protobuf length-delimited field is out of bounds."); - long left = checked((long)length); - Span scratch = stackalloc byte[1024]; - while (left > 0) - { - int chunk = (int)Math.Min(left, scratch.Length); - ReadExact(scratch[..chunk]); - left -= chunk; - } - } - - private int ReadByte(bool allowEof) - { - if (_remaining == 0) - { - if (allowEof) return -1; - throw new InvalidDataException("Truncated protobuf stream."); - } - int value = _source.ReadByte(); - if (value < 0) - { - if (allowEof && _remaining < 0) return -1; - throw new InvalidDataException("Truncated protobuf stream."); - } - if (_remaining > 0) _remaining--; - return value; - } - - private void ReadExact(scoped Span destination) - { - if (_remaining >= 0 && destination.Length > _remaining) - throw new InvalidDataException("Protobuf field overruns its message."); - int read = 0; - while (read < destination.Length) - { - int n = _source.Read(destination[read..]); - if (n <= 0) throw new InvalidDataException("Truncated protobuf stream."); - read += n; - } - if (_remaining > 0) _remaining -= destination.Length; - } - } -} diff --git a/src/Sdcb.SimdGcdrNet/Sdcb.SimdGcdrNet.csproj b/src/Sdcb.SimdGcdrNet/Sdcb.SimdGcdrNet.csproj index b974788..fe10bd3 100644 --- a/src/Sdcb.SimdGcdrNet/Sdcb.SimdGcdrNet.csproj +++ b/src/Sdcb.SimdGcdrNet/Sdcb.SimdGcdrNet.csproj @@ -14,4 +14,10 @@ + + + + + + diff --git a/tests/Sdcb.SimdGcdrNet.Tests/Avx512KernelTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/Avx512KernelTests.cs index eda6389..490ec79 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/Avx512KernelTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/Avx512KernelTests.cs @@ -1,5 +1,7 @@ using System.Runtime.Intrinsics.X86; -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Kernels; using Sdcb.SimdGcdrNet.Kernels; namespace Sdcb.SimdGcdrNet.Tests; @@ -32,91 +34,6 @@ private static float[] Both(Action run, int n) return avx512; } - [SkippableTheory] - [InlineData(3, 16, 3, 1, 37, 23, 1, 0f, false, 0)] // gc stem: 16-wide tail kernel, reflect, row runs - [InlineData(6, 32, 3, 1, 29, 41, 1, 0f, false, 1)] // dr stem: one block pair - [InlineData(16, 48, 3, 1, 11, 13, 0, 0.25f, true, 1)] // odd block count, pad value, residual - [InlineData(20, 20, 3, 2, 17, 19, 0, 0f, true, 2)] // stride 2, oc lane tail, sigmoid - [InlineData(32, 64, 3, 2, 12, 31, 1, 0f, false, 3)] // stride 2, reflect, gelu - [InlineData(24, 3, 3, 1, 9, 10, 0, 0f, false, 0)] // oc=3 (single partial lane block) - [InlineData(40, 36, 1, 1, 7, 33, 0, 0f, true, 1)] // 1x1 conv - [InlineData(8, 32, 5, 1, 14, 9, 1, 0f, false, 1)] // 5x5 (direct path), narrow rows - public void Conv_Direct_MatchesAvx2(int ic, int oc, int k, int stride, int h, int w, int mode, - float padValue, bool residual, int act) - { - var rng = new Random(ic * 131 + oc * 7 + k); - int pad = k / 2; - int oh = (h + 2 * pad - k) / stride + 1, ow = (w + 2 * pad - k) / stride + 1; - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * k * k), b = rng.Array(oc); - float[] res = residual ? rng.Array(oh * ow * oc) : []; - var p = new ConvP - { - Pads = [pad, pad, pad, pad], Strides = [stride, stride], Dilations = [1, 1], Kernel = [k, k], - PadMode = (PadMode)mode, PadValue = padValue, Act = (ActivationKind)act, - }; - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, k, k], b, 1, ic); // no Winograd: direct kernels - Both(dst => Nhwc.Conv(x, h, w, ic, dst, oh, ow, oc, p, kp, res), oh * ow * oc); - } - - [SkippableTheory] - [InlineData(16, 16, 9, 11, 0, 0f, false, 0)] - [InlineData(24, 20, 13, 6, 1, 0f, true, 1)] // ic % 16 != 0: 8-lane input transform - [InlineData(32, 48, 22, 18, 0, 0.5f, false, 3)] - [InlineData(64, 16, 17, 23, 1, 0f, true, 0)] - [InlineData(128, 64, 30, 41, 1, 0f, false, 1)] // several tile chunks per image - public void Conv_Winograd_MatchesAvx2(int ic, int oc, int h, int w, int mode, float padValue, bool residual, int act) - { - var rng = new Random(ic * 17 + oc); - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * 9), b = rng.Array(oc); - float[] res = residual ? rng.Array(h * w * oc) : []; - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], - PadMode = (PadMode)mode, PadValue = padValue, Act = (ActivationKind)act, - }; - Skip.IfNot(Nhwc.WinogradEligible(p, [oc, ic, 3, 3], ic), "Winograd disabled"); - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - kp.WinoU = Nhwc.PackWinograd(wt, oc, ic); - Both(dst => Nhwc.Conv(x, h, w, ic, dst, h, w, oc, p, kp, res), h * w * oc); - } - - [SkippableTheory] - [InlineData(57, 128, 128, false, 0)] - [InlineData(23, 160, 48, true, 3)] // odd block count, transB, gelu - [InlineData(5, 64, 20, false, 1)] // fewer rows than a tile, lane tail - public void Gemm_MatchesAvx2(int m, int k, int n, bool transB, int act) - { - var rng = new Random(m + k + n); - float[] a = rng.Array(m * k), bm = rng.Array(k * n), bias = rng.Array(n); - NhwcConvPlan kp = Nhwc.PackGemm(bm, k, n, transB, bias); - Both(dst => Nhwc.Gemm(a, m, k, dst, n, kp, (ActivationKind)act), m * n); - } - - [SkippableTheory] - [InlineData(32, 2, 2, 0, false)] - [InlineData(16, 3, 2, 1, true)] // padded taps, fused relu - public void MaxPool_MatchesAvx2(int c, int k, int s, int pad, bool relu) - { - var rng = new Random(c + k); - int h = 19, w = 26, oh = (h + 2 * pad - k) / s + 1, ow = (w + 2 * pad - k) / s + 1; - float[] x = rng.Array(h * w * c); - var p = new PoolP { Kernel = [k, k], Strides = [s, s], Pads = [pad, pad, pad, pad], FuseRelu = relu }; - Both(dst => Nhwc.MaxPool(x, h, w, c, dst, oh, ow, p), oh * ow * c); - } - - [SkippableTheory] - [InlineData(32, true, true, true)] - [InlineData(16, false, false, false)] - [InlineData(24, true, false, true)] // c % 16 != 0: generic resize - public void Resize_MatchesAvx2(int c, bool relu, bool addend, bool addRelu) - { - var rng = new Random(c); - int h = 13, w = 9, oh = 26, ow = 18; - float[] x = rng.Array(h * w * c), add = addend ? rng.Array(oh * ow * c) : []; - Both(dst => Nhwc.ResizeBilinear(x, h, w, c, dst, oh, ow, 2, 2, - relu ? ActivationKind.Relu : ActivationKind.None, add, addRelu), oh * ow * c); - } - [SkippableTheory] [InlineData(32, 3, true, true, 2)] [InlineData(16, 3, false, true, 0)] @@ -126,46 +43,9 @@ public void UpHead_MatchesAvx2(int c, int oc, bool resizeRelu, bool skipRelu, in var rng = new Random(c * oc); int h = 11, w = 15, oh = 22, ow = 30; // ow % 4 != 0: partial pixel group float[] x = rng.Array(h * w * c), skip = rng.Array(oh * ow * c), wt = rng.Array(oc * c), b = rng.Array(oc); - UpHeadPlan kp = Nhwc.PackUpHead(wt, oc, c, b); + UpHeadPlan kp = GcdrKernels.PackUpHead(wt, oc, c, b); var p = new UpHeadP { Scales = [1, 1, 2, 2], ResizeRelu = resizeRelu, SkipRelu = skipRelu, Act = (ActivationKind)act }; - Both(dst => Nhwc.UpHead(x, h, w, c, skip, dst, oh, ow, 2, 2, p, kp), oh * ow * oc); - } - - [SkippableTheory] - [InlineData(3, 16, false, 1, 0, 1)] // gc stem: 16-wide tail kernel, reflect, row runs - [InlineData(6, 32, false, 1, 0, 1)] // dr stem: one block pair - [InlineData(16, 48, false, 0, 0.25f, 0)] // odd block count, pad value - [InlineData(32, 64, true, 1, 0, 1)] // Winograd bands - [InlineData(24, 20, true, 0, 0, 3)] // Winograd, ic % 16 != 0, lane tail, gelu - public void ConvPool_MatchesAvx2(int ic, int oc, bool wino, int mode, float padValue, int act) - { - var rng = new Random(ic * 13 + oc); - int h = 22, w = 30; // pooled 11 x 15: odd tile counts - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * 9), b = rng.Array(oc); - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], - PadMode = (PadMode)mode, PadValue = padValue, Act = (ActivationKind)act, - Pool = new PoolP { Kernel = [2, 2], Strides = [2, 2], Pads = [0, 0, 0, 0], FuseRelu = act == 0 }, - }; - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - if (wino) - { - Skip.IfNot(Nhwc.WinogradEligible(p, [oc, ic, 3, 3], ic), "Winograd disabled"); - kp.WinoU = Nhwc.PackWinograd(wt, oc, ic); - } - float[] fused = Both(dst => Nhwc.ConvPool(x, h, w, ic, dst, h / 2, w / 2, oc, p, kp), h / 2 * (w / 2) * oc); - - // ... and equal to the separate conv -> maxpool on the AVX-512 tier - var conv = new ConvP - { - Pads = p.Pads, Strides = p.Strides, Dilations = p.Dilations, Kernel = p.Kernel, - PadMode = p.PadMode, PadValue = p.PadValue, Act = p.Act, - }; - float[] full = new float[h * w * oc], pooled = new float[h / 2 * (w / 2) * oc]; - Nhwc.Conv(x, h, w, ic, full, h, w, oc, conv, kp, default); - Nhwc.MaxPool(full, h, w, oc, pooled, h / 2, w / 2, p.Pool); - Assert.True(pooled.AsSpan().SequenceEqual(fused), "fused conv+pool differs from conv -> maxpool"); + Both(dst => GcdrKernels.UpHead(x, h, w, c, skip, dst, oh, ow, 2, 2, p, kp), oh * ow * oc); } [SkippableTheory] @@ -184,43 +64,16 @@ public void UpHead_SkipConv_MatchesAvx2(int skipC, int c, int skipAct, int act) PadMode = PadMode.Reflect, Act = (ActivationKind)skipAct, KernelPlan = Nhwc.Pack(cw, [c, skipC, 3, 3], cb, 1, skipC), }; - UpHeadPlan kp = Nhwc.PackUpHead(wt, oc, c, b); + UpHeadPlan kp = GcdrKernels.PackUpHead(wt, oc, c, b); var p = new UpHeadP { Scales = [1, 1, 2, 2], ResizeRelu = true, SkipRelu = true, Act = (ActivationKind)act, SkipConv = sc }; - float[] fused = Both(dst => Nhwc.UpHead(x, h, w, c, sx, dst, oh, ow, 2, 2, p, kp, oh, ow, skipC), oh * ow * oc); + float[] fused = Both(dst => GcdrKernels.UpHead(x, h, w, c, sx, dst, oh, ow, 2, 2, p, kp, oh, ow, skipC), oh * ow * oc); // ... and equal to UpHead over the stored skip tensor on the AVX-512 tier float[] skip = new float[oh * ow * c], want = new float[oh * ow * oc]; Nhwc.Conv(sx, oh, ow, skipC, skip, oh, ow, c, sc, (NhwcConvPlan)sc.KernelPlan!, default); var plain = new UpHeadP { Scales = p.Scales, ResizeRelu = true, SkipRelu = true, Act = p.Act }; - Nhwc.UpHead(x, h, w, c, skip, want, oh, ow, 2, 2, plain, kp); + GcdrKernels.UpHead(x, h, w, c, skip, want, oh, ow, 2, 2, plain, kp); Assert.True(want.AsSpan().SequenceEqual(fused), "skip recompute differs from the stored skip"); } - [SkippableTheory] - [InlineData("gc", 3)] - [InlineData("dr", 6)] - public void MemFusions_Avx512_MatchUnfusedAvx2(string net, int cin) - { - // whole net: memory-fused plan on the AVX-512 tier vs unfused plan on the AVX2 tier, bit for bit - Skip.IfNot(s_tier && Avx2.IsSupported && Fma.IsSupported, "AVX-512 tier not active"); - var m = Sdcb.SimdGcdrNet.Onnx.OnnxModel.Load(Path.Combine(AppContext.BaseDirectory, "testdata", "models", - net == "gc" ? "gcdrnet_gcnet_sim.onnx" : "gcdrnet_drnet_sim.onnx")); - int[] shape = [1, cin, 96, 128]; - var rnd = new Random(7); - float[] input = Enumerable.Range(0, cin * 96 * 128).Select(_ => (float)rnd.NextDouble()).ToArray(); - float[] Run(bool fuse, bool avx512) - { - ExecPlan p = GraphCompiler.Compile(m, shape, new CompileOptions(Nhwc: true, PhysicalIo: true, MemFuse: fuse)); - try - { - Nhwc.UseAvx512 = avx512; - using var s = new InferenceSession(p); - input.CopyTo(s.Input); s.Run(); - return s.Output.ToArray(); - } - finally { Nhwc.UseAvx512 = s_tier; } - } - float[] fused512 = Run(true, true), unfused2 = Run(false, false); - Assert.True(fused512.AsSpan().SequenceEqual(unfused2), $"{net}: fused AVX-512 differs from unfused AVX2"); - } } diff --git a/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs index 7cdcb99..bcd6f62 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs @@ -1,6 +1,8 @@ -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Kernels; using Sdcb.SimdGcdrNet.Kernels; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Onnx; using Xunit; namespace Sdcb.SimdGcdrNet.Tests; @@ -16,11 +18,11 @@ public void Fusion_ChannelShift_And_ResidualAdd() { // fused plan must contain the ChannelShift nodes and fused conv residual adds — // if a model export change breaks the FuseShift pattern, this fails loudly - ExecPlan dr = GraphCompiler.Compile(OnnxModel.Load(DrPath), [1, 6, 96, 128]); - Assert.Equal(8, dr.Nodes.Count(n => n.Op == ExecOp.ChannelShift)); + ExecPlan dr = GraphCompiler.Compile(OnnxModel.Load(DrPath), [1, 6, 96, 128], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }); + Assert.Equal(8, dr.Nodes.Count(n => n.Ext == ChannelShiftOp.Instance)); - ExecPlan gc = GraphCompiler.Compile(OnnxModel.Load(GcPath), [1, 3, 96, 128]); - Assert.True(gc.Nodes.Count(n => n.Op == ExecOp.ChannelShift) >= 4, + ExecPlan gc = GraphCompiler.Compile(OnnxModel.Load(GcPath), [1, 3, 96, 128], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }); + Assert.True(gc.Nodes.Count(n => n.Ext == ChannelShiftOp.Instance) >= 4, "gc plan lost its ChannelShift fusion"); } @@ -30,7 +32,7 @@ public void Fusion_Conv_Gelu_ThroughAlias() // the gelu decomposition commutes before its producer transpose, then // fuses into the 1x1 conv via the reshape Buf-alias — the plan must // contain no Gelu node and the conv must carry the epilogue - ExecPlan gc = GraphCompiler.Compile(OnnxModel.Load(GcPath), [1, 3, 96, 128]); + ExecPlan gc = GraphCompiler.Compile(OnnxModel.Load(GcPath), [1, 3, 96, 128], CompileOptions.Default with { Extensions = GcdrExtensions.Instance }); Assert.DoesNotContain(gc.Nodes, n => n.Op == ExecOp.Gelu); Assert.True(gc.Nodes.Count(n => n.Op == ExecOp.Conv && ((ConvP)n.P!).Act == ActivationKind.Gelu) >= 2, @@ -47,7 +49,7 @@ public void Nhwc_TransposesDissolve(bool physicalIo, int expected) foreach (var (path, c) in new[] { (GcPath, 3), (DrPath, 6) }) { ExecPlan p = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, 96, 128], - new CompileOptions(Nhwc: true, PhysicalIo: physicalIo)); + new CompileOptions(Nhwc: true, PhysicalIo: physicalIo, Extensions: GcdrExtensions.Instance)); int n = p.Nodes.Count(m => m.Op == ExecOp.Transpose); Assert.True(n <= 8, $"{Path.GetFileName(path)}: {n} transposes"); Assert.Equal(expected, n); @@ -65,7 +67,7 @@ public void Nhwc_MatchesNchw() OnnxModel m = OnnxModel.Load(path); float[] Run(bool nhwc) { - using var s = new InferenceSession(GraphCompiler.Compile(m, shape, new CompileOptions(nhwc))); + using var s = new InferenceSession(GraphCompiler.Compile(m, shape, new CompileOptions(nhwc, Extensions: GcdrExtensions.Instance))); input.CopyTo(s.Input); s.Run(); return s.Output.ToArray(); } @@ -87,7 +89,7 @@ public void MemFusions_BitIdentical_AndShrinkArena() float[] input = Enumerable.Range(0, c * 96 * 128).Select(_ => (float)rnd.NextDouble()).ToArray(); (float[] y, ExecPlan p) Run(bool fuse) { - ExecPlan p = GraphCompiler.Compile(m, shape, new CompileOptions(Nhwc: true, PhysicalIo: true, MemFuse: fuse)); + ExecPlan p = GraphCompiler.Compile(m, shape, new CompileOptions(Nhwc: true, PhysicalIo: true, MemFuse: fuse, Extensions: GcdrExtensions.Instance)); using var s = new InferenceSession(p); input.CopyTo(s.Input); s.Run(); return (s.Output.ToArray(), p); @@ -110,7 +112,7 @@ public void MemFusions_GpuPlans_StemOnly() foreach (var (path, c) in new[] { (GcPath, 3), (DrPath, 6) }) { ExecPlan p = GraphCompiler.Compile(OnnxModel.Load(path), [1, c, 96, 128], - new CompileOptions(Nhwc: true, PhysicalIo: true, Gpu: true)); + new CompileOptions(Nhwc: true, PhysicalIo: true, Target: PlanTarget.Vulkan, Extensions: GcdrExtensions.Instance)); Assert.Single(p.Nodes, n => n.P is ConvP { Pool: not null }); Assert.Equal(2, p.Nodes.Count(n => n.Op == ExecOp.MaxPool)); Assert.Contains(p.Nodes, n => n.P is UpHeadP { SkipConv: not null, SkipW: >= 0 }); @@ -127,7 +129,7 @@ public void NoFuse_MatchesFused() float[] Fused() { Environment.SetEnvironmentVariable("GCDRNET_NOFUSE", null); - using var s = new InferenceSession(GraphCompiler.Compile(m, shape)); + using var s = new InferenceSession(GraphCompiler.Compile(m, shape, CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); input.CopyTo(s.Input); s.Run(); return s.Output.ToArray(); } @@ -136,7 +138,7 @@ float[] Unfused() Environment.SetEnvironmentVariable("GCDRNET_NOFUSE", "1"); try { - using var s = new InferenceSession(GraphCompiler.Compile(m, shape)); + using var s = new InferenceSession(GraphCompiler.Compile(m, shape, CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); input.CopyTo(s.Input); s.Run(); return s.Output.ToArray(); } @@ -175,25 +177,6 @@ public void DivConcat_MatchesOracle_Float(string tag) Assert.True(max < 1e-3f, $"dr_in[{tag}] max diff {max} rmse {rmse}"); } - [Fact] - public void Transpose_NonIdentityLeadingPerm_Correct() - { - // perm [1,0,3,2] must NOT take the last-two-dim fast path (would be wrong); - // it should fall through to the generic permute and still be correct - var rng = new Random(5); - int d0 = 2, d1 = 3, d2 = 4, d3 = 5; - float[] src = rng.Array(d0 * d1 * d2 * d3); - float[] got = new float[src.Length]; - Ops.Transpose(src, [d0, d1, d2, d3], got, [d1, d0, d3, d2], [1, 0, 3, 2]); - for (int i0 = 0; i0 < d0; i0++) - for (int i1 = 0; i1 < d1; i1++) - for (int i2 = 0; i2 < d2; i2++) - for (int i3 = 0; i3 < d3; i3++) - Assert.Equal( - src[((i0 * d1 + i1) * d2 + i2) * d3 + i3], - got[((i1 * d0 + i0) * d3 + i3) * d2 + i2]); - } - [Fact] public void ChannelShift_OffsetBeyondWidth_Zeros() { @@ -201,93 +184,10 @@ public void ChannelShift_OffsetBeyondWidth_Zeros() float[] src = Enumerable.Repeat(1f, 2 * 4 * 5).ToArray(); // c=2,h=4,w=5 float[] dst = Enumerable.Repeat(-1f, src.Length).ToArray(); var p = new ShiftP { Axis = 3, Offsets = [5], GroupEnds = [2] }; - Ops.ChannelShift(src, [1, 2, 4, 5], dst, [1, 2, 4, 5], p); + GcdrKernels.ChannelShiftNchw(src, [1, 2, 4, 5], dst, [1, 2, 4, 5], p); Assert.All(dst, v => Assert.Equal(0f, v)); } - [Fact] - public void Conv_GroupedNonDepthwise_MatchesReference() - { - // group=2 with ic=4 (not depthwise) routes to the scalar path — check vs manual - var rng = new Random(11); - int ic = 4, oc = 4, ih = 5, iw = 6, g = 2; - float[] x = rng.Array(ic * ih * iw); - float[] w = rng.Array(oc * (ic / g) * 9); - float[] dst = new float[oc * ih * iw]; - var p = new ConvP { Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], Group = g }; - Ops.Conv(x, [1, ic, ih, iw], w, [oc, ic / g, 3, 3], ReadOnlySpan.Empty, - dst, [1, oc, ih, iw], p); - - for (int o = 0; o < oc; o++) - for (int y = 0; y < ih; y++) - for (int xx = 0; xx < iw; xx++) - { - float acc = 0; - int gi = o / (oc / g) * (ic / g); - for (int c = 0; c < ic / g; c++) - for (int ky = 0; ky < 3; ky++) - for (int kx = 0; kx < 3; kx++) - { - int sy = y + ky - 1, sx = xx + kx - 1; - if ((uint)sy < (uint)ih && (uint)sx < (uint)iw) - acc += x[(gi + c) * ih * iw + sy * iw + sx] * - w[((o * (ic / g) + c) * 3 + ky) * 3 + kx]; - } - Assert.Equal(acc, dst[o * ih * iw + y * iw + xx], 5); - } - } - - [Fact] - public void Conv_ResidualAdd_MatchesManual() - { - // unit-level check of the Conv+Add epilogue (the fused path has no model coverage) - var rng = new Random(7); - int ic = 4, oc = 8, ih = 9, iw = 11; - float[] x = rng.Array(ic * ih * iw); - float[] w = rng.Array(oc * ic * 9); - float[] b = rng.Array(oc); - float[] res = rng.Array(oc * ih * iw); - float[] want = new float[oc * ih * iw], got = new float[oc * ih * iw]; - - var p = new ConvP { Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3] }; - Ops.Conv(x, [1, ic, ih, iw], w, [oc, ic, 3, 3], b, want, [1, oc, ih, iw], p); - for (int i = 0; i < want.Length; i++) want[i] += res[i]; - - Ops.Conv(x, [1, ic, ih, iw], w, [oc, ic, 3, 3], b, got, [1, oc, ih, iw], p, res); - var (max, _, _) = Golden.Diff(want, got); - Assert.True(max < 1e-4f, $"conv+add max diff {max}"); - } - - [SkippableTheory] - [InlineData(16, 16, 9, 11, 0, 0f, false, 0)] // constant pad - [InlineData(24, 20, 13, 6, 1, 0f, true, 1)] // reflect, residual, relu, oc lane tail, odd tiles - [InlineData(32, 48, 4, 4, 0, 0.5f, false, 3)] // nonzero pad value, gelu - [InlineData(64, 16, 17, 23, 1, 0f, true, 0)] - public void Conv_Winograd_MatchesDirect(int ic, int oc, int h, int w, int mode, float padValue, - bool residual, int act) - { - var rng = new Random(ic * 31 + oc); - float[] x = rng.Array(h * w * ic); // NHWC - float[] wt = rng.Array(oc * ic * 9); - float[] b = rng.Array(oc); - float[] res = residual ? rng.Array(h * w * oc) : []; - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], - PadMode = (PadMode)mode, PadValue = padValue, Act = (ActivationKind)act, - }; - Skip.IfNot(Nhwc.WinogradEligible(p, [oc, ic, 3, 3], ic), "Winograd disabled or ISA tier without it"); - NhwcConvPlan direct = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - NhwcConvPlan wino = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - wino.WinoU = Nhwc.PackWinograd(wt, oc, ic); - float[] want = new float[h * w * oc], got = new float[h * w * oc]; - Nhwc.Conv(x, h, w, ic, want, h, w, oc, p, direct, res); - Nhwc.Conv(x, h, w, ic, got, h, w, oc, p, wino, res); - var (max, _, _) = Golden.Diff(want, got); - float scale = want.Max(MathF.Abs); - Assert.True(max < 1e-4f * scale, $"winograd max diff {max} (|out| max {scale})"); - } - [Fact] public void Pipeline_SizeSwitching_MatchesFresh_Cpu() { diff --git a/tests/Sdcb.SimdGcdrNet.Tests/GoldenTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/GoldenTests.cs index 38b7a1e..e543ce5 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/GoldenTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/GoldenTests.cs @@ -1,5 +1,6 @@ -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Onnx; using Xunit; namespace Sdcb.SimdGcdrNet.Tests; diff --git a/tests/Sdcb.SimdGcdrNet.Tests/MetalTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/MetalTests.cs index c2ab8f4..37fd3d4 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/MetalTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/MetalTests.cs @@ -1,6 +1,8 @@ +using Sdcb.SimdOnnx.Backends.Metal; using Sdcb.SimdGcdrNet.Backends.Metal; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Onnx; using Xunit; namespace Sdcb.SimdGcdrNet.Tests; @@ -22,7 +24,7 @@ public void GcNet_MatchesGolden_Metal(string tag) float[] expect = Golden.F32(tag, "shadow"); OnnxModel m = OnnxModel.Load(GcPath); - using var sess = MetalBackend.Default.NewSession(m, shape); + using var sess = MetalBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); input.CopyTo(sess.Input); sess.Run(); var (max, rmse, _) = Golden.Diff(sess.Output, expect); @@ -42,7 +44,7 @@ public void DrNet_MatchesGolden_Metal(string tag) float[] expect = Golden.F32(tag, "output"); OnnxModel m = OnnxModel.Load(DrPath); - using var sess = MetalBackend.Default.NewSession(m, shape); + using var sess = MetalBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); input.CopyTo(sess.Input); sess.Run(); var (max, rmse, _) = Golden.Diff(sess.Output, expect); diff --git a/tests/Sdcb.SimdGcdrNet.Tests/MtlKernelTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/MtlKernelTests.cs deleted file mode 100644 index 45659eb..0000000 --- a/tests/Sdcb.SimdGcdrNet.Tests/MtlKernelTests.cs +++ /dev/null @@ -1,221 +0,0 @@ -using Sdcb.SimdGcdrNet.Backends.Metal; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Kernels; -using Xunit; - -namespace Sdcb.SimdGcdrNet.Tests; - -/// Standalone Metal kernel vs CPU kernel checks. -public unsafe class MtlKernelTests -{ - private static float[] RunGpu(MtlDevice dev, string shader, float[][] ins, int outLen, byte[] pc, uint groups, int[]? pcs = null) - { - using var pool = AutoReleasePool.Create(); - var bufs = new List(); - foreach (var f in ins) - { - var h = new Half[f.Length]; - for (int i = 0; i < f.Length; i++) h[i] = (Half)f[i]; - fixed (Half* hp = h) bufs.Add(dev.NewBuffer(hp, (nuint)(h.Length * 2))); - } - // extra inputs that must stay fp32 (bias, layernorm params) are passed via pcs tag - var bufsF = new List(); - var ob = dev.NewBuffer((nuint)(outLen * 2 + 64)); - var enc = dev.BeginCommands(); - enc.SetPipeline(dev.Pipe(shader)); - int bi = 0; - foreach (var b in bufs) { enc.SetBuffer(b, 0, bi++); } - enc.SetBuffer(ob, 0, bi++); - if (pcs != null) { /* placeholder */ } - fixed (byte* p = pc) enc.SetPc(p, (nuint)pc.Length, bi); - enc.Dispatch(groups, 1, 1, 256, 1, 1); - enc.End(); - enc.CommitAndWait(); - var dst = new float[outLen]; - var sp = new ReadOnlySpan(ob.Contents.ToPointer(), outLen); - for (int i = 0; i < outLen; i++) dst[i] = (float)sp[i]; - foreach (var b in bufs) b.Dispose(); - ob.Dispose(); - return dst; - } - - // generic: ins as fp16 tensors plus an fp32 tensor slot for bias - private static float[] RunGpuMixed(MtlDevice dev, string shader, - (float[] data, bool f32)[] ins, int outLen, byte[] pc, uint gx, uint gy = 1, uint gz = 1) - { - using var pool = AutoReleasePool.Create(); - var bufs = new List(); - foreach (var (f, f32) in ins) - { - if (f32) - fixed (float* fp = f) bufs.Add(dev.NewBuffer(fp, (nuint)(f.Length * 4))); - else - { - var h = new Half[f.Length]; - for (int i = 0; i < f.Length; i++) h[i] = (Half)f[i]; - fixed (Half* hp = h) bufs.Add(dev.NewBuffer(hp, (nuint)(h.Length * 2))); - } - } - var ob = dev.NewBuffer((nuint)(outLen * 2 + 64)); - var enc = dev.BeginCommands(); - enc.SetPipeline(dev.Pipe(shader)); - int bi = 0; - foreach (var b in bufs) enc.SetBuffer(b, 0, bi++); - enc.SetBuffer(ob, 0, bi++); - fixed (byte* p = pc) enc.SetPc(p, (nuint)pc.Length, bi); - enc.Dispatch(gx, gy, gz, 256, 1, 1); - enc.End(); - enc.CommitAndWait(); - var dst = new float[outLen]; - var sp = new ReadOnlySpan(ob.Contents.ToPointer(), outLen); - for (int i = 0; i < outLen; i++) dst[i] = (float)sp[i]; - foreach (var b in bufs) b.Dispose(); - ob.Dispose(); - return dst; - } - - private static float[] Rand(Random r, int n) - { - var a = new float[n]; - for (int i = 0; i < n; i++) a[i] = r.NextSingle() * 2 - 1; - return a; - } - - [SkippableTheory] - [InlineData("convpool", 3, 16, true, 0f, 1)] - [InlineData("convpool_k3", 3, 16, true, 0f, 1)] - [InlineData("convpool_k3", 6, 32, false, 0f, 0)] - [InlineData("convpool_k3", 7, 8, false, 0.25f, 1)] - [InlineData("convpool_k3", 1, 24, true, 0f, 0)] - public void ConvPool_MatchesCpu(string shader, int ic, int oc, bool reflect, float padValue, int act) - { - Skip.IfNot(MetalOk(), "no Metal device"); - var dev = MetalBackend.Default.Dev; - var rng = new Random(ic * 13 + oc); - int h = 22, w = 30, ph = h / 2, pw = w / 2; - float[] x = Rand(rng, h * w * ic), wt = Rand(rng, oc * ic * 9), b = Rand(rng, oc); - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], - PadMode = reflect ? PadMode.Reflect : PadMode.Constant, PadValue = padValue, - Act = act == 1 ? ActivationKind.Relu : ActivationKind.None, - Pool = new PoolP { Kernel = [2, 2], Strides = [2, 2], Pads = [0, 0, 0, 0], FuseRelu = act == 0 }, - }; - float[] cpu = new float[ph * pw * oc]; - Nhwc.ConvPool(x, h, w, ic, cpu, ph, pw, oc, p, Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic)); - float[] wk = new float[9 * ic * oc]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < 9; t++) wk[(t * ic + c) * oc + o] = wt[(o * ic + c) * 9 + t]; - uint flags = 1u | (reflect ? 2u : 0u) | (act == 1 ? 4u : 0u) | (act == 0 ? 8u : 0u); - byte[] pc = new byte[48]; - Buffer.BlockCopy(new uint[] { (uint)ic, (uint)h, (uint)w, (uint)oc, (uint)ph, (uint)pw, 3u, 3u, 1u, 1u, flags }, 0, pc, 0, 44); - Buffer.BlockCopy(new float[] { reflect ? 0f : padValue }, 0, pc, 44, 4); - float[] got = RunGpuMixed(dev, shader, [(x, false), (wk, false), (b, true)], cpu.Length, pc, - (uint)(ph * pw * (oc / 8) + 255) / 256); - AssertClose(got, cpu, 2e-2f, shader); - } - - - [SkippableTheory] - [InlineData(6, 32, 1, false, "conv_mm_32_1", false, 1)] // dr stem (scalar k gather) - [InlineData(3, 16, 1, true, "conv_mm_32_1", false, 1)] // gc stem, reflect - [InlineData(32, 64, 1, true, "conv_mm_64_0", false, 1)] - [InlineData(64, 128, 2, false, "conv_mm_128_0", false, 1)] - [InlineData(16, 32, 1, false, "conv_mm_32_0", false, 1)] - [InlineData(32, 64, 1, true, "conv_mm_64_0", true, 2)] // residual, gelu - [InlineData(64, 128, 2, false, "conv_mm_128_0", true, 0)] - [InlineData(16, 160, 1, false, "conv_mm_32_0", false, 1)] - [InlineData(32, 256, 1, false, "conv_mm_128_0", true, 1)] - [InlineData(16, 16, 1, true, "conv_mm_32_0", false, 1)] - [InlineData(16, 48, 1, false, "conv_mm_32_0", true, 2)] - [InlineData(8, 64, 2, true, "conv_mm_64_0", true, 1)] - [InlineData(3, 32, 1, false, "conv_mm_32_1", true, 2)] - [InlineData(128, 128, 1, true, "conv_mm_128_0", true, 2)] - public void ConvMm_MatchesCpu(int ic, int oc, int stride, bool reflect, string shader, bool residual, int act) - { - Skip.IfNot(MetalOk(), "no Metal device"); - var dev = MetalBackend.Default.Dev; - var rng = new Random(ic * 7 + oc); - int ih = 29, iw = 37, oh = (ih + 2 - 3) / stride + 1, ow = (iw + 2 - 3) / stride + 1; - float[] x = Rand(rng, ih * iw * ic).Select(v => (float)(Half)v).ToArray(); - float[] w = Rand(rng, oc * ic * 9).Select(v => (float)(Half)(v * 0.25f)).ToArray(); - float[] b = Rand(rng, oc); - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [stride, stride], Dilations = [1, 1], Kernel = [3, 3], - PadMode = reflect ? PadMode.Reflect : PadMode.Constant, - Act = act == 1 ? ActivationKind.Relu : act == 2 ? ActivationKind.Gelu : ActivationKind.None, - }; - float[] res = residual ? Rand(rng, oh * ow * oc).Select(v => (float)(Half)v).ToArray() : []; - float[] cpu = new float[oh * ow * oc]; - NhwcConvPlan kp = Nhwc.Pack(w, [oc, ic, 3, 3], b, 1, ic); - Nhwc.Conv(x, ih, iw, ic, cpu, oh, ow, oc, p, kp, res); - - int kr = 9 * ic, kd = (kr + 7) & ~7; - float[] wt = new float[oc * kd]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < 9; t++) wt[o * kd + t * ic + c] = w[(o * ic + c) * 9 + t]; - uint flags = 1u | (residual ? 2u : 0u) | (reflect ? 4u : 0u) | ((uint)act << 4); - uint m = (uint)(oh * ow); - byte[] pc = new byte[56]; - Buffer.BlockCopy(new uint[] { m, (uint)oc, (uint)kd, flags, (uint)ow, (uint)iw, (uint)ih, (uint)ic, 3u, - (uint)stride, (uint)stride, 1u, 1u, (uint)kr }, 0, pc, 0, 56); - uint bn = shader.StartsWith("conv_mm_128") ? 128u : shader.StartsWith("conv_mm_64") ? 64u : 32u; - float[] got = RunGpuMixed(dev, shader, - [(x, false), (wt, false), (b, true), (residual ? res : new float[8], false)], - cpu.Length, pc, (m + 63) / 64, ((uint)oc + bn - 1) / bn); - float scale = Math.Max(1f, cpu.Max(MathF.Abs)); - AssertClose(got, cpu, 2e-3f * scale, shader); - } - - - [SkippableTheory] - [InlineData(160)] - [InlineData(256)] - [InlineData(64)] - [InlineData(96)] - public void LayerNorm_MatchesCpu(int len) - { - Skip.IfNot(MetalOk(), "no Metal device"); - var dev = MetalBackend.Default.Dev; - var rng = new Random(len); - int rows = 97; - float[] x = Rand(rng, rows * len).Select(v => (float)(Half)v).ToArray(); - float[] w = Rand(rng, len), b = Rand(rng, len); - float eps = 1e-5f; - float[] want = new float[rows * len]; - for (int r = 0; r < rows; r++) - { - double m = 0, q = 0; - for (int i = 0; i < len; i++) { m += x[r * len + i]; q += x[r * len + i] * x[r * len + i]; } - m /= len; q = Math.Sqrt(q / len - m * m + eps); - for (int i = 0; i < len; i++) want[r * len + i] = (float)((x[r * len + i] - m) / q * w[i] + b[i]); - } - byte[] pc = new byte[12]; - Buffer.BlockCopy(new uint[] { (uint)len, (uint)rows }, 0, pc, 0, 8); - Buffer.BlockCopy(new float[] { eps }, 0, pc, 8, 4); - float[] got = RunGpuMixed(dev, "layernorm", [(x, false), (w, true), (b, true)], want.Length, pc, (uint)((rows + 7) / 8)); - AssertClose(got, want, 1e-2f, "layernorm"); - } - - private static void AssertClose(float[] got, float[] want, float tol, string tag) - { - double mx = 0, rmse = 0; - for (int i = 0; i < want.Length; i++) - { - double d = Math.Abs(got[i] - want[i]); - if (d > mx) mx = d; - rmse += d * d; - } - rmse = Math.Sqrt(rmse / want.Length); - Assert.True(mx < tol, $"{tag}: max {mx} rmse {rmse} (tol {tol})"); - } - - private static bool MetalOk() - { - try { return GcdrEnhancer.MetalUsable; } - catch { return false; } - } -} diff --git a/tests/Sdcb.SimdGcdrNet.Tests/NeonKernelTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/NeonKernelTests.cs index de23894..dc541b7 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/NeonKernelTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/NeonKernelTests.cs @@ -1,5 +1,7 @@ using System.Runtime.Intrinsics.Arm; -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Kernels; using Sdcb.SimdGcdrNet.Kernels; namespace Sdcb.SimdGcdrNet.Tests; @@ -46,75 +48,6 @@ private static float[] Both(Action run, int n, float tol = 2e-5f) PadMode = (PadMode)mode, PadValue = padValue, Act = (ActivationKind)act, }; - [SkippableTheory] - [InlineData(3, 16, 3, 1, 37, 23, 1, 0f, false, 0)] // gc stem: run of 9 (2 by-element groups + 1 broadcast) - [InlineData(6, 32, 3, 1, 29, 41, 1, 0f, false, 1)] // dr stem: run of 18 - [InlineData(16, 48, 3, 1, 11, 13, 0, 0.25f, true, 1)] // odd block count, pad value, residual - [InlineData(20, 20, 3, 2, 17, 19, 0, 0f, true, 2)] // stride 2, oc lane tail, sigmoid - [InlineData(32, 64, 3, 2, 12, 31, 1, 0f, false, 3)] // stride 2, reflect, gelu - [InlineData(24, 3, 3, 1, 9, 10, 0, 0f, false, 0)] // oc=3 (single partial lane block) - [InlineData(40, 36, 1, 1, 7, 33, 0, 0f, true, 1)] // 1x1 conv - [InlineData(5, 32, 5, 1, 14, 9, 1, 0f, false, 3)] // 5x5, reflect: border runs gathered, gelu - [InlineData(7, 16, 3, 1, 4, 3, 0, -0.5f, true, 2)] // tiny image: every run crosses a border - public void Conv_Direct_MatchesVector(int ic, int oc, int k, int stride, int h, int w, int mode, - float padValue, bool residual, int act) - { - var rng = new Random(ic * 131 + oc * 7 + k); - int pad = k / 2; - int oh = (h + 2 * pad - k) / stride + 1, ow = (w + 2 * pad - k) / stride + 1; - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * k * k), b = rng.Array(oc); - float[] res = residual ? rng.Array(oh * ow * oc) : []; - ConvP p = Conv3(k, stride, mode, padValue, act); - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, k, k], b, 1, ic); // no Winograd: direct kernels - Both(dst => Nhwc.Conv(x, h, w, ic, dst, oh, ow, oc, p, kp, res), oh * ow * oc); - } - - [SkippableTheory] - [InlineData(16, 16, 9, 11, 0, 0f, false, 0)] - [InlineData(24, 20, 13, 6, 1, 0f, true, 1)] - [InlineData(32, 48, 22, 18, 0, 0.5f, false, 3)] - [InlineData(64, 16, 17, 23, 1, 0f, true, 2)] - [InlineData(128, 64, 30, 41, 1, 0f, false, 1)] // several tile chunks per image - public void Conv_Winograd_MatchesVector(int ic, int oc, int h, int w, int mode, float padValue, bool residual, int act) - { - var rng = new Random(ic * 17 + oc); - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * 9), b = rng.Array(oc); - float[] res = residual ? rng.Array(h * w * oc) : []; - ConvP p = Conv3(3, 1, mode, padValue, act); - Skip.IfNot(Nhwc.WinogradEligible(p, [oc, ic, 3, 3], ic), "Winograd disabled"); - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - kp.WinoU = Nhwc.PackWinograd(wt, oc, ic); - Both(dst => Nhwc.Conv(x, h, w, ic, dst, h, w, oc, p, kp, res), h * w * oc, 2e-4f); - } - - [SkippableTheory] - [InlineData(16, 0, 0f, false, 0)] - [InlineData(20, 1, 0f, true, 3)] // c % 16 != 0, residual, gelu - [InlineData(22, 0, 0.5f, false, 2)] // c % 4 != 0: scalar channel tail, sigmoid - public void Depthwise_MatchesVector(int c, int mode, float padValue, bool residual, int act) - { - var rng = new Random(c * 5 + act); - int h = 13, w = 17; - float[] x = rng.Array(h * w * c), wt = rng.Array(c * 9), b = rng.Array(c); - float[] res = residual ? rng.Array(h * w * c) : []; - ConvP p = Conv3(3, 1, mode, padValue, act); - p.Group = c; - NhwcConvPlan kp = Nhwc.Pack(wt, [c, 1, 3, 3], b, c, c); - Both(dst => Nhwc.Conv(x, h, w, c, dst, h, w, c, p, kp, res), h * w * c); - } - - [SkippableTheory] - [InlineData(57, 128, 128, false, 0)] - [InlineData(23, 160, 48, true, 3)] // odd block count, transB, gelu - [InlineData(5, 66, 20, false, 1)] // fewer rows than a tile, k % 4 != 0, lane tail - public void Gemm_MatchesVector(int m, int k, int n, bool transB, int act) - { - var rng = new Random(m + k + n); - float[] a = rng.Array(m * k), bm = rng.Array(k * n), bias = rng.Array(n); - NhwcConvPlan kp = Nhwc.PackGemm(bm, k, n, transB, bias); - Both(dst => Nhwc.Gemm(a, m, k, dst, n, kp, (ActivationKind)act), m * n); - } - [SkippableTheory] [InlineData(32, 3, true, true, 2)] [InlineData(16, 3, false, true, 0)] @@ -125,48 +58,9 @@ public void UpHead_MatchesVector(int c, int oc, bool resizeRelu, bool skipRelu, var rng = new Random(c * oc); int h = 11, w = 15, oh = 22, ow = 30; float[] x = rng.Array(h * w * c), skip = rng.Array(oh * ow * c), wt = rng.Array(oc * c), b = rng.Array(oc); - UpHeadPlan kp = Nhwc.PackUpHead(wt, oc, c, b); + UpHeadPlan kp = GcdrKernels.PackUpHead(wt, oc, c, b); var p = new UpHeadP { Scales = [1, 1, 2, 2], ResizeRelu = resizeRelu, SkipRelu = skipRelu, Act = (ActivationKind)act }; - Both(dst => Nhwc.UpHead(x, h, w, c, skip, dst, oh, ow, 2, 2, p, kp), oh * ow * oc); - } - - [SkippableTheory] - [InlineData(128, 64)] - [InlineData(37, 18)] // c % 4 != 0 - public void LayerNorm_MatchesVector(int c, int rows) - { - var rng = new Random(c + rows); - float[] x = rng.Array(rows * c), wt = rng.Array(c), b = rng.Array(c); - Both(dst => Ops.LayerNorm(x, [rows, c], wt, b, dst, 1e-5f), rows * c); - } - - [SkippableTheory] - [InlineData(3, 16, false, 1, 0f, 1)] // gc stem - [InlineData(6, 32, false, 1, 0f, 1)] // dr stem - [InlineData(16, 48, false, 0, 0.25f, 0)] // odd block count, pad value - [InlineData(32, 64, true, 1, 0f, 1)] // Winograd bands - [InlineData(24, 20, true, 0, 0f, 3)] // Winograd, lane tail, gelu - public void ConvPool_MatchesVector_AndUnfused(int ic, int oc, bool wino, int mode, float padValue, int act) - { - var rng = new Random(ic * 13 + oc); - int h = 22, w = 30; - float[] x = rng.Array(h * w * ic), wt = rng.Array(oc * ic * 9), b = rng.Array(oc); - ConvP p = Conv3(3, 1, mode, padValue, act); - p.Pool = new PoolP { Kernel = [2, 2], Strides = [2, 2], Pads = [0, 0, 0, 0], FuseRelu = act == 0 }; - NhwcConvPlan kp = Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic); - if (wino) - { - Skip.IfNot(Nhwc.WinogradEligible(p, [oc, ic, 3, 3], ic), "Winograd disabled"); - kp.WinoU = Nhwc.PackWinograd(wt, oc, ic); - } - float[] fused = Both(dst => Nhwc.ConvPool(x, h, w, ic, dst, h / 2, w / 2, oc, p, kp), h / 2 * (w / 2) * oc, wino ? 2e-4f : 2e-5f); - - // ... and bit-identical to the separate conv -> maxpool on the NEON tier - ConvP conv = Conv3(3, 1, mode, padValue, act); - float[] full = new float[h * w * oc], pooled = new float[h / 2 * (w / 2) * oc]; - Nhwc.Conv(x, h, w, ic, full, h, w, oc, conv, kp, default); - Nhwc.MaxPool(full, h, w, oc, pooled, h / 2, w / 2, p.Pool); - Assert.True(pooled.AsSpan().SequenceEqual(fused), "fused conv+pool differs from conv -> maxpool"); + Both(dst => GcdrKernels.UpHead(x, h, w, c, skip, dst, oh, ow, 2, 2, p, kp), oh * ow * oc); } [SkippableTheory] @@ -181,39 +75,18 @@ public void UpHead_SkipConv_MatchesVector_AndStoredSkip(int skipC, int c, int sk float[] cw = rng.Array(c * skipC * 9), cb = rng.Array(c), wt = rng.Array(oc * c), b = rng.Array(oc); ConvP sc = Conv3(3, 1, (int)PadMode.Reflect, 0f, skipAct); sc.KernelPlan = Nhwc.Pack(cw, [c, skipC, 3, 3], cb, 1, skipC); - UpHeadPlan kp = Nhwc.PackUpHead(wt, oc, c, b); + UpHeadPlan kp = GcdrKernels.PackUpHead(wt, oc, c, b); var p = new UpHeadP { Scales = [1, 1, 2, 2], ResizeRelu = true, SkipRelu = true, Act = (ActivationKind)act, SkipConv = sc }; - float[] fused = Both(dst => Nhwc.UpHead(x, h, w, c, sx, dst, oh, ow, 2, 2, p, kp, oh, ow, skipC), oh * ow * oc); + float[] fused = Both(dst => GcdrKernels.UpHead(x, h, w, c, sx, dst, oh, ow, 2, 2, p, kp, oh, ow, skipC), oh * ow * oc); // ... and bit-identical to UpHead over the stored skip tensor on the NEON tier float[] skip = new float[oh * ow * c], want = new float[oh * ow * oc]; Nhwc.Conv(sx, oh, ow, skipC, skip, oh, ow, c, sc, (NhwcConvPlan)sc.KernelPlan!, default); var plain = new UpHeadP { Scales = p.Scales, ResizeRelu = true, SkipRelu = true, Act = p.Act }; - Nhwc.UpHead(x, h, w, c, skip, want, oh, ow, 2, 2, plain, kp); + GcdrKernels.UpHead(x, h, w, c, skip, want, oh, ow, 2, 2, plain, kp); Assert.True(want.AsSpan().SequenceEqual(fused), "skip recompute differs from the stored skip"); } - [SkippableTheory] - [InlineData("gc", 3)] - [InlineData("dr", 6)] - public void MemFusions_Neon_MatchUnfused(string net, int cin) - { - // whole net on the NEON tier: memory-fused plan vs unfused plan, bit for bit - SkipUnlessNeon(); - var m = Sdcb.SimdGcdrNet.Onnx.OnnxModel.Load(Path.Combine(AppContext.BaseDirectory, "testdata", "models", - net == "gc" ? "gcdrnet_gcnet_sim.onnx" : "gcdrnet_drnet_sim.onnx")); - int[] shape = [1, cin, 96, 128]; - var rnd = new Random(7); - float[] input = Enumerable.Range(0, cin * 96 * 128).Select(_ => (float)rnd.NextDouble()).ToArray(); - float[] Run(bool fuse) - { - ExecPlan p = GraphCompiler.Compile(m, shape, new CompileOptions(Nhwc: true, PhysicalIo: true, MemFuse: fuse)); - using var s = new InferenceSession(p); - input.CopyTo(s.Input); s.Run(); - return s.Output.ToArray(); - } - Assert.True(Run(true).AsSpan().SequenceEqual(Run(false)), $"{net}: fused NEON differs from unfused NEON"); - } } /// Tests that flip a process-wide kernel tier switch must not overlap any other test. diff --git a/tests/Sdcb.SimdGcdrNet.Tests/PixelFormatTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/PixelFormatTests.cs index 7368aa8..99d5e7d 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/PixelFormatTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/PixelFormatTests.cs @@ -1,4 +1,6 @@ +using Sdcb.SimdOnnx.Backends.Metal; using Sdcb.SimdGcdrNet.Backends.Metal; +using Sdcb.SimdOnnx.Backends.Vulkan; using Sdcb.SimdGcdrNet.Backends.Vulkan; using Xunit; diff --git a/tests/Sdcb.SimdGcdrNet.Tests/VkDebugTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/VkDebugTests.cs deleted file mode 100644 index 17737f5..0000000 --- a/tests/Sdcb.SimdGcdrNet.Tests/VkDebugTests.cs +++ /dev/null @@ -1,86 +0,0 @@ -using Sdcb.SimdGcdrNet.Backends.Vulkan; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; -using Xunit; - -namespace Sdcb.SimdGcdrNet.Tests; - -/// Node-by-node GPU vs CPU comparison. Enable with GCDRNET_VKDBG=1. -public class VkDebugTests -{ - private static string GcPath => Path.Combine(Golden.Dir, "models", "gcdrnet_gcnet_sim.onnx"); - private static string DrPath => Path.Combine(Golden.Dir, "models", "gcdrnet_drnet_sim.onnx"); - - [Theory] - [InlineData("gc", "small", "input")] - [InlineData("dr", "small", "dr_in")] - public void Nodes_Match(string which, string tag, string inputName) - { - if (Environment.GetEnvironmentVariable("GCDRNET_VKDBG") != "1") return; - try { _ = VkBackend.Default.Dev.DeviceName; } - catch { return; } // no Vulkan device — env-gated debug test, skip quietly - - OnnxModel m = OnnxModel.Load(which == "gc" ? GcPath : DrPath); - float[] input = Golden.F32(tag, inputName); - int[] shape = Golden.ShapeOf(tag, inputName); - - ExecPlan plan = GraphCompiler.Compile(m, shape, VkBackend.PlanOptions); - using var cpu = new InferenceSession(plan); - using var gpu = VkBackend.Default.NewSession(m, shape); - - input.CopyTo(cpu.Input); - input.CopyTo(gpu.Input); - gpu.DebugUploadInput(); - - var log = new StreamWriter(Path.Combine(Path.GetTempPath(), $"vkdbg_{which}_{tag}.txt")); - int bad = 0; - // fp16 storage: per-node error measured up to ~4e-2 on deep convs (relative ~1e-3 of activations) - float thr = VkBackend.Default.F16 ? 6e-2f : 1e-3f; - for (int i = 0; i < plan.Nodes.Length; i++) - { - ExecNode n = plan.Nodes[i]; - cpu.ExecNode(i); - var (ds, dc) = VkBackend.Default.GraphFor(m, shape).NodeRanges[i]; - gpu.DebugRunRange(ds, dc); - int ov = n.Out[0]; - CValue v = plan.Values[ov]; - float[] got = new float[v.ElementCount]; - gpu.DebugDownload(ov, got); - float[] want = cpu.ValueSpan(ov).ToArray(); - float maxd = 0; int at = -1; - for (int j = 0; j < want.Length; j++) - { - float d = Math.Abs(got[j] - want[j]); - if (d > maxd) { maxd = d; at = j; } - } - string status = maxd > thr ? " <-- BAD" : ""; - if (maxd > thr) - { - bad++; - if (bad <= 2) - { - int c = (int)v.ElementCount; - int[] sh = v.Shape; - int cw = sh[^1], chh = sh.Length >= 2 ? sh[^2] : 1; - for (int j = 0; j < c; j++) - if (Math.Abs(got[j] - want[j]) > thr) - { - int y = chh > 1 && cw > 0 ? (j / cw) % chh : j / cw; - int x = cw > 0 ? j % cw : 0; - int ch = cw > 0 && chh > 0 ? j / (cw * chh) : 0; - log.WriteLine($" [{j}] (c{ch},y{y},x{x}) gpu={got[j],12:g6} cpu={want[j],12:g6}"); - } - int badCnt = 0; - for (int j = 0; j < c; j++) - if (Math.Abs(got[j] - want[j]) > thr) badCnt++; - log.WriteLine($" badElems={badCnt}/{c}"); - for (int j = Math.Max(0, at - 2); j < Math.Min(c, at + 3); j++) - log.WriteLine($" [{j}] gpu={got[j],12:g6} cpu={want[j],12:g6}"); - } - } - log.WriteLine($"node[{i,3}] {n.Op,-14} {n.Name,-24} out={string.Join('x', v.Shape)} maxd={maxd:e3} @{at}{status}"); - } - log.Close(); - Assert.True(bad == 0, $"{bad} nodes diverged — see %TEMP%/vkdbg_{which}_{tag}.txt"); - } -} diff --git a/tests/Sdcb.SimdGcdrNet.Tests/VkKernelTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/VkKernelTests.cs index b6d6c33..5e4726e 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/VkKernelTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/VkKernelTests.cs @@ -1,5 +1,8 @@ +using Sdcb.SimdOnnx.Backends.Vulkan; using Sdcb.SimdGcdrNet.Backends.Vulkan; -using Sdcb.SimdGcdrNet.Engine; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Kernels; using Sdcb.SimdGcdrNet.Kernels; using Xunit; @@ -8,56 +11,6 @@ namespace Sdcb.SimdGcdrNet.Tests; /// Standalone GPU kernel vs CPU kernel checks (bilinear, fused upsample tail). public unsafe class VkKernelTests { - [SkippableTheory] - [InlineData(0u, "bilinear")] // plain - [InlineData(7u, "bilinear")] // relu(resize) + relu(addend) - [InlineData(2u, "bilinear")] // resize + addend - [InlineData(0u, "bilinear4")] // 4-channel variant (integrated GPUs) - [InlineData(7u, "bilinear4")] - public void Bilinear_MatchesCpu(uint flags, string shader) - { - Skip.IfNot(VkOk(), "no Vulkan device"); - var dev = VkBackend.Default.Dev; - var rng = new Random(42); - int c = 4, ih = 3, iw = 4, oh = 6, ow = 8; - float[] src = Rand(rng, c * ih * iw); - float[] add = Rand(rng, c * oh * ow); - float[] dstCpu = new float[c * oh * ow]; - bool hasAdd = (flags & 2u) != 0; - // shaders address NHWC storage: src is [ih][iw][c] - Nhwc.ResizeBilinear(src, ih, iw, c, dstCpu, oh, ow, 2f, 2f, - (flags & 1u) != 0 ? ActivationKind.Relu : ActivationKind.None, - hasAdd ? add : default, (flags & 4u) != 0); - - // PC layout: 5 uints + flags uint + 2 floats - byte[] pcb = new byte[32]; - Buffer.BlockCopy(new uint[] { (uint)c, (uint)ih, (uint)iw, (uint)oh, (uint)ow, flags }, 0, pcb, 0, 24); - Buffer.BlockCopy(new float[] { 2f, 2f }, 0, pcb, 24, 8); - uint lanes = shader == "bilinear4" ? 4u : 1u; - float[] got = RunGpu(dev, shader, [src, add], dstCpu.Length, pcb, ((uint)dstCpu.Length / lanes + 255) / 256); - AssertClose(got, dstCpu, 1e-4f, shader); - } - - [SkippableTheory] - [InlineData("maxpool", 2, 2, 0, 0u)] - [InlineData("maxpool4", 2, 2, 0, 0u)] - [InlineData("maxpool4", 3, 2, 1, 1u)] // padded taps, fused relu - public void MaxPool_MatchesCpu(string shader, int k, int st, int pad, uint relu) - { - Skip.IfNot(VkOk(), "no Vulkan device"); - var dev = VkBackend.Default.Dev; - var rng = new Random(5); - int c = 8, ih = 9, iw = 11, oh = (ih + 2 * pad - k) / st + 1, ow = (iw + 2 * pad - k) / st + 1; - float[] x = Rand(rng, ih * iw * c), cpu = new float[oh * ow * c]; - Nhwc.MaxPool(x, ih, iw, c, cpu, oh, ow, - new PoolP { Kernel = [k, k], Strides = [st, st], Pads = [pad, pad, pad, pad], FuseRelu = relu != 0 }); - byte[] pc = new byte[48]; - Buffer.BlockCopy(new uint[] { (uint)c, (uint)ih, (uint)iw, (uint)oh, (uint)ow, (uint)k, (uint)k, (uint)st, (uint)st, - (uint)pad, (uint)pad, relu }, 0, pc, 0, 48); - uint lanes = shader == "maxpool4" ? 4u : 1u; - float[] got = RunGpu(dev, shader, [x], cpu.Length, pc, ((uint)cpu.Length / lanes + 255) / 256); - AssertClose(got, cpu, 1e-6f, shader); - } [SkippableTheory] [InlineData("shift", 2)] @@ -71,7 +24,7 @@ public void Shift_MatchesCpu(string shader, int axis) int c = 16, h = 7, w = 9; int[] ge = [4, 8, 12, 16], of = [-2, -1, 1, 2]; float[] x = Rand(rng, h * w * c), cpu = new float[h * w * c]; - Nhwc.ChannelShift(x, h, w, c, cpu, new ShiftP { Axis = axis, Offsets = of, GroupEnds = ge }); + GcdrKernels.ChannelShift(x, h, w, c, cpu, new ShiftP { Axis = axis, Offsets = of, GroupEnds = ge }); var pcu = new List { (uint)c, (uint)h, (uint)w, (uint)axis }; for (int i = 0; i < 5; i++) pcu.Add((uint)(i < ge.Length ? ge[i] : c)); for (int i = 0; i < 5; i++) pcu.Add((uint)(i < of.Length ? of[i] : 0)); @@ -82,35 +35,6 @@ public void Shift_MatchesCpu(string shader, int axis) AssertClose(got, cpu, 1e-6f, shader); } - [SkippableTheory] - [InlineData("depthwise", true, 1)] - [InlineData("depthwise4", true, 1)] - [InlineData("depthwise4", false, 3)] // constant pad, gelu - public void Depthwise_MatchesCpu(string shader, bool reflect, int act) - { - Skip.IfNot(VkOk(), "no Vulkan device"); - var dev = VkBackend.Default.Dev; - var rng = new Random(8); - int c = 12, h = 10, w = 13; - float[] x = Rand(rng, h * w * c), wt = Rand(rng, c * 9), b = Rand(rng, c), cpu = new float[h * w * c]; - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], Group = c, - PadMode = reflect ? PadMode.Reflect : PadMode.Constant, Act = (ActivationKind)act, - }; - Nhwc.Conv(x, h, w, c, cpu, h, w, c, p, Nhwc.Pack(wt, [c, 1, 3, 3], b, c, c), default); - float[] wp = new float[9 * c]; // [tap][C] - for (int ch = 0; ch < c; ch++) - for (int t = 0; t < 9; t++) wp[t * c + ch] = wt[ch * 9 + t]; - uint flags = 1u | (reflect ? 2u : 0u) | (act == 1 ? 4u : 0u) | (act == 3 ? 16u : 0u); - byte[] pc = new byte[52]; - Buffer.BlockCopy(new uint[] { (uint)c, (uint)h, (uint)w, (uint)h, (uint)w, 3u, 3u, 1u, 1u, 1u, 1u, flags }, 0, pc, 0, 48); - Buffer.BlockCopy(new float[] { 0f }, 0, pc, 48, 4); - uint lanes = shader == "depthwise4" ? 4u : 1u; - float[] got = RunGpu(dev, shader, [x, wp, b], cpu.Length, pc, ((uint)cpu.Length / lanes + 255) / 256); - AssertClose(got, cpu, 2e-3f, shader); // GPU erf polynomial vs CPU erf for gelu - } - [SkippableTheory] [InlineData(3, 16, 2u, 2u)] // gc tail: relu(skip), bias, sigmoid [InlineData(3, 32, 3u, 2u)] // + relu(resize) @@ -130,7 +54,7 @@ public void UpHead_MatchesCpu(int oc, int c, uint reluFlags, uint act) Act = act == 2u ? ActivationKind.Sigmoid : act == 1u ? ActivationKind.Relu : ActivationKind.None, }; float[] cpu = new float[oh * ow * oc]; - Nhwc.UpHead(x, ih, iw, c, skip, cpu, oh, ow, 2f, 2f, p, Nhwc.PackUpHead(w, oc, c, b)); + GcdrKernels.UpHead(x, ih, iw, c, skip, cpu, oh, ow, 2f, 2f, p, GcdrKernels.PackUpHead(w, oc, c, b)); uint flags = reluFlags | 4u | (act << 3); byte[] pcb = new byte[36]; @@ -140,132 +64,6 @@ public void UpHead_MatchesCpu(int oc, int c, uint reluFlags, uint act) AssertClose(got, cpu, 1e-4f, "uphead"); } - // stem conv + 2x2/2 max pool; pooled 11 x 15 leaves a partial last workgroup - [SkippableTheory] - [InlineData("convpool", 3, 16, true, 0f, 1)] - [InlineData("convpool_k3", 3, 16, true, 0f, 1)] // gc stem: reflect, conv relu - [InlineData("convpool_k3", 6, 32, false, 0f, 0)] // dr stem: zero pad, pool relu - [InlineData("convpool_k3", 7, 8, false, 0.25f, 1)] // ic 7, one channel group, pad value - [InlineData("convpool_k3", 1, 24, true, 0f, 0)] // ic 1, three channel groups - public void ConvPool_MatchesCpu(string shader, int ic, int oc, bool reflect, float padValue, int act) - { - Skip.IfNot(VkOk(), "no Vulkan device"); - var dev = VkBackend.Default.Dev; - var rng = new Random(ic * 13 + oc); - int h = 22, w = 30, ph = h / 2, pw = w / 2; - float[] x = Rand(rng, h * w * ic), wt = Rand(rng, oc * ic * 9), b = Rand(rng, oc); - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [1, 1], Dilations = [1, 1], Kernel = [3, 3], - PadMode = reflect ? PadMode.Reflect : PadMode.Constant, PadValue = padValue, - Act = act == 1 ? ActivationKind.Relu : ActivationKind.None, - Pool = new PoolP { Kernel = [2, 2], Strides = [2, 2], Pads = [0, 0, 0, 0], FuseRelu = act == 0 }, - }; - float[] cpu = new float[ph * pw * oc]; - Nhwc.ConvPool(x, h, w, ic, cpu, ph, pw, oc, p, Nhwc.Pack(wt, [oc, ic, 3, 3], b, 1, ic)); - float[] wk = new float[9 * ic * oc]; // [K][oc], k = tap * ic + ci - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < 9; t++) wk[(t * ic + c) * oc + o] = wt[(o * ic + c) * 9 + t]; - uint flags = 1u | (reflect ? 2u : 0u) | (act == 1 ? 4u : 0u) | (act == 0 ? 8u : 0u); - byte[] pc = new byte[48]; - Buffer.BlockCopy(new uint[] { (uint)ic, (uint)h, (uint)w, (uint)oc, (uint)ph, (uint)pw, 3u, 3u, 1u, 1u, flags }, 0, pc, 0, 44); - Buffer.BlockCopy(new float[] { reflect ? 0f : padValue }, 0, pc, 44, 4); - float[] got = RunGpu(dev, shader, [x, wk, b], cpu.Length, pc, (uint)(ph * pw * (oc / 8) + 255) / 256); - AssertClose(got, cpu, 1e-4f, shader); - } - - // lite: false = the device's variant (LITE on wave64 / 32 KB parts), true = force the LITE build - [SkippableTheory] - [InlineData(6, 32, 1, false, "conv_cm_n32s", false, false, 1)] // dr stem (scalar k gather) - [InlineData(3, 16, 1, true, "conv_cm_n32s", false, false, 1)] // gc stem, reflect - [InlineData(32, 64, 1, true, "conv_cm_n64", false, false, 1)] - [InlineData(64, 128, 2, false, "conv_cm", false, false, 1)] // stride 2 - [InlineData(16, 32, 1, false, "conv_cm_n32", false, false, 1)] - [InlineData(6, 32, 1, false, "conv_cm_n32s", true, false, 1)] - [InlineData(32, 64, 1, true, "conv_cm_n64", true, true, 2)] // residual, gelu - [InlineData(64, 128, 2, false, "conv_cm", true, true, 0)] - [InlineData(16, 160, 1, false, "conv_cm_n32", true, false, 1)] // 5 N tiles: partial raster group - [InlineData(32, 256, 1, false, "conv_cm", false, true, 1)] - // shapes aimed at the SG16 build (8-row fragments, 16 lanes): half-used N tile (oc 16), - // partial N tile, minimal quad cin, any-cin stem with the epilogue, deep K (36 k-slices) - [InlineData(16, 16, 1, true, "conv_cm_n32", false, false, 1)] - [InlineData(16, 48, 1, false, "conv_cm_n32", false, true, 2)] - [InlineData(8, 64, 2, true, "conv_cm_n64", false, true, 1)] - [InlineData(3, 32, 1, false, "conv_cm_n32s", false, true, 2)] - [InlineData(128, 128, 1, true, "conv_cm", false, true, 2)] - public void ConvCm_MatchesCpu(int ic, int oc, int stride, bool reflect, string shader, bool lite, bool residual, int act) - { - Skip.IfNot(VkOk() && VkBackend.Default.Dev.CoopMat16, "no coopmat device"); - var dev = VkBackend.Default.Dev; - var rng = new Random(ic * 7 + oc); - int ih = 29, iw = 37, oh = (ih + 2 - 3) / stride + 1, ow = (iw + 2 - 3) / stride + 1; - // fp16-representable inputs so CPU and GPU see identical operands - float[] x = Rand(rng, ih * iw * ic).Select(v => (float)(Half)v).ToArray(); - float[] w = Rand(rng, oc * ic * 9).Select(v => (float)(Half)(v * 0.25f)).ToArray(); - float[] b = Rand(rng, oc); - var p = new ConvP - { - Pads = [1, 1, 1, 1], Strides = [stride, stride], Dilations = [1, 1], Kernel = [3, 3], - PadMode = reflect ? PadMode.Reflect : PadMode.Constant, - Act = act == 1 ? ActivationKind.Relu : act == 2 ? ActivationKind.Gelu : ActivationKind.None, - }; - float[] res = residual ? Rand(rng, oh * ow * oc).Select(v => (float)(Half)v).ToArray() : []; - float[] cpu = new float[oh * ow * oc]; - NhwcConvPlan kp = Nhwc.Pack(w, [oc, ic, 3, 3], b, 1, ic); - Nhwc.Conv(x, ih, iw, ic, cpu, oh, ow, oc, p, kp, res); - - int kr = 9 * ic, kd = (kr + 7) & ~7; - float[] wt = new float[oc * kd]; - for (int o = 0; o < oc; o++) - for (int c = 0; c < ic; c++) - for (int t = 0; t < 9; t++) wt[o * kd + t * ic + c] = w[(o * ic + c) * 9 + t]; - uint flags = 1u | (residual ? 2u : 0u) | (reflect ? 4u : 0u) | ((uint)act << 4); - uint m = (uint)(oh * ow); - byte[] pc = new byte[56]; - Buffer.BlockCopy(new uint[] { m, (uint)oc, (uint)kd, flags, (uint)ow, (uint)iw, (uint)ih, (uint)ic, 3u, - (uint)stride, (uint)stride, 1u, 1u, (uint)kr }, 0, pc, 0, 56); - uint bn = shader == "conv_cm" ? 128u : shader == "conv_cm_n64" ? 64u : 32u; - var bufs = new List { Half16(dev, x), Half16(dev, wt), F32(dev, b), Half16(dev, residual ? res : new float[8]) }; - VkBuffer ob = dev.NewStorageBuffer((ulong)cpu.Length * 2 + 16, true, preferHost: true); - bufs.Add(ob); - VkPipeline pipe = dev.GetCoopPipeline(shader, 5, 56, lite ? true : null); - IntPtr cmd = dev.NewCommandBuffer(); - IntPtr fence = dev.NewFence(); - var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; - Vk.Check(Vk.vkBeginCommandBuffer(cmd, &begin), "begin"); - VkInferenceSession.RecordOne(dev, cmd, pipe, bufs.Select(bb => (bb, 0UL, bb.Size)).ToArray(), pc, - (m + 127) / 128, ((uint)oc + bn - 1) / bn); - Vk.Check(Vk.vkEndCommandBuffer(cmd), "end"); - dev.Submit(cmd, fence); dev.WaitFence(fence); - dev.DestroyFence(fence); dev.FreeCommandBuffer(cmd); - Half[] gh = new Half[cpu.Length]; - fixed (Half* gp = gh) dev.Download(ob, gp, (ulong)gh.Length * 2); - foreach (VkBuffer bb in bufs) bb.Free(); - float[] got = gh.Select(h => (float)h).ToArray(); - float scale = Math.Max(1f, cpu.Max(MathF.Abs)); - AssertClose(got, cpu, 2e-3f * scale, shader); - } - - /// The SG16 build (8x16x16, 16 lanes) is only ever chosen with its pin in place and never - /// next to the 32-lane builds; devices that cannot run 16-lane subgroups never select it. - [SkippableFact] - public void CoopGate_Consistent() - { - Skip.IfNot(VkOk(), "no Vulkan device"); - var dev = VkBackend.Default.Dev; - if (dev.CoopM8) - { - Assert.True(dev.CoopMat16); - Assert.Equal(16u, dev.CoopSubgroupSize); - Assert.False(dev.CoopLite); - Assert.True(dev.SubgroupMin <= 16 && dev.SubgroupMax >= 16); - } - else if (dev.CoopMat16) - Assert.True(dev.CoopSubgroupSize is 0u or 32u); - if (dev.SubgroupMin > 16) Assert.False(dev.CoopM8); - } - private static VkBuffer Half16(VkDevice dev, float[] a) { Half[] h = a.Select(v => (Half)v).ToArray(); @@ -303,7 +101,7 @@ private static float[] RunGpu(VkDevice dev, string shader, float[][] inputs, int } VkBuffer ob = dev.NewStorageBuffer((ulong)outLen * 4, true, preferHost: true); bufs.Add(ob); - VkPipeline pipe = dev.GetPipeline(shader, bufs.Count, pc.Length); + VkPipeline pipe = dev.GetPipeline(shader, bufs.Count, pc.Length, owner: GcdrExtensions.Owner); IntPtr cmd = dev.NewCommandBuffer(); IntPtr fence = dev.NewFence(); var begin = new Vk.VkCommandBufferBeginInfo { SType = VkConst.StCommandBufferBeginInfo }; diff --git a/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs index 24e5f93..7c8caf3 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs @@ -1,6 +1,8 @@ +using Sdcb.SimdOnnx.Backends.Vulkan; using Sdcb.SimdGcdrNet.Backends.Vulkan; -using Sdcb.SimdGcdrNet.Engine; -using Sdcb.SimdGcdrNet.Onnx; +using Sdcb.SimdOnnx.Engine; +using Sdcb.SimdGcdrNet.Extensions; +using Sdcb.SimdOnnx.Onnx; using Xunit; namespace Sdcb.SimdGcdrNet.Tests; @@ -22,7 +24,7 @@ public void GcNet_MatchesGolden_Vulkan(string tag) float[] expect = Golden.F32(tag, "shadow"); OnnxModel m = OnnxModel.Load(GcPath); - using var sess = VkBackend.Default.NewSession(m, shape); + using var sess = VkBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); input.CopyTo(sess.Input); sess.Run(); var (max, rmse, _) = Golden.Diff(sess.Output, expect); @@ -42,7 +44,7 @@ public void DrNet_MatchesGolden_Vulkan(string tag) float[] expect = Golden.F32(tag, "output"); OnnxModel m = OnnxModel.Load(DrPath); - using var sess = VkBackend.Default.NewSession(m, shape); + using var sess = VkBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); input.CopyTo(sess.Input); sess.Run(); var (max, rmse, _) = Golden.Diff(sess.Output, expect); @@ -167,8 +169,8 @@ public void ConvDotRoute_MatchesGolden(string which, string tag, string inName, { // a fresh model instance: graphs are cached per model, so this compiles a new one OnnxModel m = OnnxModel.Load(which == "gc" ? GcPath : DrPath); - using var sess = VkBackend.Default.NewSession(m, shape); - Assert.Contains(VkBackend.Default.GraphFor(m, shape).Dispatches, d => d.Name!.StartsWith("conv_dot", StringComparison.Ordinal)); + using var sess = VkBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); + Assert.Contains(VkBackend.Default.GraphFor(m, shape, ext: GcdrExtensions.Instance).Dispatches, d => d.Name!.StartsWith("conv_dot", StringComparison.Ordinal)); input.CopyTo(sess.Input); sess.Run(); var (max, rmse, _) = Golden.Diff(sess.Output, expect); From 1f2eb773fc21bd6141e94a0bfa972693639c7e72 Mon Sep 17 00:00:00 2001 From: sdcb Date: Wed, 14 Oct 2026 11:29:13 +0800 Subject: [PATCH 2/9] Pin Sdcb.SimdOnnx 0.1.0-preview.2 from the ProGet test feed (nuget.config); README engine note --- Directory.Build.props | 2 +- README.md | 4 ++-- nuget.config | 18 ++++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 nuget.config diff --git a/Directory.Build.props b/Directory.Build.props index 7e66b66..745818e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -8,7 +8,7 @@ 1.0.0 - 0.1.0-preview.1 + 0.1.0-preview.2 $(MSBuildThisFileDirectory)..\Sdcb.SimdOnnx\src\Sdcb.SimdOnnx\Sdcb.SimdOnnx.csproj diff --git a/README.md b/README.md index 48d63a2..8795359 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ **Sdcb.SimdGcdrNet** is a pure C# implementation of [GCDRNet](https://github.com/ZZZHANG-jx/GCDRNet) (*Appearance Enhancement for Camera-captured Document Images in the Wild*, IEEE TAI 2023) — document-image shadow removal and appearance enhancement **without onnxruntime or any other -third-party inference engine**. It ships its own compiled-model inference engine, inspired by -`Sdcb.SimdPaddleOCR`. +third-party inference engine**. Inference runs on [Sdcb.SimdOnnx](https://github.com/sdcb/SimdOnnx), +a pure C# engine shared with `Sdcb.SimdPaddleOCR` (it grew out of this repository's own engine). The model is a two-stage pipeline: diff --git a/nuget.config b/nuget.config new file mode 100644 index 0000000..bae9f79 --- /dev/null +++ b/nuget.config @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + From 5e81a2ac21558f132d7733917e21384d7e057752 Mon Sep 17 00:00:00 2001 From: sdcb Date: Wed, 14 Oct 2026 13:41:26 +0800 Subject: [PATCH 3/9] Retry an image once after a Vulkan device loss; tests: no process-wide NOFUSE / shared-device ConvDot toggling (fixes flaky Vulkan byte-exact tests), opt-in device-loss test --- src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs | 23 ++++++++---- tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs | 21 ++++------- .../VulkanDeviceLossTests.cs | 34 ++++++++++++++++++ tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs | 36 ++++++++++--------- 4 files changed, 76 insertions(+), 38 deletions(-) create mode 100644 tests/Sdcb.SimdGcdrNet.Tests/VulkanDeviceLossTests.cs diff --git a/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs b/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs index a83518f..514c494 100644 --- a/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs +++ b/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs @@ -5,6 +5,7 @@ using Sdcb.SimdOnnx.Engine; using Sdcb.SimdOnnx.Onnx; using VkBackend = Sdcb.SimdOnnx.Backends.Vulkan.VkBackend; +using VkDeviceLostException = Sdcb.SimdOnnx.Backends.Vulkan.VkDeviceLostException; using MetalBackend = Sdcb.SimdOnnx.Backends.Metal.MetalBackend; namespace Sdcb.SimdGcdrNet; @@ -201,15 +202,23 @@ public void Enhance(ReadOnlySpan src, int width, int height, int srcStride || (_backend == GcdrBackend.Auto && VulkanOk())); if (useVk) { - var pool = VkPool(); - var pipe = pool.Rent(); - try + for (int attempt = 0; ; attempt++) { - pipe.Enhance(src, height, width, srcStride, dst, dstStride, format); - DebugAfterEnhance?.Invoke(pipe); + var pool = VkPool(); + var pipe = pool.Rent(); + try + { + pipe.Enhance(src, height, width, srcStride, dst, dstStride, format); + DebugAfterEnhance?.Invoke(pipe); + return; + } + catch (VkDeviceLostException) when (attempt == 0) + { + // the backend re-creates its device on next use; this pool's pipelines are dead + Interlocked.CompareExchange(ref _vkPool, null, pool); + } + finally { pool.Return(pipe); } } - finally { pool.Return(pipe); } - return; } int ph = (-height) & 31, pw = (-width) & 31; int h = height + ph, w = width + pw; diff --git a/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs index bcd6f62..f92bf5a 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/EngineTests.cs @@ -126,26 +126,17 @@ public void NoFuse_MatchesFused() int[] shape = Golden.ShapeOf("small", "input"); OnnxModel m = OnnxModel.Load(GcPath); - float[] Fused() + // per-compile option, not GCDRNET_NOFUSE: the env var would also unfuse graphs that tests + // running in parallel compile meanwhile (byte-exact Vulkan comparisons then fail) + float[] Run(bool fuse) { - Environment.SetEnvironmentVariable("GCDRNET_NOFUSE", null); - using var s = new InferenceSession(GraphCompiler.Compile(m, shape, CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); + using var s = new InferenceSession(GraphCompiler.Compile(m, shape, + CompileOptions.Default with { Extensions = GcdrExtensions.Instance, Fuse = fuse })); input.CopyTo(s.Input); s.Run(); return s.Output.ToArray(); } - float[] Unfused() - { - Environment.SetEnvironmentVariable("GCDRNET_NOFUSE", "1"); - try - { - using var s = new InferenceSession(GraphCompiler.Compile(m, shape, CompileOptions.Default with { Extensions = GcdrExtensions.Instance })); - input.CopyTo(s.Input); s.Run(); - return s.Output.ToArray(); - } - finally { Environment.SetEnvironmentVariable("GCDRNET_NOFUSE", null); } - } - var (max, rmse, _) = Golden.Diff(Fused(), Unfused()); + var (max, rmse, _) = Golden.Diff(Run(fuse: true), Run(fuse: false)); Assert.True(max < 1e-5f, $"fused vs unfused max diff {max} rmse {rmse}"); } diff --git a/tests/Sdcb.SimdGcdrNet.Tests/VulkanDeviceLossTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/VulkanDeviceLossTests.cs new file mode 100644 index 0000000..49a0a87 --- /dev/null +++ b/tests/Sdcb.SimdGcdrNet.Tests/VulkanDeviceLossTests.cs @@ -0,0 +1,34 @@ +using Sdcb.SimdOnnx.Backends.Vulkan; + +namespace Sdcb.SimdGcdrNet.Tests; + +/// +/// Marks the process-wide Vulkan device lost, which disturbs every other Vulkan test running in the +/// process: opt-in (SIMDONNX_TEST_DEVICELOSS=1), run it on its own (--filter DeviceLoss). +/// +public class VulkanDeviceLossTests +{ + private static string GcPath => Path.Combine(Golden.Dir, "models", "gcdrnet_gcnet_sim.onnx"); + private static string DrPath => Path.Combine(Golden.Dir, "models", "gcdrnet_drnet_sim.onnx"); + + [SkippableFact] + public void Enhance_RecoversFromDeviceLoss_Vulkan() + { + Skip.IfNot(Environment.GetEnvironmentVariable("SIMDONNX_TEST_DEVICELOSS") == "1", "opt-in: SIMDONNX_TEST_DEVICELOSS=1"); + VkBackend before; + try { before = VkBackend.Default; } + catch { throw new SkipException("no Vulkan device"); } + var (_, _, _, _, oh, ow) = Golden.Dims("small"); + byte[] src = Golden.Bgr("small"); + + using var enh = GcdrEnhancer.FromFiles(GcPath, DrPath, GcdrBackend.Vulkan); + byte[] first = new byte[ow * oh * 3], second = new byte[ow * oh * 3]; + enh.Enhance(src, ow, oh, ow * 3, first, ow * 3); + + before.Dev.Lost = true; + enh.Enhance(src, ow, oh, ow * 3, second, ow * 3); + + Assert.NotSame(before, VkBackend.Default); + Assert.Equal(first, second); + } +} diff --git a/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs b/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs index 7c8caf3..fbd11b7 100644 --- a/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs +++ b/tests/Sdcb.SimdGcdrNet.Tests/VulkanTests.cs @@ -155,27 +155,31 @@ public class VulkanConvDotTests [InlineData("dr", "mid", "dr_in", "output")] public void ConvDotRoute_MatchesGolden(string which, string tag, string inName, string outName) { - VkDevice dev; - try { dev = VkBackend.Default.Dev; } - catch { dev = null!; } - Skip.If(dev is null || !VkBackend.Default.F16, "needs a Vulkan device with fp16 storage"); + VkBackend? be = s_dotBackend.Value; + Skip.If(be is null || !be.F16, "needs a Vulkan device with fp16 storage"); float[] input = Golden.F32(tag, inName); int[] shape = Golden.ShapeOf(tag, inName); float[] expect = Golden.F32(tag, outName); - bool saved = dev.ConvDot; - dev.ConvDot = true; + OnnxModel m = OnnxModel.Load(which == "gc" ? GcPath : DrPath); + using var sess = be.NewSession(m, shape, GcdrExtensions.Instance); + Assert.Contains(be.GraphFor(m, shape, ext: GcdrExtensions.Instance).Dispatches, d => d.Name!.StartsWith("conv_dot", StringComparison.Ordinal)); + input.CopyTo(sess.Input); + sess.Run(); + var (max, rmse, _) = Golden.Diff(sess.Output, expect); + Assert.True(max < 2e-2f, $"{which}-convdot[{tag}] max diff {max} rmse {rmse}"); + } + + // a private device with the conv_dot route forced on: flipping it on the shared default device + // would re-route graphs other tests compile concurrently (byte-exact comparisons then fail) + private static readonly Lazy s_dotBackend = new(() => + { try { - // a fresh model instance: graphs are cached per model, so this compiles a new one - OnnxModel m = OnnxModel.Load(which == "gc" ? GcPath : DrPath); - using var sess = VkBackend.Default.NewSession(m, shape, GcdrExtensions.Instance); - Assert.Contains(VkBackend.Default.GraphFor(m, shape, ext: GcdrExtensions.Instance).Dispatches, d => d.Name!.StartsWith("conv_dot", StringComparison.Ordinal)); - input.CopyTo(sess.Input); - sess.Run(); - var (max, rmse, _) = Golden.Diff(sess.Output, expect); - Assert.True(max < 2e-2f, $"{which}-convdot[{tag}] max diff {max} rmse {rmse}"); + var be = new VkBackend(); + be.Dev.ConvDot = true; + return be; } - finally { dev.ConvDot = saved; } - } + catch { return null; } + }); } From a2a0bc91fb554eb2e59686248dc6387225dac632 Mon Sep 17 00:00:00 2001 From: sdcb Date: Wed, 14 Oct 2026 13:43:26 +0800 Subject: [PATCH 4/9] Pin Sdcb.SimdOnnx 0.1.0-preview.3 --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 745818e..84e0ada 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -8,7 +8,7 @@ 1.0.0 - 0.1.0-preview.2 + 0.1.0-preview.3 $(MSBuildThisFileDirectory)..\Sdcb.SimdOnnx\src\Sdcb.SimdOnnx\Sdcb.SimdOnnx.csproj From 95018214a76809a4fe5d389abcebd86eeb52639d Mon Sep 17 00:00:00 2001 From: sdcb Date: Thu, 15 Oct 2026 15:37:56 +0800 Subject: [PATCH 5/9] Vulkan: stack the gc/dr arenas by each graph's full arena (scratch / tile-read slack included); fused pipeline opts into the staged SG16 tensor-core convs (CompileOptions.StagedCoopConv); CI builds against sdcb/SimdOnnx main from source Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test.yml | 10 ++++++++++ Directory.Build.props | 2 +- .../Backends/Vulkan/VkPipeline2.cs | 14 +++++++++++--- src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs | 11 ++++++++--- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1bb3856..7314fca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,6 +33,10 @@ concurrency: env: DOTNET_NOLOGO: 1 + # until Sdcb.SimdOnnx ships on nuget.org the engine builds from source (sdcb/SimdOnnx main), + # next to this checkout; MSBuild reads both as properties + UseLocalSimdOnnx: 'true' + SimdOnnxProject: ${{ github.workspace }}/_simdonnx/src/Sdcb.SimdOnnx/Sdcb.SimdOnnx.csproj DOTNET_CLI_TELEMETRY_OPTOUT: 1 DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 @@ -77,6 +81,8 @@ jobs: - { name: linux-arm64-scalar, os: ubuntu-24.04-arm, count: 2, maxmp: 1, isa: DOTNET_EnableHWIntrinsic=0 } steps: - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + with: { repository: sdcb/SimdOnnx, path: _simdonnx } - uses: actions/download-artifact@v8 with: { name: realdae, path: tests/testdata/realdae } - name: Build @@ -149,6 +155,8 @@ jobs: replica: [1, 2, 3, 4, 5, 6] steps: - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + with: { repository: sdcb/SimdOnnx, path: _simdonnx } - uses: actions/download-artifact@v8 with: { name: realdae, path: tests/testdata/realdae } - name: Benchmark @@ -172,6 +180,8 @@ jobs: matrix: { replica: [1, 2, 3] } steps: - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + with: { repository: sdcb/SimdOnnx, path: _simdonnx } - uses: actions/download-artifact@v8 with: { name: realdae, path: tests/testdata/realdae } - name: Benchmark diff --git a/Directory.Build.props b/Directory.Build.props index 84e0ada..b7a0640 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -9,7 +9,7 @@ 0.1.0-preview.3 - $(MSBuildThisFileDirectory)..\Sdcb.SimdOnnx\src\Sdcb.SimdOnnx\Sdcb.SimdOnnx.csproj + $(MSBuildThisFileDirectory)..\Sdcb.SimdOnnx\src\Sdcb.SimdOnnx\Sdcb.SimdOnnx.csproj diff --git a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs index 83c6359..154a0a3 100644 --- a/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs +++ b/src/Sdcb.SimdGcdrNet/Backends/Vulkan/VkPipeline2.cs @@ -82,9 +82,9 @@ internal void Bind(int ih, int iw, GcdrPixelFormat fmt = GcdrPixelFormat.Bgr24) if (_gcG is null || h != _h || w != _w) { // a new size compiles two plans + graphs: build them side by side - var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], physIo: true, GcdrExtensions.Instance)); + var tg = Task.Run(() => _be.AcquireGraph(_gcModel, [1, 3, h, w], GraphOptions)); VkGraphModel gd; - try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], physIo: true, GcdrExtensions.Instance); } + try { gd = _be.AcquireGraph(_drModel, [1, 6, h, w], GraphOptions); } catch { try { tg.GetAwaiter().GetResult().Release(); } catch { } throw; } VkGraphModel gg; try { gg = tg.GetAwaiter().GetResult(); } @@ -102,7 +102,9 @@ internal void Bind(int ih, int iw, GcdrPixelFormat fmt = GcdrPixelFormat.Bgr24) ulong es = (ulong)Es; // gc base aligned to 256 B (any minStorageBufferOffsetAlignment) - var (gcOff, total) = GcdrEnhancer.StackArenas(GcPlan, DrPlan, (long)(256 / es)); + // the graphs' full arenas: the plan's values plus scratch / tile-read slack past them + var (gcOff, total) = GcdrEnhancer.StackArenas((long)((_gcG!.ArenaBytes + es - 1) / es), DrPlan, + (long)((_drG!.ArenaBytes + es - 1) / es), (long)(256 / es)); _gcBase = (ulong)gcOff * es; ulong need = (ulong)total * es + 64; // the arena only grows while within ArenaBudget; an over-budget one (huge image) @@ -120,6 +122,12 @@ internal void Bind(int ih, int iw, GcdrPixelFormat fmt = GcdrPixelFormat.Bgr24) _recorded = true; } + /// Fused-pipeline graphs: physical I/O, the GcdrNet extension ops, and the staged + /// tensor-core convs on SG16 (Intel Xe2) devices — twice as fast here as the direct family and + /// stress-clean on this model (the OCR models are not; see CompileOptions.StagedCoopConv). + private static CompileOptions GraphOptions { get; } = + VkBackend.PipeOptions with { Extensions = GcdrExtensions.Instance, StagedCoopConv = true }; + /// Per-worker device-arena budget (gc + dr). Within it the arena is grow-only /// (12.5% headroom on the first allocation, 25% on regrowth) so mixed sizes don't reallocate. /// 3 GB; on integrated GPUs the "device-local" heap is system RAM shared with the CPU, so diff --git a/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs b/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs index 514c494..acb4f12 100644 --- a/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs +++ b/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs @@ -273,12 +273,17 @@ public void Enhance(ReadOnlySpan src, int width, int height, int srcStride /// stage-2 glue that writes the dr input, and dr then reuses everything above. /// Returns the gc arena offset and the total size, in floats. /// - internal static (long gcOff, long total) StackArenas(ExecPlan gc, ExecPlan dr, long align = 16) + internal static (long gcOff, long total) StackArenas(ExecPlan gc, ExecPlan dr, long align = 16) => + StackArenas(gc.ArenaFloats, dr, dr.ArenaFloats, align); + + /// over each stage's full arena + /// (GPU graphs append scratch / tile-read slack past the plan's values). + internal static (long gcOff, long total) StackArenas(long gcArena, ExecPlan dr, long drArena, long align = 16) { CValue din = dr.Values[dr.InputValue]; long o = dr.BufOffset[din.Buf], end = (o + din.ElementCount + align - 1) / align * align; - long gcOff = o >= gc.ArenaFloats ? 0 : end; - return (gcOff, Math.Max(dr.ArenaFloats, gcOff + gc.ArenaFloats)); + long gcOff = o >= gcArena ? 0 : end; + return (gcOff, Math.Max(drArena, gcOff + gcArena)); } // browser/WASI: unlinked P/Invokes (vulkan-1, libobjc) trap the wasm module with From 38cb18fc354f9c26f7c21c9682a2c4aa247be85a Mon Sep 17 00:00:00 2001 From: sdcb Date: Thu, 15 Oct 2026 16:25:32 +0800 Subject: [PATCH 6/9] Unambiguous StackArenas cref (pack -warnaserror) Co-Authored-By: Claude Opus 5.5 --- src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs b/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs index acb4f12..4028547 100644 --- a/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs +++ b/src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs @@ -329,7 +329,7 @@ public void Dispose() /// /// Size-agnostic CPU worker: one native arena holding both stages (see - /// ), borrowed by per-call sessions. Within + /// ), borrowed by per-call sessions. Within /// it only grows (25% headroom on regrowth), so mixed sizes /// neither reallocate nor re-fault pages; an over-budget arena (one huge image) drops /// to exact size once a size that fits comes along. From c28e4271db652f60dee79a05ab421576f8779ecf Mon Sep 17 00:00:00 2001 From: sdcb Date: Thu, 15 Oct 2026 21:19:20 +0800 Subject: [PATCH 7/9] Docs for 2.0: README layout describes what stays here (GcdrEnhancer, extensions, PixelOps, fused pipelines) and points to Sdcb.SimdOnnx for the engine; migration note for the engine types that moved to Sdcb.SimdOnnx.*; package description; rk3399 doc links the NEON kernels in SimdOnnx Co-Authored-By: Claude Opus 5.5 --- Directory.Build.props | 2 +- README.md | 37 ++++++++++++++++++++----------------- docs/rockchip-rk3399.md | 4 ++-- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index b7a0640..be1b026 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -43,7 +43,7 @@ Sdcb.SimdGcdrNet $(SdcbSimdGcdrNetVersion) true - Pure C# GCDRNet inference: document image shadow removal / appearance enhancement. Self-contained ONNX graph engine with CPU SIMD (AVX-512/AVX2/NEON/Vector) and GPU (Vulkan, Metal) backends; no onnxruntime dependency. Accepts interleaved BGR24 / RGB24 / BGRA32 / RGBA32. Models ship separately in Sdcb.SimdGcdrNet.Models. + Pure C# GCDRNet inference: document image shadow removal / appearance enhancement. Runs on the pure C# Sdcb.SimdOnnx engine (CPU SIMD AVX-512/AVX2/NEON/Vector, GPU Vulkan/Metal) with GCDRNet-specific fused kernels; no onnxruntime dependency. Accepts interleaved BGR24 / RGB24 / BGRA32 / RGBA32. Models ship separately in Sdcb.SimdGcdrNet.Models. Sdcb;GCDRNet;document;document-enhancement;shadow-removal;image-enhancement;onnx;inference;deep-learning;SIMD;AVX512;NEON;wasm;AOT;Vulkan;Metal;linqpad-samples diff --git a/README.md b/README.md index 8795359..883ba42 100644 --- a/README.md +++ b/README.md @@ -110,19 +110,20 @@ dotnet run --project examples/PaddleOCR.Console -- [image] [output.jpg] ### Layout -- `src/Sdcb.SimdGcdrNet/Onnx` — minimal protobuf reader + ONNX model/graph objects -- `src/Sdcb.SimdGcdrNet/Engine` — `GraphCompiler` (specialize → fold → fuse → arena-plan), - `InferenceSession` -- `src/Sdcb.SimdGcdrNet/Kernels` — Scalar / `Vector` / AVX2 / AVX-512 kernels (`Ops`, `Avx2Kernels`, `Nhwc`, - `PixelOps`, `SimdMath`) -- `src/Sdcb.SimdGcdrNet/Backends/Vulkan` — `Vk` P/Invoke layer, `VkDevice`, `VkShaders` (embedded - SPIR-V), `VkGraph` dispatch lowering, `VkSession`, `VkPipeline2` (fused pipeline), `VkBackend` -- `src/Sdcb.SimdGcdrNet/Backends/Vulkan/Shaders` — GLSL sources; `tools/build-shaders.bat` - recompiles `.comp` → `.spv` (committed; dev-time only, not part of `dotnet build`) -- `src/Sdcb.SimdGcdrNet/Backends/Metal` — `ObjC`/`Metal` P/Invoke layer, `MtlGraph` dispatch - lowering, `MtlSession`, `MtlPipeline` (fused pipeline), `MetalBackend` -- `src/Sdcb.SimdGcdrNet/Backends/Metal/Shaders` — MSL sources (`.metal`, embedded resources, - compiled to PSOs at run time — no pre-build step) +The ONNX engine (protobuf reader, graph compiler, CPU SIMD kernels, Vulkan / Metal backends) is +[Sdcb.SimdOnnx](https://github.com/sdcb/SimdOnnx); this library holds the GCDRNet-specific parts: + +- `src/Sdcb.SimdGcdrNet/GcdrEnhancer.cs` — public API: backend selection, per-size session pools, + gc → dr stitching, arena stacking +- `src/Sdcb.SimdGcdrNet/Extensions` — engine extensions (`GcdrExtensions`): fused operators + (`UpHeadOp`, `ChannelShiftOp`) with their CPU kernels (Vector / AVX-512 / NEON / WASM SIMD) and + graph passes +- `src/Sdcb.SimdGcdrNet/Kernels` — `PixelOps` (interleaved pixels ↔ tensors, padding) +- `src/Sdcb.SimdGcdrNet/Backends/Vulkan` — `VkPipeline2` (fused gc + dr pipeline on SimdOnnx's + Vulkan backend); `Shaders` holds the GCDRNet GLSL kernels, `tools/build-shaders.bat` recompiles + `.comp` → `.spv` (committed; dev-time only, not part of `dotnet build`) +- `src/Sdcb.SimdGcdrNet/Backends/Metal` — `MtlPipeline` (fused pipeline on SimdOnnx's Metal + backend); `Shaders` holds MSL sources (embedded, compiled to PSOs at run time) - `bench/Sdcb.SimdGcdrNet.Benchmark` — `--check`, `--bench` (CPU), `--benchvk` (Vulkan e2e), `--benchmtl` / `--checkgpu` / `--mtlprof` / `--mtrace` (Metal e2e, diff check, per-shader profile, CPU-vs-Metal per-node trace), `--vkprof` (per-shader/per-dispatch GPU timing), `--vkcheck` / `--vknodes` @@ -173,8 +174,10 @@ runs two waves in parallel, then merges them into one report (the run's summary - Metal path requires macOS on Apple Silicon with Apple7+ (simdgroup-MMA, i.e. M1+; an x64 process under Rosetta also qualifies); `Auto` checks `SupportsApple7` before choosing it, `GcdrBackend.Metal` selects it unconditionally. Set `GCDRNET_MTL_VERBOSE=1` to print the chosen device. -- Thread-safety: `GcdrEnhancer` is thread-safe (CPU: per-size session pools; Vulkan: size-agnostic worker pool); individual - `InferenceSession`/`VkInferenceSession` objects are not. +- Thread-safety: `GcdrEnhancer` is thread-safe (CPU: per-size session pools; Vulkan: size-agnostic worker pool). +- 2.0 moved the engine out: `InferenceSession`, `CompiledModel`, `OnnxModel`, `VkInferenceSession` and + `MtlInferenceSession` are no longer in `Sdcb.SimdGcdrNet.*`; use the same types from + `Sdcb.SimdOnnx.*` (a dependency of this package). `GcdrEnhancer`'s API is unchanged. - Not covered: batched inference (N>1), fp16/int8. - Native AOT works (`IsAotCompatible`, no trim/AOT warnings; CPU and Vulkan output verified identical to JIT). On **x64** the executable project must set the instruction set, otherwise ILC compiles for the SSE baseline, folds @@ -202,8 +205,8 @@ are from the RealDAE dataset of the same paper. ## 中文 **Sdcb.SimdGcdrNet** 是 [GCDRNet](https://github.com/ZZZHANG-jx/GCDRNet)(相机拍摄文档去阴影/外观增强, -IEEE TAI 2023)的**纯 C# 推理库**——不依赖 onnxruntime 等第三方推理引擎,像 `Sdcb.SimdPaddleOCR` -一样自带 CompiledModel 推理引擎。 +IEEE TAI 2023)的**纯 C# 推理库**——不依赖 onnxruntime 等第三方推理引擎,与 `Sdcb.SimdPaddleOCR` +共用纯 C# 推理引擎 [Sdcb.SimdOnnx](https://github.com/sdcb/SimdOnnx)(2.0 起引擎类型移到 `Sdcb.SimdOnnx.*`)。 模型为两级串联:**GC-Net** 估计逐像素光照/阴影图 `I_sm` → `I_GC = clamp(I_s/I_sm)` 除法校正 → **DR-Net** 消费 6 通道 `[I_s; I_GC]` 输出增强图。 diff --git a/docs/rockchip-rk3399.md b/docs/rockchip-rk3399.md index ecf8a9a..1b05efb 100644 --- a/docs/rockchip-rk3399.md +++ b/docs/rockchip-rk3399.md @@ -84,7 +84,7 @@ CPU 档位(`--bench 512 512`,gc+dr 推理,锁 1008 MHz): NEON 档位(`Nhwc.UseNeon = AdvSimd.Arm64.IsSupported`)沿用 x86 各档的算子结构,以下是只在 NEON 上走的部分: -1. **6×16 micro-kernel `TileNeonK`**(`Kernels/Nhwc.Neon.cs`) +1. **6×16 micro-kernel `TileNeonK`**(Sdcb.SimdOnnx `src/Sdcb.SimdOnnx/Kernels/Nhwc.Neon.cs`) - 每像素一次 128 位读入 4 个输入通道,`fmla` by element 累加到 24 个寄存器累加器(共 32 个向量寄存器)。 - 下一组权重向量提前一组读入,照顾按序执行的 A53。`ic % 4` 的余数用 `ld1r` 广播。 - 单核微基准(`taps × ic` 次累加,L1 内):A72 6.3–7.1 GFLOP/s,约为峰值 7.5 的 85–95%;A53 3.5–3.8 GFLOP/s,约为峰值的 55%。 @@ -97,7 +97,7 @@ NEON 档位(`Nhwc.UseNeon = AdvSimd.Arm64.IsSupported`)沿用 x86 各档的 3. **Winograd** - GEMM 走 `TileNeonK`,输出 epilogue 改为向量化。 - V 块预算降到 16 KB(ic ≥ 32 时 T = 6)。4 个 A53 共享 512 KB L2,块大了会互相挤出。 - - U(36·oc·ic)超过 2 MB 的深层改走三阶段分组调度 `WinogradGrouped`(`Kernels/NhwcWinograd.Neon.cs`):输入变换 → 每个 (xi, oc 块) 一个 GEMM 任务,每块 U 每组只读一次 → 输出变换。原来是每 6 个 tile 就把整块 U 从 DRAM 流一遍。 + - U(36·oc·ic)超过 2 MB 的深层改走三阶段分组调度 `WinogradGrouped`(Sdcb.SimdOnnx `src/Sdcb.SimdOnnx/Kernels/NhwcWinograd.Neon.cs`):输入变换 → 每个 (xi, oc 块) 一个 GEMM 任务,每块 U 每组只读一次 → 输出变换。原来是每 6 个 tile 就把整块 U 从 DRAM 流一遍。 - 分组调度与原调度逐位相同:变换、k 序都一样。512→256 层(1600×1152,锁频)从 105 ms 降到 83 ms,256→128 从 120 ms 降到 93 ms。 4. **向量化激活**:`SimdMath` 增加 Vector128 版 Exp / Sigmoid / Erf / GELU,公式与 AVX2 版一致,用 FMA。 - 深度卷积 + GELU 原来在非 AVX 档逐元素走标量 `ErfScalar`,现在向量化:gc 128 通道那一层从 21.6 ms 降到 8.5 ms(512²,锁频)。 From 45180b3edcdbeec1d115c30e9dbc43e6bfb7f04b Mon Sep 17 00:00:00 2001 From: sdcb Date: Fri, 16 Oct 2026 18:38:28 +0800 Subject: [PATCH 8/9] Pin Sdcb.SimdOnnx 0.1.0-preview.4 (ProGet test feed); CI builds, tests and packs against the pinned package (what ships: the nupkg now depends on exactly [0.1.0-preview.4]) and keeps one job on sdcb/SimdOnnx main from source; NU5104 accepted while the engine is a preview Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test.yml | 16 ++++++++-------- Directory.Build.props | 5 ++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7314fca..682e7d9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ name: test # smoke - correctness: dotnet test + corpus smoke on every platform / ISA tier (+ Metal) # bench - performance: corpus benchmark (.github/scripts/bench.ps1), several replicas per platform # report - merges every result into the run summary (tests/Sdcb.SimdGcdrNet.TestReport) -# Job budget: 35 jobs (40 concurrent); macOS: 2 smoke + 3 bench (5 concurrent). +# Job budget: 36 jobs (40 concurrent); macOS: 2 smoke + 3 bench (5 concurrent). on: push: @@ -33,9 +33,8 @@ concurrency: env: DOTNET_NOLOGO: 1 - # until Sdcb.SimdOnnx ships on nuget.org the engine builds from source (sdcb/SimdOnnx main), - # next to this checkout; MSBuild reads both as properties - UseLocalSimdOnnx: 'true' + # jobs build against the pinned Sdcb.SimdOnnx package (what ships); the engine-head smoke builds + # sdcb/SimdOnnx main from source next to this checkout (MSBuild reads env vars as properties) SimdOnnxProject: ${{ github.workspace }}/_simdonnx/src/Sdcb.SimdOnnx/Sdcb.SimdOnnx.csproj DOTNET_CLI_TELEMETRY_OPTOUT: 1 DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 @@ -61,11 +60,15 @@ jobs: name: smoke ${{ matrix.name }} runs-on: ${{ matrix.os }} timeout-minutes: 30 + env: + UseLocalSimdOnnx: ${{ matrix.enginehead && 'true' || 'false' }} strategy: fail-fast: false matrix: include: - { name: win-x64, os: windows-2025, count: 5 } + # the same tests against sdcb/SimdOnnx main from source: engine changes that break this library + - { name: linux-x64-engine-head, os: ubuntu-24.04, count: 2, enginehead: true } - { name: win-arm64, os: windows-11-arm, count: 5 } - { name: linux-x64, os: ubuntu-24.04, count: 5, pack: true } - { name: linux-arm64, os: ubuntu-24.04-arm, count: 5 } @@ -82,6 +85,7 @@ jobs: steps: - uses: actions/checkout@v7 - uses: actions/checkout@v7 + if: matrix.enginehead with: { repository: sdcb/SimdOnnx, path: _simdonnx } - uses: actions/download-artifact@v8 with: { name: realdae, path: tests/testdata/realdae } @@ -155,8 +159,6 @@ jobs: replica: [1, 2, 3, 4, 5, 6] steps: - uses: actions/checkout@v7 - - uses: actions/checkout@v7 - with: { repository: sdcb/SimdOnnx, path: _simdonnx } - uses: actions/download-artifact@v8 with: { name: realdae, path: tests/testdata/realdae } - name: Benchmark @@ -180,8 +182,6 @@ jobs: matrix: { replica: [1, 2, 3] } steps: - uses: actions/checkout@v7 - - uses: actions/checkout@v7 - with: { repository: sdcb/SimdOnnx, path: _simdonnx } - uses: actions/download-artifact@v8 with: { name: realdae, path: tests/testdata/realdae } - name: Benchmark diff --git a/Directory.Build.props b/Directory.Build.props index be1b026..914a541 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -8,8 +8,11 @@ 1.0.0 - 0.1.0-preview.3 + 0.1.0-preview.4 $(MSBuildThisFileDirectory)..\Sdcb.SimdOnnx\src\Sdcb.SimdOnnx\Sdcb.SimdOnnx.csproj + + $(NoWarn);NU5104 From 9eaf0277f70f9bb847ec4fe41121451bd061c22d Mon Sep 17 00:00:00 2001 From: sdcb Date: Fri, 16 Oct 2026 19:03:03 +0800 Subject: [PATCH 9/9] nuget.config keeps the ProGet test feed until the engine is on nuget.org; wasm doc links the PackedSimd kernels in SimdOnnx Co-Authored-By: Claude Opus 5.5 --- docs/wasm.md | 2 +- nuget.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/wasm.md b/docs/wasm.md index e64fa47..606c68e 100644 --- a/docs/wasm.md +++ b/docs/wasm.md @@ -89,7 +89,7 @@ Transpose 现在是 gc / dr 里仅次于 Conv 的算子(未优化,两档相 各平台的 CPU 档位:x64 上 AVX-512 / AVX2,arm64 上 NEON,browser-wasm 上 **PackedSimd**;没有专门内核的算子(Transpose、Resize、MaxPool 等)仍走 `Vector`,再往下是标量。PackedSimd 档(`Nhwc.UsePacked = PackedSimd.IsSupported`)参照 SimdPaddleOCR 的 WASM 优化(`Nhwc.Conv.PackedSimd`),结构上对应本仓库 NEON 档已有的特化点: -1. **4 像素 × 8 输出通道 micro-kernel `TilePkK`**(`Kernels/Nhwc.PackedSimd.cs`) +1. **4 像素 × 8 输出通道 micro-kernel `TilePkK`**(Sdcb.SimdOnnx `src/Sdcb.SimdOnnx/Kernels/Nhwc.PackedSimd.cs`) - V8 在 x64 上最多给 v128 分 15 个 XMM 寄存器,6 行 tile 会在内层循环溢出。改为 8 个累加器 + 2 个权重 + 1 个 splat,每个 16 通道打包块跑两遍。 - 输入直接 `v128.load32_splat` 从内存广播,乘、加分开(WASM SIMD 无 FMA),按 k 升序累加:与 `Vector` 档逐 lane 运算顺序相同。 - 用于 1×1 / Gemm(`GemmPk`)、直接卷积(`DenseRowPk`)和 Winograd 的 36 个小 GEMM(tile 行数随之改为 4)。 diff --git a/nuget.config b/nuget.config index bae9f79..9085df2 100644 --- a/nuget.config +++ b/nuget.config @@ -1,6 +1,6 @@ + nuget.org; remove the "simdonnx-preview" source (and its mapping) once it does. -->