From 08970c4e11ef83f30bfe10637c225a8d73b13c10 Mon Sep 17 00:00:00 2001 From: mirek190 Date: Sat, 24 Oct 2026 21:50:51 +0100 Subject: [PATCH 1/5] perf(moss): use optional fused SiTU activation --- docs/moss_image.md | 4 ++++ src/model/diffusion/moss_image.hpp | 22 +++++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/moss_image.md b/docs/moss_image.md index 27ab26f804..fb86182443 100644 --- a/docs/moss_image.md +++ b/docs/moss_image.md @@ -35,3 +35,7 @@ Pass the external `tokenizer.json` with `--tokenizer`; Qwen3.5 uses a different Use dimensions divisible by 16. The default discrete scheduler uses the upstream time shift of 5. Use `--flow-shift` to override it. The negative CFG branch skips DiT layer 22 (zero-based), matching the upstream pipeline. + +## Optional fused SiTU + +When ggml advertises GGML_HAS_SITU and the selected backend supports the operation, MOSS uses fused SiTU. Older headers, upstream mode and unsupported devices retain the original composite activation. Acceleration requires compatible ggml SiTU support; this patch does not update the submodule. diff --git a/src/model/diffusion/moss_image.hpp b/src/model/diffusion/moss_image.hpp index f1cbbc7f37..26851cedcd 100644 --- a/src/model/diffusion/moss_image.hpp +++ b/src/model/diffusion/moss_image.hpp @@ -53,7 +53,14 @@ namespace MOSSImage { } }; - inline ggml_tensor* situ(ggml_context* c, ggml_tensor* hidden, ggml_tensor* gate) { + inline ggml_tensor* situ(GGMLRunnerContext* ctx, ggml_tensor* hidden, ggml_tensor* gate) { + auto c = ctx->ggml_ctx; +#if !defined(SD_USE_UPSTREAM_GGML) && defined(GGML_HAS_SITU) + auto out = ggml_glu_split(c, gate, hidden, GGML_GLU_OP_SITU); + if (ctx->backend != nullptr && ggml_backend_supports_op(ctx->backend, out)) { + return out; + } +#endif auto a = ggml_scale(c, ggml_tanh(c, ggml_scale(c, gate, 0.25f)), 4.f); a = ggml_mul(c, a, ggml_sigmoid(c, gate)); auto b = ggml_scale(c, ggml_tanh(c, ggml_scale(c, hidden, 1.f / 25.f)), 25.f); @@ -92,7 +99,7 @@ namespace MOSSImage { y = linear("proj", y); x = ggml_add(c, x, y); y = norm("norm2", x); - y = situ(c, linear("ffn_up", y), linear("ffn_gate", y)); + y = situ(ctx, linear("ffn_up", y), linear("ffn_gate", y)); y = linear("ffn_down", y); return ggml_add(c, x, y); } @@ -134,7 +141,16 @@ namespace MOSSImage { y = norm("norm2", x); y = ggml_add(c, ggml_add(c, y, ggml_mul(c, y, part(4))), part(3)); auto up_gate = linear("proj.ffn_up_gate", y); - y = situ(c, ggml_ext_slice(c, up_gate, 0, 0, config.ffn_dim), ggml_ext_slice(c, up_gate, 0, config.ffn_dim, 2 * config.ffn_dim)); + y = nullptr; +#if !defined(SD_USE_UPSTREAM_GGML) && defined(GGML_HAS_SITU) + auto activation = ggml_glu(c, up_gate, GGML_GLU_OP_SITU, true); + if (ctx->backend != nullptr && ggml_backend_supports_op(ctx->backend, activation)) { + y = activation; + } +#endif + if (y == nullptr) { + y = situ(ctx, ggml_ext_slice(c, up_gate, 0, 0, config.ffn_dim), ggml_ext_slice(c, up_gate, 0, config.ffn_dim, 2 * config.ffn_dim)); + } y = linear("proj.ffn_down", y); y = ggml_mul(c, y, part(5)); return ggml_add(c, x, y); From 451ec773010d7a899883c108f87204ebe98725c0 Mon Sep 17 00:00:00 2001 From: mirek190 Date: Sat, 24 Oct 2026 22:44:11 +0100 Subject: [PATCH 2/5] docs(moss): remove unnecessary optimization documentation --- docs/moss_image.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/moss_image.md b/docs/moss_image.md index fb86182443..27ab26f804 100644 --- a/docs/moss_image.md +++ b/docs/moss_image.md @@ -35,7 +35,3 @@ Pass the external `tokenizer.json` with `--tokenizer`; Qwen3.5 uses a different Use dimensions divisible by 16. The default discrete scheduler uses the upstream time shift of 5. Use `--flow-shift` to override it. The negative CFG branch skips DiT layer 22 (zero-based), matching the upstream pipeline. - -## Optional fused SiTU - -When ggml advertises GGML_HAS_SITU and the selected backend supports the operation, MOSS uses fused SiTU. Older headers, upstream mode and unsupported devices retain the original composite activation. Acceleration requires compatible ggml SiTU support; this patch does not update the submodule. From a84e21e91ad8c85bbb5d7dc2dfa3e2f504fb6358 Mon Sep 17 00:00:00 2001 From: mirek190 Date: Wed, 28 Oct 2026 13:06:14 +0000 Subject: [PATCH 3/5] fix(moss): guard SiTU with the upstream ggml build option Retain backend capability checks and the composite fallback without requiring a per-operator availability macro. --- src/model/diffusion/moss_image.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/model/diffusion/moss_image.hpp b/src/model/diffusion/moss_image.hpp index 26851cedcd..27e627c20c 100644 --- a/src/model/diffusion/moss_image.hpp +++ b/src/model/diffusion/moss_image.hpp @@ -55,7 +55,7 @@ namespace MOSSImage { inline ggml_tensor* situ(GGMLRunnerContext* ctx, ggml_tensor* hidden, ggml_tensor* gate) { auto c = ctx->ggml_ctx; -#if !defined(SD_USE_UPSTREAM_GGML) && defined(GGML_HAS_SITU) +#ifndef SD_USE_UPSTREAM_GGML auto out = ggml_glu_split(c, gate, hidden, GGML_GLU_OP_SITU); if (ctx->backend != nullptr && ggml_backend_supports_op(ctx->backend, out)) { return out; @@ -142,7 +142,7 @@ namespace MOSSImage { y = ggml_add(c, ggml_add(c, y, ggml_mul(c, y, part(4))), part(3)); auto up_gate = linear("proj.ffn_up_gate", y); y = nullptr; -#if !defined(SD_USE_UPSTREAM_GGML) && defined(GGML_HAS_SITU) +#ifndef SD_USE_UPSTREAM_GGML auto activation = ggml_glu(c, up_gate, GGML_GLU_OP_SITU, true); if (ctx->backend != nullptr && ggml_backend_supports_op(ctx->backend, activation)) { y = activation; From ca1107bc722e1812835a7b67797ae7ee19d31ab6 Mon Sep 17 00:00:00 2001 From: mirek190 Date: Wed, 28 Oct 2026 17:04:25 +0000 Subject: [PATCH 4/5] fix(moss): discover optional SiTU with older ggml headers Resolve the optional GLU operation by its existing public name API, then retain backend support checks and the composite fallback. This keeps the unchanged pinned backend buildable without per-operator availability macros. --- docs/moss_image.md | 5 +++++ src/model/diffusion/moss_image.hpp | 35 +++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/docs/moss_image.md b/docs/moss_image.md index 27ab26f804..4d146be6aa 100644 --- a/docs/moss_image.md +++ b/docs/moss_image.md @@ -35,3 +35,8 @@ Pass the external `tokenizer.json` with `--tokenizer`; Qwen3.5 uses a different Use dimensions divisible by 16. The default discrete scheduler uses the upstream time shift of 5. Use `--flow-shift` to override it. The negative CFG branch skips DiT layer 22 (zero-based), matching the upstream pipeline. + +MOSS uses native SiTU when ggml lists the operation and the selected backend +supports it. Older pinned ggml revisions, upstream builds and unsupported +backends retain the composite activation. No per-operator availability macro +or submodule update is required. diff --git a/src/model/diffusion/moss_image.hpp b/src/model/diffusion/moss_image.hpp index 27e627c20c..20617c459e 100644 --- a/src/model/diffusion/moss_image.hpp +++ b/src/model/diffusion/moss_image.hpp @@ -3,6 +3,7 @@ #include #include +#include #include "model/common/ggml_block.hpp" #include "model/common/rope.hpp" @@ -53,12 +54,31 @@ namespace MOSSImage { } }; +#ifndef SD_USE_UPSTREAM_GGML + inline ggml_glu_op situ_op() { + // Older pinned headers can lack the optional SiTU enumerator. + static const auto op = [] { + for (int i = 0; i < GGML_GLU_OP_COUNT; ++i) { + auto candidate = static_cast(i); + if (std::strcmp(ggml_glu_op_name(candidate), "SITU") == 0) { + return candidate; + } + } + return GGML_GLU_OP_COUNT; + }(); + return op; + } +#endif + inline ggml_tensor* situ(GGMLRunnerContext* ctx, ggml_tensor* hidden, ggml_tensor* gate) { auto c = ctx->ggml_ctx; #ifndef SD_USE_UPSTREAM_GGML - auto out = ggml_glu_split(c, gate, hidden, GGML_GLU_OP_SITU); - if (ctx->backend != nullptr && ggml_backend_supports_op(ctx->backend, out)) { - return out; + const auto op = situ_op(); + if (ctx->backend != nullptr && op != GGML_GLU_OP_COUNT) { + auto out = ggml_glu_split(c, gate, hidden, op); + if (ggml_backend_supports_op(ctx->backend, out)) { + return out; + } } #endif auto a = ggml_scale(c, ggml_tanh(c, ggml_scale(c, gate, 0.25f)), 4.f); @@ -143,9 +163,12 @@ namespace MOSSImage { auto up_gate = linear("proj.ffn_up_gate", y); y = nullptr; #ifndef SD_USE_UPSTREAM_GGML - auto activation = ggml_glu(c, up_gate, GGML_GLU_OP_SITU, true); - if (ctx->backend != nullptr && ggml_backend_supports_op(ctx->backend, activation)) { - y = activation; + const auto op = situ_op(); + if (ctx->backend != nullptr && op != GGML_GLU_OP_COUNT) { + auto activation = ggml_glu(c, up_gate, op, true); + if (ggml_backend_supports_op(ctx->backend, activation)) { + y = activation; + } } #endif if (y == nullptr) { From 03774ed82dcb6eec1978b0c4eddf7e6e309a0177 Mon Sep 17 00:00:00 2001 From: leejet Date: Tue, 3 Nov 2026 23:10:03 +0800 Subject: [PATCH 5/5] refactor: centralize SiTU activation in ggml_ext_situ --- docs/moss_image.md | 5 --- ggml | 2 +- src/core/ggml_extend.cpp | 21 +++++++++++++ src/core/ggml_extend.h | 3 ++ src/model/diffusion/moss_image.hpp | 50 ++---------------------------- 5 files changed, 27 insertions(+), 54 deletions(-) diff --git a/docs/moss_image.md b/docs/moss_image.md index 4d146be6aa..27ab26f804 100644 --- a/docs/moss_image.md +++ b/docs/moss_image.md @@ -35,8 +35,3 @@ Pass the external `tokenizer.json` with `--tokenizer`; Qwen3.5 uses a different Use dimensions divisible by 16. The default discrete scheduler uses the upstream time shift of 5. Use `--flow-shift` to override it. The negative CFG branch skips DiT layer 22 (zero-based), matching the upstream pipeline. - -MOSS uses native SiTU when ggml lists the operation and the selected backend -supports it. Older pinned ggml revisions, upstream builds and unsupported -backends retain the composite activation. No per-operator availability macro -or submodule update is required. diff --git a/ggml b/ggml index d49c49e1dd..ccb88f810b 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit d49c49e1dd361ba3c0a77c90b5b1404b83969aa8 +Subproject commit ccb88f810be05bccc9621e706354611ccc2462c7 diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp index ca1b231c86..34d35056a9 100644 --- a/src/core/ggml_extend.cpp +++ b/src/core/ggml_extend.cpp @@ -144,6 +144,27 @@ ggml_tensor* ggml_ext_silu_act(ggml_context* ctx, ggml_tensor* x, bool gate_firs return gate_first ? ggml_swiglu(ctx, x) : ggml_swiglu_swapped(ctx, x); } +ggml_tensor* ggml_ext_situ(GGMLRunnerContext* ctx, ggml_tensor* hidden, ggml_tensor* gate) { +#ifndef SD_USE_UPSTREAM_GGML + if (ctx->backend != nullptr) { + auto out = gate ? ggml_glu_split(ctx->ggml_ctx, gate, hidden, GGML_GLU_OP_SITU) + : ggml_glu(ctx->ggml_ctx, hidden, GGML_GLU_OP_SITU, true); + if (ggml_backend_supports_op(ctx->backend, out)) { + return out; + } + } +#endif + if (!gate) { + const auto dim = hidden->ne[0] / 2; + gate = ggml_ext_slice(ctx->ggml_ctx, hidden, 0, dim, 2 * dim); + hidden = ggml_ext_slice(ctx->ggml_ctx, hidden, 0, 0, dim); + } + auto a = ggml_scale(ctx->ggml_ctx, ggml_tanh(ctx->ggml_ctx, ggml_scale(ctx->ggml_ctx, gate, 0.25f)), 4.f); + a = ggml_mul(ctx->ggml_ctx, a, ggml_sigmoid(ctx->ggml_ctx, gate)); + auto b = ggml_scale(ctx->ggml_ctx, ggml_tanh(ctx->ggml_ctx, ggml_scale(ctx->ggml_ctx, hidden, 1.f / 25.f)), 25.f); + return ggml_mul(ctx->ggml_ctx, a, b); +} + ggml_tensor* ggml_ext_group_norm_32(ggml_context* ctx, ggml_tensor* a) { const float eps = 1e-6f; // default eps parameter diff --git a/src/core/ggml_extend.h b/src/core/ggml_extend.h index 9ddfb0ad61..791155780f 100644 --- a/src/core/ggml_extend.h +++ b/src/core/ggml_extend.h @@ -52,6 +52,9 @@ std::vector ggml_ext_chunk(ggml_context* ctx, ggml_tensor* ggml_ext_silu_act(ggml_context* ctx, ggml_tensor* x, bool gate_first = true); +// Without gate, hidden contains [hidden, gate] packed along ne[0]. +ggml_tensor* ggml_ext_situ(GGMLRunnerContext* ctx, ggml_tensor* hidden, ggml_tensor* gate = nullptr); + ggml_tensor* ggml_ext_group_norm_32(ggml_context* ctx, ggml_tensor* a); diff --git a/src/model/diffusion/moss_image.hpp b/src/model/diffusion/moss_image.hpp index 9004cedb61..36d8d8328b 100644 --- a/src/model/diffusion/moss_image.hpp +++ b/src/model/diffusion/moss_image.hpp @@ -4,7 +4,6 @@ #include #include #include -#include #include "core/pe_cache.h" @@ -57,39 +56,6 @@ namespace MOSSImage { } }; -#ifndef SD_USE_UPSTREAM_GGML - inline ggml_glu_op situ_op() { - // Older pinned headers can lack the optional SiTU enumerator. - static const auto op = [] { - for (int i = 0; i < GGML_GLU_OP_COUNT; ++i) { - auto candidate = static_cast(i); - if (std::strcmp(ggml_glu_op_name(candidate), "SITU") == 0) { - return candidate; - } - } - return GGML_GLU_OP_COUNT; - }(); - return op; - } -#endif - - inline ggml_tensor* situ(GGMLRunnerContext* ctx, ggml_tensor* hidden, ggml_tensor* gate) { - auto c = ctx->ggml_ctx; -#ifndef SD_USE_UPSTREAM_GGML - const auto op = situ_op(); - if (ctx->backend != nullptr && op != GGML_GLU_OP_COUNT) { - auto out = ggml_glu_split(c, gate, hidden, op); - if (ggml_backend_supports_op(ctx->backend, out)) { - return out; - } - } -#endif - auto a = ggml_scale(c, ggml_tanh(c, ggml_scale(c, gate, 0.25f)), 4.f); - a = ggml_mul(c, a, ggml_sigmoid(c, gate)); - auto b = ggml_scale(c, ggml_tanh(c, ggml_scale(c, hidden, 1.f / 25.f)), 25.f); - return ggml_mul(c, a, b); - } - struct TextBlock : public GGMLBlock { MOSSImageConfig config; @@ -122,7 +88,7 @@ namespace MOSSImage { y = linear("proj", y); x = ggml_add(c, x, y); y = norm("norm2", x); - y = situ(ctx, linear("ffn_up", y), linear("ffn_gate", y)); + y = ggml_ext_situ(ctx, linear("ffn_up", y), linear("ffn_gate", y)); y = linear("ffn_down", y); return ggml_add(c, x, y); } @@ -165,19 +131,7 @@ namespace MOSSImage { y = norm("norm2", x); y = ggml_add(c, ggml_add(c, y, ggml_mul(c, y, part(4))), part(3)); auto up_gate = linear("proj.ffn_up_gate", y); - y = nullptr; -#ifndef SD_USE_UPSTREAM_GGML - const auto op = situ_op(); - if (ctx->backend != nullptr && op != GGML_GLU_OP_COUNT) { - auto activation = ggml_glu(c, up_gate, op, true); - if (ggml_backend_supports_op(ctx->backend, activation)) { - y = activation; - } - } -#endif - if (y == nullptr) { - y = situ(ctx, ggml_ext_slice(c, up_gate, 0, 0, config.ffn_dim), ggml_ext_slice(c, up_gate, 0, config.ffn_dim, 2 * config.ffn_dim)); - } + y = ggml_ext_situ(ctx, up_gate); y = linear("proj.ffn_down", y); y = ggml_mul(c, y, part(5)); return ggml_add(c, x, y);