Апстрим master-1031-c6dfa9d

docs/backend.md

@@ -247,6 +247,9 @@ These are backend-level defaults. GGML still chooses the actual kernel based on
 weight type, shape, and hardware; enabling direct convolution does not guarantee
 that every operation uses implicit GEMM. The CUDA-family implicit GEMM paths
 require F16 weights and can use ordinary direct kernels for other cases.
+For 3D convolution, a supported direct kernel is also used as a fallback when
+the im2col decomposition is unsupported for the actual tensor sizes, including
+when an intermediate reshape or matrix multiplication exceeds backend limits.
 This option does not change the separately created image upscaler context.
 
 Repeating `--conv-direct` replaces the previous assignment string. Existing

docs/performance.md

@@ -159,6 +159,25 @@ allocator/scheduler capacity. Compatible gallocr reservations are reused across
 graphs; idle workspaces can be reclaimed under pressure and are freed at runner
 completion. Cross-graph caches survive individual graphs, but cut buffers do not.
 
+On discrete Vulkan devices, runners reuse released device buffers of the same
+size to reduce allocation overhead during segmented execution. This idle buffer
+cache is limited to 1 GiB per device and is reclaimed before allocating a new
+size or at runner completion. It is outside the managed `--max-vram` budget,
+so physical VRAM usage can be higher. Set `GGML_VK_REUSE_BUFFERS=0` to disable
+it. Unified-memory and host-memory modes do not use this cache.
+
+Discrete NVIDIA and non-GCN AMD Vulkan devices use a separate transfer queue by
+default when available and graphics queue use is disabled, so weight prefetch can overlap compute. Set
+`GGML_VK_ASYNC_USE_TRANSFER_QUEUE=0` to disable it.
+
+On discrete NVIDIA devices, allocation chunks can grow up to 4 GiB, bounded by the device's buffer and allocation
+limits, to reduce fragmentation within large graph workspaces. Other devices
+retain the 1 GiB default. `GGML_VK_SUBALLOCATION_BLOCK_SIZE` overrides this limit
+in bytes on any vendor; for example, `1073741824` selects 1 GiB and `4294967296`
+selects 4 GiB, subject to device limits. This is a soft chunk limit, not a reservation
+or a change to `--max-vram`: a single oversized tensor or exhausting the allocator's
+chunk count can require a larger allocation, which may fail.
+
 The recommended shape for "biggest model my card can host" is:
 
 ```shell

src/core/compute_workspace.cpp

@@ -92,7 +92,8 @@ namespace sd {
     ComputeWorkspace::Measurement ComputeWorkspace::measure(
         ggml_cgraph* graph,
         const std::function<ggml_backend_t(const ggml_tensor*)>& external_backend,
-        const AssignNodes& assign_nodes) {
+        const AssignNodes& assign_nodes,
+        bool cache_measurement) {
         std::vector<const ggml_tensor*> tensors;
         std::unordered_set<const ggml_tensor*> seen;
         auto visit = [&](const ggml_tensor* tensor) {
@@ -112,6 +113,52 @@ namespace sd {
                 visit(source);
             }
         }
+        std::vector<uint8_t> key;
+        if (cache_measurement) {
+            key.reserve(tensors.size() * sizeof(ggml_tensor));
+            auto append = [&](const auto& value) {
+                const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+                key.insert(key.end(), bytes, bytes + sizeof(value));
+            };
+            append(graph->n_nodes);
+            append(graph->n_leafs);
+            append(graph->order);
+            for (auto tensor : tensors) {
+                append(tensor);
+                append(tensor->type);
+                append(tensor->ne);
+                append(tensor->nb);
+                append(tensor->op);
+                append(tensor->op_params);
+                append(tensor->flags);
+                append(tensor->src);
+                append(tensor->view_src);
+                append(tensor->view_offs);
+                append(tensor->name);
+                const auto external = external_backend(tensor);
+                append(external);
+                if (external == nullptr || tensor->view_src != nullptr) {
+                    append(tensor->buffer);
+                    append(tensor->data);
+                    append(tensor->extra);
+                    if (tensor->buffer != nullptr) {
+                        const auto usage = ggml_backend_buffer_get_usage(tensor->buffer);
+                        const auto type  = ggml_backend_buffer_get_type(tensor->buffer);
+                        append(usage);
+                        append(type);
+                    }
+                }
+                const size_t index = ggml_hash_find(&graph->visited_hash_set, tensor);
+                const int32_t uses = index != GGML_HASHSET_FULL && ggml_bitset_get(graph->visited_hash_set.used, index)
+                                         ? graph->use_counts[index]
+                                         : 0;
+                append(uses);
+            }
+            // Match the full allocation metadata, including topology and external placement.
+            if (key == measurement_key_) {
+                return {measured_buffers_};
+            }
+        }
         const size_t graph_size = tensors.size() + 8;
         auto context            = ggml_init({tensors.size() * ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false), nullptr, true});
         if (context == nullptr) {
@@ -178,6 +225,10 @@ namespace sd {
             ggml_backend_buffer_free(entry.second);
         }
         ggml_free(context);
+        if (cache_measurement && !result.buffers.empty()) {
+            measurement_key_  = std::move(key);
+            measured_buffers_ = result.buffers;
+        }
         return result;
     }
 
@@ -212,6 +263,8 @@ namespace sd {
         if (active_) {
             return false;
         }
+        measurement_key_.clear();
+        measured_buffers_.clear();
         ggml_backend_sched_free(scheduler_);
         scheduler_          = nullptr;
         scheduler_capacity_ = 0;

src/core/compute_workspace.h

@@ -19,6 +19,8 @@ namespace sd {
         ggml_backend_sched_t scheduler_ = nullptr;
         size_t scheduler_capacity_      = 0;
         bool active_                    = false;
+        std::vector<uint8_t> measurement_key_;
+        std::vector<BackendBufferSize> measured_buffers_;
 
         ggml_backend_sched_t make_scheduler(size_t graph_size);
 
@@ -41,7 +43,8 @@ namespace sd {
         Measurement measure(
             ggml_cgraph* graph,
             const std::function<ggml_backend_t(const ggml_tensor*)>& external_backend,
-            const AssignNodes& assign_nodes);
+            const AssignNodes& assign_nodes,
+            bool cache_measurement = false);
         void synchronize() const;
         bool segment_end() noexcept;
         bool release();

src/core/ggml_extend.cpp

@@ -3,6 +3,7 @@
 #include <cmath>
 #include <stdexcept>
 #include <utility>
+#include <vector>
 
 #include "core/ggml_extend_backend.h"
 
@@ -135,21 +136,11 @@ ggml_tensor* ggml_ext_silu_act(ggml_context* ctx, ggml_tensor* x, bool gate_firs
     // x: [ne3, ne2, ne1, ne0]
     // return: [ne3, ne2, ne1, ne0/2]
 
-    auto x_vec = ggml_ext_chunk(ctx, x, 2, 0, false);
-    ggml_tensor* gate;
-    if (gate_first) {
-        gate = x_vec[0];
-        x    = x_vec[1];
-    } else {
-        x    = x_vec[0];
-        gate = x_vec[1];
+    GGML_ASSERT(x->ne[0] % 2 == 0);
+    if (!ggml_is_contiguous_1(x)) {
+        x = ggml_cont(ctx, x);
     }
-    gate = ggml_cont(ctx, gate);
-    gate = ggml_silu_inplace(ctx, gate);
-
-    x = ggml_mul(ctx, x, gate);  // [ne3, ne2, ne1, ne0/2]
-
-    return x;
+    return gate_first ? ggml_swiglu(ctx, x) : ggml_swiglu_swapped(ctx, x);
 }
 
 ggml_tensor* ggml_ext_group_norm_32(ggml_context* ctx,
@@ -488,21 +479,33 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx,
         x          = ggml_cont(ctx, ggml_permute(ctx, x, 0, 1, 3, 2));
         x          = ggml_reshape_4d(ctx, x, im2col->ne[1], im2col->ne[2], OD, OC * N);
     } else {
-        // ggml_conv_3d decomposes into GGML_OP_IM2COL_3D, which some backends
-        // (e.g. Metal, see #850) do not implement. Fall back to
-        // GGML_OP_CONV_3D on those backends.
-        bool im2col_3d_supported = true;
+        auto convolution = ggml_conv_3d(ctx, w, x, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2);
         if (backend != nullptr) {
-            ggml_tensor* im2col = ggml_im2col_3d(ctx, w, x, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, w->type);
-            im2col_3d_supported = ggml_backend_supports_op(backend, im2col);
-        }
-        if (im2col_3d_supported) {
-            x = ggml_conv_3d(ctx, w, x, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2);
-        } else {
-            int64_t OC = w->ne[3] / IC;
-            int64_t N  = x->ne[3] / IC;
-            x          = ggml_conv_3d_direct(ctx, w, x, s0, s1, s2, p0, p1, p2, d0, d1, d2, (int)IC, (int)N, (int)OC);
+            // im2col can support large buffers that its reshape/GEMM consumers cannot.
+            std::vector<ggml_tensor*> nodes{convolution};
+            bool supported = true;
+            for (size_t i = 0; i < nodes.size() && supported; ++i) {
+                auto node = nodes[i];
+                if (node == x || node == w) {
+                    continue;
+                }
+                supported = ggml_backend_supports_op(backend, node);
+                for (auto source : node->src) {
+                    if (source != nullptr) {
+                        nodes.push_back(source);
+                    }
+                }
+            }
+            if (!supported) {
+                int64_t OC    = w->ne[3] / IC;
+                int64_t N     = x->ne[3] / IC;
+                auto fallback = ggml_conv_3d_direct(ctx, w, x, s0, s1, s2, p0, p1, p2, d0, d1, d2, (int)IC, (int)N, (int)OC);
+                if (ggml_backend_supports_op(backend, fallback)) {
+                    convolution = fallback;
+                }
+            }
         }
+        x = convolution;
     }
 
     if (scale != 1.f) {

src/core/ggml_runner.cpp

@@ -735,7 +735,7 @@ ComputeWorkspace::Measurement GGMLRunner::measure(ggml_cgraph* graph, const std:
     auto assign_nodes = [&](ggml_backend_sched_t scheduler, ggml_cgraph* copy) {
         pin_multi_device_nodes(scheduler, copy, graph);
     };
-    return workspace_.measure(graph, external_backend, assign_nodes);
+    return workspace_.measure(graph, external_backend, assign_nodes, !is_multi_device());
 }
 
 std::vector<DeviceMemoryRequest> GGMLRunner::memory_requests(

src/model/diffusion/ernie_image.hpp

@@ -204,9 +204,8 @@ namespace ErnieImage {
             auto linear_fc2 = std::dynamic_pointer_cast<Linear>(blocks["linear_fc2"]);
 
             auto gate = gate_proj->forward(ctx, x);
-            gate      = ggml_ext_gelu(ctx->ggml_ctx, gate);
             x         = up_proj->forward(ctx, x);
-            x         = ggml_mul(ctx->ggml_ctx, x, gate);
+            x         = ggml_geglu_split(ctx->ggml_ctx, gate, x);
             x         = linear_fc2->forward(ctx, x);
             return x;
         }

src/model/diffusion/flux.hpp

@@ -358,9 +358,8 @@ namespace Flux {
             auto down_proj = std::dynamic_pointer_cast<Linear>(blocks["down_proj"]);
 
             auto gate = gate_proj->forward(ctx, x);
-            gate      = ggml_silu_inplace(ctx->ggml_ctx, gate);
             x         = up_proj->forward(ctx, x);
-            x         = ggml_mul(ctx->ggml_ctx, x, gate);
+            x         = ggml_swiglu_split(ctx->ggml_ctx, gate, x);
             x         = down_proj->forward(ctx, x);
             return x;
         }
@@ -850,12 +849,10 @@ namespace Flux {
             auto res_x = x;
             x          = norm->forward(ctx, x);  // [batch_size, n_token, hidden_size_x]
 
-            auto x1 = ggml_mul_mat(ctx->ggml_ctx, fc1_gate, x);  // [batch_size, n_token, hidden_size_x*mlp_ratio]
-            x1      = ggml_silu_inplace(ctx->ggml_ctx, x1);
-
+            auto x1 = ggml_mul_mat(ctx->ggml_ctx, fc1_gate, x);   // [batch_size, n_token, hidden_size_x*mlp_ratio]
             auto x2 = ggml_mul_mat(ctx->ggml_ctx, fc1_value, x);  // [batch_size, n_token, hidden_size_x*mlp_ratio]
 
-            x = ggml_mul_inplace(ctx->ggml_ctx, x1, x2);  // [batch_size, n_token, hidden_size_x*mlp_ratio]
+            x = ggml_swiglu_split(ctx->ggml_ctx, x1, x2);  // [batch_size, n_token, hidden_size_x*mlp_ratio]
 
             x = ggml_mul_mat(ctx->ggml_ctx, fc2, x);  // [batch_size, n_token, hidden_size_x]
 

src/model/diffusion/ideogram4.hpp

@@ -244,9 +244,9 @@ namespace Ideogram4 {
             auto w2 = std::dynamic_pointer_cast<Linear>(blocks["w2"]);
             auto w3 = std::dynamic_pointer_cast<Linear>(blocks["w3"]);
 
-            auto x1 = ggml_silu(ctx->ggml_ctx, w1->forward(ctx, x));
+            auto x1 = w1->forward(ctx, x);
             auto x3 = w3->forward(ctx, x);
-            x       = ggml_mul(ctx->ggml_ctx, x1, x3);
+            x       = ggml_swiglu_split(ctx->ggml_ctx, x1, x3);
             x       = w2->forward(ctx, x);
             return x;
         }

src/model/diffusion/krea2.hpp

@@ -208,9 +208,9 @@ namespace Krea2 {
             auto up   = std::dynamic_pointer_cast<Linear>(blocks["up"]);
             auto down = std::dynamic_pointer_cast<Linear>(blocks["down"]);
 
-            auto gated = ggml_silu(ctx->ggml_ctx, gate->forward(ctx, x));
-            auto up_x  = up->forward(ctx, x);
-            x          = ggml_mul(ctx->ggml_ctx, gated, up_x);
+            auto gate_x = gate->forward(ctx, x);
+            auto up_x   = up->forward(ctx, x);
+            x           = ggml_swiglu_split(ctx->ggml_ctx, gate_x, up_x);
             return down->forward(ctx, x);
         }
     };

src/model/diffusion/lens.hpp

@@ -104,9 +104,9 @@ namespace Lens {
             auto w2 = std::dynamic_pointer_cast<Linear>(blocks["w2"]);
             auto w3 = std::dynamic_pointer_cast<Linear>(blocks["w3"]);
 
-            auto gate = ggml_silu(ctx->ggml_ctx, w1->forward(ctx, x));
+            auto gate = w1->forward(ctx, x);
             auto up   = w3->forward(ctx, x);
-            x         = ggml_mul(ctx->ggml_ctx, gate, up);
+            x         = ggml_swiglu_split(ctx->ggml_ctx, gate, up);
             return w2->forward(ctx, x);
         }
     };

src/model/diffusion/lingbot_video.hpp

@@ -231,9 +231,8 @@ namespace LingBotVideo {
             auto down_proj = std::dynamic_pointer_cast<Linear>(blocks["down_proj"]);
 
             auto gate = gate_proj->forward(ctx, x);
-            gate      = ggml_silu_inplace(ctx->ggml_ctx, gate);
             auto up   = up_proj->forward(ctx, x);
-            x         = ggml_mul(ctx->ggml_ctx, gate, up);
+            x         = ggml_swiglu_split(ctx->ggml_ctx, gate, up);
             x         = down_proj->forward(ctx, x);
             return x;
         }
@@ -406,9 +405,8 @@ namespace LingBotVideo {
 
             x         = ggml_reshape_3d(ctx->ggml_ctx, x, hidden_size, 1, n_token_total);
             auto gate = expert_linear(ctx, "experts.w1", x, selected_experts);
-            gate      = ggml_silu_inplace(ctx->ggml_ctx, gate);
             auto up   = expert_linear(ctx, "experts.w3", x, selected_experts);
-            auto act  = ggml_mul(ctx->ggml_ctx, gate, up);
+            auto act  = ggml_swiglu_split(ctx->ggml_ctx, gate, up);
             auto out  = expert_linear(ctx, "experts.w2", act, selected_experts);
             out       = ggml_mul(ctx->ggml_ctx, out, weights);
 

src/model/diffusion/minit2i.hpp

@@ -207,9 +207,9 @@ namespace MiniT2I {
             auto w1   = std::dynamic_pointer_cast<Linear>(blocks["w1"]);
             auto w3   = std::dynamic_pointer_cast<Linear>(blocks["w3"]);
             auto w2   = std::dynamic_pointer_cast<Linear>(blocks["w2"]);
-            auto gate = ggml_silu(ctx->ggml_ctx, w1->forward(ctx, x));
+            auto gate = w1->forward(ctx, x);
             auto up   = w3->forward(ctx, x);
-            return w2->forward(ctx, ggml_mul(ctx->ggml_ctx, gate, up));
+            return w2->forward(ctx, ggml_swiglu_split(ctx->ggml_ctx, gate, up));
         }
     };
 

src/model/diffusion/pid.hpp

@@ -253,8 +253,8 @@ namespace Pid {
             auto w1 = std::dynamic_pointer_cast<Linear>(blocks["w1"]);
             auto w2 = std::dynamic_pointer_cast<Linear>(blocks["w2"]);
             auto w3 = std::dynamic_pointer_cast<Linear>(blocks["w3"]);
-            auto h  = ggml_silu_inplace(ctx->ggml_ctx, w1->forward(ctx, x));
-            h       = ggml_mul_inplace(ctx->ggml_ctx, h, w3->forward(ctx, x));
+            auto h  = w1->forward(ctx, x);
+            h       = ggml_swiglu_split(ctx->ggml_ctx, h, w3->forward(ctx, x));
             return w2->forward(ctx, h);
         }
     };

src/model/te/llm.hpp

@@ -517,13 +517,13 @@ namespace LLM {
             auto up_proj   = std::dynamic_pointer_cast<Linear>(blocks["up_proj"]);
             auto down_proj = std::dynamic_pointer_cast<Linear>(blocks["down_proj"]);
 
-            auto h = gate_proj->forward(ctx, x);
+            auto h  = gate_proj->forward(ctx, x);
+            auto up = up_proj->forward(ctx, x);
             if (activation == MLPActivation::GELU_TANH) {
-                h = ggml_ext_gelu(ctx->ggml_ctx, h, true);
+                h = ggml_geglu_split(ctx->ggml_ctx, h, up);
             } else {
-                h = ggml_silu_inplace(ctx->ggml_ctx, h);
+                h = ggml_swiglu_split(ctx->ggml_ctx, h, up);
             }
-            h = ggml_mul_inplace(ctx->ggml_ctx, h, up_proj->forward(ctx, x));
             h = down_proj->forward(ctx, h);
             return h;
         }
@@ -1027,8 +1027,7 @@ namespace LLM {
                 auto up_proj   = std::dynamic_pointer_cast<Linear>(blocks["up_proj"]);
                 auto down_proj = std::dynamic_pointer_cast<Linear>(blocks["down_proj"]);
                 auto h         = gate_proj->forward(ctx, x);
-                h              = ggml_silu_inplace(ctx->ggml_ctx, h);
-                h              = ggml_mul_inplace(ctx->ggml_ctx, h, up_proj->forward(ctx, x));
+                h              = ggml_swiglu_split(ctx->ggml_ctx, h, up_proj->forward(ctx, x));
                 x              = down_proj->forward(ctx, h);
             }
             return x;
@@ -1520,7 +1519,7 @@ namespace LLM {
             out        = ggml_view_3d(c, out, head_dim, value_heads, n, head_dim * sizeof(float), head_dim * value_heads * sizeof(float), 0);
             out        = std::dynamic_pointer_cast<LLMRMSNorm>(blocks["norm"])->forward(ctx, out);
             auto z     = ggml_reshape_3d(c, linear("in_proj_z"), head_dim, value_heads, n);
-            out        = ggml_mul(c, out, ggml_silu(c, z));
+            out        = ggml_swiglu_split(c, z, out);
             out        = ggml_reshape_3d(c, out, head_dim * value_heads, n, 1);
             return std::dynamic_pointer_cast<Linear>(blocks["out_proj"])->forward(ctx, out);
         }

src/model/te/t5.hpp

@@ -156,9 +156,9 @@ struct T5DenseGatedActDense : public UnaryBlock {
         auto wi_1 = std::dynamic_pointer_cast<Linear>(blocks["wi_1"]);
         auto wo   = std::dynamic_pointer_cast<Linear>(blocks["wo"]);
 
-        auto hidden_gelu   = ggml_ext_gelu(ctx->ggml_ctx, wi_0->forward(ctx, x), true);
+        auto hidden_gate   = wi_0->forward(ctx, x);
         auto hidden_linear = wi_1->forward(ctx, x);
-        x                  = ggml_mul_inplace(ctx->ggml_ctx, hidden_gelu, hidden_linear);
+        x                  = ggml_geglu_split(ctx->ggml_ctx, hidden_gate, hidden_linear);
         x                  = wo->forward(ctx, x);
         return x;
     }