From e239e4b0fd3de49834cec67031a44796ca465abc Mon Sep 17 00:00:00 2001 From: leejet Date: Fri, 30 Oct 2026 01:32:46 +0800 Subject: [PATCH 1/3] refactor: replace manual gated activations with native ggml GLU (#2250) --- src/core/ggml_extend.cpp | 18 ++++-------------- src/model/diffusion/ernie_image.hpp | 3 +-- src/model/diffusion/flux.hpp | 9 +++------ src/model/diffusion/ideogram4.hpp | 4 ++-- src/model/diffusion/krea2.hpp | 6 +++--- src/model/diffusion/lens.hpp | 4 ++-- src/model/diffusion/lingbot_video.hpp | 6 ++---- src/model/diffusion/minit2i.hpp | 4 ++-- src/model/diffusion/pid.hpp | 4 ++-- src/model/te/llm.hpp | 13 ++++++------- src/model/te/t5.hpp | 4 ++-- src/model/vae/minimax_h3_audio_vae.hpp | 4 ++-- src/model/vae/minimax_h3_vae.hpp | 7 ++----- 13 files changed, 33 insertions(+), 53 deletions(-) diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp index bd5f6f63d0..d3548f46e9 100644 --- a/src/core/ggml_extend.cpp +++ b/src/core/ggml_extend.cpp @@ -135,21 +135,11 @@ ggml_tensor* ggml_ext_silu_act(ggml_context* ctx, ggml_tensor* x, bool gate_firs // x: [ne3, ne2, ne1, ne0] // return: [ne3, ne2, ne1, ne0/2] - auto x_vec = ggml_ext_chunk(ctx, x, 2, 0, false); - ggml_tensor* gate; - if (gate_first) { - gate = x_vec[0]; - x = x_vec[1]; - } else { - x = x_vec[0]; - gate = x_vec[1]; + GGML_ASSERT(x->ne[0] % 2 == 0); + if (!ggml_is_contiguous_1(x)) { + x = ggml_cont(ctx, x); } - gate = ggml_cont(ctx, gate); - gate = ggml_silu_inplace(ctx, gate); - - x = ggml_mul(ctx, x, gate); // [ne3, ne2, ne1, ne0/2] - - return x; + return gate_first ? ggml_swiglu(ctx, x) : ggml_swiglu_swapped(ctx, x); } ggml_tensor* ggml_ext_group_norm_32(ggml_context* ctx, diff --git a/src/model/diffusion/ernie_image.hpp b/src/model/diffusion/ernie_image.hpp index d4f774fc61..59b06cc2f1 100644 --- a/src/model/diffusion/ernie_image.hpp +++ b/src/model/diffusion/ernie_image.hpp @@ -204,9 +204,8 @@ namespace ErnieImage { auto linear_fc2 = std::dynamic_pointer_cast(blocks["linear_fc2"]); auto gate = gate_proj->forward(ctx, x); - gate = ggml_ext_gelu(ctx->ggml_ctx, gate); x = up_proj->forward(ctx, x); - x = ggml_mul(ctx->ggml_ctx, x, gate); + x = ggml_geglu_split(ctx->ggml_ctx, gate, x); x = linear_fc2->forward(ctx, x); return x; } diff --git a/src/model/diffusion/flux.hpp b/src/model/diffusion/flux.hpp index 3199d0fded..c9e3f5faab 100644 --- a/src/model/diffusion/flux.hpp +++ b/src/model/diffusion/flux.hpp @@ -358,9 +358,8 @@ namespace Flux { auto down_proj = std::dynamic_pointer_cast(blocks["down_proj"]); auto gate = gate_proj->forward(ctx, x); - gate = ggml_silu_inplace(ctx->ggml_ctx, gate); x = up_proj->forward(ctx, x); - x = ggml_mul(ctx->ggml_ctx, x, gate); + x = ggml_swiglu_split(ctx->ggml_ctx, gate, x); x = down_proj->forward(ctx, x); return x; } @@ -850,12 +849,10 @@ namespace Flux { auto res_x = x; x = norm->forward(ctx, x); // [batch_size, n_token, hidden_size_x] - auto x1 = ggml_mul_mat(ctx->ggml_ctx, fc1_gate, x); // [batch_size, n_token, hidden_size_x*mlp_ratio] - x1 = ggml_silu_inplace(ctx->ggml_ctx, x1); - + auto x1 = ggml_mul_mat(ctx->ggml_ctx, fc1_gate, x); // [batch_size, n_token, hidden_size_x*mlp_ratio] auto x2 = ggml_mul_mat(ctx->ggml_ctx, fc1_value, x); // [batch_size, n_token, hidden_size_x*mlp_ratio] - x = ggml_mul_inplace(ctx->ggml_ctx, x1, x2); // [batch_size, n_token, hidden_size_x*mlp_ratio] + x = ggml_swiglu_split(ctx->ggml_ctx, x1, x2); // [batch_size, n_token, hidden_size_x*mlp_ratio] x = ggml_mul_mat(ctx->ggml_ctx, fc2, x); // [batch_size, n_token, hidden_size_x] diff --git a/src/model/diffusion/ideogram4.hpp b/src/model/diffusion/ideogram4.hpp index 664c827d54..5a5cb000cc 100644 --- a/src/model/diffusion/ideogram4.hpp +++ b/src/model/diffusion/ideogram4.hpp @@ -244,9 +244,9 @@ namespace Ideogram4 { auto w2 = std::dynamic_pointer_cast(blocks["w2"]); auto w3 = std::dynamic_pointer_cast(blocks["w3"]); - auto x1 = ggml_silu(ctx->ggml_ctx, w1->forward(ctx, x)); + auto x1 = w1->forward(ctx, x); auto x3 = w3->forward(ctx, x); - x = ggml_mul(ctx->ggml_ctx, x1, x3); + x = ggml_swiglu_split(ctx->ggml_ctx, x1, x3); x = w2->forward(ctx, x); return x; } diff --git a/src/model/diffusion/krea2.hpp b/src/model/diffusion/krea2.hpp index 1558e4e9bf..d21574b63b 100644 --- a/src/model/diffusion/krea2.hpp +++ b/src/model/diffusion/krea2.hpp @@ -208,9 +208,9 @@ namespace Krea2 { auto up = std::dynamic_pointer_cast(blocks["up"]); auto down = std::dynamic_pointer_cast(blocks["down"]); - auto gated = ggml_silu(ctx->ggml_ctx, gate->forward(ctx, x)); - auto up_x = up->forward(ctx, x); - x = ggml_mul(ctx->ggml_ctx, gated, up_x); + auto gate_x = gate->forward(ctx, x); + auto up_x = up->forward(ctx, x); + x = ggml_swiglu_split(ctx->ggml_ctx, gate_x, up_x); return down->forward(ctx, x); } }; diff --git a/src/model/diffusion/lens.hpp b/src/model/diffusion/lens.hpp index 76f7bee1f5..62ad6d3732 100644 --- a/src/model/diffusion/lens.hpp +++ b/src/model/diffusion/lens.hpp @@ -104,9 +104,9 @@ namespace Lens { auto w2 = std::dynamic_pointer_cast(blocks["w2"]); auto w3 = std::dynamic_pointer_cast(blocks["w3"]); - auto gate = ggml_silu(ctx->ggml_ctx, w1->forward(ctx, x)); + auto gate = w1->forward(ctx, x); auto up = w3->forward(ctx, x); - x = ggml_mul(ctx->ggml_ctx, gate, up); + x = ggml_swiglu_split(ctx->ggml_ctx, gate, up); return w2->forward(ctx, x); } }; diff --git a/src/model/diffusion/lingbot_video.hpp b/src/model/diffusion/lingbot_video.hpp index 194bf79126..1ea7b1c0cf 100644 --- a/src/model/diffusion/lingbot_video.hpp +++ b/src/model/diffusion/lingbot_video.hpp @@ -231,9 +231,8 @@ namespace LingBotVideo { auto down_proj = std::dynamic_pointer_cast(blocks["down_proj"]); auto gate = gate_proj->forward(ctx, x); - gate = ggml_silu_inplace(ctx->ggml_ctx, gate); auto up = up_proj->forward(ctx, x); - x = ggml_mul(ctx->ggml_ctx, gate, up); + x = ggml_swiglu_split(ctx->ggml_ctx, gate, up); x = down_proj->forward(ctx, x); return x; } @@ -406,9 +405,8 @@ namespace LingBotVideo { x = ggml_reshape_3d(ctx->ggml_ctx, x, hidden_size, 1, n_token_total); auto gate = expert_linear(ctx, "experts.w1", x, selected_experts); - gate = ggml_silu_inplace(ctx->ggml_ctx, gate); auto up = expert_linear(ctx, "experts.w3", x, selected_experts); - auto act = ggml_mul(ctx->ggml_ctx, gate, up); + auto act = ggml_swiglu_split(ctx->ggml_ctx, gate, up); auto out = expert_linear(ctx, "experts.w2", act, selected_experts); out = ggml_mul(ctx->ggml_ctx, out, weights); diff --git a/src/model/diffusion/minit2i.hpp b/src/model/diffusion/minit2i.hpp index 6dc9c1798d..c6e90dae1c 100644 --- a/src/model/diffusion/minit2i.hpp +++ b/src/model/diffusion/minit2i.hpp @@ -207,9 +207,9 @@ namespace MiniT2I { auto w1 = std::dynamic_pointer_cast(blocks["w1"]); auto w3 = std::dynamic_pointer_cast(blocks["w3"]); auto w2 = std::dynamic_pointer_cast(blocks["w2"]); - auto gate = ggml_silu(ctx->ggml_ctx, w1->forward(ctx, x)); + auto gate = w1->forward(ctx, x); auto up = w3->forward(ctx, x); - return w2->forward(ctx, ggml_mul(ctx->ggml_ctx, gate, up)); + return w2->forward(ctx, ggml_swiglu_split(ctx->ggml_ctx, gate, up)); } }; diff --git a/src/model/diffusion/pid.hpp b/src/model/diffusion/pid.hpp index 6c5a59be6b..0e502371a8 100644 --- a/src/model/diffusion/pid.hpp +++ b/src/model/diffusion/pid.hpp @@ -253,8 +253,8 @@ namespace Pid { auto w1 = std::dynamic_pointer_cast(blocks["w1"]); auto w2 = std::dynamic_pointer_cast(blocks["w2"]); auto w3 = std::dynamic_pointer_cast(blocks["w3"]); - auto h = ggml_silu_inplace(ctx->ggml_ctx, w1->forward(ctx, x)); - h = ggml_mul_inplace(ctx->ggml_ctx, h, w3->forward(ctx, x)); + auto h = w1->forward(ctx, x); + h = ggml_swiglu_split(ctx->ggml_ctx, h, w3->forward(ctx, x)); return w2->forward(ctx, h); } }; diff --git a/src/model/te/llm.hpp b/src/model/te/llm.hpp index 85f7511693..add8b6ff9a 100644 --- a/src/model/te/llm.hpp +++ b/src/model/te/llm.hpp @@ -517,13 +517,13 @@ namespace LLM { auto up_proj = std::dynamic_pointer_cast(blocks["up_proj"]); auto down_proj = std::dynamic_pointer_cast(blocks["down_proj"]); - auto h = gate_proj->forward(ctx, x); + auto h = gate_proj->forward(ctx, x); + auto up = up_proj->forward(ctx, x); if (activation == MLPActivation::GELU_TANH) { - h = ggml_ext_gelu(ctx->ggml_ctx, h, true); + h = ggml_geglu_split(ctx->ggml_ctx, h, up); } else { - h = ggml_silu_inplace(ctx->ggml_ctx, h); + h = ggml_swiglu_split(ctx->ggml_ctx, h, up); } - h = ggml_mul_inplace(ctx->ggml_ctx, h, up_proj->forward(ctx, x)); h = down_proj->forward(ctx, h); return h; } @@ -1027,8 +1027,7 @@ namespace LLM { auto up_proj = std::dynamic_pointer_cast(blocks["up_proj"]); auto down_proj = std::dynamic_pointer_cast(blocks["down_proj"]); auto h = gate_proj->forward(ctx, x); - h = ggml_silu_inplace(ctx->ggml_ctx, h); - h = ggml_mul_inplace(ctx->ggml_ctx, h, up_proj->forward(ctx, x)); + h = ggml_swiglu_split(ctx->ggml_ctx, h, up_proj->forward(ctx, x)); x = down_proj->forward(ctx, h); } return x; @@ -1520,7 +1519,7 @@ namespace LLM { out = ggml_view_3d(c, out, head_dim, value_heads, n, head_dim * sizeof(float), head_dim * value_heads * sizeof(float), 0); out = std::dynamic_pointer_cast(blocks["norm"])->forward(ctx, out); auto z = ggml_reshape_3d(c, linear("in_proj_z"), head_dim, value_heads, n); - out = ggml_mul(c, out, ggml_silu(c, z)); + out = ggml_swiglu_split(c, z, out); out = ggml_reshape_3d(c, out, head_dim * value_heads, n, 1); return std::dynamic_pointer_cast(blocks["out_proj"])->forward(ctx, out); } diff --git a/src/model/te/t5.hpp b/src/model/te/t5.hpp index f445fe8005..7e7ac30984 100644 --- a/src/model/te/t5.hpp +++ b/src/model/te/t5.hpp @@ -156,9 +156,9 @@ struct T5DenseGatedActDense : public UnaryBlock { auto wi_1 = std::dynamic_pointer_cast(blocks["wi_1"]); auto wo = std::dynamic_pointer_cast(blocks["wo"]); - auto hidden_gelu = ggml_ext_gelu(ctx->ggml_ctx, wi_0->forward(ctx, x), true); + auto hidden_gate = wi_0->forward(ctx, x); auto hidden_linear = wi_1->forward(ctx, x); - x = ggml_mul_inplace(ctx->ggml_ctx, hidden_gelu, hidden_linear); + x = ggml_geglu_split(ctx->ggml_ctx, hidden_gate, hidden_linear); x = wo->forward(ctx, x); return x; } diff --git a/src/model/vae/minimax_h3_audio_vae.hpp b/src/model/vae/minimax_h3_audio_vae.hpp index 02829a4d0d..2dc8c07e2d 100644 --- a/src/model/vae/minimax_h3_audio_vae.hpp +++ b/src/model/vae/minimax_h3_audio_vae.hpp @@ -134,8 +134,8 @@ namespace MiniMaxH3 { auto w1 = std::dynamic_pointer_cast(blocks["w1"]); auto w2 = std::dynamic_pointer_cast(blocks["w2"]); x = norm->forward(ctx, x); - auto gate = ggml_ext_gelu(ctx->ggml_ctx, w0->forward(ctx, x), true); - return w2->forward(ctx, ggml_mul(ctx->ggml_ctx, gate, w1->forward(ctx, x))); + auto gate = w0->forward(ctx, x); + return w2->forward(ctx, ggml_geglu_split(ctx->ggml_ctx, gate, w1->forward(ctx, x))); } }; diff --git a/src/model/vae/minimax_h3_vae.hpp b/src/model/vae/minimax_h3_vae.hpp index d397a40df9..1e06558a19 100644 --- a/src/model/vae/minimax_h3_vae.hpp +++ b/src/model/vae/minimax_h3_vae.hpp @@ -316,11 +316,8 @@ namespace MiniMaxH3VAE { ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { auto w1 = std::dynamic_pointer_cast(blocks["w1"]); auto w2 = std::dynamic_pointer_cast(blocks["w2"]); - auto gate = ggml_ext_chunk(ctx->ggml_ctx, w1->forward(ctx, x), 2, 0); - return w2->forward(ctx, - ggml_mul(ctx->ggml_ctx, - ggml_silu(ctx->ggml_ctx, gate[0]), - gate[1])); + auto gate = ggml_swiglu(ctx->ggml_ctx, w1->forward(ctx, x)); + return w2->forward(ctx, gate); } }; From 0a067120b388bfefb9a87b96960cb548b97e9157 Mon Sep 17 00:00:00 2001 From: leejet Date: Sat, 31 Oct 2026 14:55:58 +0800 Subject: [PATCH 2/3] perf: optimize Vulkan kernels, graph fusion, and workspace reuse (#2251) --- docs/backend.md | 3 ++ ggml | 2 +- src/core/compute_workspace.cpp | 55 +++++++++++++++++++++++++++++++++- src/core/compute_workspace.h | 5 +++- src/core/ggml_extend.cpp | 39 ++++++++++++++++-------- src/core/ggml_runner.cpp | 2 +- 6 files changed, 89 insertions(+), 17 deletions(-) diff --git a/docs/backend.md b/docs/backend.md index c5d98f0e89..49ca0632b6 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -247,6 +247,9 @@ These are backend-level defaults. GGML still chooses the actual kernel based on weight type, shape, and hardware; enabling direct convolution does not guarantee that every operation uses implicit GEMM. The CUDA-family implicit GEMM paths require F16 weights and can use ordinary direct kernels for other cases. +For 3D convolution, a supported direct kernel is also used as a fallback when +the im2col decomposition is unsupported for the actual tensor sizes, including +when an intermediate reshape or matrix multiplication exceeds backend limits. This option does not change the separately created image upscaler context. Repeating `--conv-direct` replaces the previous assignment string. Existing diff --git a/ggml b/ggml index 7b2b435da0..6a89493756 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit 7b2b435da087614be772ee6f5ea80e68eca81de6 +Subproject commit 6a89493756a18bd9cd8875bc4db17e329818b13f diff --git a/src/core/compute_workspace.cpp b/src/core/compute_workspace.cpp index 462a5e3f3c..de1cea8631 100644 --- a/src/core/compute_workspace.cpp +++ b/src/core/compute_workspace.cpp @@ -92,7 +92,8 @@ namespace sd { ComputeWorkspace::Measurement ComputeWorkspace::measure( ggml_cgraph* graph, const std::function& external_backend, - const AssignNodes& assign_nodes) { + const AssignNodes& assign_nodes, + bool cache_measurement) { std::vector tensors; std::unordered_set seen; auto visit = [&](const ggml_tensor* tensor) { @@ -112,6 +113,52 @@ namespace sd { visit(source); } } + std::vector key; + if (cache_measurement) { + key.reserve(tensors.size() * sizeof(ggml_tensor)); + auto append = [&](const auto& value) { + const auto* bytes = reinterpret_cast(&value); + key.insert(key.end(), bytes, bytes + sizeof(value)); + }; + append(graph->n_nodes); + append(graph->n_leafs); + append(graph->order); + for (auto tensor : tensors) { + append(tensor); + append(tensor->type); + append(tensor->ne); + append(tensor->nb); + append(tensor->op); + append(tensor->op_params); + append(tensor->flags); + append(tensor->src); + append(tensor->view_src); + append(tensor->view_offs); + append(tensor->name); + const auto external = external_backend(tensor); + append(external); + if (external == nullptr || tensor->view_src != nullptr) { + append(tensor->buffer); + append(tensor->data); + append(tensor->extra); + if (tensor->buffer != nullptr) { + const auto usage = ggml_backend_buffer_get_usage(tensor->buffer); + const auto type = ggml_backend_buffer_get_type(tensor->buffer); + append(usage); + append(type); + } + } + const size_t index = ggml_hash_find(&graph->visited_hash_set, tensor); + const int32_t uses = index != GGML_HASHSET_FULL && ggml_bitset_get(graph->visited_hash_set.used, index) + ? graph->use_counts[index] + : 0; + append(uses); + } + // Match the full allocation metadata, including topology and external placement. + if (key == measurement_key_) { + return {measured_buffers_}; + } + } const size_t graph_size = tensors.size() + 8; auto context = ggml_init({tensors.size() * ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false), nullptr, true}); if (context == nullptr) { @@ -178,6 +225,10 @@ namespace sd { ggml_backend_buffer_free(entry.second); } ggml_free(context); + if (cache_measurement && !result.buffers.empty()) { + measurement_key_ = std::move(key); + measured_buffers_ = result.buffers; + } return result; } @@ -212,6 +263,8 @@ namespace sd { if (active_) { return false; } + measurement_key_.clear(); + measured_buffers_.clear(); ggml_backend_sched_free(scheduler_); scheduler_ = nullptr; scheduler_capacity_ = 0; diff --git a/src/core/compute_workspace.h b/src/core/compute_workspace.h index b0491c0910..8dee5da556 100644 --- a/src/core/compute_workspace.h +++ b/src/core/compute_workspace.h @@ -19,6 +19,8 @@ namespace sd { ggml_backend_sched_t scheduler_ = nullptr; size_t scheduler_capacity_ = 0; bool active_ = false; + std::vector measurement_key_; + std::vector measured_buffers_; ggml_backend_sched_t make_scheduler(size_t graph_size); @@ -41,7 +43,8 @@ namespace sd { Measurement measure( ggml_cgraph* graph, const std::function& external_backend, - const AssignNodes& assign_nodes); + const AssignNodes& assign_nodes, + bool cache_measurement = false); void synchronize() const; bool segment_end() noexcept; bool release(); diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp index d3548f46e9..6d72caedfb 100644 --- a/src/core/ggml_extend.cpp +++ b/src/core/ggml_extend.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "core/ggml_extend_backend.h" @@ -478,21 +479,33 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx, x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 1, 3, 2)); x = ggml_reshape_4d(ctx, x, im2col->ne[1], im2col->ne[2], OD, OC * N); } else { - // ggml_conv_3d decomposes into GGML_OP_IM2COL_3D, which some backends - // (e.g. Metal, see #850) do not implement. Fall back to - // GGML_OP_CONV_3D on those backends. - bool im2col_3d_supported = true; + auto convolution = ggml_conv_3d(ctx, w, x, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2); if (backend != nullptr) { - ggml_tensor* im2col = ggml_im2col_3d(ctx, w, x, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, w->type); - im2col_3d_supported = ggml_backend_supports_op(backend, im2col); - } - if (im2col_3d_supported) { - x = ggml_conv_3d(ctx, w, x, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2); - } else { - int64_t OC = w->ne[3] / IC; - int64_t N = x->ne[3] / IC; - x = ggml_conv_3d_direct(ctx, w, x, s0, s1, s2, p0, p1, p2, d0, d1, d2, (int)IC, (int)N, (int)OC); + // im2col can support large buffers that its reshape/GEMM consumers cannot. + std::vector nodes{convolution}; + bool supported = true; + for (size_t i = 0; i < nodes.size() && supported; ++i) { + auto node = nodes[i]; + if (node == x || node == w) { + continue; + } + supported = ggml_backend_supports_op(backend, node); + for (auto source : node->src) { + if (source != nullptr) { + nodes.push_back(source); + } + } + } + if (!supported) { + int64_t OC = w->ne[3] / IC; + int64_t N = x->ne[3] / IC; + auto fallback = ggml_conv_3d_direct(ctx, w, x, s0, s1, s2, p0, p1, p2, d0, d1, d2, (int)IC, (int)N, (int)OC); + if (ggml_backend_supports_op(backend, fallback)) { + convolution = fallback; + } + } } + x = convolution; } if (scale != 1.f) { diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp index 7d8aeb08e2..8c2c5ba218 100644 --- a/src/core/ggml_runner.cpp +++ b/src/core/ggml_runner.cpp @@ -735,7 +735,7 @@ ComputeWorkspace::Measurement GGMLRunner::measure(ggml_cgraph* graph, const std: auto assign_nodes = [&](ggml_backend_sched_t scheduler, ggml_cgraph* copy) { pin_multi_device_nodes(scheduler, copy, graph); }; - return workspace_.measure(graph, external_backend, assign_nodes); + return workspace_.measure(graph, external_backend, assign_nodes, !is_multi_device()); } std::vector GGMLRunner::memory_requests( From c6dfa9def12fd0cb36ac7149ba30a75792bedbf0 Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 1 Nov 2026 00:26:59 +0800 Subject: [PATCH 3/3] perf(vulkan): update ggml for H3 inference optimizations (#2252) --- docs/performance.md | 19 +++++++++++++++++++ ggml | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/performance.md b/docs/performance.md index 98c785681e..f347dc1721 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -159,6 +159,25 @@ allocator/scheduler capacity. Compatible gallocr reservations are reused across graphs; idle workspaces can be reclaimed under pressure and are freed at runner completion. Cross-graph caches survive individual graphs, but cut buffers do not. +On discrete Vulkan devices, runners reuse released device buffers of the same +size to reduce allocation overhead during segmented execution. This idle buffer +cache is limited to 1 GiB per device and is reclaimed before allocating a new +size or at runner completion. It is outside the managed `--max-vram` budget, +so physical VRAM usage can be higher. Set `GGML_VK_REUSE_BUFFERS=0` to disable +it. Unified-memory and host-memory modes do not use this cache. + +Discrete NVIDIA and non-GCN AMD Vulkan devices use a separate transfer queue by +default when available and graphics queue use is disabled, so weight prefetch can overlap compute. Set +`GGML_VK_ASYNC_USE_TRANSFER_QUEUE=0` to disable it. + +On discrete NVIDIA devices, allocation chunks can grow up to 4 GiB, bounded by the device's buffer and allocation +limits, to reduce fragmentation within large graph workspaces. Other devices +retain the 1 GiB default. `GGML_VK_SUBALLOCATION_BLOCK_SIZE` overrides this limit +in bytes on any vendor; for example, `1073741824` selects 1 GiB and `4294967296` +selects 4 GiB, subject to device limits. This is a soft chunk limit, not a reservation +or a change to `--max-vram`: a single oversized tensor or exhausting the allocator's +chunk count can require a larger allocation, which may fail. + The recommended shape for "biggest model my card can host" is: ```shell diff --git a/ggml b/ggml index 6a89493756..c57b71a344 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit 6a89493756a18bd9cd8875bc4db17e329818b13f +Subproject commit c57b71a3448809d07d49360273201bb5a5e7cafb