From 5c3081d39d4484ee97e0fdab425df95f5ee89565 Mon Sep 17 00:00:00 2001 From: farawayso Date: Thu, 29 Oct 2026 21:56:06 +0800 Subject: [PATCH 01/10] feat: MiniMax-H3 latent hires second pass Adds --hires support for MiniMax-H3 vid_gen, i.e. the draft -> latent upscale -> refine workflow: the first pass runs at the draft resolution, the packed AV latent is then upscaled in latent space (interpolating or model-backed learned upscaler), and a second img2img-style sampling pass runs at the target resolution using the existing hires sigma schedule. The time axis is never resampled, so the packed audio latent length stays valid across the stage change and the soundtrack is preserved. Refs #2204 --- docs/minimax_h3.md | 48 ++ src/model/upscaler/h3_latent_upscaler.hpp | 560 ++++++++++++++++++++++ src/pipeline/video.cpp | 386 +++++++++++++-- 3 files changed, 967 insertions(+), 27 deletions(-) create mode 100644 src/model/upscaler/h3_latent_upscaler.hpp diff --git a/docs/minimax_h3.md b/docs/minimax_h3.md index 0c17ba0e01..a397634d06 100644 --- a/docs/minimax_h3.md +++ b/docs/minimax_h3.md @@ -99,6 +99,54 @@ The C API exposes the same inputs through `ref_images`, `ref_videos`, and `ref_audios` in `sd_vid_gen_params_t`. Each `sd_ref_video_t` supplies its own frame rate and optional soundtrack; non-24-fps inputs are resampled internally. +## Latent hires second pass (two-stage sampling) + +MiniMax-H3 supports `--hires` with two interchangeable second-stage upscalers: + +- an interpolating latent upscaler (`Latent`, `Latent (nearest)`, + `Latent (bicubic)`, ...): after the first pass finishes, the packed AV latent + is resampled in latent space to `--hires-width` x `--hires-height` and + refined by a second sampling pass; +- a model-backed upscaler (`--hires-upscaler ` + `--hires-upscalers-dir`): + a H3-shaped learned latent upscaler checkpoint runs between the passes, + like the LTX spatial upsampler. This is the sd.cpp equivalent of the + two-sampler workflows used for H3 in node-based frontends (e.g. the + community 3D-conv latent upscaler from + [LBH-123-AI/Minimax_h3_latent_Upscaler](https://huggingface.co/LBH-123-AI/Minimax_h3_latent_Upscaler)). + +Draft at a low megapixel count (e.g. 0.4 MP) and refine at a higher one +(e.g. 0.8 MP) instead of paying for the full resolution in every step: + +```sh +.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "a lovely cat" --cfg-scale 1.0 -v -W 864 -H 480 --diffusion-fa --offload-to-cpu --rng cpu --video-frames 56 --hires --hires-upscaler "Latent (bicubic)" --hires-width 1216 --hires-height 672 --hires-steps 4 --hires-denoising-strength 0.4 +``` + +or with the learned upscaler: + +```sh +.\bin\Release\sd-cli.exe -M vid_gen ... --hires --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler h3-latent-upscaler-3d --hires-width 1216 --hires-height 672 --hires-steps 4 --hires-denoising-strength 0.4 +``` + +Notes for this mode: + +- The frame count is never changed by the second stage, so the packed audio + latent length stays valid and the generated soundtrack is preserved. +- Width and height still have to be multiples of 32; the hires target is + aligned like any other resolution. The learned upscaler additionally wants + an even latent size on both axes, which 32-pixel targets provide. +- `--hires-sigmas` can replace `--hires-denoising-strength` when the second + stage should resume from an explicit sigma schedule, which is how most + two-stage H3 workflows split the sampler. +- Keyframe conditioning (`--init-img` / `--end-img`) is not available together + with hires yet: those latents are encoded at the first stage resolution and + would have to be re-encoded for the hires canvas. Use T2VA or Ref2VA. +- The second stage runs at the larger latent, so it is the memory-hungry half; + combine it with quantization, `--vae-tiling` and `--offload-to-cpu` on + small cards. +- The learned upscaler is scale-conditional: the target-to-source ratio is + fed into the network as an embedding, 1.2-2.0x is the trained domain, and + downscaling is rejected. + ## Shape and runtime notes - Width and height are aligned upward to a multiple of 32. diff --git a/src/model/upscaler/h3_latent_upscaler.hpp b/src/model/upscaler/h3_latent_upscaler.hpp new file mode 100644 index 0000000000..94a9627dd6 --- /dev/null +++ b/src/model/upscaler/h3_latent_upscaler.hpp @@ -0,0 +1,560 @@ +#ifndef __SD_MODEL_UPSCALER_H3_LATENT_UPSCALER_HPP__ +#define __SD_MODEL_UPSCALER_H3_LATENT_UPSCALER_HPP__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/ggml_extend.h" +#include "core/ggml_graph_cut.h" +#include "core/ggml_runner.h" +#include "core/tensor.hpp" +#include "core/util.h" +#include "model/common/ggml_block.hpp" +#include "model/upscaler/ltx_latent_upscaler.hpp" +#include "model_loader.h" + +// MiniMax-H3 learned latent upscaler (H3-shaped 3D-conv LatentResizer). +// Ported from the community ComfyUI node LBH-123-AI/Comfyui_Minimax_h3_latent_Upscaler +// (MIT) with checkpoint LBH-123-AI/Minimax_h3_latent_Upscaler (Apache-2.0): +// +// h = conv_in(x) Conv3d 24 -> 512, k3 +// for N in checkpoint module-list order: +// h = in_blocks.N(h, emb) ResBlock (AdaLN) or TemporalConv +// m = spatial_bilinear(h, target_hw) time axis untouched +// for N in checkpoint module-list order: +// g = out_blocks.N(g, emb) +// y = conv_out(silu(norm_out(g))) +// +// The per-channel latent (de)normalization of the community model is applied on the +// host around the two GPU stages, mirroring the LTX flow's un_normalize/normalize. +// Depthwise temporal convs (kernel (k,1,1)) are shipped as full block-diagonal +// Conv3d weights by the converter script, so only stock Conv3d ops are required. +// The middle spatial resample runs on the host between the two GPU graphs; the +// time axis is never resampled, so the packed audio latent length stays valid. + +namespace H3VUpsampler { + constexpr int H3_UPSAMPLER_GRAPH_SIZE = 8192; + + // Per-channel H3 video-latent statistics (24 channels) from the community checkpoint. + struct H3LatentStats { + static constexpr int kChannels = 24; + static constexpr float kMean[24] = { + 0.858090341091156f, -0.9606591463088989f, 1.0661640167236328f, -0.5090325474739075f, + -0.2727581858634949f, -1.3675414323806763f, -0.2553254961967468f, -0.26907554268836975f, + -0.5376840829849243f, -0.0464097298681736f, 0.6657370328903198f, 0.19690127670764923f, + -0.5460608005523682f, -0.4035342037677765f, -0.23683024942874908f, 0.25928452610969543f, + -0.30133944749832153f, 0.211341992020607f, -1.1206848621368408f, 0.3581933379173279f, + -0.04225143790245056f, 0.2604829967021942f, 0.22864092886447906f, 0.7056031823158264f}; + static constexpr float kStd[24] = { + 1.2223774194717407f, 1.2767263650894165f, 1.6831774711608887f, 1.7549455165863037f, + 1.5636216402053833f, 2.194143533706665f, 0.9653137922286987f, 1.0569885969161987f, + 0.841948926448822f, 0.7729952931404114f, 1.8955937623977661f, 0.946841835975647f, + 0.7996809482574463f, 0.44988900423049927f, 0.7197399735450745f, 0.6936293244361877f, + 2.961095094680786f, 2.7694199085235596f, 3.0496184825897217f, 2.1088054180145264f, + 3.276226282119751f, 3.1627357006073f, 2.2816812992095947f, 2.6127843856811523f}; + }; + + struct H3LatentUpscalerConfig { + int64_t in_channels = 24; + int64_t mid_channels = 512; + int embed_dim = 64; + int temporal_kernel = 5; + // Ordered module-list indices in checkpoint order (res and temporal blocks + // are interleaved in a single list in the state dict). + std::vector in_res, in_temporal, out_res, out_temporal; + bool valid = false; + + static H3LatentUpscalerConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, + const std::string& prefix = "") { + H3LatentUpscalerConfig config; + std::string base = prefix; + if (!base.empty()) { + base += "."; + } + auto find_weight = [&](const std::string& suffix) -> const TensorStorage* { + auto iter = tensor_storage_map.find(base + suffix); + if (iter == tensor_storage_map.end()) { + return nullptr; + } + return &iter->second; + }; + + const TensorStorage* conv_in = find_weight("conv_in.weight"); + if (conv_in == nullptr) { + LOG_ERROR("h3 latent upsampler: 'conv_in.weight' not found under prefix '%s'", prefix.c_str()); + return config; + } + // Conv weights are either PyTorch 5D [OC, IC, k, k, k] or sdcpp 4D [k, k, k, OC*IC]; + // the biases are 1D in both layouts, so read channel counts from the biases. + const TensorStorage* conv_in_bias = find_weight("conv_in.bias"); + if (conv_in_bias != nullptr) { + config.mid_channels = conv_in_bias->ne[0]; + } else if (conv_in->n_dims == 5) { + config.mid_channels = conv_in->ne[0]; + } + const TensorStorage* conv_out_bias = find_weight("conv_out.bias"); + if (conv_out_bias != nullptr) { + config.in_channels = conv_out_bias->ne[0]; + } else if (conv_in->n_dims == 5) { + config.in_channels = conv_in->ne[1]; + } + if (config.mid_channels <= 0 || config.in_channels <= 0) { + LOG_ERROR("h3 latent upsampler: could not infer channel counts from the checkpoint"); + return config; + } + + const TensorStorage* embed_mid = find_weight("embed.2.weight"); + if (embed_mid != nullptr && embed_mid->n_dims >= 1) { + config.embed_dim = static_cast(embed_mid->ne[0]); + } + + std::set in_res_idx, in_temporal_idx, out_res_idx, out_temporal_idx; + for (const auto& [name, _] : tensor_storage_map) { + if (!starts_with(name, base)) { + continue; + } + const std::string rel = name.substr(base.size()); + const bool is_in = starts_with(rel, "in_blocks."); + const bool is_out = starts_with(rel, "out_blocks."); + if (!is_in && !is_out) { + continue; + } + const std::string rest = rel.substr(is_in ? 10 : 11); + const size_t dot = rest.find('.'); + if (dot == std::string::npos) { + continue; + } + int idx = 0; + try { + idx = std::stoi(rest.substr(0, dot)); + } catch (...) { + continue; + } + const std::string part = rest.substr(dot + 1); + if (starts_with(part, "dwconv.") || starts_with(part, "pwconv.")) { + (is_in ? in_temporal_idx : out_temporal_idx).insert(idx); + } else if (starts_with(part, "in_layers.") || starts_with(part, "out_norm.") || + starts_with(part, "out_layers.") || starts_with(part, "emb_layers.")) { + (is_in ? in_res_idx : out_res_idx).insert(idx); + } else if (starts_with(part, "attn.")) { + LOG_ERROR("h3 latent upsampler: attention blocks are not supported by the v1 runner"); + return config; + } + } + config.in_res = std::vector(in_res_idx.begin(), in_res_idx.end()); + config.in_temporal = std::vector(in_temporal_idx.begin(), in_temporal_idx.end()); + config.out_res = std::vector(out_res_idx.begin(), out_res_idx.end()); + config.out_temporal = std::vector(out_temporal_idx.begin(), out_temporal_idx.end()); + if (config.in_res.empty() || config.out_res.empty()) { + LOG_ERROR("h3 latent upsampler: no res blocks found in the checkpoint"); + return config; + } + + auto temporal_kernel_of = [&](std::vector idxs, const char* stage) -> int { + for (int idx : idxs) { + const TensorStorage* dw = find_weight(std::string(stage) + "." + std::to_string(idx) + ".dwconv.weight"); + if (dw != nullptr) { + // Expanded full-conv layout: PyTorch 5D [C, C, k, 1, 1] or sdcpp 4D [k, 1, 1, C*C]. + return static_cast(dw->n_dims == 5 ? dw->ne[2] : dw->ne[0]); + } + } + return 0; + }; + const int k_in = temporal_kernel_of(config.in_temporal, "in_blocks"); + const int k_out = k_in > 0 ? k_in : temporal_kernel_of(config.out_temporal, "out_blocks"); + if (k_out > 0) { + config.temporal_kernel = k_out; + } else { + config.temporal_kernel = 1; // no temporal convs in the checkpoint + } + + config.valid = true; + LOG_VERBOSE("h3 latent upsampler: in_channels = %" PRId64 ", mid_channels = %" PRId64 + ", in_res = %zu, in_temporal = %zu, out_res = %zu, out_temporal = %zu, temporal_kernel = %d, embed_dim = %d", + config.in_channels, + config.mid_channels, + config.in_res.size(), + config.in_temporal.size(), + config.out_res.size(), + config.out_temporal.size(), + config.temporal_kernel, + config.embed_dim); + return config; + } + }; + + // H3 res block: in_layers (GN -> SiLU -> Conv3d k3); AdaLN modulation + // out_norm(h) * (1 + scale) + shift from the scale-embedding MLP; out_layers + // (SiLU -> zero-init Conv3d k3); identity skip. + class H3ResBlock : public GGMLBlock { + protected: + int64_t channels; + int64_t embed_dim; + + void init_params(ggml_context* ctx, + const String2TensorStorage& tensor_storage_map, + const std::string& prefix) override { + auto make_conv = [&](const std::string& name) { + blocks[name] = std::shared_ptr(new Conv3d(channels, + channels, + {3, 3, 3}, + {1, 1, 1}, + {1, 1, 1})); + }; + make_conv("in_layers.2"); + make_conv("out_layers.2"); + blocks["in_layers.0"] = std::shared_ptr(new LTXVUpsampler::VideoGroupNorm(32, channels)); + blocks["out_norm"] = std::shared_ptr(new LTXVUpsampler::VideoGroupNorm(32, channels)); + blocks["emb_layers.1"] = std::shared_ptr(new Linear(embed_dim, 2 * channels, true)); + } + + public: + H3ResBlock(int64_t channels, int64_t embed_dim) + : channels(channels), + embed_dim(embed_dim) {} + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* emb) { + auto in_layers = std::dynamic_pointer_cast(blocks["in_layers.2"]); + auto in_norm = std::dynamic_pointer_cast(blocks["in_layers.0"]); + auto out_layers = std::dynamic_pointer_cast(blocks["out_layers.2"]); + auto out_norm = std::dynamic_pointer_cast(blocks["out_norm"]); + auto emb_layers = std::dynamic_pointer_cast(blocks["emb_layers.1"]); + + // in_layers: GN -> SiLU -> Conv3d (in_norm is the GroupNorm, in_layers is the Conv3d) + ggml_tensor* h = in_norm->forward(ctx, x); + h = ggml_silu_inplace(ctx->ggml_ctx, h); + h = in_layers->forward(ctx, h); + + // AdaLN: emb_out = emb_layers(SiLU(emb)) -> split [scale, shift] + ggml_tensor* emb_silu = ggml_silu_inplace(ctx->ggml_ctx, emb); + ggml_tensor* emb_out = emb_layers->forward(ctx, emb_silu); + ggml_tensor* emb_1d = ggml_reshape_1d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, emb_out)); + ggml_tensor* scale = ggml_reshape_4d(ctx->ggml_ctx, ggml_ext_slice(ctx->ggml_ctx, emb_1d, 0, 0, channels), 1, 1, 1, channels); + ggml_tensor* shift = ggml_reshape_4d(ctx->ggml_ctx, ggml_ext_slice(ctx->ggml_ctx, emb_1d, 0, channels, 2 * channels), 1, 1, 1, channels); + ggml_tensor* one = ggml_new_tensor_1d(ctx->ggml_ctx, GGML_TYPE_F32, 1); + ggml_set_f32_1d(ctx->ggml_ctx, one, 0, 1.0f); + ggml_tensor* gain = ggml_add(ctx->ggml_ctx, scale, one); + + // out_norm * (1 + scale) + shift, then SiLU -> out_layers + h = out_norm->forward(ctx, h); + h = ggml_mul_inplace(ctx->ggml_ctx, h, gain); + h = ggml_add_inplace(ctx->ggml_ctx, h, shift); + h = ggml_silu_inplace(ctx->ggml_ctx, h); + h = out_layers->forward(ctx, h); + return ggml_add(ctx->ggml_ctx, h, x); // identity skip + } + }; + + // Temporal conv: GN -> SiLU -> depthwise Conv3d (k,1,1) shipped as a full + // block-diagonal Conv3d -> pointwise Conv3d (zero-init at training) + identity. + class H3TemporalConv : public GGMLBlock { + protected: + int64_t channels; + int temporal_kernel; + + void init_params(ggml_context* ctx, + const String2TensorStorage& tensor_storage_map, + const std::string& prefix) override { + const int pad = temporal_kernel / 2; + blocks["dwconv"] = std::shared_ptr(new Conv3d(channels, + channels, + {temporal_kernel, 1, 1}, + {1, 1, 1}, + {pad, 0, 0})); + blocks["pwconv"] = std::shared_ptr(new Conv3d(channels, + channels, + {1, 1, 1}, + {1, 1, 1}, + {0, 0, 0})); + blocks["norm"] = std::shared_ptr(new LTXVUpsampler::VideoGroupNorm(32, channels)); + } + + public: + H3TemporalConv(int64_t channels, int temporal_kernel) + : channels(channels), + temporal_kernel(temporal_kernel) {} + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { + auto dwconv = std::dynamic_pointer_cast(blocks["dwconv"]); + auto pwconv = std::dynamic_pointer_cast(blocks["pwconv"]); + auto norm = std::dynamic_pointer_cast(blocks["norm"]); + + ggml_tensor* h = norm->forward(ctx, x); + h = ggml_silu_inplace(ctx->ggml_ctx, h); + h = dwconv->forward(ctx, h); + h = pwconv->forward(ctx, h); + return ggml_add(ctx->ggml_ctx, h, x); + } + }; + + // Scale-embedding MLP: Linear(1 -> E) -> SiLU -> Linear(E -> E). + class H3Embed : public GGMLBlock { + protected: + int embed_dim; + + void init_params(ggml_context* ctx, + const String2TensorStorage& tensor_storage_map, + const std::string& prefix) override { + blocks["embed.0"] = std::shared_ptr(new Linear(1, embed_dim, true)); + blocks["embed.2"] = std::shared_ptr(new Linear(embed_dim, embed_dim, true)); + } + + public: + explicit H3Embed(int embed_dim) + : embed_dim(embed_dim) {} + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* scale_input) { + auto l0 = std::dynamic_pointer_cast(blocks["embed.0"]); + auto l2 = std::dynamic_pointer_cast(blocks["embed.2"]); + ggml_tensor* h = l0->forward(ctx, ggml_reshape_2d(ctx->ggml_ctx, scale_input, 1, 1)); + h = ggml_silu_inplace(ctx->ggml_ctx, h); + h = l2->forward(ctx, h); + return ggml_silu_inplace(ctx->ggml_ctx, h); + } + }; + + class H3LatentUpscaler : public GGMLBlock { + public: + H3LatentUpscalerConfig config; + + explicit H3LatentUpscaler(H3LatentUpscalerConfig config_) + : config(std::move(config_)) { + for (int idx : config.in_res) { + blocks["in_blocks." + std::to_string(idx)] = std::shared_ptr(new H3ResBlock(config.mid_channels, + config.embed_dim)); + } + for (int idx : config.in_temporal) { + blocks["in_blocks." + std::to_string(idx)] = std::shared_ptr(new H3TemporalConv(config.mid_channels, + config.temporal_kernel)); + } + for (int idx : config.out_res) { + blocks["out_blocks." + std::to_string(idx)] = std::shared_ptr(new H3ResBlock(config.mid_channels, + config.embed_dim)); + } + for (int idx : config.out_temporal) { + blocks["out_blocks." + std::to_string(idx)] = std::shared_ptr(new H3TemporalConv(config.mid_channels, + config.temporal_kernel)); + } + blocks["conv_in"] = std::shared_ptr(new Conv3d(config.in_channels, + config.mid_channels, + {3, 3, 3}, + {1, 1, 1}, + {1, 1, 1})); + blocks["norm_out"] = std::shared_ptr(new LTXVUpsampler::VideoGroupNorm(32, config.mid_channels)); + blocks["conv_out"] = std::shared_ptr(new Conv3d(config.mid_channels, + config.in_channels, + {3, 3, 3}, + {1, 1, 1}, + {1, 1, 1})); + blocks["embed"] = std::shared_ptr(new H3Embed(config.embed_dim)); + } + + // Stage A: conv_in + in-block chain at the original spatial size. + ggml_tensor* forward_stage_a(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* scale_input) { + auto conv_in = std::dynamic_pointer_cast(blocks["conv_in"]); + auto embed = std::dynamic_pointer_cast(blocks["embed"]); + + ggml_tensor* h = conv_in->forward(ctx, x); + sd::ggml_graph_cut::mark_graph_cut(h, "h3_latent_upscaler.conv_in", "x"); + + ggml_tensor* emb = embed->forward(ctx, scale_input); + std::set temporal_set(config.in_temporal.begin(), config.in_temporal.end()); + int max_idx = 0; + for (int idx : config.in_res) { + max_idx = std::max(max_idx, idx); + } + for (int idx : config.in_temporal) { + max_idx = std::max(max_idx, idx); + } + for (int i = 0; i <= max_idx; ++i) { + if (temporal_set.count(i)) { + auto block = std::dynamic_pointer_cast(blocks["in_blocks." + std::to_string(i)]); + h = block->forward(ctx, h); + } else { + auto block = std::dynamic_pointer_cast(blocks["in_blocks." + std::to_string(i)]); + h = block->forward(ctx, h, emb); + } + sd::ggml_graph_cut::mark_graph_cut(h, "h3_latent_upscaler.in_blocks." + std::to_string(i), "x"); + } + return h; + } + + // Stage B: out-block chain + norm_out + SiLU + conv_out at the target spatial size. + ggml_tensor* forward_stage_b(GGMLRunnerContext* ctx, ggml_tensor* m, ggml_tensor* scale_input) { + auto conv_out = std::dynamic_pointer_cast(blocks["conv_out"]); + auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); + auto embed = std::dynamic_pointer_cast(blocks["embed"]); + + ggml_tensor* g = m; + ggml_tensor* emb = embed->forward(ctx, scale_input); + std::set temporal_set(config.out_temporal.begin(), config.out_temporal.end()); + int max_idx = 0; + for (int idx : config.out_res) { + max_idx = std::max(max_idx, idx); + } + for (int idx : config.out_temporal) { + max_idx = std::max(max_idx, idx); + } + for (int i = 0; i <= max_idx; ++i) { + if (temporal_set.count(i)) { + auto block = std::dynamic_pointer_cast(blocks["out_blocks." + std::to_string(i)]); + g = block->forward(ctx, g); + } else { + auto block = std::dynamic_pointer_cast(blocks["out_blocks." + std::to_string(i)]); + g = block->forward(ctx, g, emb); + } + sd::ggml_graph_cut::mark_graph_cut(g, "h3_latent_upscaler.out_blocks." + std::to_string(i), "x"); + } + + g = norm_out->forward(ctx, g); + g = ggml_silu_inplace(ctx->ggml_ctx, g); + g = conv_out->forward(ctx, g); + sd::ggml_graph_cut::mark_graph_cut(g, "h3_latent_upscaler.conv_out", "x"); + return g; + } + }; + + struct H3LatentUpscalerRunner : public GGMLRunner { + H3LatentUpscalerConfig config; + std::unique_ptr model; + std::string weight_prefix; + + H3LatentUpscalerRunner(ggml_backend_t backend, + const String2TensorStorage& tensor_storage_map, + const std::string& prefix = "", + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), + config(H3LatentUpscalerConfig::detect_from_weights(tensor_storage_map, prefix)), + weight_prefix(prefix) { + if (!config.valid || config.embed_dim <= 0 || config.mid_channels <= 0) { + LOG_ERROR("unsupported H3 latent upsampler config: in_channels=%" PRId64 " mid_channels=%" PRId64 " embed_dim=%d temporal_kernel=%d", + config.in_channels, + config.mid_channels, + config.embed_dim, + config.temporal_kernel); + return; + } + model = std::make_unique(config); + model->init(params_ctx, tensor_storage_map, prefix); + } + + std::string get_desc() override { + return "h3_latent_upscaler"; + } + + void get_param_tensors(std::map& tensors) { + if (model) { + model->get_param_tensors(tensors, weight_prefix); + } + } + + // x: normalized H3 video latent [W, H, T, C] (or 5D batch 1); returns the + // denormalized upscaled latent at the target spatial size. The scale-1 + // embedding input is a graph leaf so both stage graphs carry it. + sd::Tensor compute(const int n_threads, + const sd::Tensor& x, + int64_t target_w, + int64_t target_h) { + if (model == nullptr) { + LOG_ERROR("H3 latent upsampler is not loaded"); + return {}; + } + if (x.dim() != 4 && x.dim() != 5) { + LOG_ERROR("H3 latent upsampler expects a 4D or 5D video latent, got dim=%lld", (long long)x.dim()); + return {}; + } + if (x.dim() == 5 && x.shape()[4] != 1) { + LOG_ERROR("H3 latent upsampler currently supports batch size 1, got batch=%lld", (long long)x.shape()[4]); + return {}; + } + if (x.shape()[3] != config.in_channels) { + LOG_ERROR("H3 latent upsampler expected %" PRId64 " channels, got %lld", config.in_channels, (long long)x.shape()[3]); + return {}; + } + const int64_t in_w = x.shape()[0]; + const int64_t in_h = x.shape()[1]; + const int64_t T = x.shape()[2]; + if (target_w % 2 != 0 || target_h % 2 != 0) { + LOG_ERROR("H3 latent upsampler target latent size must be even (pixel size a multiple of 32), got %" PRId64 "x%" PRId64, + target_w, + target_h); + return {}; + } + // Non-isotropic targets: the single scale-embedding input uses the mean of the + // width/height scale factors. + const float scale = (static_cast(target_w) / static_cast(in_w) + + static_cast(target_h) / static_cast(in_h)) / + 2.0f; + if (scale < 1.0f - 1e-3f) { + LOG_ERROR("H3 latent upsampler only supports upscaling (scale >= 1.0), got %.3f", scale); + return {}; + } + + const size_t expected_dim = static_cast(x.dim()); + // Stage A: conv_in + in-block chain at the original spatial size. + auto get_graph_a = [this, &x, scale]() -> ggml_cgraph* { + ggml_cgraph* gf = new_graph_custom(H3_UPSAMPLER_GRAPH_SIZE); + ggml_tensor* x_t = make_input(x); + ggml_tensor* scale_input = ggml_new_tensor_1d(gf->ctx, GGML_TYPE_F32, 1); + ggml_set_f32_1d(gf->ctx, scale_input, 0, scale - 1.0f); + auto runner_ctx = get_context(gf); + ggml_tensor* out = model->forward_stage_a(&runner_ctx, x_t, scale_input); + ggml_build_forward_expand(gf, out); + return gf; + }; + std::optional> mid_opt = GGMLRunner::compute(get_graph_a, n_threads, false, false); + runner_end(); + if (!mid_opt.has_value() || mid_opt->empty()) { + LOG_ERROR("H3 latent upsampler stage A (in-block chain) failed"); + return {}; + } + sd::Tensor mid = std::move(*mid_opt); + + // Middle spatial resample on the host; the time axis is never touched. + std::vector mid_shape = mid.shape(); + mid_shape[0] = target_w; + mid_shape[1] = target_h; + sd::Tensor mid_up = sd::ops::interpolate(mid, mid_shape, sd::ops::InterpolateMode::Bilinear, false, false); + if (mid_up.empty()) { + LOG_ERROR("H3 latent upsampler middle spatial interpolate failed: %" PRId64 "x%" PRId64 " -> %" PRId64 "x%" PRId64, + in_w, in_h, target_w, target_h); + return {}; + } + + // Stage B: out-block chain + conv_out at the target spatial size. + auto get_graph_b = [this, &mid_up, scale]() -> ggml_cgraph* { + ggml_cgraph* gf = new_graph_custom(H3_UPSAMPLER_GRAPH_SIZE); + ggml_tensor* m_t = make_input(mid_up); + ggml_tensor* scale_input = ggml_new_tensor_1d(gf->ctx, GGML_TYPE_F32, 1); + ggml_set_f32_1d(gf->ctx, scale_input, 0, scale - 1.0f); + auto runner_ctx = get_context(gf); + ggml_tensor* out = model->forward_stage_b(&runner_ctx, m_t, scale_input); + ggml_build_forward_expand(gf, out); + return gf; + }; + std::optional> out_opt = GGMLRunner::compute(get_graph_b, n_threads, false, false); + runner_end(); + if (!out_opt.has_value() || out_opt->empty()) { + LOG_ERROR("H3 latent upsampler stage B (out-block chain) failed"); + return {}; + } + sd::Tensor out = std::move(*out_opt); + if (out.shape()[0] != target_w || out.shape()[1] != target_h || out.shape()[2] != T) { + LOG_ERROR("H3 latent upsampler output shape mismatch: got %" PRId64 "x%" PRId64 "x%" PRId64 " expected %" PRId64 "x%" PRId64 "x%" PRId64, + out.shape()[0], out.shape()[1], out.shape()[2], target_w, target_h, T); + return {}; + } + return restore_trailing_singleton_dims(std::move(out), expected_dim); + } + }; + +} // namespace H3VUpsampler + +#endif // __SD_MODEL_UPSCALER_H3_LATENT_UPSCALER_HPP__ diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index a07b93175c..ab860087e0 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -12,6 +12,7 @@ #include "core/rng_philox.hpp" #include "diffusion_engine.h" #include "model/diffusion/minimax_h3.hpp" +#include "model/upscaler/h3_latent_upscaler.hpp" #include "model/upscaler/ltx_latent_upscaler.hpp" #include "model/vae/audio_vae.hpp" #include "model/vae/vae.hpp" @@ -1317,6 +1318,280 @@ namespace sd::pipeline { return result_images; } + // Video models without a dedicated latent upsampler checkpoint (MiniMax-H3) + // resample the packed [w, h, f, c] latent itself with an interpolating kernel. + static bool hires_upscaler_is_latent_interpolation(enum sd_hires_upscaler_t upscaler) { + switch (upscaler) { + case SD_HIRES_UPSCALER_LATENT: + case SD_HIRES_UPSCALER_LATENT_NEAREST: + case SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT: + case SD_HIRES_UPSCALER_LATENT_ANTIALIASED: + case SD_HIRES_UPSCALER_LATENT_BICUBIC: + case SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED: + return true; + default: + return false; + } + } + + static sd::ops::InterpolateMode hires_latent_interpolate_mode(enum sd_hires_upscaler_t upscaler, + bool* antialias) { + if (antialias != nullptr) { + *antialias = false; + } + switch (upscaler) { + case SD_HIRES_UPSCALER_LATENT_NEAREST: + return sd::ops::InterpolateMode::Nearest; + case SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT: + return sd::ops::InterpolateMode::NearestExact; + case SD_HIRES_UPSCALER_LATENT_BICUBIC: + return sd::ops::InterpolateMode::Bicubic; + case SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED: + if (antialias != nullptr) { + *antialias = true; + } + return sd::ops::InterpolateMode::Bicubic; + case SD_HIRES_UPSCALER_LATENT_ANTIALIASED: + if (antialias != nullptr) { + *antialias = true; + } + return sd::ops::InterpolateMode::Bilinear; + default: + // SD_HIRES_UPSCALER_LATENT + return sd::ops::InterpolateMode::Bilinear; + } + } + + // Per-channel H3 video-latent (de)normalization, as applied by the community + // learned upscaler around its network. + static sd::Tensor h3_normalize_video_latent(const sd::Tensor& x) { + if (x.empty() || x.shape()[3] < H3VUpsampler::H3LatentStats::kChannels) { + // Defensive no-op: the learned runner validates the channel count before this runs. + return x; + } + sd::Tensor y = x; + const int64_t c = H3VUpsampler::H3LatentStats::kChannels; + const float* mean = H3VUpsampler::H3LatentStats::kMean; + const float* std_ = H3VUpsampler::H3LatentStats::kStd; + for (int64_t i = 0; i < y.numel(); ++i) { + const int64_t channel = i % c; + y.data()[i] = (y.data()[i] - mean[channel]) / std_[channel]; + } + return y; + } + + static sd::Tensor h3_denormalize_video_latent(const sd::Tensor& x) { + if (x.empty() || x.shape()[3] < H3VUpsampler::H3LatentStats::kChannels) { + return x; + } + sd::Tensor y = x; + const int64_t c = H3VUpsampler::H3LatentStats::kChannels; + const float* mean = H3VUpsampler::H3LatentStats::kMean; + const float* std_ = H3VUpsampler::H3LatentStats::kStd; + for (int64_t i = 0; i < y.numel(); ++i) { + const int64_t channel = i % c; + y.data()[i] = y.data()[i] * std_[channel] + mean[channel]; + } + return y; + } + + // Spatial-only latent upscale that keeps the video time axis untouched, so the + // packed audio latent length stays valid. Used as the hires second stage for + // video models that have no latent upsampler checkpoint (currently MiniMax-H3). + static sd::Tensor upscale_spatial_video_latent(StableDiffusionGGML* sd, + const GenerationRequest& request, + const sd::Tensor& packed_latent, + int audio_length) { + if (sd == nullptr || packed_latent.empty()) { + return {}; + } + if (packed_latent.dim() != 4 && packed_latent.dim() != 5) { + LOG_ERROR("video latent spatial upscale expects a 4D or 5D latent"); + return {}; + } + int64_t target_width = request.hires.target_width / request.vae_scale_factor; + int64_t target_height = request.hires.target_height / request.vae_scale_factor; + if (target_width <= 0 || target_height <= 0) { + LOG_ERROR("video latent spatial upscale target must be positive: %dx%d", + request.hires.target_width, + request.hires.target_height); + return {}; + } + + int latent_channels = sd->get_latent_channel(); + sd::Tensor video_latent = packed_latent; + sd::Tensor audio_latent; + if (packed_latent.shape()[3] > latent_channels) { + video_latent = sd::ops::slice(packed_latent, 3, 0, latent_channels); + audio_latent = sd_version_is_minimax_h3(sd->version) + ? unpack_minimax_h3_audio_latent(packed_latent, audio_length, latent_channels) + : unpack_ltxav_audio_latent(packed_latent, audio_length, latent_channels); + if (audio_latent.empty()) { + LOG_ERROR("failed to unpack audio latent before spatial latent upscale"); + return {}; + } + } + + bool antialias = false; + auto mode = hires_latent_interpolate_mode(request.hires.upscaler, &antialias); + std::vector target_shape = video_latent.shape(); + target_shape[0] = target_width; + target_shape[1] = target_height; + + LOG_INFO("video latent %s spatial upscale: %" PRId64 "x%" PRId64 "x%" PRId64 "x%" PRId64 " -> %" PRId64 "x%" PRId64 "x%" PRId64 "x%" PRId64, + sd_hires_upscaler_name(request.hires.upscaler), + (int64_t)video_latent.shape()[0], + (int64_t)video_latent.shape()[1], + (int64_t)video_latent.shape()[2], + (int64_t)video_latent.shape()[3], + target_shape[0], + target_shape[1], + target_shape[2], + target_shape[3]); + + sd::Tensor upscaled = sd::ops::interpolate(video_latent, target_shape, mode, false, antialias); + if (upscaled.empty()) { + LOG_ERROR("video latent spatial upscale failed"); + return {}; + } + if (audio_latent.empty()) { + return upscaled; + } + // The audio latent is re-packed for the new spatial size; its own length + // is unchanged because no frame was added or dropped. + return pack_ltxav_audio_and_video_latents(upscaled, audio_latent); + } + + // Model-backed H3 hires second stage: run the H3-shaped learned latent upscaler + // (e.g. the community 3D-conv checkpoint) between the low-resolution pass and + // the high-resolution refine pass. Audio is unpacked around the network and + // re-packed for the new spatial size. + sd::Tensor upscale_h3_learned_video_latent(StableDiffusionGGML* sd, + const char* model_path, + const sd::Tensor& packed_latent, + int audio_length, + int64_t target_width, + int64_t target_height) { + if (sd == nullptr || sd->model_manager == nullptr || packed_latent.empty()) { + return {}; + } + if (strlen(SAFE_STR(model_path)) == 0) { + LOG_ERROR("H3 learned latent upscale requires a hires model path"); + return {}; + } + if (!sd->ensure_backend_pair(SDBackendModule::UPSCALER)) { + return {}; + } + + int latent_channels = sd->get_latent_channel(); + sd::Tensor video_latent = packed_latent; + sd::Tensor audio_latent; + if (packed_latent.shape()[3] > latent_channels) { + video_latent = sd::ops::slice(packed_latent, 3, 0, latent_channels); + audio_latent = unpack_minimax_h3_audio_latent(packed_latent, audio_length, latent_channels); + if (audio_latent.empty()) { + LOG_ERROR("failed to unpack H3 audio latent before learned latent upscale"); + return {}; + } + } + + LOG_INFO("H3 learned latent spatial upscale: latent %dx%dx%dx%d -> %" PRId64 "x%" PRId64 " model output", + (int)video_latent.shape()[0], + (int)video_latent.shape()[1], + (int)video_latent.shape()[2], + (int)video_latent.shape()[3], + target_width, + target_height); + + sd::Tensor normalized = h3_normalize_video_latent(video_latent); + if (sd->first_stage_model) { + sd->first_stage_model->runner_end(); + } + if (normalized.empty()) { + LOG_ERROR("H3 latent normalization failed before learned spatial upscale"); + return {}; + } + + auto model_manager = sd->model_manager; + struct UpsamplerScope { + ModelManager& manager; + ModelLoader::FileId owned_source = 0; + std::unique_ptr runner; + std::vector params; + + ~UpsamplerScope() { + if (runner) { + runner->runner_end(); + } + GGML_ASSERT(manager.unregister_param_tensors(params)); + if (owned_source != 0) { + GGML_ASSERT(manager.del_file(owned_source)); + } + } + } scope{*model_manager}; + + const std::string prefix = "h3_latent_upscaler"; + ModelLoader candidate = model_manager->loader(); + ModelLoader::FileId source_file = 0; + if (!candidate.add_file(model_path, prefix + ".", &source_file)) { + LOG_ERROR("init H3 latent upsampler model loader from file failed: '%s'", model_path); + return {}; + } + const bool owns_source = model_manager->loader().file_revision(source_file) == 0; + if (!model_manager->set_loader(std::move(candidate))) { + return {}; + } + scope.owned_source = owns_source ? source_file : 0; + + auto& upsampler = scope.runner; + upsampler = std::make_unique(sd->backend_for(SDBackendModule::UPSCALER), + model_manager->loader().get_tensor_storage_map(), + prefix, + model_manager); + const size_t max_graph_vram_bytes = sd->max_graph_vram_bytes_for_module(SDBackendModule::UPSCALER); + upsampler->set_max_graph_vram_bytes(max_graph_vram_bytes); + if (upsampler->model == nullptr) { + LOG_ERROR("init H3 latent upsampler from metadata failed"); + return {}; + } + + std::map tensors; + upsampler->get_param_tensors(tensors); + for (const auto& entry : tensors) { + scope.params.push_back(entry.second); + } + if (!model_manager->register_param_tensors(ModelComponent::LatentUpsampler, + std::move(tensors), + ModelManager::ResidencyMode::ParamBackend, + sd->backend_for(SDBackendModule::UPSCALER), + sd->params_backend_for(SDBackendModule::UPSCALER)) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register H3 latent upsampler tensors with model manager failed"); + return {}; + } + + sd::Tensor upscaled = upsampler->compute(sd->n_threads, normalized, target_width, target_height); + upsampler->runner_end(); + if (upscaled.empty()) { + LOG_ERROR("H3 learned latent spatial upscale failed"); + return {}; + } + + upscaled = h3_denormalize_video_latent(upscaled); + if (sd->first_stage_model) { + sd->first_stage_model->runner_end(); + } + if (upscaled.empty()) { + LOG_ERROR("H3 latent denormalization failed after learned spatial upscale"); + return {}; + } + + if (!audio_latent.empty()) { + upscaled = pack_ltxav_audio_and_video_latents(upscaled, audio_latent); + } + return upscaled; + } + sd::Tensor upscale_ltx_spatial_video_latent(StableDiffusionGGML* sd, const char* model_path, const sd::Tensor& packed_latent, @@ -1630,17 +1905,45 @@ namespace sd::pipeline { bool latent_upscale_enabled = request.hires.enabled; GenerationRequest hires_request = request; if (latent_upscale_enabled) { - if (!sd_version_is_ltxav(sd->version)) { - LOG_ERROR("LTX latent spatial upscale is only supported for LTX video models"); + const bool is_ltxav = sd_version_is_ltxav(sd->version); + const bool is_h3 = sd_version_is_minimax_h3(sd->version); + if (!is_ltxav && !is_h3) { + LOG_ERROR("video latent spatial upscale is only supported for LTX and MiniMax-H3 video models"); return false; } - if (request.hires.upscaler != SD_HIRES_UPSCALER_MODEL) { - LOG_ERROR("LTX latent spatial upscale currently requires hires upscaler MODEL"); - return false; - } - if (strlen(SAFE_STR(request.hires.model_path)) == 0) { - LOG_ERROR("LTX latent spatial upscale is enabled but hires model path was not provided"); - return false; + if (is_ltxav) { + if (request.hires.upscaler != SD_HIRES_UPSCALER_MODEL) { + LOG_ERROR("LTX latent spatial upscale currently requires hires upscaler MODEL"); + return false; + } + if (strlen(SAFE_STR(request.hires.model_path)) == 0) { + LOG_ERROR("LTX latent spatial upscale is enabled but hires model path was not provided"); + return false; + } + } else { + // MiniMax-H3 ships no official latent upsampler checkpoint, so the second + // stage either resamples the packed latent with a latent interpolator or, + // when a model-backed upscaler is selected, runs a H3-shaped learned + // upscaler checkpoint from --hires-upscalers-dir. + const bool model_upscaler = request.hires.upscaler == SD_HIRES_UPSCALER_MODEL; + if (model_upscaler) { + if (strlen(SAFE_STR(request.hires.model_path)) == 0) { + LOG_ERROR("MiniMax-H3 learned hires upscaler is enabled but hires model path was not provided"); + return false; + } + } else if (!hires_upscaler_is_latent_interpolation(request.hires.upscaler)) { + LOG_ERROR("MiniMax-H3 hires requires an interpolating latent upscaler (e.g. \"Latent\" or \"Latent (bicubic)\") or a MODEL upscaler checkpoint"); + return false; + } + if (sd_vid_gen_params->init_image.data != nullptr || + sd_vid_gen_params->end_image.data != nullptr) { + LOG_ERROR("MiniMax-H3 hires does not support --init-img/--end-img keyframes yet; use T2VA or Ref2VA"); + return false; + } + if (request.hires.target_width <= 0 || request.hires.target_height <= 0) { + LOG_ERROR("MiniMax-H3 hires requires a positive hires target size"); + return false; + } } } @@ -1668,7 +1971,7 @@ namespace sd::pipeline { } ImageGenerationEmbeds embeds = std::move(*embeds_opt); if (latent_upscale_enabled) { - LOG_INFO("generate_video %dx%dx%d -> LTX latent spatial upscale", + LOG_INFO("generate_video %dx%dx%d -> video latent spatial upscale", request.width, request.height, request.frames); @@ -1780,16 +2083,31 @@ namespace sd::pipeline { LOG_ERROR("cancelling generation before latent upscale"); return false; } - int64_t upscale_start = ggml_time_ms(); - sd::Tensor upscaled_latent = upscale_ltx_spatial_video_latent(sd, - request.hires.model_path, - final_latent, - latents.audio_length); - int64_t upscale_end = ggml_time_ms(); + int64_t upscale_start = ggml_time_ms(); + sd::Tensor upscaled_latent; + if (sd_version_is_ltxav(sd->version)) { + upscaled_latent = upscale_ltx_spatial_video_latent(sd, + request.hires.model_path, + final_latent, + latents.audio_length); + } else if (request.hires.upscaler == SD_HIRES_UPSCALER_MODEL) { + upscaled_latent = upscale_h3_learned_video_latent(sd, + request.hires.model_path, + final_latent, + latents.audio_length, + request.hires.target_width / request.vae_scale_factor, + request.hires.target_height / request.vae_scale_factor); + } else { + upscaled_latent = upscale_spatial_video_latent(sd, + request, + final_latent, + latents.audio_length); + } + int64_t upscale_end = ggml_time_ms(); if (upscaled_latent.empty()) { return false; } - LOG_INFO("LTX latent spatial upscale completed, taking %.2fs", + LOG_INFO("video latent spatial upscale completed, taking %.2fs", (upscale_end - upscale_start) * 1.0f / 1000); x_t = std::move(upscaled_latent); @@ -1798,7 +2116,7 @@ namespace sd::pipeline { int upscaled_latent_frames = static_cast(x_t.shape()[2]); int upscaled_frames = sd->latent_frames_to_video_frames(upscaled_latent_frames); if (upscaled_frames != hires_request.frames) { - LOG_INFO("LTX latent upsampler output latent frames %d, frames %d -> %d", + LOG_INFO("video latent upsampler output latent frames %d, frames %d -> %d", upscaled_latent_frames, hires_request.frames, upscaled_frames); @@ -1829,7 +2147,8 @@ namespace sd::pipeline { latents.audio_length = target_audio_length; } } - if ((request.hires.target_width > 0 || request.hires.target_height > 0) && + if (sd_version_is_ltxav(sd->version) && + (request.hires.target_width > 0 || request.hires.target_height > 0) && (request.hires.target_width != hires_request.width || request.hires.target_height != hires_request.height)) { LOG_WARN("LTX latent spatial upsampler output is %dx%d; ignoring hires target %dx%d", hires_request.width, @@ -1843,13 +2162,26 @@ namespace sd::pipeline { LOG_ERROR("cancelling generation before latent upscale refine"); return false; } - if (!apply_ltxv_refine_image_conditioning(sd, - sd_vid_gen_params, - hires_request, - latents, - &x_t, - &hires_denoise_mask, - &hires_video_positions)) { + if (sd_version_is_ltxav(sd->version) && + !apply_ltxv_refine_image_conditioning(sd, + sd_vid_gen_params, + hires_request, + latents, + &x_t, + &hires_denoise_mask, + &hires_video_positions)) { + return false; + } + // TODO(keyframes): MiniMax-H3 keyframe latents are encoded at the first + // stage resolution. Supporting --init-img/--end-img hires requires + // re-encoding them at hires_request.width/height and replacing the + // matching entries of embeds.cond/uncond.c_ref_images, so the hires + // stage rejects them for now (see the guard before sampling). + if (sd_version_is_minimax_h3(sd->version) && + !latents.denoise_mask.empty() && + (latents.denoise_mask.shape()[0] != x_t.shape()[0] || + latents.denoise_mask.shape()[1] != x_t.shape()[1])) { + LOG_ERROR("MiniMax-H3 hires cannot rescale an existing video denoise mask"); return false; } noise = sd::Tensor::randn_like(x_t, sd->rng); @@ -1872,7 +2204,7 @@ namespace sd::pipeline { hires_sample_method); LOG_VERBOSE("sample(latent upscale) %dx%dx%d", W, H, T); - LOG_INFO("LTX latent spatial upscale refine: scheduler_steps=%d, denoising_strength=%.2f, sampler=%s, sigma_sched_size=%zu%s", + LOG_INFO("video latent spatial upscale refine: scheduler_steps=%d, denoising_strength=%.2f, sampler=%s, sigma_sched_size=%zu%s", hires_scheduler_steps, request.hires.denoising_strength, sampling_methods_str[hires_sample_method], From b0af1b56b35b459582e5ffddf54c268d899a2d7e Mon Sep 17 00:00:00 2001 From: farawayso Date: Thu, 29 Oct 2026 21:57:07 +0800 Subject: [PATCH 02/10] fix: address H3 latent channels as the planar (slowest) axis sd::Tensor stores shape[0] as the fastest-varying dimension, so the channel axis (shape[3]) is a contiguous run of numel()/channels values, not an interleaved i % channels stride. The previous per-element loop therefore applied mismatched mean/std to every element instead of per channel. Fold both directions into one per-channel affine helper and iterate channel blocks explicitly. --- src/pipeline/video.cpp | 57 +++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index ab860087e0..b478349c99 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -1362,37 +1362,48 @@ namespace sd::pipeline { } } - // Per-channel H3 video-latent (de)normalization, as applied by the community - // learned upscaler around its network. - static sd::Tensor h3_normalize_video_latent(const sd::Tensor& x) { - if (x.empty() || x.shape()[3] < H3VUpsampler::H3LatentStats::kChannels) { + // Per-channel affine transform over the H3 video latent, as applied by the + // community learned upscaler around its network. sd::Tensor keeps the channel + // axis (shape[3]) as the slowest-varying dimension, so each channel occupies a + // contiguous run of numel()/channels values. + static sd::Tensor h3_apply_channel_affine(const sd::Tensor& x, + const float* scale, + const float* bias) { + const int64_t c = H3VUpsampler::H3LatentStats::kChannels; + if (x.empty() || x.dim() < 4 || x.shape()[3] != c) { // Defensive no-op: the learned runner validates the channel count before this runs. return x; } - sd::Tensor y = x; - const int64_t c = H3VUpsampler::H3LatentStats::kChannels; - const float* mean = H3VUpsampler::H3LatentStats::kMean; - const float* std_ = H3VUpsampler::H3LatentStats::kStd; - for (int64_t i = 0; i < y.numel(); ++i) { - const int64_t channel = i % c; - y.data()[i] = (y.data()[i] - mean[channel]) / std_[channel]; + const int64_t spatial = x.numel() / c; + sd::Tensor y = x; + for (int64_t channel = 0; channel < c; ++channel) { + const float s = scale[channel]; + const float b = bias[channel]; + float* p = y.data() + channel * spatial; + for (int64_t i = 0; i < spatial; ++i) { + p[i] = p[i] * s + b; + } } return y; } + static sd::Tensor h3_normalize_video_latent(const sd::Tensor& x) { + constexpr int64_t c = H3VUpsampler::H3LatentStats::kChannels; + float scale[c]; + float bias[c]; + for (int64_t i = 0; i < c; ++i) { + const float mean = H3VUpsampler::H3LatentStats::kMean[i]; + const float std_ = H3VUpsampler::H3LatentStats::kStd[i]; + scale[i] = 1.f / std_; + bias[i] = -mean / std_; + } + return h3_apply_channel_affine(x, scale, bias); + } + static sd::Tensor h3_denormalize_video_latent(const sd::Tensor& x) { - if (x.empty() || x.shape()[3] < H3VUpsampler::H3LatentStats::kChannels) { - return x; - } - sd::Tensor y = x; - const int64_t c = H3VUpsampler::H3LatentStats::kChannels; - const float* mean = H3VUpsampler::H3LatentStats::kMean; - const float* std_ = H3VUpsampler::H3LatentStats::kStd; - for (int64_t i = 0; i < y.numel(); ++i) { - const int64_t channel = i % c; - y.data()[i] = y.data()[i] * std_[channel] + mean[channel]; - } - return y; + return h3_apply_channel_affine(x, + H3VUpsampler::H3LatentStats::kStd, + H3VUpsampler::H3LatentStats::kMean); } // Spatial-only latent upscale that keeps the video time axis untouched, so the From 0c978cadcee22b25a32c8cc667afb2462efe69c0 Mon Sep 17 00:00:00 2001 From: farawayso Date: Thu, 29 Oct 2026 23:09:11 +0800 Subject: [PATCH 03/10] fix: use valid ggml APIs in the H3 upscaler graph Addresses the review findings: - ggml_reshape_1d needs the output length; pass 2 * channels explicitly. - ggml_cgraph has no ctx member in the pinned ggml, so drop ggml_new_tensor_1d on gf->ctx and bind the scale-1 embedding through make_input like the latent inputs. This also removes ggml_set_f32_1d, which takes no context argument. - Build 1 + scale with ggml_scale_bias instead of a host-filled constant tensor. - Register the H3Embed children as "0"/"2": GGMLBlock prefixes child names, so "embed.0" nested under blocks["embed"] resolved to embed.embed.0.* and never matched the checkpoint's embed.0.* weights. --- src/model/upscaler/h3_latent_upscaler.hpp | 30 ++++++++++++----------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/model/upscaler/h3_latent_upscaler.hpp b/src/model/upscaler/h3_latent_upscaler.hpp index 94a9627dd6..0f01145dc5 100644 --- a/src/model/upscaler/h3_latent_upscaler.hpp +++ b/src/model/upscaler/h3_latent_upscaler.hpp @@ -235,12 +235,12 @@ namespace H3VUpsampler { // AdaLN: emb_out = emb_layers(SiLU(emb)) -> split [scale, shift] ggml_tensor* emb_silu = ggml_silu_inplace(ctx->ggml_ctx, emb); ggml_tensor* emb_out = emb_layers->forward(ctx, emb_silu); - ggml_tensor* emb_1d = ggml_reshape_1d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, emb_out)); + ggml_tensor* emb_1d = ggml_reshape_1d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, emb_out), 2 * channels); ggml_tensor* scale = ggml_reshape_4d(ctx->ggml_ctx, ggml_ext_slice(ctx->ggml_ctx, emb_1d, 0, 0, channels), 1, 1, 1, channels); ggml_tensor* shift = ggml_reshape_4d(ctx->ggml_ctx, ggml_ext_slice(ctx->ggml_ctx, emb_1d, 0, channels, 2 * channels), 1, 1, 1, channels); - ggml_tensor* one = ggml_new_tensor_1d(ctx->ggml_ctx, GGML_TYPE_F32, 1); - ggml_set_f32_1d(ctx->ggml_ctx, one, 0, 1.0f); - ggml_tensor* gain = ggml_add(ctx->ggml_ctx, scale, one); + // 1 + scale as a graph op; a host-filled constant tensor would have to be + // allocated and bound outside the graph context. + ggml_tensor* gain = ggml_scale_bias(ctx->ggml_ctx, scale, 1.f, 1.f); // out_norm * (1 + scale) + shift, then SiLU -> out_layers h = out_norm->forward(ctx, h); @@ -302,8 +302,10 @@ namespace H3VUpsampler { void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map, const std::string& prefix) override { - blocks["embed.0"] = std::shared_ptr(new Linear(1, embed_dim, true)); - blocks["embed.2"] = std::shared_ptr(new Linear(embed_dim, embed_dim, true)); + // Child names are relative to the "embed" block prefix, so these bind as + // embed.0.* / embed.2.* to match the checkpoint. + blocks["0"] = std::shared_ptr(new Linear(1, embed_dim, true)); + blocks["2"] = std::shared_ptr(new Linear(embed_dim, embed_dim, true)); } public: @@ -311,8 +313,8 @@ namespace H3VUpsampler { : embed_dim(embed_dim) {} ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* scale_input) { - auto l0 = std::dynamic_pointer_cast(blocks["embed.0"]); - auto l2 = std::dynamic_pointer_cast(blocks["embed.2"]); + auto l0 = std::dynamic_pointer_cast(blocks["0"]); + auto l2 = std::dynamic_pointer_cast(blocks["2"]); ggml_tensor* h = l0->forward(ctx, ggml_reshape_2d(ctx->ggml_ctx, scale_input, 1, 1)); h = ggml_silu_inplace(ctx->ggml_ctx, h); h = l2->forward(ctx, h); @@ -498,12 +500,13 @@ namespace H3VUpsampler { } const size_t expected_dim = static_cast(x.dim()); + // scale - 1 as one host tensor bound as a graph input for both stages. + const sd::Tensor scale_tensor = sd::Tensor::from_vector({scale - 1.0f}); // Stage A: conv_in + in-block chain at the original spatial size. - auto get_graph_a = [this, &x, scale]() -> ggml_cgraph* { + auto get_graph_a = [this, &x, &scale_tensor]() -> ggml_cgraph* { ggml_cgraph* gf = new_graph_custom(H3_UPSAMPLER_GRAPH_SIZE); ggml_tensor* x_t = make_input(x); - ggml_tensor* scale_input = ggml_new_tensor_1d(gf->ctx, GGML_TYPE_F32, 1); - ggml_set_f32_1d(gf->ctx, scale_input, 0, scale - 1.0f); + ggml_tensor* scale_input = make_input(scale_tensor); auto runner_ctx = get_context(gf); ggml_tensor* out = model->forward_stage_a(&runner_ctx, x_t, scale_input); ggml_build_forward_expand(gf, out); @@ -529,11 +532,10 @@ namespace H3VUpsampler { } // Stage B: out-block chain + conv_out at the target spatial size. - auto get_graph_b = [this, &mid_up, scale]() -> ggml_cgraph* { + auto get_graph_b = [this, &mid_up, &scale_tensor]() -> ggml_cgraph* { ggml_cgraph* gf = new_graph_custom(H3_UPSAMPLER_GRAPH_SIZE); ggml_tensor* m_t = make_input(mid_up); - ggml_tensor* scale_input = ggml_new_tensor_1d(gf->ctx, GGML_TYPE_F32, 1); - ggml_set_f32_1d(gf->ctx, scale_input, 0, scale - 1.0f); + ggml_tensor* scale_input = make_input(scale_tensor); auto runner_ctx = get_context(gf); ggml_tensor* out = model->forward_stage_b(&runner_ctx, m_t, scale_input); ggml_build_forward_expand(gf, out); From f9a2f57f949a53bdf4838840361396de4c3ed5cf Mon Sep 17 00:00:00 2001 From: farawayso Date: Thu, 29 Oct 2026 23:30:47 +0800 Subject: [PATCH 04/10] fix: match the reference activation semantics in the H3 upscaler Two conditioning bugs found while diffing against the reference PyTorch implementation (ComfyUI MiniMax-H3 latent upscaler): - H3ResBlock activated the shared embedding in place. The stage builds emb once and hands the same tensor to every residual block, so the first block's in-place SiLU left the buffer activated and the following blocks applied SiLU to an already activated embedding. Use the out-of-place ggml_silu. - H3Embed returned an extra SiLU. The checkpoint's embed is Sequential(Linear, SiLU, Linear) and ends at the second linear layer; each residual block supplies its own activation, so the trailing SiLU activated the conditioning twice. Return the linear output directly. --- src/model/upscaler/h3_latent_upscaler.hpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/model/upscaler/h3_latent_upscaler.hpp b/src/model/upscaler/h3_latent_upscaler.hpp index 0f01145dc5..56a774c3d4 100644 --- a/src/model/upscaler/h3_latent_upscaler.hpp +++ b/src/model/upscaler/h3_latent_upscaler.hpp @@ -232,8 +232,11 @@ namespace H3VUpsampler { h = ggml_silu_inplace(ctx->ggml_ctx, h); h = in_layers->forward(ctx, h); - // AdaLN: emb_out = emb_layers(SiLU(emb)) -> split [scale, shift] - ggml_tensor* emb_silu = ggml_silu_inplace(ctx->ggml_ctx, emb); + // AdaLN: emb_out = emb_layers(SiLU(emb)) -> split [scale, shift]. + // emb is shared by every residual block of the stage, so the activation has + // to be out-of-place: an in-place SiLU would leave the buffer activated and + // the following blocks would re-activate an already activated embedding. + ggml_tensor* emb_silu = ggml_silu(ctx->ggml_ctx, emb); ggml_tensor* emb_out = emb_layers->forward(ctx, emb_silu); ggml_tensor* emb_1d = ggml_reshape_1d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, emb_out), 2 * channels); ggml_tensor* scale = ggml_reshape_4d(ctx->ggml_ctx, ggml_ext_slice(ctx->ggml_ctx, emb_1d, 0, 0, channels), 1, 1, 1, channels); @@ -315,10 +318,11 @@ namespace H3VUpsampler { ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* scale_input) { auto l0 = std::dynamic_pointer_cast(blocks["0"]); auto l2 = std::dynamic_pointer_cast(blocks["2"]); + // Sequential(Linear, SiLU, Linear): the reference checkpoint ends at the + // second linear layer, each residual block activates the embedding itself. ggml_tensor* h = l0->forward(ctx, ggml_reshape_2d(ctx->ggml_ctx, scale_input, 1, 1)); h = ggml_silu_inplace(ctx->ggml_ctx, h); - h = l2->forward(ctx, h); - return ggml_silu_inplace(ctx->ggml_ctx, h); + return l2->forward(ctx, h); } }; From 38eb38f2974c9a7bc095d0b460eff19673424dd4 Mon Sep 17 00:00:00 2001 From: farawayso Date: Thu, 29 Oct 2026 23:34:51 +0800 Subject: [PATCH 05/10] fix: skip the latent upscale when the target equals the draft size The learned network is not an identity mapping at scale 1, so a hires target that matches the draft latent size must return the packed latent untouched instead of resampling through the model. The reference node short-circuits the same way. Add the guard to both the interpolate and the learned paths, and correct the runner's doc comment, which claimed to return a denormalized latent although the per-channel (de)normalization is applied by the caller. --- src/model/upscaler/h3_latent_upscaler.hpp | 8 +++++--- src/pipeline/video.cpp | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/model/upscaler/h3_latent_upscaler.hpp b/src/model/upscaler/h3_latent_upscaler.hpp index 56a774c3d4..2b390965ae 100644 --- a/src/model/upscaler/h3_latent_upscaler.hpp +++ b/src/model/upscaler/h3_latent_upscaler.hpp @@ -461,9 +461,11 @@ namespace H3VUpsampler { } } - // x: normalized H3 video latent [W, H, T, C] (or 5D batch 1); returns the - // denormalized upscaled latent at the target spatial size. The scale-1 - // embedding input is a graph leaf so both stage graphs carry it. + // x: H3 video latent [W, H, T, C] (or 5D batch 1) in the normalized domain + // the checkpoint expects. Returns the upscaled latent at the target spatial + // size, still in that domain; the caller applies the per-channel (de)normalization + // around this call. The scale-1 embedding input is a graph leaf so both stage + // graphs carry it. sd::Tensor compute(const int n_threads, const sd::Tensor& x, int64_t target_w, diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index b478349c99..34ecfa60b2 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -1428,6 +1428,14 @@ namespace sd::pipeline { request.hires.target_height); return {}; } + // Nothing to do at 1x. This matters for the learned upscaler: the network is not + // an identity mapping at scale 1, so running it would alter a latent the caller + // asked to keep at the draft size. Matches the reference node, which returns the + // input latent unchanged when the target size equals the input size. + if (packed_latent.shape()[0] == target_width && packed_latent.shape()[1] == target_height) { + LOG_DEBUG("video latent spatial upscale: target equals the draft latent size, skipping"); + return packed_latent; + } int latent_channels = sd->get_latent_channel(); sd::Tensor video_latent = packed_latent; @@ -1490,6 +1498,13 @@ namespace sd::pipeline { LOG_ERROR("H3 learned latent upscale requires a hires model path"); return {}; } + // The network is not an identity mapping at scale 1, so a target equal to the + // draft latent size must skip it instead of resampling through the model. The + // reference node returns the input latent unchanged in this case. + if (packed_latent.shape()[0] == target_width && packed_latent.shape()[1] == target_height) { + LOG_DEBUG("H3 learned latent upscale: target equals the draft latent size, skipping"); + return packed_latent; + } if (!sd->ensure_backend_pair(SDBackendModule::UPSCALER)) { return {}; } From 09227ca35878d5ede352d0feb3fe8fb411bc1830 Mon Sep 17 00:00:00 2001 From: farawayso Date: Mon, 2 Nov 2026 10:18:01 +0800 Subject: [PATCH 06/10] fix: build the temporal conv as a real depthwise conv, and match init_params Two defects found once a compiler was available (MinGW g++ 16.2.0): - The three init_params overrides took 'const std::string&' while GGMLBlock declares the prefix by value, so none of them overrode anything and the translation unit did not compile. This is the first actual build of the branch. - H3TemporalConv built its depthwise kernel as a dense Conv3d(C, C, (k,1,1)). The reference module is Conv3d(C, C, (k,1,1), groups=C), so the checkpoint stores [C, 1, k, 1, 1]; the loader folds it to [1, 1, k, C] (2,560 values at C=512), while a dense Conv3d expects [1, 1, k, C*C] (1,310,720). Every other tensor in the checkpoint already matched, so the upscaler could never have loaded. Expanding the kernel to a block-diagonal dense one would restore loading but multiply the conv cost by 512x, which is not affordable for a 3D conv on an iGPU; implement the depthwise convolution directly as k shifted multiply-adds along T instead (elementwise ops only, so every backend supports it). Verified against the released bf16 checkpoint: all 322 tensors now match the shapes this code expects. --- src/model/upscaler/h3_latent_upscaler.hpp | 101 +++++++++++++++++++--- 1 file changed, 89 insertions(+), 12 deletions(-) diff --git a/src/model/upscaler/h3_latent_upscaler.hpp b/src/model/upscaler/h3_latent_upscaler.hpp index 2b390965ae..2c51556190 100644 --- a/src/model/upscaler/h3_latent_upscaler.hpp +++ b/src/model/upscaler/h3_latent_upscaler.hpp @@ -200,7 +200,7 @@ namespace H3VUpsampler { void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map, - const std::string& prefix) override { + const std::string prefix) override { auto make_conv = [&](const std::string& name) { blocks[name] = std::shared_ptr(new Conv3d(channels, channels, @@ -255,8 +255,90 @@ namespace H3VUpsampler { } }; - // Temporal conv: GN -> SiLU -> depthwise Conv3d (k,1,1) shipped as a full - // block-diagonal Conv3d -> pointwise Conv3d (zero-init at training) + identity. + // Depthwise convolution over the time axis only. + // + // The reference implementation declares this as Conv3d(C, C, (k,1,1), groups=C), + // so the checkpoint stores a genuine depthwise kernel: PyTorch [C, 1, k, 1, 1], + // which the safetensors loader folds to [1, 1, k, C] (2,560 values at C=512). + // sdcpp's Conv3d is dense-only and would expect C*C*k values (512x more), so it + // cannot host this weight. Expanding the kernel to a block-diagonal dense one + // would also inflate the conv cost by the same 512x, which for a 3D conv on an + // iGPU is not affordable. Implement it directly instead: k shifted + // multiply-adds along T, all elementwise, so every backend supports it and the + // FLOP count stays at the depthwise level. + class H3TemporalDWConv3d : public UnaryBlock { + protected: + int64_t channels; + int kernel; + bool has_bias; + std::string prefix; + + void init_params(ggml_context* ctx, + const String2TensorStorage& tensor_storage_map, + const std::string prefix) override { + this->prefix = prefix; + params["weight"] = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 1, 1, kernel, channels); + if (has_bias) { + params["bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, channels); + } + } + + public: + H3TemporalDWConv3d(int64_t channels, int kernel, bool has_bias = true) + : channels(channels), + kernel(kernel), + has_bias(has_bias) {} + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { + ggml_tensor* w = params["weight"]; + if (ctx->weight_adapter) { + w = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, w, prefix + "weight"); + } + // Activations in this network are F32 (conv outputs are F32); the kernel + // is stored as F16, so match the activation type before multiplying. + ggml_tensor* w32 = ggml_cast(ctx->ggml_ctx, w, GGML_TYPE_F32); + ggml_tensor* xa = x; + if (xa->type != GGML_TYPE_F32) { + xa = ggml_cast(ctx->ggml_ctx, xa, GGML_TYPE_F32); + } + + const int64_t T = xa->ne[2]; + const int pad = kernel / 2; + + // Zero-pad both ends of T. ggml_pad only pads the end of each dimension, + // so build a [W, H, pad, C] zero tensor from a zeroed slice of the input + // and concatenate it on both sides. + ggml_tensor* xp = xa; + if (pad > 0) { + ggml_tensor* zero = ggml_scale(ctx->ggml_ctx, ggml_ext_slice(ctx->ggml_ctx, xa, 2, 0, 1), 0.0f); + ggml_tensor* lead = zero; + for (int i = 1; i < pad; ++i) { + lead = ggml_concat(ctx->ggml_ctx, lead, zero, 2); + } + xp = ggml_concat(ctx->ggml_ctx, lead, xa, 2); + xp = ggml_concat(ctx->ggml_ctx, xp, lead, 2); + } + + ggml_tensor* acc = nullptr; + for (int j = 0; j < kernel; ++j) { + ggml_tensor* xs = ggml_ext_slice(ctx->ggml_ctx, xp, 2, j, j + T); + ggml_tensor* wj = ggml_ext_slice(ctx->ggml_ctx, w32, 2, j, j + 1); // [1, 1, 1, C] + ggml_tensor* term = ggml_mul(ctx->ggml_ctx, xs, wj); + acc = acc == nullptr ? term : ggml_add(ctx->ggml_ctx, acc, term); + } + if (has_bias) { + ggml_tensor* b = params["bias"]; + if (ctx->weight_adapter) { + b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias"); + } + acc = ggml_add(ctx->ggml_ctx, acc, ggml_reshape_4d(ctx->ggml_ctx, b, 1, 1, 1, channels)); + } + return acc; + } + }; + + // Temporal conv: GN -> SiLU -> depthwise Conv3d (k,1,1) -> pointwise Conv3d + // (zero-init at training) + identity. class H3TemporalConv : public GGMLBlock { protected: int64_t channels; @@ -264,13 +346,8 @@ namespace H3VUpsampler { void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map, - const std::string& prefix) override { - const int pad = temporal_kernel / 2; - blocks["dwconv"] = std::shared_ptr(new Conv3d(channels, - channels, - {temporal_kernel, 1, 1}, - {1, 1, 1}, - {pad, 0, 0})); + const std::string prefix) override { + blocks["dwconv"] = std::shared_ptr(new H3TemporalDWConv3d(channels, temporal_kernel)); blocks["pwconv"] = std::shared_ptr(new Conv3d(channels, channels, {1, 1, 1}, @@ -285,7 +362,7 @@ namespace H3VUpsampler { temporal_kernel(temporal_kernel) {} ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { - auto dwconv = std::dynamic_pointer_cast(blocks["dwconv"]); + auto dwconv = std::dynamic_pointer_cast(blocks["dwconv"]); auto pwconv = std::dynamic_pointer_cast(blocks["pwconv"]); auto norm = std::dynamic_pointer_cast(blocks["norm"]); @@ -304,7 +381,7 @@ namespace H3VUpsampler { void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map, - const std::string& prefix) override { + const std::string prefix) override { // Child names are relative to the "embed" block prefix, so these bind as // embed.0.* / embed.2.* to match the checkpoint. blocks["0"] = std::shared_ptr(new Linear(1, embed_dim, true)); From 7f98042c72214e4fb55d92d2a99e95a270d2d988 Mon Sep 17 00:00:00 2001 From: farawayso Date: Mon, 2 Nov 2026 10:35:15 +0800 Subject: [PATCH 07/10] fix: read the temporal kernel from ne[2] when probing the depthwise weights detect_from_weights read ne[0] for the folded 4D form. The loader folds a PyTorch 5D kernel [OC, IC, kD, kH, kW] to [OC*IC, kD, kH, kW] and then reverse_ne turns it into [kW, kH, kD, OC*IC], so ne[0] is the spatial extent (1) and the temporal extent sits at index 2. The probe therefore derived temporal_kernel = 1 for this checkpoint, built 1-tap depthwise convolutions and then failed the model-manager shape check on every dwconv weight with 'got [1, 1, 5, 512], expected [1, 1, 1, 512]'. Both layouts that can reach this code keep kD at index 2, so read ne[2] unconditionally. Verified end to end on CPU with the released bf16 checkpoint: the learned path now loads all 322 tensors, upscales latent 16x16x7x24 to 32x32 (T preserved), refines, and decodes a 512x512x22 video with audio. --- src/model/upscaler/h3_latent_upscaler.hpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/model/upscaler/h3_latent_upscaler.hpp b/src/model/upscaler/h3_latent_upscaler.hpp index 2c51556190..bc411511a6 100644 --- a/src/model/upscaler/h3_latent_upscaler.hpp +++ b/src/model/upscaler/h3_latent_upscaler.hpp @@ -161,8 +161,13 @@ namespace H3VUpsampler { for (int idx : idxs) { const TensorStorage* dw = find_weight(std::string(stage) + "." + std::to_string(idx) + ".dwconv.weight"); if (dw != nullptr) { - // Expanded full-conv layout: PyTorch 5D [C, C, k, 1, 1] or sdcpp 4D [k, 1, 1, C*C]. - return static_cast(dw->n_dims == 5 ? dw->ne[2] : dw->ne[0]); + // The temporal extent is ne[2] in both layouts that can reach here: + // an unfolded PyTorch 5D kernel [OC, IC, kD, kH, kW] keeps kD at + // index 2, and the loader's folded form [OC*IC, kD, kH, kW] followed + // by reverse_ne becomes [kW, kH, kD, OC*IC], which also puts kD at + // index 2. Reading ne[0] instead picked up the spatial extent (1) and + // silently collapsed the depthwise kernel to a 1-tap convolution. + return static_cast(dw->ne[2]); } } return 0; From 9892e908038e44b37fa3c6f6ae24c3825155142b Mon Sep 17 00:00:00 2001 From: leejet Date: Wed, 4 Nov 2026 23:05:25 +0800 Subject: [PATCH 08/10] fix: preserve diffusion latent normalization in H3 learned upscaler --- src/model/upscaler/h3_latent_upscaler.hpp | 31 ++---------- src/pipeline/video.cpp | 60 +---------------------- 2 files changed, 6 insertions(+), 85 deletions(-) diff --git a/src/model/upscaler/h3_latent_upscaler.hpp b/src/model/upscaler/h3_latent_upscaler.hpp index bc411511a6..747b3af8ae 100644 --- a/src/model/upscaler/h3_latent_upscaler.hpp +++ b/src/model/upscaler/h3_latent_upscaler.hpp @@ -32,8 +32,9 @@ // g = out_blocks.N(g, emb) // y = conv_out(silu(norm_out(g))) // -// The per-channel latent (de)normalization of the community model is applied on the -// host around the two GPU stages, mirroring the LTX flow's un_normalize/normalize. +// The network uses normalized H3 diffusion latents for both input and output. +// sd.cpp's sampler already uses this domain, so no additional VAE mean/std +// conversion is needed around the network. // Depthwise temporal convs (kernel (k,1,1)) are shipped as full block-diagonal // Conv3d weights by the converter script, so only stock Conv3d ops are required. // The middle spatial resample runs on the host between the two GPU graphs; the @@ -42,25 +43,6 @@ namespace H3VUpsampler { constexpr int H3_UPSAMPLER_GRAPH_SIZE = 8192; - // Per-channel H3 video-latent statistics (24 channels) from the community checkpoint. - struct H3LatentStats { - static constexpr int kChannels = 24; - static constexpr float kMean[24] = { - 0.858090341091156f, -0.9606591463088989f, 1.0661640167236328f, -0.5090325474739075f, - -0.2727581858634949f, -1.3675414323806763f, -0.2553254961967468f, -0.26907554268836975f, - -0.5376840829849243f, -0.0464097298681736f, 0.6657370328903198f, 0.19690127670764923f, - -0.5460608005523682f, -0.4035342037677765f, -0.23683024942874908f, 0.25928452610969543f, - -0.30133944749832153f, 0.211341992020607f, -1.1206848621368408f, 0.3581933379173279f, - -0.04225143790245056f, 0.2604829967021942f, 0.22864092886447906f, 0.7056031823158264f}; - static constexpr float kStd[24] = { - 1.2223774194717407f, 1.2767263650894165f, 1.6831774711608887f, 1.7549455165863037f, - 1.5636216402053833f, 2.194143533706665f, 0.9653137922286987f, 1.0569885969161987f, - 0.841948926448822f, 0.7729952931404114f, 1.8955937623977661f, 0.946841835975647f, - 0.7996809482574463f, 0.44988900423049927f, 0.7197399735450745f, 0.6936293244361877f, - 2.961095094680786f, 2.7694199085235596f, 3.0496184825897217f, 2.1088054180145264f, - 3.276226282119751f, 3.1627357006073f, 2.2816812992095947f, 2.6127843856811523f}; - }; - struct H3LatentUpscalerConfig { int64_t in_channels = 24; int64_t mid_channels = 512; @@ -543,11 +525,8 @@ namespace H3VUpsampler { } } - // x: H3 video latent [W, H, T, C] (or 5D batch 1) in the normalized domain - // the checkpoint expects. Returns the upscaled latent at the target spatial - // size, still in that domain; the caller applies the per-channel (de)normalization - // around this call. The scale-1 embedding input is a graph leaf so both stage - // graphs carry it. + // x: H3 diffusion latent [W, H, T, C] (or 5D batch 1). The output stays + // in the same normalized domain for the next sampling pass. sd::Tensor compute(const int n_threads, const sd::Tensor& x, int64_t target_w, diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index 34ecfa60b2..97e1131f81 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -1362,50 +1362,6 @@ namespace sd::pipeline { } } - // Per-channel affine transform over the H3 video latent, as applied by the - // community learned upscaler around its network. sd::Tensor keeps the channel - // axis (shape[3]) as the slowest-varying dimension, so each channel occupies a - // contiguous run of numel()/channels values. - static sd::Tensor h3_apply_channel_affine(const sd::Tensor& x, - const float* scale, - const float* bias) { - const int64_t c = H3VUpsampler::H3LatentStats::kChannels; - if (x.empty() || x.dim() < 4 || x.shape()[3] != c) { - // Defensive no-op: the learned runner validates the channel count before this runs. - return x; - } - const int64_t spatial = x.numel() / c; - sd::Tensor y = x; - for (int64_t channel = 0; channel < c; ++channel) { - const float s = scale[channel]; - const float b = bias[channel]; - float* p = y.data() + channel * spatial; - for (int64_t i = 0; i < spatial; ++i) { - p[i] = p[i] * s + b; - } - } - return y; - } - - static sd::Tensor h3_normalize_video_latent(const sd::Tensor& x) { - constexpr int64_t c = H3VUpsampler::H3LatentStats::kChannels; - float scale[c]; - float bias[c]; - for (int64_t i = 0; i < c; ++i) { - const float mean = H3VUpsampler::H3LatentStats::kMean[i]; - const float std_ = H3VUpsampler::H3LatentStats::kStd[i]; - scale[i] = 1.f / std_; - bias[i] = -mean / std_; - } - return h3_apply_channel_affine(x, scale, bias); - } - - static sd::Tensor h3_denormalize_video_latent(const sd::Tensor& x) { - return h3_apply_channel_affine(x, - H3VUpsampler::H3LatentStats::kStd, - H3VUpsampler::H3LatentStats::kMean); - } - // Spatial-only latent upscale that keeps the video time axis untouched, so the // packed audio latent length stays valid. Used as the hires second stage for // video models that have no latent upsampler checkpoint (currently MiniMax-H3). @@ -1529,14 +1485,9 @@ namespace sd::pipeline { target_width, target_height); - sd::Tensor normalized = h3_normalize_video_latent(video_latent); if (sd->first_stage_model) { sd->first_stage_model->runner_end(); } - if (normalized.empty()) { - LOG_ERROR("H3 latent normalization failed before learned spatial upscale"); - return {}; - } auto model_manager = sd->model_manager; struct UpsamplerScope { @@ -1596,22 +1547,13 @@ namespace sd::pipeline { return {}; } - sd::Tensor upscaled = upsampler->compute(sd->n_threads, normalized, target_width, target_height); + sd::Tensor upscaled = upsampler->compute(sd->n_threads, video_latent, target_width, target_height); upsampler->runner_end(); if (upscaled.empty()) { LOG_ERROR("H3 learned latent spatial upscale failed"); return {}; } - upscaled = h3_denormalize_video_latent(upscaled); - if (sd->first_stage_model) { - sd->first_stage_model->runner_end(); - } - if (upscaled.empty()) { - LOG_ERROR("H3 latent denormalization failed after learned spatial upscale"); - return {}; - } - if (!audio_latent.empty()) { upscaled = pack_ltxav_audio_and_video_latents(upscaled, audio_latent); } From fb8e04f740a2601803b0fa64db7032a3cde02432 Mon Sep 17 00:00:00 2001 From: leejet Date: Thu, 5 Nov 2026 01:53:48 +0800 Subject: [PATCH 09/10] fix: restore second normalization in H3 latent upscaler --- docs/minimax_h3.md | 3 +++ src/model/upscaler/h3_latent_upscaler.hpp | 31 +++++++++++++++++------ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/docs/minimax_h3.md b/docs/minimax_h3.md index a397634d06..db554ddb03 100644 --- a/docs/minimax_h3.md +++ b/docs/minimax_h3.md @@ -129,6 +129,9 @@ or with the learned upscaler: Notes for this mode: +- The learned upscaler expects a second normalization of the diffusion latent, + matching its training convention. The runner applies and reverses this extra + transform internally; callers should pass the usual H3 diffusion latents. - The frame count is never changed by the second stage, so the packed audio latent length stays valid and the generated soundtrack is preserved. - Width and height still have to be multiples of 32; the hires target is diff --git a/src/model/upscaler/h3_latent_upscaler.hpp b/src/model/upscaler/h3_latent_upscaler.hpp index 747b3af8ae..25c1cf1ced 100644 --- a/src/model/upscaler/h3_latent_upscaler.hpp +++ b/src/model/upscaler/h3_latent_upscaler.hpp @@ -32,9 +32,9 @@ // g = out_blocks.N(g, emb) // y = conv_out(silu(norm_out(g))) // -// The network uses normalized H3 diffusion latents for both input and output. -// sd.cpp's sampler already uses this domain, so no additional VAE mean/std -// conversion is needed around the network. +// The runner accepts and returns normalized H3 diffusion latents. The checkpoint +// was trained with a second normalization using the VAE statistics, applied before +// the network and inverted afterwards; this is separate from VAE encode/decode. // Depthwise temporal convs (kernel (k,1,1)) are shipped as full block-diagonal // Conv3d weights by the converter script, so only stock Conv3d ops are required. // The middle spatial resample runs on the host between the two GPU graphs; the @@ -543,8 +543,8 @@ namespace H3VUpsampler { LOG_ERROR("H3 latent upsampler currently supports batch size 1, got batch=%lld", (long long)x.shape()[4]); return {}; } - if (x.shape()[3] != config.in_channels) { - LOG_ERROR("H3 latent upsampler expected %" PRId64 " channels, got %lld", config.in_channels, (long long)x.shape()[3]); + if (config.in_channels != 24 || x.shape()[3] != config.in_channels) { + LOG_ERROR("H3 latent upsampler requires 24 channels, got model=%" PRId64 " input=%lld", config.in_channels, (long long)x.shape()[3]); return {}; } const int64_t in_w = x.shape()[0]; @@ -567,12 +567,27 @@ namespace H3VUpsampler { } const size_t expected_dim = static_cast(x.dim()); + const sd::Tensor mean({1, 1, 1, 24}, + {0.858090341091156f, -0.9606591463088989f, 1.0661640167236328f, -0.5090325474739075f, + -0.2727581858634949f, -1.3675414323806763f, -0.2553254961967468f, -0.26907554268836975f, + -0.5376840829849243f, -0.0464097298681736f, 0.6657370328903198f, 0.19690127670764923f, + -0.5460608005523682f, -0.4035342037677765f, -0.23683024942874908f, 0.25928452610969543f, + -0.30133944749832153f, 0.211341992020607f, -1.1206848621368408f, 0.3581933379173279f, + -0.04225143790245056f, 0.2604829967021942f, 0.22864092886447906f, 0.7056031823158264f}); + const sd::Tensor stddev({1, 1, 1, 24}, + {1.2223774194717407f, 1.2767263650894165f, 1.6831774711608887f, 1.7549455165863037f, + 1.5636216402053833f, 2.194143533706665f, 0.9653137922286987f, 1.0569885969161987f, + 0.841948926448822f, 0.7729952931404114f, 1.8955937623977661f, 0.946841835975647f, + 0.7996809482574463f, 0.44988900423049927f, 0.7197399735450745f, 0.6936293244361877f, + 2.961095094680786f, 2.7694199085235596f, 3.0496184825897217f, 2.1088054180145264f, + 3.276226282119751f, 3.1627357006073f, 2.2816812992095947f, 2.6127843856811523f}); + const sd::Tensor normalized = (x - mean) / stddev; // scale - 1 as one host tensor bound as a graph input for both stages. const sd::Tensor scale_tensor = sd::Tensor::from_vector({scale - 1.0f}); // Stage A: conv_in + in-block chain at the original spatial size. - auto get_graph_a = [this, &x, &scale_tensor]() -> ggml_cgraph* { + auto get_graph_a = [this, &normalized, &scale_tensor]() -> ggml_cgraph* { ggml_cgraph* gf = new_graph_custom(H3_UPSAMPLER_GRAPH_SIZE); - ggml_tensor* x_t = make_input(x); + ggml_tensor* x_t = make_input(normalized); ggml_tensor* scale_input = make_input(scale_tensor); auto runner_ctx = get_context(gf); ggml_tensor* out = model->forward_stage_a(&runner_ctx, x_t, scale_input); @@ -620,7 +635,7 @@ namespace H3VUpsampler { out.shape()[0], out.shape()[1], out.shape()[2], target_w, target_h, T); return {}; } - return restore_trailing_singleton_dims(std::move(out), expected_dim); + return restore_trailing_singleton_dims(out * stddev + mean, expected_dim); } }; From 97b84530c3063fad7a2d37d7430559d4a974156f Mon Sep 17 00:00:00 2001 From: leejet Date: Thu, 5 Nov 2026 02:06:55 +0800 Subject: [PATCH 10/10] chore: trim redundant comments in H3 latent upscaling --- src/model/upscaler/h3_latent_upscaler.hpp | 174 +++++++--------------- src/pipeline/video.cpp | 62 +++----- 2 files changed, 75 insertions(+), 161 deletions(-) diff --git a/src/model/upscaler/h3_latent_upscaler.hpp b/src/model/upscaler/h3_latent_upscaler.hpp index 25c1cf1ced..006415081f 100644 --- a/src/model/upscaler/h3_latent_upscaler.hpp +++ b/src/model/upscaler/h3_latent_upscaler.hpp @@ -20,25 +20,8 @@ #include "model/upscaler/ltx_latent_upscaler.hpp" #include "model_loader.h" -// MiniMax-H3 learned latent upscaler (H3-shaped 3D-conv LatentResizer). // Ported from the community ComfyUI node LBH-123-AI/Comfyui_Minimax_h3_latent_Upscaler -// (MIT) with checkpoint LBH-123-AI/Minimax_h3_latent_Upscaler (Apache-2.0): -// -// h = conv_in(x) Conv3d 24 -> 512, k3 -// for N in checkpoint module-list order: -// h = in_blocks.N(h, emb) ResBlock (AdaLN) or TemporalConv -// m = spatial_bilinear(h, target_hw) time axis untouched -// for N in checkpoint module-list order: -// g = out_blocks.N(g, emb) -// y = conv_out(silu(norm_out(g))) -// -// The runner accepts and returns normalized H3 diffusion latents. The checkpoint -// was trained with a second normalization using the VAE statistics, applied before -// the network and inverted afterwards; this is separate from VAE encode/decode. -// Depthwise temporal convs (kernel (k,1,1)) are shipped as full block-diagonal -// Conv3d weights by the converter script, so only stock Conv3d ops are required. -// The middle spatial resample runs on the host between the two GPU graphs; the -// time axis is never resampled, so the packed audio latent length stays valid. +// (MIT) with checkpoint LBH-123-AI/Minimax_h3_latent_Upscaler (Apache-2.0). namespace H3VUpsampler { constexpr int H3_UPSAMPLER_GRAPH_SIZE = 8192; @@ -48,13 +31,12 @@ namespace H3VUpsampler { int64_t mid_channels = 512; int embed_dim = 64; int temporal_kernel = 5; - // Ordered module-list indices in checkpoint order (res and temporal blocks - // are interleaved in a single list in the state dict). + // Residual and temporal blocks share interleaved checkpoint indices. std::vector in_res, in_temporal, out_res, out_temporal; - bool valid = false; + bool valid = false; static H3LatentUpscalerConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, - const std::string& prefix = "") { + const std::string& prefix = "") { H3LatentUpscalerConfig config; std::string base = prefix; if (!base.empty()) { @@ -73,8 +55,7 @@ namespace H3VUpsampler { LOG_ERROR("h3 latent upsampler: 'conv_in.weight' not found under prefix '%s'", prefix.c_str()); return config; } - // Conv weights are either PyTorch 5D [OC, IC, k, k, k] or sdcpp 4D [k, k, k, OC*IC]; - // the biases are 1D in both layouts, so read channel counts from the biases. + // Bias shapes preserve channel counts when the loader folds 5D conv weights. const TensorStorage* conv_in_bias = find_weight("conv_in.bias"); if (conv_in_bias != nullptr) { config.mid_channels = conv_in_bias->ne[0]; @@ -102,9 +83,9 @@ namespace H3VUpsampler { if (!starts_with(name, base)) { continue; } - const std::string rel = name.substr(base.size()); - const bool is_in = starts_with(rel, "in_blocks."); - const bool is_out = starts_with(rel, "out_blocks."); + const std::string rel = name.substr(base.size()); + const bool is_in = starts_with(rel, "in_blocks."); + const bool is_out = starts_with(rel, "out_blocks."); if (!is_in && !is_out) { continue; } @@ -143,12 +124,7 @@ namespace H3VUpsampler { for (int idx : idxs) { const TensorStorage* dw = find_weight(std::string(stage) + "." + std::to_string(idx) + ".dwconv.weight"); if (dw != nullptr) { - // The temporal extent is ne[2] in both layouts that can reach here: - // an unfolded PyTorch 5D kernel [OC, IC, kD, kH, kW] keeps kD at - // index 2, and the loader's folded form [OC*IC, kD, kH, kW] followed - // by reverse_ne becomes [kW, kH, kD, OC*IC], which also puts kD at - // index 2. Reading ne[0] instead picked up the spatial extent (1) and - // silently collapsed the depthwise kernel to a 1-tap convolution. + // kD stays at ne[2] in both [OC, IC, kD, kH, kW] and [kW, kH, kD, OC*IC]. return static_cast(dw->ne[2]); } } @@ -159,7 +135,7 @@ namespace H3VUpsampler { if (k_out > 0) { config.temporal_kernel = k_out; } else { - config.temporal_kernel = 1; // no temporal convs in the checkpoint + config.temporal_kernel = 1; } config.valid = true; @@ -177,9 +153,6 @@ namespace H3VUpsampler { } }; - // H3 res block: in_layers (GN -> SiLU -> Conv3d k3); AdaLN modulation - // out_norm(h) * (1 + scale) + shift from the scale-embedding MLP; out_layers - // (SiLU -> zero-init Conv3d k3); identity skip. class H3ResBlock : public GGMLBlock { protected: int64_t channels; @@ -190,10 +163,10 @@ namespace H3VUpsampler { const std::string prefix) override { auto make_conv = [&](const std::string& name) { blocks[name] = std::shared_ptr(new Conv3d(channels, - channels, - {3, 3, 3}, - {1, 1, 1}, - {1, 1, 1})); + channels, + {3, 3, 3}, + {1, 1, 1}, + {1, 1, 1})); }; make_conv("in_layers.2"); make_conv("out_layers.2"); @@ -214,45 +187,29 @@ namespace H3VUpsampler { auto out_norm = std::dynamic_pointer_cast(blocks["out_norm"]); auto emb_layers = std::dynamic_pointer_cast(blocks["emb_layers.1"]); - // in_layers: GN -> SiLU -> Conv3d (in_norm is the GroupNorm, in_layers is the Conv3d) ggml_tensor* h = in_norm->forward(ctx, x); - h = ggml_silu_inplace(ctx->ggml_ctx, h); - h = in_layers->forward(ctx, h); + h = ggml_silu_inplace(ctx->ggml_ctx, h); + h = in_layers->forward(ctx, h); - // AdaLN: emb_out = emb_layers(SiLU(emb)) -> split [scale, shift]. - // emb is shared by every residual block of the stage, so the activation has - // to be out-of-place: an in-place SiLU would leave the buffer activated and - // the following blocks would re-activate an already activated embedding. + // Keep emb intact: every residual block uses the same embedding. ggml_tensor* emb_silu = ggml_silu(ctx->ggml_ctx, emb); ggml_tensor* emb_out = emb_layers->forward(ctx, emb_silu); ggml_tensor* emb_1d = ggml_reshape_1d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, emb_out), 2 * channels); ggml_tensor* scale = ggml_reshape_4d(ctx->ggml_ctx, ggml_ext_slice(ctx->ggml_ctx, emb_1d, 0, 0, channels), 1, 1, 1, channels); ggml_tensor* shift = ggml_reshape_4d(ctx->ggml_ctx, ggml_ext_slice(ctx->ggml_ctx, emb_1d, 0, channels, 2 * channels), 1, 1, 1, channels); - // 1 + scale as a graph op; a host-filled constant tensor would have to be - // allocated and bound outside the graph context. ggml_tensor* gain = ggml_scale_bias(ctx->ggml_ctx, scale, 1.f, 1.f); - // out_norm * (1 + scale) + shift, then SiLU -> out_layers - h = out_norm->forward(ctx, h); - h = ggml_mul_inplace(ctx->ggml_ctx, h, gain); - h = ggml_add_inplace(ctx->ggml_ctx, h, shift); - h = ggml_silu_inplace(ctx->ggml_ctx, h); - h = out_layers->forward(ctx, h); - return ggml_add(ctx->ggml_ctx, h, x); // identity skip + h = out_norm->forward(ctx, h); + h = ggml_mul_inplace(ctx->ggml_ctx, h, gain); + h = ggml_add_inplace(ctx->ggml_ctx, h, shift); + h = ggml_silu_inplace(ctx->ggml_ctx, h); + h = out_layers->forward(ctx, h); + return ggml_add(ctx->ggml_ctx, h, x); } }; - // Depthwise convolution over the time axis only. - // - // The reference implementation declares this as Conv3d(C, C, (k,1,1), groups=C), - // so the checkpoint stores a genuine depthwise kernel: PyTorch [C, 1, k, 1, 1], - // which the safetensors loader folds to [1, 1, k, C] (2,560 values at C=512). - // sdcpp's Conv3d is dense-only and would expect C*C*k values (512x more), so it - // cannot host this weight. Expanding the kernel to a block-diagonal dense one - // would also inflate the conv cost by the same 512x, which for a 3D conv on an - // iGPU is not affordable. Implement it directly instead: k shifted - // multiply-adds along T, all elementwise, so every backend supports it and the - // FLOP count stays at the depthwise level. + // Shifted multiply-adds preserve depthwise weights [1, 1, k, C] without + // expanding them into the dense kernels required by Conv3d. class H3TemporalDWConv3d : public UnaryBlock { protected: int64_t channels; @@ -281,8 +238,6 @@ namespace H3VUpsampler { if (ctx->weight_adapter) { w = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, w, prefix + "weight"); } - // Activations in this network are F32 (conv outputs are F32); the kernel - // is stored as F16, so match the activation type before multiplying. ggml_tensor* w32 = ggml_cast(ctx->ggml_ctx, w, GGML_TYPE_F32); ggml_tensor* xa = x; if (xa->type != GGML_TYPE_F32) { @@ -292,9 +247,7 @@ namespace H3VUpsampler { const int64_t T = xa->ne[2]; const int pad = kernel / 2; - // Zero-pad both ends of T. ggml_pad only pads the end of each dimension, - // so build a [W, H, pad, C] zero tensor from a zeroed slice of the input - // and concatenate it on both sides. + // ggml_pad only pads the end; concatenate zeros for symmetric padding. ggml_tensor* xp = xa; if (pad > 0) { ggml_tensor* zero = ggml_scale(ctx->ggml_ctx, ggml_ext_slice(ctx->ggml_ctx, xa, 2, 0, 1), 0.0f); @@ -308,10 +261,10 @@ namespace H3VUpsampler { ggml_tensor* acc = nullptr; for (int j = 0; j < kernel; ++j) { - ggml_tensor* xs = ggml_ext_slice(ctx->ggml_ctx, xp, 2, j, j + T); - ggml_tensor* wj = ggml_ext_slice(ctx->ggml_ctx, w32, 2, j, j + 1); // [1, 1, 1, C] + ggml_tensor* xs = ggml_ext_slice(ctx->ggml_ctx, xp, 2, j, j + T); + ggml_tensor* wj = ggml_ext_slice(ctx->ggml_ctx, w32, 2, j, j + 1); // [1, 1, 1, C] ggml_tensor* term = ggml_mul(ctx->ggml_ctx, xs, wj); - acc = acc == nullptr ? term : ggml_add(ctx->ggml_ctx, acc, term); + acc = acc == nullptr ? term : ggml_add(ctx->ggml_ctx, acc, term); } if (has_bias) { ggml_tensor* b = params["bias"]; @@ -324,8 +277,6 @@ namespace H3VUpsampler { } }; - // Temporal conv: GN -> SiLU -> depthwise Conv3d (k,1,1) -> pointwise Conv3d - // (zero-init at training) + identity. class H3TemporalConv : public GGMLBlock { protected: int64_t channels; @@ -336,11 +287,11 @@ namespace H3VUpsampler { const std::string prefix) override { blocks["dwconv"] = std::shared_ptr(new H3TemporalDWConv3d(channels, temporal_kernel)); blocks["pwconv"] = std::shared_ptr(new Conv3d(channels, - channels, - {1, 1, 1}, - {1, 1, 1}, - {0, 0, 0})); - blocks["norm"] = std::shared_ptr(new LTXVUpsampler::VideoGroupNorm(32, channels)); + channels, + {1, 1, 1}, + {1, 1, 1}, + {0, 0, 0})); + blocks["norm"] = std::shared_ptr(new LTXVUpsampler::VideoGroupNorm(32, channels)); } public: @@ -354,14 +305,13 @@ namespace H3VUpsampler { auto norm = std::dynamic_pointer_cast(blocks["norm"]); ggml_tensor* h = norm->forward(ctx, x); - h = ggml_silu_inplace(ctx->ggml_ctx, h); - h = dwconv->forward(ctx, h); - h = pwconv->forward(ctx, h); + h = ggml_silu_inplace(ctx->ggml_ctx, h); + h = dwconv->forward(ctx, h); + h = pwconv->forward(ctx, h); return ggml_add(ctx->ggml_ctx, h, x); } }; - // Scale-embedding MLP: Linear(1 -> E) -> SiLU -> Linear(E -> E). class H3Embed : public GGMLBlock { protected: int embed_dim; @@ -369,8 +319,6 @@ namespace H3VUpsampler { void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map, const std::string prefix) override { - // Child names are relative to the "embed" block prefix, so these bind as - // embed.0.* / embed.2.* to match the checkpoint. blocks["0"] = std::shared_ptr(new Linear(1, embed_dim, true)); blocks["2"] = std::shared_ptr(new Linear(embed_dim, embed_dim, true)); } @@ -380,10 +328,8 @@ namespace H3VUpsampler { : embed_dim(embed_dim) {} ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* scale_input) { - auto l0 = std::dynamic_pointer_cast(blocks["0"]); - auto l2 = std::dynamic_pointer_cast(blocks["2"]); - // Sequential(Linear, SiLU, Linear): the reference checkpoint ends at the - // second linear layer, each residual block activates the embedding itself. + auto l0 = std::dynamic_pointer_cast(blocks["0"]); + auto l2 = std::dynamic_pointer_cast(blocks["2"]); ggml_tensor* h = l0->forward(ctx, ggml_reshape_2d(ctx->ggml_ctx, scale_input, 1, 1)); h = ggml_silu_inplace(ctx->ggml_ctx, h); return l2->forward(ctx, h); @@ -398,19 +344,19 @@ namespace H3VUpsampler { : config(std::move(config_)) { for (int idx : config.in_res) { blocks["in_blocks." + std::to_string(idx)] = std::shared_ptr(new H3ResBlock(config.mid_channels, - config.embed_dim)); + config.embed_dim)); } for (int idx : config.in_temporal) { blocks["in_blocks." + std::to_string(idx)] = std::shared_ptr(new H3TemporalConv(config.mid_channels, - config.temporal_kernel)); + config.temporal_kernel)); } for (int idx : config.out_res) { blocks["out_blocks." + std::to_string(idx)] = std::shared_ptr(new H3ResBlock(config.mid_channels, - config.embed_dim)); + config.embed_dim)); } for (int idx : config.out_temporal) { blocks["out_blocks." + std::to_string(idx)] = std::shared_ptr(new H3TemporalConv(config.mid_channels, - config.temporal_kernel)); + config.temporal_kernel)); } blocks["conv_in"] = std::shared_ptr(new Conv3d(config.in_channels, config.mid_channels, @@ -419,14 +365,13 @@ namespace H3VUpsampler { {1, 1, 1})); blocks["norm_out"] = std::shared_ptr(new LTXVUpsampler::VideoGroupNorm(32, config.mid_channels)); blocks["conv_out"] = std::shared_ptr(new Conv3d(config.mid_channels, - config.in_channels, - {3, 3, 3}, - {1, 1, 1}, - {1, 1, 1})); + config.in_channels, + {3, 3, 3}, + {1, 1, 1}, + {1, 1, 1})); blocks["embed"] = std::shared_ptr(new H3Embed(config.embed_dim)); } - // Stage A: conv_in + in-block chain at the original spatial size. ggml_tensor* forward_stage_a(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* scale_input) { auto conv_in = std::dynamic_pointer_cast(blocks["conv_in"]); auto embed = std::dynamic_pointer_cast(blocks["embed"]); @@ -449,14 +394,13 @@ namespace H3VUpsampler { h = block->forward(ctx, h); } else { auto block = std::dynamic_pointer_cast(blocks["in_blocks." + std::to_string(i)]); - h = block->forward(ctx, h, emb); + h = block->forward(ctx, h, emb); } sd::ggml_graph_cut::mark_graph_cut(h, "h3_latent_upscaler.in_blocks." + std::to_string(i), "x"); } return h; } - // Stage B: out-block chain + norm_out + SiLU + conv_out at the target spatial size. ggml_tensor* forward_stage_b(GGMLRunnerContext* ctx, ggml_tensor* m, ggml_tensor* scale_input) { auto conv_out = std::dynamic_pointer_cast(blocks["conv_out"]); auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); @@ -478,14 +422,14 @@ namespace H3VUpsampler { g = block->forward(ctx, g); } else { auto block = std::dynamic_pointer_cast(blocks["out_blocks." + std::to_string(i)]); - g = block->forward(ctx, g, emb); + g = block->forward(ctx, g, emb); } sd::ggml_graph_cut::mark_graph_cut(g, "h3_latent_upscaler.out_blocks." + std::to_string(i), "x"); } - g = norm_out->forward(ctx, g); - g = ggml_silu_inplace(ctx->ggml_ctx, g); - g = conv_out->forward(ctx, g); + g = norm_out->forward(ctx, g); + g = ggml_silu_inplace(ctx->ggml_ctx, g); + g = conv_out->forward(ctx, g); sd::ggml_graph_cut::mark_graph_cut(g, "h3_latent_upscaler.conv_out", "x"); return g; } @@ -556,8 +500,6 @@ namespace H3VUpsampler { target_h); return {}; } - // Non-isotropic targets: the single scale-embedding input uses the mean of the - // width/height scale factors. const float scale = (static_cast(target_w) / static_cast(in_w) + static_cast(target_h) / static_cast(in_h)) / 2.0f; @@ -581,11 +523,11 @@ namespace H3VUpsampler { 0.7996809482574463f, 0.44988900423049927f, 0.7197399735450745f, 0.6936293244361877f, 2.961095094680786f, 2.7694199085235596f, 3.0496184825897217f, 2.1088054180145264f, 3.276226282119751f, 3.1627357006073f, 2.2816812992095947f, 2.6127843856811523f}); - const sd::Tensor normalized = (x - mean) / stddev; - // scale - 1 as one host tensor bound as a graph input for both stages. + // Training applies the VAE normalization a second time to diffusion latents. + // Undo only this extra transform after the network. + const sd::Tensor normalized = (x - mean) / stddev; const sd::Tensor scale_tensor = sd::Tensor::from_vector({scale - 1.0f}); - // Stage A: conv_in + in-block chain at the original spatial size. - auto get_graph_a = [this, &normalized, &scale_tensor]() -> ggml_cgraph* { + auto get_graph_a = [this, &normalized, &scale_tensor]() -> ggml_cgraph* { ggml_cgraph* gf = new_graph_custom(H3_UPSAMPLER_GRAPH_SIZE); ggml_tensor* x_t = make_input(normalized); ggml_tensor* scale_input = make_input(scale_tensor); @@ -602,18 +544,16 @@ namespace H3VUpsampler { } sd::Tensor mid = std::move(*mid_opt); - // Middle spatial resample on the host; the time axis is never touched. std::vector mid_shape = mid.shape(); mid_shape[0] = target_w; mid_shape[1] = target_h; - sd::Tensor mid_up = sd::ops::interpolate(mid, mid_shape, sd::ops::InterpolateMode::Bilinear, false, false); + sd::Tensor mid_up = sd::ops::interpolate(mid, mid_shape, sd::ops::InterpolateMode::Bilinear, false, false); if (mid_up.empty()) { LOG_ERROR("H3 latent upsampler middle spatial interpolate failed: %" PRId64 "x%" PRId64 " -> %" PRId64 "x%" PRId64, in_w, in_h, target_w, target_h); return {}; } - // Stage B: out-block chain + conv_out at the target spatial size. auto get_graph_b = [this, &mid_up, &scale_tensor]() -> ggml_cgraph* { ggml_cgraph* gf = new_graph_custom(H3_UPSAMPLER_GRAPH_SIZE); ggml_tensor* m_t = make_input(mid_up); diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index 97e1131f81..8c911b143a 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -1318,8 +1318,6 @@ namespace sd::pipeline { return result_images; } - // Video models without a dedicated latent upsampler checkpoint (MiniMax-H3) - // resample the packed [w, h, f, c] latent itself with an interpolating kernel. static bool hires_upscaler_is_latent_interpolation(enum sd_hires_upscaler_t upscaler) { switch (upscaler) { case SD_HIRES_UPSCALER_LATENT: @@ -1357,14 +1355,10 @@ namespace sd::pipeline { } return sd::ops::InterpolateMode::Bilinear; default: - // SD_HIRES_UPSCALER_LATENT return sd::ops::InterpolateMode::Bilinear; } } - // Spatial-only latent upscale that keeps the video time axis untouched, so the - // packed audio latent length stays valid. Used as the hires second stage for - // video models that have no latent upsampler checkpoint (currently MiniMax-H3). static sd::Tensor upscale_spatial_video_latent(StableDiffusionGGML* sd, const GenerationRequest& request, const sd::Tensor& packed_latent, @@ -1384,10 +1378,6 @@ namespace sd::pipeline { request.hires.target_height); return {}; } - // Nothing to do at 1x. This matters for the learned upscaler: the network is not - // an identity mapping at scale 1, so running it would alter a latent the caller - // asked to keep at the draft size. Matches the reference node, which returns the - // input latent unchanged when the target size equals the input size. if (packed_latent.shape()[0] == target_width && packed_latent.shape()[1] == target_height) { LOG_DEBUG("video latent spatial upscale: target equals the draft latent size, skipping"); return packed_latent; @@ -1407,8 +1397,8 @@ namespace sd::pipeline { } } - bool antialias = false; - auto mode = hires_latent_interpolate_mode(request.hires.upscaler, &antialias); + bool antialias = false; + auto mode = hires_latent_interpolate_mode(request.hires.upscaler, &antialias); std::vector target_shape = video_latent.shape(); target_shape[0] = target_width; target_shape[1] = target_height; @@ -1432,15 +1422,9 @@ namespace sd::pipeline { if (audio_latent.empty()) { return upscaled; } - // The audio latent is re-packed for the new spatial size; its own length - // is unchanged because no frame was added or dropped. return pack_ltxav_audio_and_video_latents(upscaled, audio_latent); } - // Model-backed H3 hires second stage: run the H3-shaped learned latent upscaler - // (e.g. the community 3D-conv checkpoint) between the low-resolution pass and - // the high-resolution refine pass. Audio is unpacked around the network and - // re-packed for the new spatial size. sd::Tensor upscale_h3_learned_video_latent(StableDiffusionGGML* sd, const char* model_path, const sd::Tensor& packed_latent, @@ -1454,9 +1438,7 @@ namespace sd::pipeline { LOG_ERROR("H3 learned latent upscale requires a hires model path"); return {}; } - // The network is not an identity mapping at scale 1, so a target equal to the - // draft latent size must skip it instead of resampling through the model. The - // reference node returns the input latent unchanged in this case. + // The learned network is not an identity mapping at scale 1. if (packed_latent.shape()[0] == target_width && packed_latent.shape()[1] == target_height) { LOG_DEBUG("H3 learned latent upscale: target equals the draft latent size, skipping"); return packed_latent; @@ -1520,11 +1502,11 @@ namespace sd::pipeline { } scope.owned_source = owns_source ? source_file : 0; - auto& upsampler = scope.runner; - upsampler = std::make_unique(sd->backend_for(SDBackendModule::UPSCALER), - model_manager->loader().get_tensor_storage_map(), - prefix, - model_manager); + auto& upsampler = scope.runner; + upsampler = std::make_unique(sd->backend_for(SDBackendModule::UPSCALER), + model_manager->loader().get_tensor_storage_map(), + prefix, + model_manager); const size_t max_graph_vram_bytes = sd->max_graph_vram_bytes_for_module(SDBackendModule::UPSCALER); upsampler->set_max_graph_vram_bytes(max_graph_vram_bytes); if (upsampler->model == nullptr) { @@ -1889,10 +1871,6 @@ namespace sd::pipeline { return false; } } else { - // MiniMax-H3 ships no official latent upsampler checkpoint, so the second - // stage either resamples the packed latent with a latent interpolator or, - // when a model-backed upscaler is selected, runs a H3-shaped learned - // upscaler checkpoint from --hires-upscalers-dir. const bool model_upscaler = request.hires.upscaler == SD_HIRES_UPSCALER_MODEL; if (model_upscaler) { if (strlen(SAFE_STR(request.hires.model_path)) == 0) { @@ -2055,9 +2033,9 @@ namespace sd::pipeline { sd::Tensor upscaled_latent; if (sd_version_is_ltxav(sd->version)) { upscaled_latent = upscale_ltx_spatial_video_latent(sd, - request.hires.model_path, - final_latent, - latents.audio_length); + request.hires.model_path, + final_latent, + latents.audio_length); } else if (request.hires.upscaler == SD_HIRES_UPSCALER_MODEL) { upscaled_latent = upscale_h3_learned_video_latent(sd, request.hires.model_path, @@ -2132,19 +2110,15 @@ namespace sd::pipeline { } if (sd_version_is_ltxav(sd->version) && !apply_ltxv_refine_image_conditioning(sd, - sd_vid_gen_params, - hires_request, - latents, - &x_t, - &hires_denoise_mask, - &hires_video_positions)) { + sd_vid_gen_params, + hires_request, + latents, + &x_t, + &hires_denoise_mask, + &hires_video_positions)) { return false; } - // TODO(keyframes): MiniMax-H3 keyframe latents are encoded at the first - // stage resolution. Supporting --init-img/--end-img hires requires - // re-encoding them at hires_request.width/height and replacing the - // matching entries of embeds.cond/uncond.c_ref_images, so the hires - // stage rejects them for now (see the guard before sampling). + // H3 keyframe conditioning must be re-encoded at the hires resolution. if (sd_version_is_minimax_h3(sd->version) && !latents.denoise_mask.empty() && (latents.denoise_mask.shape()[0] != x_t.shape()[0] ||