From d234ac62faad6c1fa0bb72c40970634e3df3e42d Mon Sep 17 00:00:00 2001 From: Wagner Bruna Date: Sun, 25 Oct 2026 04:58:35 -0300 Subject: [PATCH 01/11] feat: print sigma values before inference (#2205) --- src/pipeline/diffusion_engine.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index d7a0f005cb..d161d9e5be 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2871,6 +2871,18 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr::max_digits10); + for (size_t i = 0; i < sigmas.size(); ++i) { + if (i != 0) { + oss << ","; + } + oss << sigmas[i]; + } + LOG_DEBUG("sigmas: %s", oss.str().c_str()); + } + auto x0_opt = sample_k_diffusion(method, denoise, x_t, sigmas, sampler_rng, eta, is_flow_denoiser, extra_sample_args, denoiser); if (x0_opt.empty()) { LOG_ERROR("Diffusion model sampling failed"); From 2a5f81095d8fee00b3290a471ece49b28da11667 Mon Sep 17 00:00:00 2001 From: Mirek Date: Sun, 25 Oct 2026 08:00:26 +0000 Subject: [PATCH 02/11] perf: reuse shared modulation slices for moss image (#2208) --- src/model/diffusion/moss_image.hpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/model/diffusion/moss_image.hpp b/src/model/diffusion/moss_image.hpp index 0eace334ec..7aaf9f3771 100644 --- a/src/model/diffusion/moss_image.hpp +++ b/src/model/diffusion/moss_image.hpp @@ -2,6 +2,7 @@ #define __SD_MODEL_DIFFUSION_MOSS_IMAGE_HPP__ #include +#include #include #include "core/pe_cache.h" @@ -114,11 +115,11 @@ namespace MOSSImage { blocks["proj.ffn_down"] = std::make_shared(config.ffn_dim, config.dim, false); } - ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe, ggml_tensor* mod) { + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe, const std::array& mod) { auto c = ctx->ggml_ctx; auto linear = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast(blocks[name])->forward(ctx, h); }; auto norm = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast(blocks[name])->forward(ctx, h); }; - auto part = [&](int i) { return ggml_ext_slice(c, mod, 0, i * config.dim, (i + 1) * config.dim); }; + auto part = [&](int i) { return mod[i]; }; auto y = norm("norm1", x); y = ggml_add(c, ggml_add(c, y, ggml_mul(c, y, part(1))), part(0)); auto qkv = linear("proj.qkv", y); @@ -176,10 +177,17 @@ namespace MOSSImage { time = linear("time_embedding.2", ggml_silu(c, linear("time_embedding.0", time))); auto mod = linear("shared_block_modulation", time); auto head = linear("video_head_modulation", time); + std::array parts; + for (int i = 0; i < 6; ++i) { + parts[i] = ggml_ext_slice(c, mod, 0, i * config.dim, (i + 1) * config.dim); + if (ctx->graph != nullptr) { + ggml_build_forward_expand(ctx->graph, parts[i]); + } + } x = ggml_concat(c, x, text, 1); for (int64_t i = 0; i < config.depth; ++i) { if (std::find(skip.begin(), skip.end(), i) == skip.end()) { - x = std::dynamic_pointer_cast(blocks["layers." + std::to_string(i)])->forward(ctx, x, pe, mod); + x = std::dynamic_pointer_cast(blocks["layers." + std::to_string(i)])->forward(ctx, x, pe, parts); sd::ggml_graph_cut::mark_graph_cut(x, "moss_image.layers." + std::to_string(i), "x"); } } @@ -255,7 +263,7 @@ namespace MOSSImage { } auto time = ggml_new_tensor_3d(compute_ctx, GGML_TYPE_F32, config.freq_dim, 1, 1); set_backend_tensor_data(time, time_data.data()); - auto ctx = get_context(); + auto ctx = get_context(gf); ggml_build_forward_expand(gf, model.forward(&ctx, x, time, text, pe, skip)); return gf; }; From e6370bf093c50260190159237ac80954634cdf3b Mon Sep 17 00:00:00 2001 From: Mirek Date: Sun, 25 Oct 2026 08:54:51 +0000 Subject: [PATCH 03/11] perf: avoid redundant MOSS QKV and value layout copies (#2209) Co-authored-by: leejet --- src/core/ggml_extend.cpp | 14 ++++++++------ src/model/diffusion/moss_image.hpp | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp index 1099f4b037..bd5f6f63d0 100644 --- a/src/core/ggml_extend.cpp +++ b/src/core/ggml_extend.cpp @@ -678,21 +678,23 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx, // CUDA FA MMA starts at 64 channels; keep the original head's attention scale. q_in = ggml_pad(ctx, q_in, static_cast(64 - d_head), 0, 0, 0); k_in = ggml_pad(ctx, k_in, static_cast(64 - d_head), 0, 0, 0); + v_in = ggml_pad(ctx, v_in, static_cast(64 - d_head), 0, 0, 0); } if (kv_scale != 1.0f) { k_in = ggml_ext_scale(ctx, k_in, kv_scale); } k_in = ggml_cast(ctx, k_in, GGML_TYPE_F16); - v_in = ggml_ext_cont(ctx, ggml_permute(ctx, v_in, 0, 2, 1, 3)); - v_in = ggml_reshape_3d(ctx, v_in, d_head, L_k, n_kv_head * N); - if (pad_head) { - v_in = ggml_pad(ctx, v_in, static_cast(64 - d_head), 0, 0, 0); - } if (kv_scale != 1.0f) { v_in = ggml_ext_scale(ctx, v_in, kv_scale); } - v_in = ggml_cast(ctx, v_in, GGML_TYPE_F16); + v_in = ggml_permute(ctx, v_in, 0, 2, 1, 3); + auto v_f16 = ggml_cast(ctx, v_in, GGML_TYPE_F16); + if (!ggml_backend_supports_op(backend, v_f16)) { + v_f16 = ggml_cast(ctx, ggml_ext_cont(ctx, v_in), GGML_TYPE_F16); + } + v_in = v_f16; + v_in = ggml_reshape_3d(ctx, v_in, v_in->ne[0], L_k, n_kv_head * N); if (mask_in != nullptr) { // ggml_flash_attn_ext expects the mask as a contiguous F16 tensor shaped diff --git a/src/model/diffusion/moss_image.hpp b/src/model/diffusion/moss_image.hpp index 7aaf9f3771..a854c3275f 100644 --- a/src/model/diffusion/moss_image.hpp +++ b/src/model/diffusion/moss_image.hpp @@ -124,7 +124,8 @@ namespace MOSSImage { y = ggml_add(c, ggml_add(c, y, ggml_mul(c, y, part(1))), part(0)); auto qkv = linear("proj.qkv", y); auto split = [&](int i) { - return ggml_reshape_4d(c, ggml_ext_slice(c, qkv, 0, i * config.dim, (i + 1) * config.dim), config.head_dim, config.dim / config.head_dim, x->ne[1], x->ne[2]); + return ggml_view_4d(c, qkv, config.head_dim, config.dim / config.head_dim, x->ne[1], x->ne[2], + config.head_dim * qkv->nb[0], qkv->nb[1], qkv->nb[2], i * config.dim * qkv->nb[0]); }; auto q = norm("q_norm", split(0)); auto k = norm("k_norm", split(1)); From bb6a0ccfa31ef24d776d951580a51599797d8cc0 Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 25 Oct 2026 16:55:45 +0800 Subject: [PATCH 04/11] feat: support FLUX.3 Diffusers LoRA tensor names (#2214) --- src/name_conversion.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp index f37c4ddbe3..57a41c5c39 100644 --- a/src/name_conversion.cpp +++ b/src/name_conversion.cpp @@ -708,6 +708,24 @@ std::string convert_diffusers_dit_to_original_flux(std::string name) { return name; } +static std::string convert_diffusers_dit_to_original_flux3(std::string name) { + static const std::vector> prefix_map = { + {"single_transformer_blocks.", "single_blocks."}, + {"mode_blocks.", "content_mode_blocks."}, + }; + static const std::vector> name_map = { + {".attn.to_q.", ".q_proj."}, + {".attn.to_k.", ".k_proj."}, + {".attn.to_v.", ".v_proj."}, + {".attn.to_out.0.", ".attn_out."}, + {".ff.linear_in.", ".mlp_in."}, + {".ff.linear_out.", ".mlp_out."}, + }; + replace_with_prefix_map(name, prefix_map); + replace_with_name_map(name, name_map); + return name; +} + std::string convert_hunyuan_video_to_original_flux(std::string name) { int num_layers = 54; int num_single_layers = 0; @@ -1077,6 +1095,8 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S name = convert_diffusers_dit_to_original_sd3(name); } else if (sd_version_is_flux(version) || sd_version_is_flux2(version) || sd_version_is_longcat(version) || sd_version_is_sefi_image(version)) { name = convert_diffusers_dit_to_original_flux(name); + } else if (sd_version_is_flux3(version)) { + name = convert_diffusers_dit_to_original_flux3(name); } else if (sd_version_is_hunyuan_video(version)) { name = convert_hunyuan_video_to_original_flux(name); } else if (version == VERSION_MOSS_IMAGE) { From 0f76bf5c1ec8944b24997617f20a8eada33ad9de Mon Sep 17 00:00:00 2001 From: Crystal Sight Date: Sun, 25 Oct 2026 04:18:47 -0500 Subject: [PATCH 05/11] feat(server): accept ref_videos, ref_audios, ref_video_audios in sdcpp vid_gen (#2194) --- examples/common/common.cpp | 93 ++++++++++++++- examples/common/media_io.cpp | 108 +++++++++++------- examples/common/media_io.h | 7 ++ examples/server/api.md | 3 + examples/server/model_catalog.cpp | 7 ++ .../server/recipes/minimax-h3-ref2va.json | 5 +- examples/server/recipes/wan22-s2v.json | 3 +- examples/server/routes_sdcpp.cpp | 11 +- 8 files changed, 186 insertions(+), 51 deletions(-) diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 00301dc9c9..637bbb659f 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -1959,6 +1959,7 @@ static bool parse_image_json_field(const json& parent, return decode_base64_image(parent.at(key).get(), channels, 0, 0, out_image); } +/** Parses a JSON array of base64 image strings into owned images. */ static bool parse_image_array_json_field(const json& parent, const char* key, int channels, @@ -1988,6 +1989,84 @@ static bool parse_image_array_json_field(const json& parent, return true; } +/** Parses a JSON array of base64 WAV strings into owned audio clips. */ +static bool parse_audio_array_json_field(const json& parent, + const char* key, + std::vector& out_audios) { + if (!parent.contains(key)) { + return true; + } + if (parent.at(key).is_null()) { + out_audios.clear(); + return true; + } + if (!parent.at(key).is_array()) { + return false; + } + + out_audios.clear(); + for (const auto& item : parent.at(key)) { + if (!item.is_string()) { + return false; + } + std::string encoded = item.get(); + auto comma_pos = encoded.find(','); + if (comma_pos != std::string::npos) { + encoded = encoded.substr(comma_pos + 1); + } + std::vector audio_bytes = decode_base64_bytes(encoded); + if (audio_bytes.empty()) { + return false; + } + std::vector samples; + uint32_t sample_rate = 0; + uint32_t channels = 0; + if (!load_wav_from_memory(audio_bytes.data(), audio_bytes.size(), samples, sample_rate, channels)) { + return false; + } + SDAudioOwner audio; + audio.reset(std::move(samples), sample_rate, channels); + out_audios.push_back(std::move(audio)); + } + return true; +} + +/** Parses a JSON array of per-video base64 frame arrays into owned videos. */ +static bool parse_video_array_json_field(const json& parent, + const char* key, + std::vector>& out_videos) { + if (!parent.contains(key)) { + return true; + } + if (parent.at(key).is_null()) { + out_videos.clear(); + return true; + } + if (!parent.at(key).is_array()) { + return false; + } + + out_videos.clear(); + for (const auto& video : parent.at(key)) { + if (!video.is_array() || video.empty()) { + return false; + } + std::vector frames; + for (const auto& frame : video) { + if (!frame.is_string()) { + return false; + } + SDImageOwner image; + if (!decode_base64_image(frame.get(), 0, 0, 0, image)) { + return false; + } + frames.push_back(std::move(image)); + } + out_videos.push_back(std::move(frames)); + } + return true; +} + static bool parse_lora_json_field(const json& parent, const std::function& lora_path_resolver, std::map& lora_map, @@ -2102,6 +2181,7 @@ bool SDGenerationParams::parse_image_preprocess_json(const std::string& json_str return true; } +/** Populates generation params from a JSON payload (server APIs); returns false on any malformed field. */ bool SDGenerationParams::from_json_str( const std::string& json_str, const std::function& lora_path_resolver) { @@ -2330,8 +2410,11 @@ bool SDGenerationParams::from_json_str( return false; } if (!parse_image_array_json_field(j, "ref_images", 0, ref_images) || - !parse_image_array_json_field(j, "control_frames", 3, control_frames)) { - LOG_ERROR("invalid input image array"); + !parse_image_array_json_field(j, "control_frames", 3, control_frames) || + !parse_audio_array_json_field(j, "ref_audios", ref_audios) || + !parse_audio_array_json_field(j, "ref_video_audios", ref_video_audios) || + !parse_video_array_json_field(j, "ref_videos", ref_videos)) { + LOG_ERROR("invalid input image/audio/video array"); return false; } @@ -2583,6 +2666,7 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s return true; } +/** Validates params for a generation mode; enforces mode-specific input rules. */ bool SDGenerationParams::validate(SDMode mode) { if (!image_preprocess.empty() && mode != IMG_GEN && mode != VID_GEN) { LOG_ERROR("--image-preprocess requires img_gen or vid_gen mode"); @@ -2636,12 +2720,13 @@ bool SDGenerationParams::validate(SDMode mode) { return false; } - if (ref_video_audio_paths.size() > ref_video_paths.size()) { + if (ref_video_audio_paths.size() + ref_video_audios.size() > ref_video_paths.size() + ref_videos.size()) { LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video"); return false; } - if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty())) { + if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty() || + !ref_videos.empty() || !ref_video_audios.empty() || !ref_audios.empty())) { LOG_ERROR("error: reference video and audio inputs require vid_gen mode"); return false; } diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp index f09c2166ee..6c03a2a104 100644 --- a/examples/common/media_io.cpp +++ b/examples/common/media_io.cpp @@ -1437,70 +1437,94 @@ static uint32_t read_le32(const uint8_t* data) { (static_cast(data[3]) << 24); } +/** Loads a WAV file from disk and delegates decoding to load_wav_from_memory(). */ bool load_wav_from_file(const std::string& path, std::vector& interleaved_samples, uint32_t& sample_rate, uint32_t& channels) { + std::ifstream file(fs::u8path(path), std::ios::binary); + if (!file) { + interleaved_samples.clear(); + sample_rate = 0; + channels = 0; + return false; + } + file.seekg(0, std::ios::end); + const std::streamoff file_size = file.tellg(); + file.seekg(0, std::ios::beg); + if (file_size < 0) { + interleaved_samples.clear(); + sample_rate = 0; + channels = 0; + return false; + } + std::vector buffer(static_cast(file_size)); + if (file_size > 0 && !file.read(reinterpret_cast(buffer.data()), file_size)) { + interleaved_samples.clear(); + sample_rate = 0; + channels = 0; + return false; + } + return load_wav_from_memory(buffer.data(), buffer.size(), interleaved_samples, sample_rate, channels); +} + +/** Parses a RIFF/WAVE buffer in place; compares chunk sizes against the remaining + * bytes so payload-controlled sizes cannot wrap size_t on 32-bit builds. */ +bool load_wav_from_memory(const uint8_t* data, + size_t size, + std::vector& interleaved_samples, + uint32_t& sample_rate, + uint32_t& channels) { interleaved_samples.clear(); sample_rate = 0; channels = 0; - std::ifstream file(fs::u8path(path), std::ios::binary); - uint8_t riff_header[12]; - if (!file.read(reinterpret_cast(riff_header), sizeof(riff_header)) || - std::memcmp(riff_header, "RIFF", 4) != 0 || - std::memcmp(riff_header + 8, "WAVE", 4) != 0) { + if (data == nullptr || + size < 12 || + std::memcmp(data, "RIFF", 4) != 0 || + std::memcmp(data + 8, "WAVE", 4) != 0) { return false; } uint16_t audio_format = 0; uint16_t bits_per_sample = 0; uint16_t block_align = 0; - std::streampos data_pos = std::streampos(-1); + size_t data_pos = SIZE_MAX; uint32_t data_size = 0; - while (file.good()) { - uint8_t chunk_header[8]; - if (!file.read(reinterpret_cast(chunk_header), sizeof(chunk_header))) { - break; - } - uint32_t chunk_size = read_le32(chunk_header + 4); - std::streampos chunk_data_pos = file.tellg(); - - if (std::memcmp(chunk_header, "fmt ", 4) == 0) { - if (chunk_size < 16) { - return false; - } - std::vector fmt(chunk_size); - if (!file.read(reinterpret_cast(fmt.data()), chunk_size)) { + size_t pos = 12; + while (pos + 8 <= size) { + uint32_t chunk_size = read_le32(data + pos + 4); + size_t chunk_data_pos = pos + 8; + const size_t remaining = size - chunk_data_pos; // safe: chunk_data_pos <= size + // Chunk sizes come from the payload, so compare against the remaining bytes + // instead of adding them to offsets (avoids size_t wrap on 32-bit builds). + const bool data_fits = static_cast(chunk_size) <= remaining; + + if (std::memcmp(data + pos, "fmt ", 4) == 0) { + if (chunk_size < 16 || !data_fits) { return false; } - audio_format = read_le16(fmt.data()); - channels = read_le16(fmt.data() + 2); - sample_rate = read_le32(fmt.data() + 4); - block_align = read_le16(fmt.data() + 12); - bits_per_sample = read_le16(fmt.data() + 14); + audio_format = read_le16(data + chunk_data_pos); + channels = read_le16(data + chunk_data_pos + 2); + sample_rate = read_le32(data + chunk_data_pos + 4); + block_align = read_le16(data + chunk_data_pos + 12); + bits_per_sample = read_le16(data + chunk_data_pos + 14); if (audio_format == 0xfffe && chunk_size >= 40) { - audio_format = read_le16(fmt.data() + 24); + audio_format = read_le16(data + chunk_data_pos + 24); } - } else if (std::memcmp(chunk_header, "data", 4) == 0) { + } else if (std::memcmp(data + pos, "data", 4) == 0) { data_pos = chunk_data_pos; data_size = chunk_size; - file.seekg(chunk_size, std::ios::cur); - } else { - file.seekg(chunk_size, std::ios::cur); - } - - if (!file.good()) { - break; - } - if ((chunk_size & 1) != 0) { - file.seekg(1, std::ios::cur); } + // Only advance past a chunk whose size and pad byte fully fit (keeps the sum + // overflow-free); a chunk truncated at the buffer end terminates the walk. + const bool chunk_fits = data_fits && ((chunk_size & 1) == 0 || static_cast(chunk_size) < remaining); + pos = chunk_fits ? chunk_data_pos + static_cast(chunk_size) + (chunk_size & 1) : size; } const uint32_t bytes_per_sample = (bits_per_sample + 7) / 8; - if (data_pos == std::streampos(-1) || data_size == 0 || channels == 0 || sample_rate == 0 || + if (data_pos == SIZE_MAX || data_size == 0 || channels == 0 || sample_rate == 0 || block_align == 0 || bytes_per_sample == 0 || block_align < channels * bytes_per_sample || (audio_format != 1 && audio_format != 3)) { return false; @@ -1510,16 +1534,14 @@ bool load_wav_from_file(const std::string& path, if (frame_count == 0 || frame_count > SIZE_MAX / channels) { return false; } - std::vector pcm(data_size); - file.clear(); - file.seekg(data_pos); - if (!file.read(reinterpret_cast(pcm.data()), data_size)) { + if (static_cast(data_size) > size - data_pos) { return false; } + const uint8_t* pcm = data + data_pos; interleaved_samples.resize(static_cast(frame_count * channels)); for (uint64_t frame = 0; frame < frame_count; ++frame) { - const uint8_t* frame_data = pcm.data() + frame * block_align; + const uint8_t* frame_data = pcm + frame * block_align; for (uint32_t channel = 0; channel < channels; ++channel) { const uint8_t* sample_data = frame_data + channel * bytes_per_sample; float sample = 0.0f; diff --git a/examples/common/media_io.h b/examples/common/media_io.h index 18042a8ba4..7feb0ada25 100644 --- a/examples/common/media_io.h +++ b/examples/common/media_io.h @@ -126,4 +126,11 @@ bool load_wav_from_file(const std::string& path, uint32_t& sample_rate, uint32_t& channels); +/** Decodes a WAV buffer (PCM 8/16/24/32 or IEEE float 32/64) into interleaved samples. */ +bool load_wav_from_memory(const uint8_t* data, + size_t size, + std::vector& interleaved_samples, + uint32_t& sample_rate, + uint32_t& channels); + #endif // __MEDIA_IO_H__ diff --git a/examples/server/api.md b/examples/server/api.md index 1258b1af58..b5e0250c4a 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -183,6 +183,9 @@ Default-enabled features are `img2img`, `mask_image`, `lora`, `vae_tiling`, | `clip_skip` | CLIP Skip | | `img2img`, `mask_image` | Image initialization/strength and masks | | `ref_images` | Reference image inputs | +| `ref_videos` | Reference videos as frame arrays | +| `ref_audios` | Standalone reference audio; runtime requires an Audio VAE or Audio Encoder | +| `ref_video_audios` | Soundtracks paired with reference videos; runtime requires an Audio VAE | | `controlnet` | Control image and ControlNet strength | | `control_image` | Control image without a separate ControlNet (FLUX Canny/Depth) | | `ip_adapter` | IP-Adapter image and strength | diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp index eef685614c..1041dea6ac 100644 --- a/examples/server/model_catalog.cpp +++ b/examples/server/model_catalog.cpp @@ -291,6 +291,7 @@ static void apply_recipe_context(const json& settings, SDContextParams& params) } } +// Known recipe feature names with their defaults; recipes may flip any of these. static const std::map k_recipe_features = { {"img2img", true}, {"mask_image", true}, @@ -312,6 +313,9 @@ static const std::map k_recipe_features = { {"init_image", false}, {"end_image", false}, {"control_frames", false}, + {"ref_videos", false}, + {"ref_audios", false}, + {"ref_video_audios", false}, {"vace", false}, {"high_noise", false}, {"cache_unet", false}, @@ -369,6 +373,9 @@ json model_generation_features(const ServerRuntime& runtime) { features["controlnet"] = features.value("controlnet", false) && !runtime.ctx_params->control_net_path.empty(); features["high_noise"] = features.value("high_noise", false) && !runtime.ctx_params->high_noise_diffusion_model_path.empty(); features["preview_tae"] = features.value("preview_tae", false) && !runtime.ctx_params->taesd_path.empty(); + features["ref_audios"] = features.value("ref_audios", false) && + (!runtime.ctx_params->audio_vae_path.empty() || !runtime.ctx_params->audio_encoder_path.empty()); + features["ref_video_audios"] = features.value("ref_video_audios", false) && !runtime.ctx_params->audio_vae_path.empty(); if (!runtime.ctx_params->pulid_weights_path.empty()) features["slg"] = false; return features; diff --git a/examples/server/recipes/minimax-h3-ref2va.json b/examples/server/recipes/minimax-h3-ref2va.json index 085f5d5fed..2fb0417806 100644 --- a/examples/server/recipes/minimax-h3-ref2va.json +++ b/examples/server/recipes/minimax-h3-ref2va.json @@ -45,7 +45,10 @@ }, "features": [ "flow_shift", - "ref_images" + "ref_images", + "ref_videos", + "ref_audios", + "ref_video_audios" ], "disabled_features": [ "img2img", diff --git a/examples/server/recipes/wan22-s2v.json b/examples/server/recipes/wan22-s2v.json index ba1ee4f7c3..55440ad317 100644 --- a/examples/server/recipes/wan22-s2v.json +++ b/examples/server/recipes/wan22-s2v.json @@ -44,7 +44,8 @@ }, "features": [ "flow_shift", - "init_image" + "init_image", + "ref_audios" ], "disabled_features": [ "img2img", diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp index b972768b9e..0dda9a95be 100644 --- a/examples/server/routes_sdcpp.cpp +++ b/examples/server/routes_sdcpp.cpp @@ -212,11 +212,15 @@ static json make_img_gen_features_json() { }; } +/** Builds the default feature set advertised for vid_gen requests. */ static json make_vid_gen_features_json() { return { {"init_image", true}, {"end_image", true}, {"control_frames", true}, + {"ref_videos", true}, + {"ref_audios", true}, + {"ref_video_audios", true}, {"high_noise_sample_params", true}, {"lora", true}, {"vae_tiling", true}, @@ -235,6 +239,7 @@ static json make_preview_modes_json() { return modes; } +/** Builds the /capabilities payload: model, defaults, limits and feature maps. */ static json make_capabilities_json(ServerRuntime& runtime) { refresh_lora_cache(runtime); refresh_upscaler_cache(runtime); @@ -351,8 +356,10 @@ static json make_capabilities_json(ServerRuntime& runtime) { const json generation_features = model_generation_features(runtime); auto recipe_features = [&](json features, bool video) { features["init_image"] = generation_features.at(video ? "init_image" : "img2img"); - for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "lora", "vae_tiling", "cache", "preview"}) { - if (features.contains(key) || std::string(key) == "ref_images") + for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "ref_videos", "ref_audios", "ref_video_audios", "lora", "vae_tiling", "cache", "preview"}) { + if (features.contains(key) || std::string(key) == "ref_images" || + (video && (std::string(key) == "ref_videos" || std::string(key) == "ref_audios" || + std::string(key) == "ref_video_audios"))) features[key] = generation_features.at(key); } if (video) From b29962db7ac1b41a0acdff60b37190507b70bd24 Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 25 Oct 2026 19:30:23 +0800 Subject: [PATCH 06/11] sync: update frontend --- examples/server/frontend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/frontend b/examples/server/frontend index 34e5ad6edc..9157320a3b 160000 --- a/examples/server/frontend +++ b/examples/server/frontend @@ -1 +1 @@ -Subproject commit 34e5ad6edcaf827b7fe1e9ffa55ee86cd2fc43d2 +Subproject commit 9157320a3bdd6a345f370868a32f730c77749cb5 From a5e48c66aacb0a7be9b73fa3b84df9a6829578a1 Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 25 Oct 2026 19:37:41 +0800 Subject: [PATCH 07/11] chore: reduce large recipe defaults to 1024x1024 except PiD --- examples/server/recipes/defaults.md | 25 +++++++++++-------- examples/server/recipes/flux1-fill.json | 4 +-- examples/server/recipes/hidream-o1-dev.json | 4 +-- examples/server/recipes/krea2-turbo-edit.json | 4 +-- examples/server/recipes/krea2-turbo.json | 4 +-- examples/server/recipes/lens-turbo.json | 4 +-- examples/server/recipes/lens.json | 4 +-- examples/server/recipes/ming-image.json | 4 +-- .../server/recipes/qwen-image-21-turbo.json | 4 +-- examples/server/recipes/qwen-image-21.json | 4 +-- examples/server/recipes/qwen-image.json | 4 +-- examples/server/recipes/sensenova-u15.json | 4 +-- 12 files changed, 36 insertions(+), 33 deletions(-) diff --git a/examples/server/recipes/defaults.md b/examples/server/recipes/defaults.md index 2be4dfbf68..e20a2cd783 100644 --- a/examples/server/recipes/defaults.md +++ b/examples/server/recipes/defaults.md @@ -5,6 +5,9 @@ Reviewed against model-publisher cards and reference implementations on 2026-10- These are starting values, not limits on supported resolutions or generation settings. Built-in step defaults above 20 are reduced to 20; the cited sources may recommend more steps. Users can still choose a higher step count. +Except for PiD recipes, built-in dimensions above 1,048,576 pixels (1024 x 1024) are reduced to 1024 x 1024, even when the cited source uses a larger size. +PiD recipes retain their 2048 x 2048 defaults. +Users can still choose a higher resolution. When several aspect ratios or ranges are published, the selected example or square bucket is noted below. A dash means no recipe override: the available primary sources did not establish a fixed value, or the size depends on the input image. Ordinary DiT recipes explicitly use Euler; PiD retains its LCM sampling path. @@ -27,7 +30,7 @@ For community edit adapters and other optional extensions, these are the base-mo | `flux1-canny` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Canny-dev) | | `flux1-depth` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Depth-dev) | | `flux1-dev` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-dev) | -| `flux1-fill` | 1 | 20 | 1232 × 1632 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Fill-dev) | +| `flux1-fill` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Fill-dev) | | `flux1-kontext` | 1 | 20 | — | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/flux/pipeline_flux_kontext.py) Output size normally follows the reference image; no fixed size is imposed by this recipe. | | `flux1-pulid` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-dev) FLUX.1-dev defaults; identity conditioning is provided by PuLID. | | `flux1-schnell` | 1 | 4 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-schnell) | @@ -38,7 +41,7 @@ For community edit adapters and other optional extensions, these are the base-mo | `flux2-klein-base-9b` | 4 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B) | | `flux3-klein` | 1 | 16 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/black-forest-labs/flux-3-image-klein/main/README.md) | | `flux3-klein-base` | 4 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/black-forest-labs/flux-3-image-klein/main/src/flux3/pipeline.py) | -| `hidream-o1-dev` | 1 | 20 | 2048 × 2048 | [Source](https://huggingface.co/HiDream-ai/HiDream-O1-Image-Dev) | +| `hidream-o1-dev` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/HiDream-ai/HiDream-O1-Image-Dev) | | `hunyuan-video-15-i2v` | 6 | 20 | 1280 × 720 | [Source](https://huggingface.co/tencent/HunyuanVideo-1.5) 720p base I2V defaults; CFG/step-distilled and 480p checkpoints use different settings. | | `hunyuan-video-15-t2v` | 6 | 20 | 1280 × 720 | [Source](https://huggingface.co/tencent/HunyuanVideo-1.5) 720p base T2V defaults; CFG-distilled / 480p checkpoints need their published settings. | | `ideogram4` | 7 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/ideogram-oss/ideogram4/main/docs/inference.md) Constant-guidance pipeline defaults; this recipe does not reproduce the CLI’s per-step guidance schedule. | @@ -46,10 +49,10 @@ For community edit adapters and other optional extensions, these are the base-mo | `iris-3b` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/speridlabs/iris-3b) | | `krea2-raw` | 3.5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/krea/Krea-2-Raw) Base model defaults; edit requires a matching community LoRA with its own settings. | | `krea2-raw-edit` | 3.5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/krea/Krea-2-Raw) Base model defaults; edit requires a matching community LoRA with its own settings. | -| `krea2-turbo` | 1 | 8 | 2048 × 2048 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. | -| `krea2-turbo-edit` | 1 | 8 | 2048 × 2048 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. | -| `lens` | 5 | 20 | 1440 × 1440 | [Source](https://huggingface.co/microsoft/Lens) Official model card: RL model, default base_resolution 1440. | -| `lens-turbo` | 1 | 4 | 1440 × 1440 | [Source](https://huggingface.co/microsoft/Lens-Turbo) Official model card: 4-step distilled model, default base_resolution 1440. | +| `krea2-turbo` | 1 | 8 | 1024 × 1024 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. | +| `krea2-turbo-edit` | 1 | 8 | 1024 × 1024 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. | +| `lens` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Lens) Official model card: RL model, default base_resolution 1440. | +| `lens-turbo` | 1 | 4 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Lens-Turbo) Official model card: 4-step distilled model, default base_resolution 1440. | | `lingbot-video-dense` | 3 | 20 | 832 × 480 | [Source](https://huggingface.co/robbyant/lingbot-video-dense-1.3b) Official first-stage inference example; optional refiner is separate. | | `lingbot-video-moe` | 3 | 20 | 832 × 480 | [Source](https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b) Official inference example. | | `llada-image` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/inclusionAI/LLaDA-Image) | @@ -62,7 +65,7 @@ For community edit adapters and other optional extensions, these are the base-mo | `mage-flow` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow) | | `mage-flow-edit` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow) | | `mage-flow-turbo` | 1 | 4 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow) | -| `ming-image` | 1 | 12 | 2048 × 2048 | [Source](https://huggingface.co/inclusionAI/Ming-Image-0.1-Design) | +| `ming-image` | 1 | 12 | 1024 × 1024 | [Source](https://huggingface.co/inclusionAI/Ming-Image-0.1-Design) | | `minimax-h3-fl2va` | 1 | — | 1344 × 768 | [Source](https://huggingface.co/docs/diffusers/main/en/api/pipelines/minimax_h3) Guidance-distilled; official 768p canvas. No fixed step count is prescribed here; the upstream API counts sigma grid points rather than sd.cpp denoising steps. | | `minimax-h3-ref2va` | 1 | — | 1344 × 768 | [Source](https://huggingface.co/docs/diffusers/main/en/api/pipelines/minimax_h3) Guidance-distilled; official 768p canvas. No fixed step count is prescribed here; the upstream API counts sigma grid points rather than sd.cpp denoising steps. | | `minit2i` | 2.5 | 20 | 512 × 512 | [Source](https://huggingface.co/MiniT2I/MiniT2I/raw/main/pipeline.py) B/16 example and native model size; L/16 uses CFG 6. | @@ -81,9 +84,9 @@ For community edit adapters and other optional extensions, these are the base-mo | `pixart-alpha-1024` | 4.5 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py) | | `pixart-alpha-512` | 4.5 | 20 | 512 × 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py) | | `pixart-sigma` | 4.5 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py) | -| `qwen-image` | 4 | 20 | 1328 × 1328 | [Source](https://huggingface.co/Qwen/Qwen-Image) Official square resolution bucket; true_cfg_scale maps to cfg_scale. | -| `qwen-image-21` | 1 | 20 | 2048 × 2048 | [Source](https://raw.githubusercontent.com/QwenLM/Qwen-Image-2.1/main/README.md) Original checkpoint defaults; select the matching TAEQI2.1 to replace its VAE. | -| `qwen-image-21-turbo` | 1 | 8 | 2048 × 2048 | [Model card](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo), [fixed sigmas](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo/blob/main/model_index.json). Standalone Turbo weights; the fixed 8-step schedule is applied on model load. | +| `qwen-image` | 4 | 20 | 1024 × 1024 | [Source](https://huggingface.co/Qwen/Qwen-Image) The source uses a larger square resolution bucket; true_cfg_scale maps to cfg_scale. | +| `qwen-image-21` | 1 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/QwenLM/Qwen-Image-2.1/main/README.md) Original checkpoint sampling defaults; select the matching TAEQI2.1 to replace its VAE. | +| `qwen-image-21-turbo` | 1 | 8 | 1024 × 1024 | [Model card](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo), [fixed sigmas](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo/blob/main/model_index.json). Standalone Turbo weights; the fixed 8-step schedule is applied on model load. | | `qwen-image-edit` | 4 | 20 | — | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit) Official editing example; output dimensions follow reference-image aspect ratio. | | `qwen-image-edit-2509` | 4 | 20 | — | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit-2509) Official editing example; output dimensions follow reference-image aspect ratio. | | `qwen-image-edit-2511` | 4 | 20 | — | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit-2511) Official editing example; output dimensions follow reference-image aspect ratio. | @@ -107,7 +110,7 @@ For community edit adapters and other optional extensions, these are the base-mo | `sdxl-turbo` | 1 | 1 | 512 × 512 | [Source](https://huggingface.co/stabilityai/sdxl-turbo) | | `sdxs` | 1 | 1 | 512 × 512 | [Source](https://huggingface.co/IDKiro/sdxs-512-dreamshaper) | | `sefi-image` | 4 | 20 | — | [Source](https://huggingface.co/SeFi-Image/SeFi-Image-5B-Base) Base/RL family defaults; Turbo checkpoints require 4 steps / CFG 1. | -| `sensenova-u15` | 4 | 20 | 2048 × 2048 | [Source](https://raw.githubusercontent.com/OpenSenseNova/SenseNova-U1/main/examples/README.md) | +| `sensenova-u15` | 4 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/OpenSenseNova/SenseNova-U1/main/examples/README.md) | | `wan21-flf2v` | 5 | 20 | 1280 × 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py) | | `wan21-i2v` | 5 | 20 | 832 × 480 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py) 480p base defaults; also accepts compatible SkyReels V2 and single Wan 2.2 I2V experts. | | `wan21-t2v` | 5 | 20 | 1280 × 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py) 14B base defaults; single Wan 2.2 experts and acceleration LoRAs need matching settings. | diff --git a/examples/server/recipes/flux1-fill.json b/examples/server/recipes/flux1-fill.json index 5e6485ad10..fe27a48e48 100644 --- a/examples/server/recipes/flux1-fill.json +++ b/examples/server/recipes/flux1-fill.json @@ -39,8 +39,8 @@ "scheduler": "flux", "cfg_scale": 1, "steps": 20, - "width": 1232, - "height": 1632 + "width": 1024, + "height": 1024 }, "features": [ "clip_skip", diff --git a/examples/server/recipes/hidream-o1-dev.json b/examples/server/recipes/hidream-o1-dev.json index 8e95f5ceac..cf1c1790bb 100644 --- a/examples/server/recipes/hidream-o1-dev.json +++ b/examples/server/recipes/hidream-o1-dev.json @@ -13,8 +13,8 @@ "defaults": { "cfg_scale": 1, "steps": 20, - "width": 2048, - "height": 2048, + "width": 1024, + "height": 1024, "sampling_method": "euler" }, "features": [ diff --git a/examples/server/recipes/krea2-turbo-edit.json b/examples/server/recipes/krea2-turbo-edit.json index fa248c083b..e8c521d354 100644 --- a/examples/server/recipes/krea2-turbo-edit.json +++ b/examples/server/recipes/krea2-turbo-edit.json @@ -39,8 +39,8 @@ "sampling_method": "euler", "cfg_scale": 1, "steps": 8, - "width": 2048, - "height": 2048 + "width": 1024, + "height": 1024 }, "features": [ "flow_shift", diff --git a/examples/server/recipes/krea2-turbo.json b/examples/server/recipes/krea2-turbo.json index 23dad12fd5..f19513f77a 100644 --- a/examples/server/recipes/krea2-turbo.json +++ b/examples/server/recipes/krea2-turbo.json @@ -32,8 +32,8 @@ "defaults": { "cfg_scale": 1, "steps": 8, - "width": 2048, - "height": 2048, + "width": 1024, + "height": 1024, "sampling_method": "euler" }, "features": [ diff --git a/examples/server/recipes/lens-turbo.json b/examples/server/recipes/lens-turbo.json index e9282eafc4..2b87bb6486 100644 --- a/examples/server/recipes/lens-turbo.json +++ b/examples/server/recipes/lens-turbo.json @@ -36,8 +36,8 @@ "defaults": { "cfg_scale": 1, "steps": 4, - "width": 1440, - "height": 1440, + "width": 1024, + "height": 1024, "sampling_method": "euler" }, "features": [ diff --git a/examples/server/recipes/lens.json b/examples/server/recipes/lens.json index 0a89542293..34165ad5a8 100644 --- a/examples/server/recipes/lens.json +++ b/examples/server/recipes/lens.json @@ -36,8 +36,8 @@ "defaults": { "cfg_scale": 5, "steps": 20, - "width": 1440, - "height": 1440, + "width": 1024, + "height": 1024, "sampling_method": "euler" }, "features": [ diff --git a/examples/server/recipes/ming-image.json b/examples/server/recipes/ming-image.json index 6c368e8e1a..465a690efe 100644 --- a/examples/server/recipes/ming-image.json +++ b/examples/server/recipes/ming-image.json @@ -36,8 +36,8 @@ "defaults": { "cfg_scale": 1, "steps": 12, - "width": 2048, - "height": 2048, + "width": 1024, + "height": 1024, "sampling_method": "euler" }, "features": [ diff --git a/examples/server/recipes/qwen-image-21-turbo.json b/examples/server/recipes/qwen-image-21-turbo.json index 2ef9fa183e..99da90d7b2 100644 --- a/examples/server/recipes/qwen-image-21-turbo.json +++ b/examples/server/recipes/qwen-image-21-turbo.json @@ -40,8 +40,8 @@ "sampling_method": "euler", "cfg_scale": 1, "steps": 8, - "width": 2048, - "height": 2048, + "width": 1024, + "height": 1024, "custom_sigmas": [ 1.0, 0.978453, diff --git a/examples/server/recipes/qwen-image-21.json b/examples/server/recipes/qwen-image-21.json index 4b6c67b86e..fd487e58f8 100644 --- a/examples/server/recipes/qwen-image-21.json +++ b/examples/server/recipes/qwen-image-21.json @@ -40,8 +40,8 @@ "sampling_method": "euler", "cfg_scale": 1, "steps": 20, - "width": 2048, - "height": 2048 + "width": 1024, + "height": 1024 }, "features": [ "flow_shift", diff --git a/examples/server/recipes/qwen-image.json b/examples/server/recipes/qwen-image.json index 417a621ae2..468f8f2613 100644 --- a/examples/server/recipes/qwen-image.json +++ b/examples/server/recipes/qwen-image.json @@ -35,8 +35,8 @@ "flow_shift": 3.0, "cfg_scale": 4, "steps": 20, - "width": 1328, - "height": 1328 + "width": 1024, + "height": 1024 }, "features": [ "flow_shift" diff --git a/examples/server/recipes/sensenova-u15.json b/examples/server/recipes/sensenova-u15.json index afd20466b6..f2df865e62 100644 --- a/examples/server/recipes/sensenova-u15.json +++ b/examples/server/recipes/sensenova-u15.json @@ -14,8 +14,8 @@ "flow_shift": 3.0, "cfg_scale": 4, "steps": 20, - "width": 2048, - "height": 2048, + "width": 1024, + "height": 1024, "sampling_method": "euler" }, "features": [ From 012c7f88661fefef61ce8e5108516b3f3b719d0c Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 25 Oct 2026 19:39:32 +0800 Subject: [PATCH 08/11] ci: trigger builds on recipe JSON changes --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 54582e84ca..8fc89cd2a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,6 +26,7 @@ on: "ggml", "examples/server/frontend", "examples/server/frontend/**", + "examples/server/recipes/*.json", "start-server.sh", "start-server.bat", ] @@ -46,6 +47,7 @@ on: "ggml", "examples/server/frontend", "examples/server/frontend/**", + "examples/server/recipes/*.json", "start-server.sh", "start-server.bat", ] From e41fbcf455e0fbb45e08bd535b00d6cba865e921 Mon Sep 17 00:00:00 2001 From: leejet Date: Mon, 26 Oct 2026 01:01:43 +0800 Subject: [PATCH 09/11] feat(server): add recipe constraints and update generation workspace (#2217) --- docs/image_preprocessing.md | 11 ++-- examples/common/media_io.cpp | 2 +- examples/server/README.md | 4 +- examples/server/api.md | 60 +++++++++++++++++-- examples/server/frontend | 2 +- examples/server/model_catalog.cpp | 47 ++++++++++++--- examples/server/recipes/README.md | 32 ++++++++++ examples/server/recipes/animatediff-sd15.json | 8 +++ examples/server/recipes/chroma-radiance.json | 1 + examples/server/recipes/cosxl-edit.json | 1 + .../server/recipes/ernie-image-turbo.json | 1 + examples/server/recipes/ernie-image.json | 1 + examples/server/recipes/fibo2-dev.json | 1 + examples/server/recipes/fibo2-edit-turbo.json | 1 + examples/server/recipes/flux2-dev.json | 1 + examples/server/recipes/flux2-klein-4b.json | 1 + examples/server/recipes/flux2-klein-9b.json | 1 + .../server/recipes/flux2-klein-base-4b.json | 1 + .../server/recipes/flux2-klein-base-9b.json | 1 + examples/server/recipes/flux3-klein-base.json | 1 + examples/server/recipes/flux3-klein.json | 1 + examples/server/recipes/hidream-o1-dev.json | 1 + .../server/recipes/hunyuan-video-15-i2v.json | 7 +++ .../server/recipes/hunyuan-video-15-t2v.json | 7 +++ examples/server/recipes/ideogram4.json | 1 + examples/server/recipes/instruct-pix2pix.json | 1 + examples/server/recipes/iris-3b.json | 1 + examples/server/recipes/lens-turbo.json | 1 + examples/server/recipes/lens.json | 1 + .../server/recipes/lingbot-video-dense.json | 7 +++ .../server/recipes/lingbot-video-moe.json | 7 +++ .../server/recipes/llada-image-turbo.json | 1 + examples/server/recipes/llada-image.json | 1 + examples/server/recipes/ltx23-dev.json | 7 +++ examples/server/recipes/ltx23-distilled.json | 7 +++ examples/server/recipes/ltx25-dev.json | 7 +++ examples/server/recipes/mage-flow-edit.json | 1 + examples/server/recipes/mage-flow-turbo.json | 1 + examples/server/recipes/mage-flow.json | 1 + examples/server/recipes/ming-image.json | 1 + examples/server/recipes/minimax-h3-fl2va.json | 8 +++ .../server/recipes/minimax-h3-ref2va.json | 8 +++ examples/server/recipes/minit2i.json | 1 + examples/server/recipes/moss-image-base.json | 1 + examples/server/recipes/moss-image-sft.json | 1 + examples/server/recipes/nitrosd-realism.json | 1 + examples/server/recipes/pid-15-flux2.json | 1 + .../server/recipes/pid-15-qwen-image.json | 1 + examples/server/recipes/pid-15-sd3.json | 1 + examples/server/recipes/pid-15.json | 1 + examples/server/recipes/pid-flux2.json | 1 + examples/server/recipes/pid-qwen-image.json | 1 + examples/server/recipes/pid-sd3.json | 1 + examples/server/recipes/pid.json | 1 + .../server/recipes/pixart-alpha-1024.json | 1 + examples/server/recipes/pixart-alpha-512.json | 1 + examples/server/recipes/pixart-sigma.json | 1 + .../server/recipes/qwen-image-21-turbo.json | 1 + examples/server/recipes/qwen-image-21.json | 1 + examples/server/recipes/sd-turbo.json | 1 + examples/server/recipes/sd1-checkpoint.json | 1 + examples/server/recipes/sd1-controlnet.json | 1 + examples/server/recipes/sd1-inpainting.json | 1 + examples/server/recipes/sd1-ip-adapter.json | 1 + .../server/recipes/sd2-512-checkpoint.json | 1 + .../server/recipes/sd2-768-checkpoint.json | 1 + examples/server/recipes/sd2-inpainting.json | 1 + examples/server/recipes/sdxl-checkpoint.json | 1 + examples/server/recipes/sdxl-controlnet.json | 1 + examples/server/recipes/sdxl-inpainting.json | 1 + examples/server/recipes/sdxl-ip-adapter.json | 1 + examples/server/recipes/sdxl-photomaker.json | 1 + examples/server/recipes/sdxl-turbo.json | 1 + examples/server/recipes/sdxs.json | 1 + examples/server/recipes/sefi-image.json | 1 + examples/server/recipes/sensenova-u15.json | 1 + examples/server/recipes/wan21-flf2v.json | 7 +++ examples/server/recipes/wan21-i2v.json | 7 +++ examples/server/recipes/wan21-t2v.json | 7 +++ examples/server/recipes/wan21-vace.json | 7 +++ examples/server/recipes/wan22-i2v.json | 7 +++ examples/server/recipes/wan22-s2v.json | 8 +++ examples/server/recipes/wan22-t2v.json | 7 +++ examples/server/recipes/wan22-ti2v-5b.json | 7 +++ examples/server/recipes/z-image-l2p.json | 1 + src/model/diffusion/moss_image.hpp | 2 +- src/pipeline/image.cpp | 4 +- src/runtime/image_preprocess.cpp | 9 ++- 88 files changed, 339 insertions(+), 24 deletions(-) diff --git a/docs/image_preprocessing.md b/docs/image_preprocessing.md index 8ec236e71d..c097e5a901 100644 --- a/docs/image_preprocessing.md +++ b/docs/image_preprocessing.md @@ -100,10 +100,13 @@ for individual references, identity images, or video control frames. --image-preprocess "target=control-frame,index=2,canny=true" ``` -Init and mask sources must have the same dimensions. The mask inherits the -init crop, resize, and padding coordinates, while retaining its own filter, -padding value, and Canny setting. Conflicting mask geometry is rejected. An -omitted mask remains absent until the original pipeline creates its default mask. +If init and mask source dimensions differ, the mask is first center-cropped to +the init source aspect ratio and resized to the init source dimensions using +nearest-exact interpolation. The mask then inherits the init crop, resize, and +padding coordinates, while retaining its own filter, padding value, and Canny +setting for that transform. Explicit mask geometry is evaluated after this +alignment; conflicting geometry is rejected. An omitted mask remains absent +until the original pipeline creates its default mask. ## Downstream behavior diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp index 6c03a2a104..e878c889e6 100644 --- a/examples/common/media_io.cpp +++ b/examples/common/media_io.cpp @@ -1499,7 +1499,7 @@ bool load_wav_from_memory(const uint8_t* data, const size_t remaining = size - chunk_data_pos; // safe: chunk_data_pos <= size // Chunk sizes come from the payload, so compare against the remaining bytes // instead of adding them to offsets (avoids size_t wrap on 32-bit builds). - const bool data_fits = static_cast(chunk_size) <= remaining; + const bool data_fits = static_cast(chunk_size) <= remaining; if (std::memcmp(data + pos, "fmt ", 4) == 0) { if (chunk_size < 16 || !data_fits) { diff --git a/examples/server/README.md b/examples/server/README.md index 1bb49a59f5..8d8f687cb5 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -89,8 +89,8 @@ FLUX 1/2/3, Chroma, Qwen Image and editing variants; Z-Image, Anima, ERNIE, LENS, PiD and other supported image families; AnimateDiff, Wan, LTX, HunyuanVideo, LingBot Video and MiniMax-H3. Model-specific text encoder labels identify the expected encoder family and size. Loading a video model enables **Video Generation**; I2V also requires a -start image. The Models panel can be collapsed and remembers its open state. -Its header shows elapsed loading time from the status API and keeps the final +start image. The model button below the prompt opens model and recipe selection. +Its status summary shows elapsed loading time from the status API and keeps the final time after success or failure, including after a page refresh. Built-ins are embedded in the binary from [recipes/](recipes/), with one JSON file per recipe. Adding or diff --git a/examples/server/api.md b/examples/server/api.md index b5e0250c4a..02c83b5cc4 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -244,6 +244,7 @@ the same one-recipe-per-file schema: "id": "my-flux", "name": "My FLUX", "description": "FLUX.1 dev", + "size_multiple": 16, "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-dev", "slots": [ {"id": "diffusion_model", "required": true}, @@ -259,7 +260,8 @@ the same one-recipe-per-file schema: IDs contain 1–128 ASCII letters, digits, dots, underscores or hyphens. `name`, `schema_version: 1` and `slots` are required; `description` and -`model_url`, `required_any`, `defaults`, `context`, `features` and `disabled_features` are optional. Each slot requires only `id` and boolean `required`. +`model_url`, `required_any`, `defaults`, `context`, `features`, `disabled_features` +and `size_multiple`, `video_constraints` are optional. Each slot requires only `id` and boolean `required`. The server supplies the default label and search directories for each slot. An optional nonempty `label` overrides the display name, for example `{"id": "diffusion_model", "label": "Low-noise diffusion model", "required": true}`. @@ -267,6 +269,45 @@ Legacy `category` fields remain accepted, but must match one of the slot's supported directories; they cannot redefine or restrict its search directories. At least one required `model` or `diffusion_model` slot is necessary. +`size_multiple` is a positive integer from 1 to 4096 that declares the width and +height increment used by the web UI. It defaults to `8` when omitted; the UI also +uses `8` before a recipe is selected. For example, `"size_multiple": 16` makes +generated size presets multiples of 16. Exact dimensions that do not match the +increment require correction before submission. This is recipe metadata, not a +generation parameter, and does not change the core model's alignment behavior. + +`video_constraints` is optional metadata for the web UI's Frames / Duration +controls. It does not override the runtime's model-specific rules. For example: + +```json +"video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 +} +``` + +Legal frame counts are `frame_step * n + frame_offset`, with integer `n >= 0`, +within the inclusive `min_frames` / `max_frames` range. All fields are optional: + +| Field | Default | Accepted values | +| --- | --- | --- | +| `frame_step` | `1` | Integer 1–4096 | +| `frame_offset` | `0` | Integer 0 to `frame_step - 1` | +| `frame_rounding` | `"floor"` | `"floor"` or `"ceil"` | +| `min_frames` | `1` | Integer 1–4096 | +| `max_frames` | `4096` | Integer 1–4096, at least `min_frames` | +| `fixed_fps` | Unset | Integer 1–240; locks the FPS control | + +The range must contain at least one legal frame count. Unknown keys are rejected. +Omitting the object adds no frame alignment in the UI. `defaults.fps` only sets +the initial FPS; it does not lock it. The UI multiplies requested duration by FPS, +aligns using the declared rounding direction, then clamps to the legal range. +Duration stays in the UI; generation requests still send only `video_frames` and +`fps`. Switching input modes preserves the aligned length. Changing FPS keeps +frames in Frames mode, and keeps the requested seconds in Duration mode. + `model_url` is an optional HTTPS Hugging Face model repository link, shown beside the recipe description. Built-ins link to the model's author/publisher, not repacks. `required_any` contains groups of declared slot IDs. At least one model in each @@ -1693,15 +1734,26 @@ When omitted, backend defaults apply to these fields: ### Frame Count Semantics -`video_frames` is the requested target length, but the current core video path internally normalizes the effective frame count to the largest `4n + 1` value that does not exceed the requested count. +`video_frames` is the requested target length. The core normalizes it according +to the loaded model: + +| Model family | Frame alignment | Fixed FPS | +| --- | --- | --- | +| Wan, HunyuanVideo, LingBot | `4n + 1`, round down | Wan 2.2 S2V: 16 | +| LTX | `8n + 1`, round down | None | +| MiniMax-H3 | `17n + 5`, round up, minimum 5 | 24 | +| AnimateDiff | Whole frames, maximum 32 | None | -Examples: +Wan examples: - `video_frames = 33` stays `33` - `video_frames = 34` becomes `33` - `video_frames = 32` becomes `29` -The completed job payload includes the actual decoded `frame_count`. +At 16 FPS, a 5-second target requests 80 frames and aligns to 77 frames for Wan, +or approximately 4.813 seconds. Audio-driven generation and temporal upscaling +can change the final length. Use the completed job's actual decoded `frame_count` +and `fps` for the final playback duration. ### Completion Result diff --git a/examples/server/frontend b/examples/server/frontend index 9157320a3b..fa93429c47 160000 --- a/examples/server/frontend +++ b/examples/server/frontend @@ -1 +1 @@ -Subproject commit 9157320a3bdd6a345f370868a32f730c77749cb5 +Subproject commit fa93429c47c924054e6a9c14f8d7b8e06f57a0ec diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp index 1041dea6ac..2617d60074 100644 --- a/examples/server/model_catalog.cpp +++ b/examples/server/model_catalog.cpp @@ -369,20 +369,44 @@ json model_generation_features(const ServerRuntime& runtime) { if (runtime.models->active_recipe.value("source", "") == "command_line") return features; // Optional components only become usable once selected and loaded. - features["ip_adapter"] = features.value("ip_adapter", false) && !runtime.ctx_params->ip_adapter_path.empty(); - features["controlnet"] = features.value("controlnet", false) && !runtime.ctx_params->control_net_path.empty(); - features["high_noise"] = features.value("high_noise", false) && !runtime.ctx_params->high_noise_diffusion_model_path.empty(); - features["preview_tae"] = features.value("preview_tae", false) && !runtime.ctx_params->taesd_path.empty(); - features["ref_audios"] = features.value("ref_audios", false) && - (!runtime.ctx_params->audio_vae_path.empty() || !runtime.ctx_params->audio_encoder_path.empty()); + features["ip_adapter"] = features.value("ip_adapter", false) && !runtime.ctx_params->ip_adapter_path.empty(); + features["controlnet"] = features.value("controlnet", false) && !runtime.ctx_params->control_net_path.empty(); + features["high_noise"] = features.value("high_noise", false) && !runtime.ctx_params->high_noise_diffusion_model_path.empty(); + features["preview_tae"] = features.value("preview_tae", false) && !runtime.ctx_params->taesd_path.empty(); + features["ref_audios"] = features.value("ref_audios", false) && + (!runtime.ctx_params->audio_vae_path.empty() || !runtime.ctx_params->audio_encoder_path.empty()); features["ref_video_audios"] = features.value("ref_video_audios", false) && !runtime.ctx_params->audio_vae_path.empty(); if (!runtime.ctx_params->pulid_weights_path.empty()) features["slg"] = false; return features; } +static void validate_video_constraints(const json& constraints) { + check_keys(constraints, {"frame_step", "frame_offset", "frame_rounding", "min_frames", "max_frames", "fixed_fps"}); + for (const auto* key : {"frame_step", "frame_offset", "min_frames", "max_frames", "fixed_fps"}) { + const int minimum = std::string(key) == "frame_offset" ? 0 : 1; + const int maximum = std::string(key) == "fixed_fps" ? 240 : 4096; + if (constraints.contains(key) && + (!constraints[key].is_number_integer() || constraints[key] < minimum || constraints[key] > maximum)) { + throw std::runtime_error(std::string("video_constraints.") + key + " is outside its integer range"); + } + } + const int step = constraints.value("frame_step", 1); + const int offset = constraints.value("frame_offset", 0); + const int minimum = constraints.value("min_frames", 1); + const int maximum = constraints.value("max_frames", 4096); + if (offset >= step || minimum > maximum || + offset + std::max(0, (minimum - offset + step - 1) / step) * step > maximum) { + throw std::runtime_error("video_constraints must describe a nonempty frame range with frame_offset < frame_step"); + } + if (constraints.contains("frame_rounding") && + constraints["frame_rounding"] != "floor" && constraints["frame_rounding"] != "ceil") { + throw std::runtime_error("video_constraints.frame_rounding must be floor or ceil"); + } +} + static void validate_recipe(const json& recipe) { - check_keys(recipe, {"schema_version", "id", "name", "description", "model_url", "slots", "required_any", "defaults", "context", "features", "disabled_features"}); + check_keys(recipe, {"schema_version", "id", "name", "description", "model_url", "slots", "required_any", "defaults", "context", "features", "disabled_features", "size_multiple", "video_constraints"}); if (!recipe.at("schema_version").is_number_integer() || recipe.at("schema_version") != 1 || !valid_identifier(recipe.at("id").get()) || recipe.at("name").get().empty()) { throw std::runtime_error("invalid schema_version, id or name"); @@ -390,6 +414,13 @@ static void validate_recipe(const json& recipe) { if (recipe.contains("description") && !recipe["description"].is_string()) { throw std::runtime_error("description must be a string"); } + if (recipe.contains("size_multiple") && + (!recipe["size_multiple"].is_number_integer() || recipe["size_multiple"] < 1 || recipe["size_multiple"] > 4096)) { + throw std::runtime_error("size_multiple must be an integer from 1 to 4096"); + } + if (recipe.contains("video_constraints")) { + validate_video_constraints(recipe["video_constraints"]); + } if (recipe.contains("model_url")) { const auto url = recipe.at("model_url").get(); const std::string prefix = "https://huggingface.co/"; @@ -442,6 +473,7 @@ static void validate_recipe(const json& recipe) { static json normalize_recipe(json recipe) { validate_recipe(recipe); + recipe["size_multiple"] = recipe.value("size_multiple", 8); recipe["effective_features"] = resolve_recipe_features(recipe); for (auto& slot : recipe.at("slots")) { const auto& spec = *find_model_slot(slot.at("id").get()); @@ -605,6 +637,7 @@ void initialize_server_models(ServerRuntime& runtime) { {"description", "Uses the loaded command-line model and its generation defaults."}, {"slots", json::array()}, {"defaults", json::object()}, + {"size_multiple", 8}, {"features", features}, }; models.active_recipe["effective_features"] = resolve_recipe_features(models.active_recipe); diff --git a/examples/server/recipes/README.md b/examples/server/recipes/README.md index 5043f6d1b8..dcf9d69ab3 100644 --- a/examples/server/recipes/README.md +++ b/examples/server/recipes/README.md @@ -77,3 +77,35 @@ These are built-in catalog rules. Users may keep personal presets in `/recipes/`; they need not add those presets to the built-in list. See [the recipe schema and feature reference](../api.md#user-recipe-format) for supported fields. + +## Output dimension alignment + +Set `size_multiple` when the runtime's output dimension alignment differs from +the default of `8`. Match `GenerationRequest::align_image_size` in +[`src/pipeline/request.cpp`](../../../src/pipeline/request.cpp): the VAE scale +factor multiplied by the model definition's `diffusion_down_factor`. The scale +comes from [`VAE::get_scale_factor`](../../../src/model/vae/vae.hpp), with the +PiD override in [`diffusion_engine.cpp`](../../../src/pipeline/diffusion_engine.cpp); +model geometry is defined under [`src/model/integrations`](../../../src/model/integrations/). + +Use the runtime's output alignment, not the nominal patch size: some models pad +patches internally and crop their output back to the requested size. Pixel-space +models with an alignment of `1` explicitly use `size_multiple: 1`. MiniT2I also +has this alignment, but its separate fixed `512 × 512` output restriction still +applies. Reference-image preprocessing constraints are independent of this field. + +## Video length constraints + +Video recipes declare `video_constraints` to match `align_video_frames` in +[`diffusion_engine.cpp`](../../../src/pipeline/diffusion_engine.cpp) and the FPS +and AnimateDiff overrides in +[`request.cpp`](../../../src/pipeline/request.cpp). Wan, HunyuanVideo and LingBot +round down to `4n + 1`; LTX rounds down to `8n + 1`. MiniMax-H3 rounds up to +`17n + 5`, with at least 5 frames and fixed 24 FPS. AnimateDiff accepts whole +frames up to 32. Wan 2.2 S2V fixes FPS at 16. + +Declare fixed FPS only where the runtime enforces it; recommended FPS belongs +in `defaults.fps`. These constraints guide UI conversion and estimates, not +runtime behavior. Audio-driven output and temporal upscaling may still change +the final frame count. See the [recipe schema](../api.md#user-recipe-format) +for field defaults and validation rules. diff --git a/examples/server/recipes/animatediff-sd15.json b/examples/server/recipes/animatediff-sd15.json index a7c8338f19..e7bf07d3a8 100644 --- a/examples/server/recipes/animatediff-sd15.json +++ b/examples/server/recipes/animatediff-sd15.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "animatediff-sd15", "name": "AnimateDiff SD 1.5", + "size_multiple": 64, "description": "Use Video Generation with an SD 1.5 checkpoint and compatible motion module.", "model_url": "https://huggingface.co/guoyww/animatediff", "slots": [ @@ -22,6 +23,13 @@ "required": false } ], + "video_constraints": { + "frame_step": 1, + "frame_offset": 0, + "frame_rounding": "floor", + "min_frames": 1, + "max_frames": 32 + }, "defaults": { "sampling_method": "euler", "video_frames": 8, diff --git a/examples/server/recipes/chroma-radiance.json b/examples/server/recipes/chroma-radiance.json index 6a0be8a562..2d35f434cc 100644 --- a/examples/server/recipes/chroma-radiance.json +++ b/examples/server/recipes/chroma-radiance.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "chroma-radiance", "name": "Chroma Radiance", + "size_multiple": 1, "description": "Pixel-space generation without a VAE. Also accepts the x0 and experimental patch-32 variants.", "model_url": "https://huggingface.co/lodestones/Chroma1-Radiance", "slots": [ diff --git a/examples/server/recipes/cosxl-edit.json b/examples/server/recipes/cosxl-edit.json index 49407fccd1..eb794248ae 100644 --- a/examples/server/recipes/cosxl-edit.json +++ b/examples/server/recipes/cosxl-edit.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "cosxl-edit", "name": "CosXL Edit", + "size_multiple": 64, "description": "Supply the image to edit as a reference image.", "model_url": "https://huggingface.co/stabilityai/cosxl", "slots": [ diff --git a/examples/server/recipes/ernie-image-turbo.json b/examples/server/recipes/ernie-image-turbo.json index 5702f5efcb..90ec37df55 100644 --- a/examples/server/recipes/ernie-image-turbo.json +++ b/examples/server/recipes/ernie-image-turbo.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "ernie-image-turbo", "name": "ERNIE Image Turbo", + "size_multiple": 16, "model_url": "https://huggingface.co/baidu/ERNIE-Image-Turbo", "slots": [ { diff --git a/examples/server/recipes/ernie-image.json b/examples/server/recipes/ernie-image.json index a610000083..6373fed518 100644 --- a/examples/server/recipes/ernie-image.json +++ b/examples/server/recipes/ernie-image.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "ernie-image", "name": "ERNIE Image", + "size_multiple": 16, "model_url": "https://huggingface.co/baidu/ERNIE-Image", "slots": [ { diff --git a/examples/server/recipes/fibo2-dev.json b/examples/server/recipes/fibo2-dev.json index aa7e814602..061b9395e0 100644 --- a/examples/server/recipes/fibo2-dev.json +++ b/examples/server/recipes/fibo2-dev.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "fibo2-dev", "name": "Fibo 2.0 Dev", + "size_multiple": 16, "description": "Text-to-image generation and native reference-image editing. Select a separate vision projector for editing when it is not bundled in the text encoder.", "model_url": "https://huggingface.co/briaai/Fibo-2.0-dev", "slots": [ diff --git a/examples/server/recipes/fibo2-edit-turbo.json b/examples/server/recipes/fibo2-edit-turbo.json index b84e1cce44..a78f5d3678 100644 --- a/examples/server/recipes/fibo2-edit-turbo.json +++ b/examples/server/recipes/fibo2-edit-turbo.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "fibo2-edit-turbo", "name": "Fibo 2.0 Edit Turbo", + "size_multiple": 16, "description": "Dedicated reference-image editing checkpoint; supply reference images. Select a separate vision projector when it is not bundled in the text encoder.", "model_url": "https://huggingface.co/briaai/Fibo-2.0-edit-turbo", "slots": [ diff --git a/examples/server/recipes/flux2-dev.json b/examples/server/recipes/flux2-dev.json index 4d27ea16b6..12cf6af75c 100644 --- a/examples/server/recipes/flux2-dev.json +++ b/examples/server/recipes/flux2-dev.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "flux2-dev", "name": "FLUX.2 dev", + "size_multiple": 16, "description": "Supports text-to-image and reference-image editing.", "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-dev", "slots": [ diff --git a/examples/server/recipes/flux2-klein-4b.json b/examples/server/recipes/flux2-klein-4b.json index f09360cfbc..67281e1e7c 100644 --- a/examples/server/recipes/flux2-klein-4b.json +++ b/examples/server/recipes/flux2-klein-4b.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "flux2-klein-4b", "name": "FLUX.2 Klein 4B", + "size_multiple": 16, "description": "Supports reference-image editing. The FLUX.2 small decoder is an alternative VAE.", "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-4B", "slots": [ diff --git a/examples/server/recipes/flux2-klein-9b.json b/examples/server/recipes/flux2-klein-9b.json index ad10d6e516..1c8d50f659 100644 --- a/examples/server/recipes/flux2-klein-9b.json +++ b/examples/server/recipes/flux2-klein-9b.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "flux2-klein-9b", "name": "FLUX.2 Klein 9B", + "size_multiple": 16, "description": "Supports text-to-image and reference-image editing.", "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-9B", "slots": [ diff --git a/examples/server/recipes/flux2-klein-base-4b.json b/examples/server/recipes/flux2-klein-base-4b.json index 56c69f9801..3fff7d9680 100644 --- a/examples/server/recipes/flux2-klein-base-4b.json +++ b/examples/server/recipes/flux2-klein-base-4b.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "flux2-klein-base-4b", "name": "FLUX.2 Klein Base 4B", + "size_multiple": 16, "description": "Supports text-to-image and reference-image editing.", "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4B", "slots": [ diff --git a/examples/server/recipes/flux2-klein-base-9b.json b/examples/server/recipes/flux2-klein-base-9b.json index e9bd53c6cf..b9a9eb6dd7 100644 --- a/examples/server/recipes/flux2-klein-base-9b.json +++ b/examples/server/recipes/flux2-klein-base-9b.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "flux2-klein-base-9b", "name": "FLUX.2 Klein Base 9B", + "size_multiple": 16, "description": "Use the 9B base transformer with Qwen 3 8B. Supports reference-image editing.", "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B", "slots": [ diff --git a/examples/server/recipes/flux3-klein-base.json b/examples/server/recipes/flux3-klein-base.json index 29f6f68196..06789c2e58 100644 --- a/examples/server/recipes/flux3-klein-base.json +++ b/examples/server/recipes/flux3-klein-base.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "flux3-klein-base", "name": "FLUX.3 Image Klein Base", + "size_multiple": 16, "model_url": "https://huggingface.co/black-forest-labs/flux-3-image-klein-base", "slots": [ { diff --git a/examples/server/recipes/flux3-klein.json b/examples/server/recipes/flux3-klein.json index 164bce12ad..e63314dd1c 100644 --- a/examples/server/recipes/flux3-klein.json +++ b/examples/server/recipes/flux3-klein.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "flux3-klein", "name": "FLUX.3 Image Klein", + "size_multiple": 16, "description": "Supports text-to-image and reference-image editing.", "model_url": "https://huggingface.co/black-forest-labs/flux-3-image-klein", "slots": [ diff --git a/examples/server/recipes/hidream-o1-dev.json b/examples/server/recipes/hidream-o1-dev.json index cf1c1790bb..b5c35921ae 100644 --- a/examples/server/recipes/hidream-o1-dev.json +++ b/examples/server/recipes/hidream-o1-dev.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "hidream-o1-dev", "name": "HiDream O1 Image dev", + "size_multiple": 1, "description": "Bundled checkpoint; also supports reference-image editing.", "model_url": "https://huggingface.co/HiDream-ai/HiDream-O1-Image-Dev", "slots": [ diff --git a/examples/server/recipes/hunyuan-video-15-i2v.json b/examples/server/recipes/hunyuan-video-15-i2v.json index a4d672f7b1..5dc0509f91 100644 --- a/examples/server/recipes/hunyuan-video-15-i2v.json +++ b/examples/server/recipes/hunyuan-video-15-i2v.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "hunyuan-video-15-i2v", "name": "HunyuanVideo 1.5 I2V", + "size_multiple": 16, "description": "Select HunyuanVideo 1.5 I2V weights and supply a start image in Video Generation.", "model_url": "https://huggingface.co/tencent/HunyuanVideo-1.5", "slots": [ @@ -36,6 +37,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "video_frames": 33, diff --git a/examples/server/recipes/hunyuan-video-15-t2v.json b/examples/server/recipes/hunyuan-video-15-t2v.json index d477b8063d..81aaceee64 100644 --- a/examples/server/recipes/hunyuan-video-15-t2v.json +++ b/examples/server/recipes/hunyuan-video-15-t2v.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "hunyuan-video-15-t2v", "name": "HunyuanVideo 1.5 T2V", + "size_multiple": 16, "description": "Select HunyuanVideo 1.5 T2V weights and use Video Generation. Use the separate I2V recipe for I2V weights.", "model_url": "https://huggingface.co/tencent/HunyuanVideo-1.5", "slots": [ @@ -36,6 +37,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "video_frames": 33, diff --git a/examples/server/recipes/ideogram4.json b/examples/server/recipes/ideogram4.json index 2d5230019b..133cf5f359 100644 --- a/examples/server/recipes/ideogram4.json +++ b/examples/server/recipes/ideogram4.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "ideogram4", "name": "Ideogram 4", + "size_multiple": 16, "description": "Select the matching unconditional transformer for classifier-free guidance.", "model_url": "https://huggingface.co/ideogram-ai/ideogram-4-fp8", "slots": [ diff --git a/examples/server/recipes/instruct-pix2pix.json b/examples/server/recipes/instruct-pix2pix.json index e819509108..e1f69a8a48 100644 --- a/examples/server/recipes/instruct-pix2pix.json +++ b/examples/server/recipes/instruct-pix2pix.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "instruct-pix2pix", "name": "Instruct-Pix2Pix", + "size_multiple": 64, "description": "Supply the image to edit as a reference image.", "model_url": "https://huggingface.co/timbrooks/instruct-pix2pix", "slots": [ diff --git a/examples/server/recipes/iris-3b.json b/examples/server/recipes/iris-3b.json index c1b5d84fcd..96a617efe5 100644 --- a/examples/server/recipes/iris-3b.json +++ b/examples/server/recipes/iris-3b.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "iris-3b", "name": "Iris 3B", + "size_multiple": 16, "description": "Pixel-space model; no VAE is required.", "model_url": "https://huggingface.co/speridlabs/iris-3b", "slots": [ diff --git a/examples/server/recipes/lens-turbo.json b/examples/server/recipes/lens-turbo.json index 2b87bb6486..364320e7b8 100644 --- a/examples/server/recipes/lens-turbo.json +++ b/examples/server/recipes/lens-turbo.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "lens-turbo", "name": "LENS Turbo", + "size_multiple": 16, "model_url": "https://huggingface.co/microsoft/Lens-Turbo", "slots": [ { diff --git a/examples/server/recipes/lens.json b/examples/server/recipes/lens.json index 34165ad5a8..0e049c82f9 100644 --- a/examples/server/recipes/lens.json +++ b/examples/server/recipes/lens.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "lens", "name": "LENS", + "size_multiple": 16, "model_url": "https://huggingface.co/microsoft/Lens", "slots": [ { diff --git a/examples/server/recipes/lingbot-video-dense.json b/examples/server/recipes/lingbot-video-dense.json index 5f313a0cd9..3affb48516 100644 --- a/examples/server/recipes/lingbot-video-dense.json +++ b/examples/server/recipes/lingbot-video-dense.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "lingbot-video-dense", "name": "LingBot Video Dense 1.3B", + "size_multiple": 16, "description": "Use Video Generation.", "model_url": "https://huggingface.co/robbyant/lingbot-video-dense-1.3b", "slots": [ @@ -30,6 +31,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "video_frames": 33, "cfg_scale": 3, diff --git a/examples/server/recipes/lingbot-video-moe.json b/examples/server/recipes/lingbot-video-moe.json index cdbce61ee9..55bf59f0e3 100644 --- a/examples/server/recipes/lingbot-video-moe.json +++ b/examples/server/recipes/lingbot-video-moe.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "lingbot-video-moe", "name": "LingBot Video MoE 30B A3B", + "size_multiple": 16, "description": "Use Video Generation.", "model_url": "https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b", "slots": [ @@ -30,6 +31,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "video_frames": 33, "cfg_scale": 3, diff --git a/examples/server/recipes/llada-image-turbo.json b/examples/server/recipes/llada-image-turbo.json index 0589bb3846..4965ddc990 100644 --- a/examples/server/recipes/llada-image-turbo.json +++ b/examples/server/recipes/llada-image-turbo.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "llada-image-turbo", "name": "LLaDA Image Turbo", + "size_multiple": 16, "description": "Use matching turbo text encoder and connectors. Editing needs connectors that include the SigVQ encoder.", "model_url": "https://huggingface.co/inclusionAI/LLaDA-Image-Turbo", "slots": [ diff --git a/examples/server/recipes/llada-image.json b/examples/server/recipes/llada-image.json index 27887c849e..e16feda062 100644 --- a/examples/server/recipes/llada-image.json +++ b/examples/server/recipes/llada-image.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "llada-image", "name": "LLaDA Image (base)", + "size_multiple": 16, "description": "Use matching base transformer, text encoder and connectors; do not mix them with Turbo components. Editing needs connectors with SigVQ and 1024x1024 output.", "model_url": "https://huggingface.co/inclusionAI/LLaDA-Image", "slots": [ diff --git a/examples/server/recipes/ltx23-dev.json b/examples/server/recipes/ltx23-dev.json index 83bb934e86..f0bf3f2883 100644 --- a/examples/server/recipes/ltx23-dev.json +++ b/examples/server/recipes/ltx23-dev.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "ltx23-dev", "name": "LTX 2.3 dev", + "size_multiple": 32, "description": "Supports text, start-image and start/end-image video. Audio VAE is optional for silent video. Configure latent upscaling separately in generation settings. Optional distillation LoRAs use this recipe; select the LoRA and adjust the sampling settings separately.", "model_url": "https://huggingface.co/Lightricks/LTX-2.3", "slots": [ @@ -39,6 +40,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 8, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "video_frames": 33, diff --git a/examples/server/recipes/ltx23-distilled.json b/examples/server/recipes/ltx23-distilled.json index a2a6ac33c2..3e4d08e9db 100644 --- a/examples/server/recipes/ltx23-distilled.json +++ b/examples/server/recipes/ltx23-distilled.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "ltx23-distilled", "name": "LTX 2.3 Distilled 1.1", + "size_multiple": 32, "description": "Supports text, start-image and start/end-image video. Audio VAE is optional for silent video. Configure latent upscaling separately in generation settings.", "model_url": "https://huggingface.co/Lightricks/LTX-2.3", "slots": [ @@ -39,6 +40,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 8, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "video_frames": 121, diff --git a/examples/server/recipes/ltx25-dev.json b/examples/server/recipes/ltx25-dev.json index 47efbc76fd..011a49edaa 100644 --- a/examples/server/recipes/ltx25-dev.json +++ b/examples/server/recipes/ltx25-dev.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "ltx25-dev", "name": "LTX 2.5 dev", + "size_multiple": 32, "description": "Use the conv video VAE and LTX-specific Gemma 4 with bundled projection. Audio VAE is optional for silent video.", "model_url": "https://huggingface.co/Lightricks/LTX-2.5", "slots": [ @@ -35,6 +36,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 8, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "fps": 24, diff --git a/examples/server/recipes/mage-flow-edit.json b/examples/server/recipes/mage-flow-edit.json index 82d66f6bd2..e0c552457c 100644 --- a/examples/server/recipes/mage-flow-edit.json +++ b/examples/server/recipes/mage-flow-edit.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "mage-flow-edit", "name": "Mage Flow Edit", + "size_multiple": 16, "description": "Supply reference images. Select a separate vision projector when it is not bundled in the text encoder.", "model_url": "https://huggingface.co/microsoft/Mage-Flow-Edit", "slots": [ diff --git a/examples/server/recipes/mage-flow-turbo.json b/examples/server/recipes/mage-flow-turbo.json index 5cf9bb25a7..8e8a9d63d4 100644 --- a/examples/server/recipes/mage-flow-turbo.json +++ b/examples/server/recipes/mage-flow-turbo.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "mage-flow-turbo", "name": "Mage Flow Turbo", + "size_multiple": 16, "model_url": "https://huggingface.co/microsoft/Mage-Flow-Turbo", "slots": [ { diff --git a/examples/server/recipes/mage-flow.json b/examples/server/recipes/mage-flow.json index 4cbd15934c..c2751f086a 100644 --- a/examples/server/recipes/mage-flow.json +++ b/examples/server/recipes/mage-flow.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "mage-flow", "name": "Mage Flow", + "size_multiple": 16, "model_url": "https://huggingface.co/microsoft/Mage-Flow", "slots": [ { diff --git a/examples/server/recipes/ming-image.json b/examples/server/recipes/ming-image.json index 465a690efe..f100581b4b 100644 --- a/examples/server/recipes/ming-image.json +++ b/examples/server/recipes/ming-image.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "ming-image", "name": "Ming Image 0.1", + "size_multiple": 16, "model_url": "https://huggingface.co/inclusionAI/Ming-Image-0.1-Design", "slots": [ { diff --git a/examples/server/recipes/minimax-h3-fl2va.json b/examples/server/recipes/minimax-h3-fl2va.json index 3971923917..f0af22bd9d 100644 --- a/examples/server/recipes/minimax-h3-fl2va.json +++ b/examples/server/recipes/minimax-h3-fl2va.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "minimax-h3-fl2va", "name": "MiniMax H3 FL2VA", + "size_multiple": 32, "description": "Generate video and audio, optionally with start/end images. Supports full and pruned FL2VA weights. Audio VAE is optional for silent video.", "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-H3", "slots": [ @@ -35,6 +36,13 @@ "tae" ] ], + "video_constraints": { + "frame_step": 17, + "frame_offset": 5, + "frame_rounding": "ceil", + "min_frames": 5, + "fixed_fps": 24 + }, "defaults": { "video_frames": 56, "fps": 24, diff --git a/examples/server/recipes/minimax-h3-ref2va.json b/examples/server/recipes/minimax-h3-ref2va.json index 2fb0417806..9ee5f63d7d 100644 --- a/examples/server/recipes/minimax-h3-ref2va.json +++ b/examples/server/recipes/minimax-h3-ref2va.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "minimax-h3-ref2va", "name": "MiniMax H3 Ref2VA", + "size_multiple": 32, "description": "Supply image, video or audio references with compatible Ref2VA weights. Audio VAE is optional for silent video.", "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-H3", "slots": [ @@ -35,6 +36,13 @@ "tae" ] ], + "video_constraints": { + "frame_step": 17, + "frame_offset": 5, + "frame_rounding": "ceil", + "min_frames": 5, + "fixed_fps": 24 + }, "defaults": { "video_frames": 56, "fps": 24, diff --git a/examples/server/recipes/minit2i.json b/examples/server/recipes/minit2i.json index 2c7b94eba6..a82333d939 100644 --- a/examples/server/recipes/minit2i.json +++ b/examples/server/recipes/minit2i.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "minit2i", "name": "MiniT2I", + "size_multiple": 1, "description": "Pixel-space model; no VAE is required.", "model_url": "https://huggingface.co/MiniT2I/MiniT2I", "slots": [ diff --git a/examples/server/recipes/moss-image-base.json b/examples/server/recipes/moss-image-base.json index bc2cc7d9b1..273eeea7b4 100644 --- a/examples/server/recipes/moss-image-base.json +++ b/examples/server/recipes/moss-image-base.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "moss-image-base", "name": "MOSS Image Base", + "size_multiple": 16, "description": "Select the MOSS-Image-Base transformer with its matching text encoder and FLUX.2 VAE or TAE.", "model_url": "https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base", "slots": [ diff --git a/examples/server/recipes/moss-image-sft.json b/examples/server/recipes/moss-image-sft.json index e0239b0bcd..d63dd2eb27 100644 --- a/examples/server/recipes/moss-image-sft.json +++ b/examples/server/recipes/moss-image-sft.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "moss-image-sft", "name": "MOSS Image SFT", + "size_multiple": 16, "description": "Select the MOSS-Image-SFT transformer with its matching text encoder and FLUX.2 VAE or TAE.", "model_url": "https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT", "slots": [ diff --git a/examples/server/recipes/nitrosd-realism.json b/examples/server/recipes/nitrosd-realism.json index 75f2d43ff8..ae7345db72 100644 --- a/examples/server/recipes/nitrosd-realism.json +++ b/examples/server/recipes/nitrosd-realism.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "nitrosd-realism", "name": "NitroSD Realism", + "size_multiple": 64, "model_url": "https://huggingface.co/ChenDY/NitroFusion", "slots": [ { diff --git a/examples/server/recipes/pid-15-flux2.json b/examples/server/recipes/pid-15-flux2.json index 4fb3db3a72..84cc941c51 100644 --- a/examples/server/recipes/pid-15-flux2.json +++ b/examples/server/recipes/pid-15-flux2.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "pid-15-flux2", "name": "PiD 1.5 (Flux.2 VAE)", + "size_multiple": 1, "description": "Supply a reference image; the first image is used as the source latent. Select a matching Flux.2 PiD checkpoint and VAE.", "model_url": "https://huggingface.co/nvidia/PiD", "slots": [ diff --git a/examples/server/recipes/pid-15-qwen-image.json b/examples/server/recipes/pid-15-qwen-image.json index 9db9e7422b..1415ac755d 100644 --- a/examples/server/recipes/pid-15-qwen-image.json +++ b/examples/server/recipes/pid-15-qwen-image.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "pid-15-qwen-image", "name": "PiD 1.5 (Qwen Image VAE)", + "size_multiple": 1, "description": "Supply a reference image; the first image is used as the source latent. Select a matching Qwen Image PiD checkpoint and VAE. Use the Qwen Image 2D VAE.", "model_url": "https://huggingface.co/nvidia/PiD", "slots": [ diff --git a/examples/server/recipes/pid-15-sd3.json b/examples/server/recipes/pid-15-sd3.json index 3980b86710..c92e9c0388 100644 --- a/examples/server/recipes/pid-15-sd3.json +++ b/examples/server/recipes/pid-15-sd3.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "pid-15-sd3", "name": "PiD 1.5 (SD3 VAE)", + "size_multiple": 1, "description": "Supply a reference image; the first image is used as the source latent. Select a matching SD3 PiD checkpoint and VAE.", "model_url": "https://huggingface.co/nvidia/PiD", "slots": [ diff --git a/examples/server/recipes/pid-15.json b/examples/server/recipes/pid-15.json index f7913bee8e..e98d762cb7 100644 --- a/examples/server/recipes/pid-15.json +++ b/examples/server/recipes/pid-15.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "pid-15", "name": "PiD 1.5 (Flux / Z-Image VAE)", + "size_multiple": 1, "description": "Supply a reference image; the first image is used as the source latent. Select a Flux/Z-Image PiD checkpoint and matching Flux VAE.", "model_url": "https://huggingface.co/nvidia/PiD", "slots": [ diff --git a/examples/server/recipes/pid-flux2.json b/examples/server/recipes/pid-flux2.json index 9c802dc153..827b4f2ab1 100644 --- a/examples/server/recipes/pid-flux2.json +++ b/examples/server/recipes/pid-flux2.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "pid-flux2", "name": "PiD (Flux.2 VAE)", + "size_multiple": 1, "description": "Supply a reference image; the first image is used as the source latent. Select a matching Flux.2 PiD checkpoint and VAE.", "model_url": "https://huggingface.co/nvidia/PiD", "slots": [ diff --git a/examples/server/recipes/pid-qwen-image.json b/examples/server/recipes/pid-qwen-image.json index 4503b1cd69..2155260280 100644 --- a/examples/server/recipes/pid-qwen-image.json +++ b/examples/server/recipes/pid-qwen-image.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "pid-qwen-image", "name": "PiD (Qwen Image VAE)", + "size_multiple": 1, "description": "Supply a reference image; the first image is used as the source latent. Select a matching Qwen Image PiD checkpoint and VAE. Use the Qwen Image 2D VAE.", "model_url": "https://huggingface.co/nvidia/PiD", "slots": [ diff --git a/examples/server/recipes/pid-sd3.json b/examples/server/recipes/pid-sd3.json index 6dad2ca09e..2170b8fb1c 100644 --- a/examples/server/recipes/pid-sd3.json +++ b/examples/server/recipes/pid-sd3.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "pid-sd3", "name": "PiD (SD3 VAE)", + "size_multiple": 1, "description": "Supply a reference image; the first image is used as the source latent. Select a matching SD3 PiD checkpoint and VAE.", "model_url": "https://huggingface.co/nvidia/PiD", "slots": [ diff --git a/examples/server/recipes/pid.json b/examples/server/recipes/pid.json index 2479449659..77c02e969a 100644 --- a/examples/server/recipes/pid.json +++ b/examples/server/recipes/pid.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "pid", "name": "PiD (Flux / Z-Image VAE)", + "size_multiple": 1, "description": "Supply a reference image; the first image is used as the source latent. Select a Flux/Z-Image PiD checkpoint and matching Flux VAE.", "model_url": "https://huggingface.co/nvidia/PiD", "slots": [ diff --git a/examples/server/recipes/pixart-alpha-1024.json b/examples/server/recipes/pixart-alpha-1024.json index 1144f87b2a..e17bf54a0c 100644 --- a/examples/server/recipes/pixart-alpha-1024.json +++ b/examples/server/recipes/pixart-alpha-1024.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "pixart-alpha-1024", "name": "PixArt Alpha XL 1024", + "size_multiple": 16, "description": "Use the matching PixArt Alpha transformer and SD 1.x-style VAE. Resolution/aspect-ratio micro-conditioning is not currently applied by the runtime.", "model_url": "https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS", "slots": [ diff --git a/examples/server/recipes/pixart-alpha-512.json b/examples/server/recipes/pixart-alpha-512.json index 804a7a0c9e..7f6817d022 100644 --- a/examples/server/recipes/pixart-alpha-512.json +++ b/examples/server/recipes/pixart-alpha-512.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "pixart-alpha-512", "name": "PixArt Alpha XL 512", + "size_multiple": 16, "description": "Use the matching PixArt Alpha transformer and SD 1.x-style VAE. Resolution/aspect-ratio micro-conditioning is not currently applied by the runtime.", "model_url": "https://huggingface.co/PixArt-alpha/PixArt-XL-2-512x512", "slots": [ diff --git a/examples/server/recipes/pixart-sigma.json b/examples/server/recipes/pixart-sigma.json index ed4d2f0dec..dd79065ce4 100644 --- a/examples/server/recipes/pixart-sigma.json +++ b/examples/server/recipes/pixart-sigma.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "pixart-sigma", "name": "PixArt Sigma XL 1024", + "size_multiple": 16, "model_url": "https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS", "slots": [ { diff --git a/examples/server/recipes/qwen-image-21-turbo.json b/examples/server/recipes/qwen-image-21-turbo.json index 99da90d7b2..5962b92a3b 100644 --- a/examples/server/recipes/qwen-image-21-turbo.json +++ b/examples/server/recipes/qwen-image-21-turbo.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "qwen-image-21-turbo", "name": "Qwen Image 2.1 Turbo", + "size_multiple": 32, "description": "Use the standalone Turbo diffusion weights for generation and editing with the fixed 8-step schedule. Select the Qwen Image 2.1 VAE or TAEQI2.1, and supply a vision projector when needed.", "model_url": "https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo", "slots": [ diff --git a/examples/server/recipes/qwen-image-21.json b/examples/server/recipes/qwen-image-21.json index fd487e58f8..12c907d868 100644 --- a/examples/server/recipes/qwen-image-21.json +++ b/examples/server/recipes/qwen-image-21.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "qwen-image-21", "name": "Qwen Image 2.1", + "size_multiple": 32, "description": "Supports generation and editing. Select the full Qwen Image 2.1 VAE or TAEQI2.1. Supply a separate vision projector when needed. Select acceleration LoRAs and adjust their settings separately.", "model_url": "https://huggingface.co/Qwen/Qwen-Image-2.1", "slots": [ diff --git a/examples/server/recipes/sd-turbo.json b/examples/server/recipes/sd-turbo.json index 7723c81071..cd5955266e 100644 --- a/examples/server/recipes/sd-turbo.json +++ b/examples/server/recipes/sd-turbo.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sd-turbo", "name": "SD Turbo", + "size_multiple": 64, "description": "Select a single-file SD-Turbo checkpoint. Uses one step with CFG disabled (scale 1 in sd.cpp); use more steps for partial image-to-image denoising.", "model_url": "https://huggingface.co/stabilityai/sd-turbo", "slots": [ diff --git a/examples/server/recipes/sd1-checkpoint.json b/examples/server/recipes/sd1-checkpoint.json index 38d2386775..240bdf0957 100644 --- a/examples/server/recipes/sd1-checkpoint.json +++ b/examples/server/recipes/sd1-checkpoint.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sd1-checkpoint", "name": "SD 1.x", + "size_multiple": 64, "description": "Supports SD 1.x, compatible LCM checkpoints and layer-count variants. For LCM weights, select LCM sampling and adjust steps/CFG. Tiny SD Diffusers directories must first be converted into a single-file checkpoint.", "model_url": "https://huggingface.co/CompVis/stable-diffusion-v1-4", "slots": [ diff --git a/examples/server/recipes/sd1-controlnet.json b/examples/server/recipes/sd1-controlnet.json index eeaee22fb3..ddd764940c 100644 --- a/examples/server/recipes/sd1-controlnet.json +++ b/examples/server/recipes/sd1-controlnet.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sd1-controlnet", "name": "SD 1.5 ControlNet", + "size_multiple": 64, "description": "Supply a control image; input image and mask are optional for image-to-image or inpainting.", "model_url": "https://huggingface.co/lllyasviel/control_v11p_sd15_canny", "slots": [ diff --git a/examples/server/recipes/sd1-inpainting.json b/examples/server/recipes/sd1-inpainting.json index f9f3064f3f..a4c0aa2c87 100644 --- a/examples/server/recipes/sd1-inpainting.json +++ b/examples/server/recipes/sd1-inpainting.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sd1-inpainting", "name": "SD 1.5 Inpainting", + "size_multiple": 64, "description": "Select an inpainting checkpoint, then supply an input image and mask.", "slots": [ { diff --git a/examples/server/recipes/sd1-ip-adapter.json b/examples/server/recipes/sd1-ip-adapter.json index 6c0ab1a6b0..7e14c15668 100644 --- a/examples/server/recipes/sd1-ip-adapter.json +++ b/examples/server/recipes/sd1-ip-adapter.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sd1-ip-adapter", "name": "SD 1.5 IP-Adapter / Plus", + "size_multiple": 64, "description": "Select matching SD 1.5 IP-Adapter or Plus weights and CLIP Vision H, then supply an IP-Adapter image.", "model_url": "https://huggingface.co/h94/IP-Adapter", "slots": [ diff --git a/examples/server/recipes/sd2-512-checkpoint.json b/examples/server/recipes/sd2-512-checkpoint.json index 77838c2805..e5df5af305 100644 --- a/examples/server/recipes/sd2-512-checkpoint.json +++ b/examples/server/recipes/sd2-512-checkpoint.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sd2-512-checkpoint", "name": "SD 2.1 512", + "size_multiple": 64, "description": "Supports SD 2.x 512 and compatible layer-count variants. Tiny SD2 Diffusers directories must first be converted into a single-file checkpoint.", "slots": [ { diff --git a/examples/server/recipes/sd2-768-checkpoint.json b/examples/server/recipes/sd2-768-checkpoint.json index 808b187d7a..8e04418569 100644 --- a/examples/server/recipes/sd2-768-checkpoint.json +++ b/examples/server/recipes/sd2-768-checkpoint.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sd2-768-checkpoint", "name": "SD 2.1 768", + "size_multiple": 64, "slots": [ { "id": "model", diff --git a/examples/server/recipes/sd2-inpainting.json b/examples/server/recipes/sd2-inpainting.json index 4980f7f0a5..7e5e31262d 100644 --- a/examples/server/recipes/sd2-inpainting.json +++ b/examples/server/recipes/sd2-inpainting.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sd2-inpainting", "name": "SD 2 Inpainting", + "size_multiple": 64, "description": "Select an inpainting checkpoint, then supply Init Image and Mask Image.", "slots": [ { diff --git a/examples/server/recipes/sdxl-checkpoint.json b/examples/server/recipes/sdxl-checkpoint.json index 372d5a6d4a..a1fea1bd27 100644 --- a/examples/server/recipes/sdxl-checkpoint.json +++ b/examples/server/recipes/sdxl-checkpoint.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sdxl-checkpoint", "name": "SDXL", + "size_multiple": 64, "description": "Supports SDXL and compatible reduced-depth checkpoints such as SSD-1B and Segmind Vega.", "model_url": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0", "slots": [ diff --git a/examples/server/recipes/sdxl-controlnet.json b/examples/server/recipes/sdxl-controlnet.json index c50efbd2b4..3ffb9b4c83 100644 --- a/examples/server/recipes/sdxl-controlnet.json +++ b/examples/server/recipes/sdxl-controlnet.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sdxl-controlnet", "name": "SDXL ControlNet", + "size_multiple": 64, "description": "Select a compatible SDXL ControlNet and supply a preprocessed control image. Input image and mask are optional for image-to-image or inpainting.", "model_url": "https://huggingface.co/diffusers/controlnet-canny-sdxl-1.0", "slots": [ diff --git a/examples/server/recipes/sdxl-inpainting.json b/examples/server/recipes/sdxl-inpainting.json index 574423d08f..f4ef972175 100644 --- a/examples/server/recipes/sdxl-inpainting.json +++ b/examples/server/recipes/sdxl-inpainting.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sdxl-inpainting", "name": "SDXL Inpainting", + "size_multiple": 64, "description": "Select an inpainting checkpoint, then supply Init Image and Mask Image.", "model_url": "https://huggingface.co/diffusers/stable-diffusion-xl-1.0-inpainting-0.1", "slots": [ diff --git a/examples/server/recipes/sdxl-ip-adapter.json b/examples/server/recipes/sdxl-ip-adapter.json index b649fb1d5a..f0bc8cdafb 100644 --- a/examples/server/recipes/sdxl-ip-adapter.json +++ b/examples/server/recipes/sdxl-ip-adapter.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sdxl-ip-adapter", "name": "SDXL IP-Adapter / Plus", + "size_multiple": 64, "description": "Select matching SDXL ViT-H IP-Adapter or Plus weights and CLIP Vision H, then supply an IP-Adapter image.", "model_url": "https://huggingface.co/h94/IP-Adapter", "slots": [ diff --git a/examples/server/recipes/sdxl-photomaker.json b/examples/server/recipes/sdxl-photomaker.json index 025fb7668e..c8b5790256 100644 --- a/examples/server/recipes/sdxl-photomaker.json +++ b/examples/server/recipes/sdxl-photomaker.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sdxl-photomaker", "name": "SDXL PhotoMaker", + "size_multiple": 64, "description": "Select PhotoMaker v1 and supply identity images. Include the img trigger in the prompt.", "model_url": "https://huggingface.co/TencentARC/PhotoMaker", "slots": [ diff --git a/examples/server/recipes/sdxl-turbo.json b/examples/server/recipes/sdxl-turbo.json index 8650e6c581..496291d71a 100644 --- a/examples/server/recipes/sdxl-turbo.json +++ b/examples/server/recipes/sdxl-turbo.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sdxl-turbo", "name": "SDXL Turbo", + "size_multiple": 64, "model_url": "https://huggingface.co/stabilityai/sdxl-turbo", "slots": [ { diff --git a/examples/server/recipes/sdxs.json b/examples/server/recipes/sdxs.json index 19e218ac5c..f83b060a8a 100644 --- a/examples/server/recipes/sdxs.json +++ b/examples/server/recipes/sdxs.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sdxs", "name": "SDXS", + "size_multiple": 64, "model_url": "https://huggingface.co/IDKiro/sdxs-512-dreamshaper", "slots": [ { diff --git a/examples/server/recipes/sefi-image.json b/examples/server/recipes/sefi-image.json index 35e933360d..168809d1ea 100644 --- a/examples/server/recipes/sefi-image.json +++ b/examples/server/recipes/sefi-image.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sefi-image", "name": "SeFi Image 5B RL", + "size_multiple": 16, "model_url": "https://huggingface.co/SeFi-Image/SeFi-Image-5B-Base", "slots": [ { diff --git a/examples/server/recipes/sensenova-u15.json b/examples/server/recipes/sensenova-u15.json index f2df865e62..9ca386e446 100644 --- a/examples/server/recipes/sensenova-u15.json +++ b/examples/server/recipes/sensenova-u15.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "sensenova-u15", "name": "SenseNova U1.5", + "size_multiple": 32, "description": "Bundled MoT checkpoint.", "model_url": "https://huggingface.co/sensenova/SenseNova-U1.5-8B-MoT", "slots": [ diff --git a/examples/server/recipes/wan21-flf2v.json b/examples/server/recipes/wan21-flf2v.json index 02f0e80a91..6bb4c33d62 100644 --- a/examples/server/recipes/wan21-flf2v.json +++ b/examples/server/recipes/wan21-flf2v.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "wan21-flf2v", "name": "Wan 2.1 FLF2V 14B", + "size_multiple": 16, "description": "Select FLF2V weights and supply both start and end images.", "model_url": "https://huggingface.co/Wan-AI/Wan2.1-FLF2V-14B-720P", "slots": [ @@ -35,6 +36,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "flow_shift": 16, diff --git a/examples/server/recipes/wan21-i2v.json b/examples/server/recipes/wan21-i2v.json index d8d8d13ee1..32d5fc7ace 100644 --- a/examples/server/recipes/wan21-i2v.json +++ b/examples/server/recipes/wan21-i2v.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "wan21-i2v", "name": "Wan 2.1 I2V", + "size_multiple": 16, "description": "Image-to-video with Wan 2.1, compatible SkyReels V2 weights, or a single Wan 2.2 I2V expert. CLIP Vision is required by Wan 2.1 I2V and is unused by Wan 2.2. Defaults target Wan 2.1 480p.", "model_url": "https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-480P", "slots": [ @@ -35,6 +36,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "video_frames": 33, diff --git a/examples/server/recipes/wan21-t2v.json b/examples/server/recipes/wan21-t2v.json index 367611c413..998078cb07 100644 --- a/examples/server/recipes/wan21-t2v.json +++ b/examples/server/recipes/wan21-t2v.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "wan21-t2v", "name": "Wan 2.1 T2V", + "size_multiple": 16, "description": "Text-to-video with Wan 2.1 or a compatible single Wan 2.2 T2V expert. Defaults target Wan 2.1 14B; adjust settings for the selected weights and acceleration LoRAs.", "model_url": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B", "slots": [ @@ -30,6 +31,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "video_frames": 33, diff --git a/examples/server/recipes/wan21-vace.json b/examples/server/recipes/wan21-vace.json index 7b7f7e41b8..d7f834380b 100644 --- a/examples/server/recipes/wan21-vace.json +++ b/examples/server/recipes/wan21-vace.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "wan21-vace", "name": "Wan 2.1 VACE 1.3B / 14B", + "size_multiple": 16, "description": "Select matching VACE weights. Supports text, reference images and control video through Video Generation.", "model_url": "https://huggingface.co/Wan-AI/Wan2.1-VACE-14B", "slots": [ @@ -30,6 +31,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "video_frames": 33, diff --git a/examples/server/recipes/wan22-i2v.json b/examples/server/recipes/wan22-i2v.json index 62afdfce4f..c7ab2ed8db 100644 --- a/examples/server/recipes/wan22-i2v.json +++ b/examples/server/recipes/wan22-i2v.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "wan22-i2v", "name": "Wan 2.2 I2V A14B", + "size_multiple": 16, "description": "Image-to-video. Select a start image in Video Generation after loading. LightX2V acceleration LoRAs must match the I2V low- and high-noise models; configure LoRAs and sampling separately.", "model_url": "https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B", "slots": [ @@ -35,6 +36,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "video_frames": 33, diff --git a/examples/server/recipes/wan22-s2v.json b/examples/server/recipes/wan22-s2v.json index 55440ad317..641ed6e59c 100644 --- a/examples/server/recipes/wan22-s2v.json +++ b/examples/server/recipes/wan22-s2v.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "wan22-s2v", "name": "Wan 2.2 S2V 14B", + "size_multiple": 16, "description": "Supply a start image and reference audio through Video Generation.", "model_url": "https://huggingface.co/Wan-AI/Wan2.2-S2V-14B", "slots": [ @@ -35,6 +36,13 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1, + "fixed_fps": 16 + }, "defaults": { "sampling_method": "euler", "flow_shift": 3, diff --git a/examples/server/recipes/wan22-t2v.json b/examples/server/recipes/wan22-t2v.json index 905bcfa4ff..fba962fcb2 100644 --- a/examples/server/recipes/wan22-t2v.json +++ b/examples/server/recipes/wan22-t2v.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "wan22-t2v", "name": "Wan 2.2 T2V A14B", + "size_multiple": 16, "description": "Text-to-video. Use Video Generation after loading. LightX2V acceleration LoRAs must match the T2V low- and high-noise models; configure LoRAs and sampling separately.", "model_url": "https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B", "slots": [ @@ -35,6 +36,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "video_frames": 33, diff --git a/examples/server/recipes/wan22-ti2v-5b.json b/examples/server/recipes/wan22-ti2v-5b.json index 3e2aa35b88..bc433611a6 100644 --- a/examples/server/recipes/wan22-ti2v-5b.json +++ b/examples/server/recipes/wan22-ti2v-5b.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "wan22-ti2v-5b", "name": "Wan 2.2 TI2V 5B", + "size_multiple": 32, "description": "Use Wan 2.2 VAE. Supports text, start-image and start/end-image conditioning.", "model_url": "https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B", "slots": [ @@ -30,6 +31,12 @@ "tae" ] ], + "video_constraints": { + "frame_step": 4, + "frame_offset": 1, + "frame_rounding": "floor", + "min_frames": 1 + }, "defaults": { "sampling_method": "euler", "video_frames": 33, diff --git a/examples/server/recipes/z-image-l2p.json b/examples/server/recipes/z-image-l2p.json index 0375f1387b..192d547266 100644 --- a/examples/server/recipes/z-image-l2p.json +++ b/examples/server/recipes/z-image-l2p.json @@ -2,6 +2,7 @@ "schema_version": 1, "id": "z-image-l2p", "name": "Z-Image L2P", + "size_multiple": 16, "description": "Pixel-space L2P checkpoint with Qwen 3 4B; no VAE is needed. Use dimensions divisible by 16.", "model_url": "https://huggingface.co/zhen-nan/L2P", "slots": [ diff --git a/src/model/diffusion/moss_image.hpp b/src/model/diffusion/moss_image.hpp index a854c3275f..188c74196d 100644 --- a/src/model/diffusion/moss_image.hpp +++ b/src/model/diffusion/moss_image.hpp @@ -185,7 +185,7 @@ namespace MOSSImage { ggml_build_forward_expand(ctx->graph, parts[i]); } } - x = ggml_concat(c, x, text, 1); + x = ggml_concat(c, x, text, 1); for (int64_t i = 0; i < config.depth; ++i) { if (std::find(skip.begin(), skip.end(), i) == skip.end()) { x = std::dynamic_pointer_cast(blocks["layers." + std::to_string(i)])->forward(ctx, x, pe, parts); diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp index 7d264e22cd..607ec18c5e 100644 --- a/src/pipeline/image.cpp +++ b/src/pipeline/image.cpp @@ -314,8 +314,8 @@ namespace sd::pipeline { } bool skip_vae_encoding = (sd->version == VERSION_HIDREAM_O1) || !( - ref_image_params.pass_to_dit || ref_image_params.vlm_use_vae_input || - sd_version_is_unet_edit(sd->version) || sd_version_is_pid(sd->version)); + ref_image_params.pass_to_dit || ref_image_params.vlm_use_vae_input || + sd_version_is_unet_edit(sd->version) || sd_version_is_pid(sd->version)); std::vector> ref_latents; for (size_t i = 0; i < ref_images.size(); i++) { diff --git a/src/runtime/image_preprocess.cpp b/src/runtime/image_preprocess.cpp index ec649390be..445f0d1fc8 100644 --- a/src/runtime/image_preprocess.cpp +++ b/src/runtime/image_preprocess.cpp @@ -379,8 +379,13 @@ namespace sd { if (target == ImageTarget::Mask && has_init_transform_) { auto options = resolve_options(target, index); if (image.width != init_transform_.source_width || image.height != init_transform_.source_height) { - fail("mask and init source dimensions must match"); - return {}; + ImagePreprocessor alignment; + tensor = alignment.apply_geometry(tensor, target, index, init_transform_.source_width, init_transform_.source_height, + ImageResizeMode::CropResize, ops::InterpolateMode::NearestExact); + if (tensor.empty()) { + fail("could not align mask to init source dimensions"); + return {}; + } } bool geometry_override = options.count("width") || options.count("anchor") || (options.count("mode") && options.at("mode") != "auto"); From bb6b128b2daf92e2c80919a311da6267369a4039 Mon Sep 17 00:00:00 2001 From: leejet Date: Tue, 27 Oct 2026 00:24:43 +0800 Subject: [PATCH 10/11] fix: avoid repeated seeks when skipping small GGUF metadata values (#2223) --- src/model_io/gguf_reader_ext.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/model_io/gguf_reader_ext.h b/src/model_io/gguf_reader_ext.h index f02fe3c6f4..bf183f7488 100644 --- a/src/model_io/gguf_reader_ext.h +++ b/src/model_io/gguf_reader_ext.h @@ -60,7 +60,12 @@ class GGUFReader { if (count > remaining_bytes_ / element_size) return false; uint64_t size = count * element_size; - fin.seekg(static_cast(size), std::ios::cur); + // Seeking discards the read buffer, making arrays of short tokenizer strings expensive to skip. + if (size <= 64 * 1024) { + fin.ignore(static_cast(size)); + } else { + fin.seekg(static_cast(size), std::ios::cur); + } if (!fin.good()) return false; remaining_bytes_ -= size; From be486498eadc29dc3cd234d944c2180ed42b1415 Mon Sep 17 00:00:00 2001 From: leejet Date: Tue, 27 Oct 2026 00:32:18 +0800 Subject: [PATCH 11/11] fix: enable segmented execution for FLUX.3 (#2224) --- src/model/diffusion/flux3.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/model/diffusion/flux3.hpp b/src/model/diffusion/flux3.hpp index 223fc62dbf..6e4cd64084 100644 --- a/src/model/diffusion/flux3.hpp +++ b/src/model/diffusion/flux3.hpp @@ -171,12 +171,17 @@ namespace Flux3 { const std::string base = mode == "txt" ? "txt_mode_blocks." : "content_mode_blocks." + mode + "."; for (int i = 0; i < config.depth; ++i) { h = std::dynamic_pointer_cast(blocks[base + std::to_string(i)])->forward(ctx, h, mode_pe, {early}, {lengths.back()}); + if (i + 1 < config.depth) { + sd::ggml_graph_cut::mark_graph_cut(h, "flux3." + base + std::to_string(i), "h"); + } } joint = joint ? ggml_concat(gc, joint, h, 1) : h; + sd::ggml_graph_cut::mark_graph_cut(joint, "flux3." + base + "joint", "joint"); offset += lengths.back(); } for (int i = 0; i < config.single_depth; ++i) { joint = std::dynamic_pointer_cast(blocks["single_blocks." + std::to_string(i)])->forward(ctx, joint, pe, single_mods, lengths); + sd::ggml_graph_cut::mark_graph_cut(joint, "flux3.single_blocks." + std::to_string(i), "joint"); } auto out = std::dynamic_pointer_cast(blocks["final_layer.image"])->forward(ctx, token_slice(gc, joint, 0, lengths[0]), image_vec); return DiT::unpatchify(gc, out, height, width, 1, 1);