From dfd7efd5630ee4da08ee9da0be6c0d4b874ce536 Mon Sep 17 00:00:00 2001 From: losewayy <104015127+losewayy@users.noreply.github.com> Date: Sun, 25 Oct 2026 20:20:51 +0800 Subject: [PATCH 1/4] feat: support Fibo2 text-image conditioning --- CMakeLists.txt | 6 + cmake/stable-diffusion-config.cmake.in | 6 + docs/fibo2.md | 6 +- src/conditioning/conditioner.hpp | 1 + src/conditioning/conditioning_cache.h | 11 +- src/conditioning/fibo2.hpp | 61 +++++- src/conditioning/fibo2_text_images.cpp | 267 +++++++++++++++++++++++++ src/conditioning/fibo2_text_images.h | 15 ++ src/model/diffusion/model.hpp | 2 + src/pipeline/diffusion_engine.cpp | 26 ++- src/pipeline/image.cpp | 24 +++ 11 files changed, 410 insertions(+), 15 deletions(-) create mode 100644 src/conditioning/fibo2_text_images.cpp create mode 100644 src/conditioning/fibo2_text_images.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d7bfda5999..170345f830 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,6 +86,7 @@ option(SD_USE_SYSTEM_WEBP "sd: link against system libwebp" OFF) option(SD_WEBM "sd: enable WebM video output support" ${SD_WEBM_DEFAULT}) option(SD_USE_SYSTEM_WEBM "sd: link against system libwebm" OFF) option(SD_CUDA "sd: cuda backend" OFF) +option(SD_FREETYPE "sd: enable Fibo text image rendering with FreeType" OFF) option(SD_HIPBLAS "sd: rocm backend" OFF) option(SD_METAL "sd: metal backend" OFF) option(SD_VULKAN "sd: vulkan backend" OFF) @@ -338,6 +339,11 @@ target_link_libraries(${SD_LIB} PUBLIC ggml) find_package(Threads REQUIRED) target_link_libraries(${SD_LIB} PRIVATE Threads::Threads) target_link_libraries(${SD_LIB} PRIVATE onig sd-utf8proc) +if(SD_FREETYPE) + find_package(Freetype REQUIRED) + target_compile_definitions(${SD_LIB} PRIVATE SD_USE_FREETYPE) + target_link_libraries(${SD_LIB} PRIVATE Freetype::Freetype) +endif() if (SD_CUDA) find_package(CUDAToolkit REQUIRED) # Keep the driver stub on downstream link lines when no driver is installed. diff --git a/cmake/stable-diffusion-config.cmake.in b/cmake/stable-diffusion-config.cmake.in index 0dfe5ef834..de93301d0b 100644 --- a/cmake/stable-diffusion-config.cmake.in +++ b/cmake/stable-diffusion-config.cmake.in @@ -11,6 +11,9 @@ set(SD_BIN_DIR "@PACKAGE_SD_BIN_INSTALL_DIR@") include(CMakeFindDependencyMacro) find_dependency(ggml REQUIRED HINTS "${SD_LIB_DIR}/cmake") find_dependency(Threads REQUIRED) +if(@SD_FREETYPE@ AND NOT SD_SHARED_LIB) + find_dependency(Freetype REQUIRED) +endif() if(@SD_CUDA@) find_dependency(CUDAToolkit REQUIRED) endif() @@ -35,6 +38,9 @@ if(NOT TARGET stable-diffusion) if(@SD_CUDA@) set_property(TARGET stable-diffusion APPEND PROPERTY INTERFACE_LINK_LIBRARIES CUDA::cuda_driver) endif() + if(@SD_FREETYPE@ AND NOT SD_SHARED_LIB) + set_property(TARGET stable-diffusion APPEND PROPERTY INTERFACE_LINK_LIBRARIES Freetype::Freetype) + endif() if(SD_SHARED_LIB) target_compile_definitions(stable-diffusion diff --git a/docs/fibo2.md b/docs/fibo2.md index 064d800892..98847c6356 100644 --- a/docs/fibo2.md +++ b/docs/fibo2.md @@ -50,4 +50,8 @@ The automatic `fibo2` reference preset combines `resize_single_ref_to_target=tru The vision encoder defaults to `vlm_resize_mode=longest_side,vlm_max_size=768`. Override these with `--ref-image-args`, for example `vlm_max_size=512`, or use `vlm_resize_mode=none` to skip size-based scaling. Vision inputs are still aligned to the encoder's patch grid. -Plain prompts and structured JSON captions are accepted. This integration uses the caption-only text-rendering path (`render_text_images=False`); it does not generate auxiliary glyph images or expand prompts with Bria VGL. Native Fibo mask preprocessing is not connected to `--mask`. +Plain prompts and structured JSON captions are accepted. Caption-only text rendering is the default. To render auxiliary images for the caption's `text_render` entries, build with `-DSD_FREETYPE=ON` and pass `--ref-image-args "render_text_images=true,text_render_fonts=/path/to/Aileron-Regular.ttf|/path/to/DejaVuSans.ttf"`. The first font should be Pillow's default Aileron font; subsequent fonts provide missing glyphs in order. FreeType and the fonts are optional and are not bundled. This path also requires the Qwen3-VL vision projector (`--llm_vision`). Bria VGL prompt expansion is not connected. + +Each nonempty string in a top-level `text_render` entry is rendered in black on white at size 20, with whitespace collapsed and lines wrapped at 256 characters. The resulting images are passed only to the vision encoder, after any editing references. Text-to-image generation starts with the caption alone while the noise level is above 0.9, then uses the text images; editing uses them throughout. Step caches are disabled for the two-condition text-to-image path. A caption with no text to render keeps its existing behavior. + +With Pillow 10.1 or later, export its built-in primary font once with `python -c "from PIL import ImageFont; open('Aileron-Regular.ttf', 'wb').write(ImageFont.load_default(20).path.getvalue())"`. The renderer uses the basic FreeType layout, matching Pillow's default font; optional fallback fonts should also be outline fonts. diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index e3832a1d8e..a07e2d26c2 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -138,6 +138,7 @@ struct ConditionerParams { const std::vector>* ref_images = nullptr; // for qwen image edit const std::vector* minimax_h3_references = nullptr; RefImageParams ref_image_params; + const std::vector>* text_images = nullptr; }; struct Conditioner { diff --git a/src/conditioning/conditioning_cache.h b/src/conditioning/conditioning_cache.h index 34648c9ad1..5d9ed3646d 100644 --- a/src/conditioning/conditioning_cache.h +++ b/src/conditioning/conditioning_cache.h @@ -11,6 +11,7 @@ class ConditioningCache { struct Entry { ConditionerParams params; std::vector> ref_images; + std::vector> text_images; std::vector references; SDCondition condition; @@ -25,6 +26,10 @@ class ConditioningCache { references = *input.minimax_h3_references; params.minimax_h3_references = &references; } + if (input.text_images != nullptr) { + text_images = *input.text_images; + params.text_images = &text_images; + } } Entry(const Entry&) = delete; @@ -49,16 +54,20 @@ class ConditioningCache { r.pass_to_vlm, r.pass_to_dit, r.ref_index_mode, r.force_ref_timestep_zero, r.resize_before_vae, r.vae_input_max_pixels, r.vlm_resize_mode, r.vlm_min_size, r.vlm_max_size, r.resize_vae_to_target, - r.ref_image_size); + r.ref_image_size, r.render_text_images, r.text_render_fonts); }; if (fields(a) != fields(b) || (a.ref_images == nullptr) != (b.ref_images == nullptr) || + (a.text_images == nullptr) != (b.text_images == nullptr) || (a.minimax_h3_references == nullptr) != (b.minimax_h3_references == nullptr)) { return false; } if (a.ref_images != nullptr && !same_images(*a.ref_images, *b.ref_images)) { return false; } + if (a.text_images != nullptr && !same_images(*a.text_images, *b.text_images)) { + return false; + } if (a.minimax_h3_references != nullptr && !std::equal(a.minimax_h3_references->begin(), a.minimax_h3_references->end(), b.minimax_h3_references->begin(), b.minimax_h3_references->end(), diff --git a/src/conditioning/fibo2.hpp b/src/conditioning/fibo2.hpp index 43b2ee0845..5587cf9af1 100644 --- a/src/conditioning/fibo2.hpp +++ b/src/conditioning/fibo2.hpp @@ -77,8 +77,37 @@ struct Fibo2Conditioner : public LLMEmbedder { return {caption, instruction}; } - static std::string format_prompt(const std::string& text, int num_images) { + static std::vector text_render_entries(const std::string& text, int num_images) { auto [caption, instruction] = parse_prompt(text); + std::vector texts; + if (!caption.is_object() || (num_images > 0 && instruction.is_null())) { + return texts; + } + auto entries = caption.find("text_render"); + if (entries != caption.end() && entries->is_array()) { + for (const auto& entry : *entries) { + if (entry.is_object() && entry.contains("text") && entry["text"].is_string() && + !entry["text"].get_ref().empty()) { + texts.push_back(entry["text"].get()); + } + } + } + return texts; + } + + static std::string format_prompt(const std::string& text, int num_images, const std::vector>* text_images = nullptr) { + auto [caption, instruction] = parse_prompt(text); + if (text_images && caption.is_object() && (num_images == 0 || !instruction.is_null())) { + auto entries = caption.find("text_render"); + if (entries != caption.end() && entries->is_array()) { + for (auto& entry : *entries) { + if (entry.is_object() && entry.contains("text") && entry["text"].is_string() && + !entry["text"].get_ref().empty()) { + entry["image_tokens"] = text_images->empty() ? "" : "<|vision_start|><|image_pad|><|vision_end|>"; + } + } + } + } if (num_images > 0) { if (instruction.is_null()) { caption = nullptr; @@ -134,25 +163,37 @@ struct Fibo2Conditioner : public LLMEmbedder { SDCondition get_learned_condition(int n_threads, const ConditionerParams& p) override { int image_count = p.ref_images ? static_cast(p.ref_images->size()) : 0; - if (image_count && !llm->enable_vision) { - LOG_ERROR("Fibo 2.0 editing requires vision weights"); + int text_image_count = p.text_images ? static_cast(p.text_images->size()) : 0; + if ((image_count || text_image_count) && !llm->enable_vision) { + LOG_ERROR("Fibo 2.0 image conditioning requires vision weights"); return {}; } - std::string prompt = "<|im_start|>user\n" + format_prompt(p.text, image_count) + "<|im_end|>\n<|im_start|>assistant\n"; + std::string prompt = "<|im_start|>user\n" + format_prompt(p.text, image_count, p.text_images) + "<|im_end|>\n<|im_start|>assistant\n"; std::vector>> images; LLM::DeepStackImageEmbeds deepstack; std::vector grids; size_t marker_start = 0; - for (int i = 0; i < image_count; ++i) { - const auto& image = (*p.ref_images)[i]; + for (int i = 0; i < image_count + text_image_count; ++i) { + const auto& image = i < image_count ? (*p.ref_images)[i] : (*p.text_images)[i - image_count]; int factor = llm->config.vision.patch_size * llm->config.vision.spatial_merge_size; int height = static_cast(image.shape()[1]), width = static_cast(image.shape()[0]); int h = std::max(factor, static_cast(std::round(static_cast(height) / factor)) * factor); int w = std::max(factor, static_cast(std::round(static_cast(width) / factor)) * factor); - resize_image_dims(height, width, h, w, factor, - p.ref_image_params.vlm_min_size, - p.ref_image_params.vlm_max_size, - p.ref_image_params.vlm_resize_mode); + if (i < image_count) { + resize_image_dims(height, width, h, w, factor, + p.ref_image_params.vlm_min_size, + p.ref_image_params.vlm_max_size, + p.ref_image_params.vlm_resize_mode); + } else { + if (static_cast(std::max(height, width)) / std::min(height, width) > 200) { + LOG_ERROR("Fibo text image aspect ratio exceeds the Qwen3-VL limit of 200"); + return {}; + } + // Text images use Qwen3-VL's processor area limits, not the editing reference size. + h = static_cast(std::nearbyint(static_cast(height) / factor)) * factor; + w = static_cast(std::nearbyint(static_cast(width) / factor)) * factor; + resize_image_dims(height, width, h, w, factor, 65536, 16777216, RefImageResizeMode::AREA); + } auto pixels = sd::ops::interpolate(image, {w, h, image.shape()[2], image.shape()[3]}, sd::ops::InterpolateMode::Bicubic, false, true); auto rgb = sd::Tensor({w, h, 3, 1}); for (int64_t pixel = 0; pixel < int64_t(w) * h; ++pixel) { diff --git a/src/conditioning/fibo2_text_images.cpp b/src/conditioning/fibo2_text_images.cpp new file mode 100644 index 0000000000..b08d66ca25 --- /dev/null +++ b/src/conditioning/fibo2_text_images.cpp @@ -0,0 +1,267 @@ +#include "conditioning/fibo2_text_images.h" + +#include +#include +#include +#include +#include +#include + +#include "core/util.h" +#include "utf8proc/utf8proc.h" + +#ifdef SD_USE_FREETYPE +#include +#include FT_FREETYPE_H + +namespace sd { + struct FiboTextFonts { + FT_Library library = nullptr; + std::vector> data; + std::vector faces; + + ~FiboTextFonts() { + for (auto face : faces) + FT_Done_Face(face); + if (library) + FT_Done_FreeType(library); + } + }; + + struct FiboTextGlyph { + int x, y, width, height; + std::vector coverage; + bool run_start = false; + }; + + struct FiboTextLine { + int left = 0, top = 0, width = 1, height = 0; + std::vector glyphs; + }; + + static bool text_space(utf8proc_int32_t ch) { + return (ch >= 9 && ch <= 13) || (ch >= 28 && ch <= 32) || ch == 0x85 || + ch == 0xa0 || ch == 0x1680 || (ch >= 0x2000 && ch <= 0x200a) || + ch == 0x2028 || ch == 0x2029 || ch == 0x202f || ch == 0x205f || ch == 0x3000; + } + + static bool text_characters(const std::string& text, std::vector& characters) { + bool space = false; + for (size_t offset = 0; offset < text.size();) { + utf8proc_int32_t ch; + auto length = utf8proc_iterate(reinterpret_cast(text.data() + offset), text.size() - offset, &ch); + if (length <= 0) { + LOG_ERROR("Fibo text_render contains invalid UTF-8"); + return false; + } + offset += length; + if (text_space(ch)) { + space = !characters.empty(); + } else { + if (space) + characters.push_back(' '); + characters.push_back(ch); + space = false; + } + } + return true; + } + + static FT_Face text_face(const FiboTextFonts& fonts, utf8proc_int32_t ch) { + if (ch < 128) + return fonts.faces.front(); + for (auto face : fonts.faces) { + if (FT_Get_Char_Index(face, ch) != 0) + return face; + } + return fonts.faces.front(); + } + + static bool text_line(const std::vector& characters, + size_t first, + size_t last, + const FiboTextFonts& fonts, + bool mixed, + FiboTextLine& line) { + FT_Pos pen = 0; + FT_Face previous_face = nullptr; + FT_UInt previous = 0; + int right = 0, top = std::numeric_limits::max(), bottom = std::numeric_limits::min(); + for (size_t i = first; i < last; ++i) { + auto face = text_face(fonts, characters[i]); + auto index = FT_Get_Char_Index(face, characters[i]); + if (previous_face == face && previous && index && FT_HAS_KERNING(face)) { + FT_Vector kerning{}; + FT_Get_Kerning(face, previous, index, FT_KERNING_DEFAULT, &kerning); + // Pillow BASIC adds the rounded pixel delta to a 26.6 advance. + pen += (kerning.x + 32) >> 6; + } + if (FT_Load_Glyph(face, index, FT_LOAD_RENDER)) { + LOG_ERROR("could not render Fibo text glyph U+%04X", characters[i]); + return false; + } + const auto& bitmap = face->glyph->bitmap; + if (bitmap.pixel_mode != FT_PIXEL_MODE_GRAY && bitmap.width && bitmap.rows) { + LOG_ERROR("Fibo text rendering requires grayscale outline fonts"); + return false; + } + FiboTextGlyph glyph{static_cast((pen + 32) >> 6) + face->glyph->bitmap_left, + -face->glyph->bitmap_top, + static_cast(bitmap.width), + static_cast(bitmap.rows), + {}}; + glyph.run_start = previous_face != face; + glyph.coverage.resize(size_t(glyph.width) * glyph.height); + for (int y = 0; y < glyph.height; ++y) { + int row = bitmap.pitch < 0 ? glyph.height - 1 - y : y; + std::copy_n(bitmap.buffer + row * std::abs(bitmap.pitch), glyph.width, glyph.coverage.data() + size_t(y) * glyph.width); + } + line.left = std::min(line.left, glyph.x); + right = std::max(right, glyph.x + glyph.width); + if (glyph.height) { + top = std::min(top, glyph.y); + bottom = std::max(bottom, glyph.y + glyph.height); + } + pen += face->glyph->advance.x; + right = std::max(right, static_cast((pen + 32) >> 6)); + line.glyphs.push_back(std::move(glyph)); + previous_face = face; + previous = index; + } + line.top = top == std::numeric_limits::max() ? 0 : top; + line.width = std::max(1, mixed ? static_cast(std::nearbyint(pen / 64.0)) : right - line.left); + line.height = top == std::numeric_limits::max() ? 0 : bottom - top; + return true; + } + + static int text_multiply_alpha(int value, int alpha) { + int product = value * alpha + 128; + return (product + (product >> 8)) >> 8; + } +} +#endif + +bool sd::render_fibo2_text_images(const std::vector& texts, + const std::vector& font_paths, + std::vector>& images) { + images.clear(); + if (texts.empty()) + return true; +#ifndef SD_USE_FREETYPE + LOG_ERROR("Fibo text rendering requires a build with SD_FREETYPE=ON"); + return false; +#else + if (font_paths.empty()) { + LOG_ERROR("Fibo text rendering requires text_render_fonts (Aileron, then optional fallback fonts)"); + return false; + } + FiboTextFonts fonts; + if (FT_Init_FreeType(&fonts.library)) { + LOG_ERROR("could not initialize FreeType"); + return false; + } + for (const auto& path : font_paths) { + if (path.empty()) { + LOG_ERROR("Fibo text_render_fonts contains an empty font path"); + return false; + } + std::ifstream file(std::filesystem::u8path(path), std::ios::binary); + if (!file) { + LOG_ERROR("could not read text-rendering font '%s'", path.c_str()); + return false; + } + fonts.data.emplace_back(std::istreambuf_iterator(file), std::istreambuf_iterator()); + const auto& data = fonts.data.back(); + FT_Face face = nullptr; + if (data.size() > static_cast(std::numeric_limits::max()) || + FT_New_Memory_Face(fonts.library, data.data(), static_cast(data.size()), 0, &face)) { + LOG_ERROR("could not open text-rendering font '%s'", path.c_str()); + return false; + } + fonts.faces.push_back(face); + if (FT_Set_Pixel_Sizes(face, 0, 20)) { + LOG_ERROR("could not set text-rendering font size"); + return false; + } + } + FiboTextLine minimum; + if (!text_line({'A', 'g'}, 0, 2, fonts, false, minimum)) + return false; + for (const auto& text : texts) { + std::vector characters; + if (!text_characters(text, characters)) + return false; + bool mixed = std::any_of(characters.begin(), characters.end(), [&](auto ch) { + return ch >= 128 && FT_Get_Char_Index(fonts.faces.front(), ch) == 0; + }); + std::vector lines; + int width = 1, height = 0; + for (size_t first = 0; first < characters.size() || lines.empty(); first += 256) { + FiboTextLine line; + if (!text_line(characters, first, std::min(first + 256, characters.size()), fonts, mixed, line)) + return false; + if (characters.empty()) { + FiboTextLine space; + if (!text_line({' '}, 0, 1, fonts, false, space)) + return false; + line.width = space.width; + } + line.height = mixed ? static_cast((fonts.faces.front()->size->metrics.ascender - fonts.faces.front()->size->metrics.descender) >> 6) + : std::max(minimum.height, line.height); + width = std::max(width, line.width); + height += line.height + (lines.empty() ? 0 : 4); + lines.push_back(std::move(line)); + } + auto image = sd::full({width + 4, height + 4, 3, 1}, 1.f); + int row = 2; + for (const auto& line : lines) { + int x_offset = mixed ? 2 : 2 - line.left; + int baseline = mixed ? row + static_cast(fonts.faces.front()->size->metrics.ascender >> 6) : row - line.top; + for (size_t first = 0; first < line.glyphs.size();) { + size_t last = first + 1; + while (last < line.glyphs.size() && !line.glyphs[last].run_start) + ++last; + int left = width + 4, top = height + 4, right = 0, bottom = 0; + for (size_t i = first; i < last; ++i) { + const auto& glyph = line.glyphs[i]; + if (glyph.width && glyph.height) { + left = std::min(left, glyph.x); + top = std::min(top, glyph.y); + right = std::max(right, glyph.x + glyph.width); + bottom = std::max(bottom, glyph.y + glyph.height); + } + } + int run_width = std::max(0, right - left), run_height = std::max(0, bottom - top); + std::vector coverage(size_t(run_width) * run_height, 0); + // Pillow composites glyphs into a mask per font run, rounding each blend to 8 bits. + for (size_t i = first; i < last; ++i) { + const auto& glyph = line.glyphs[i]; + for (int y = 0; y < glyph.height; ++y) { + for (int x = 0; x < glyph.width; ++x) { + auto& pixel = coverage[size_t(glyph.y - top + y) * run_width + glyph.x - left + x]; + int alpha = glyph.coverage[size_t(y) * glyph.width + x]; + pixel = static_cast(alpha + text_multiply_alpha(pixel, 255 - alpha)); + } + } + } + for (int y = 0; y < run_height; ++y) { + for (int x = 0; x < run_width; ++x) { + int target_x = x_offset + left + x, target_y = baseline + top + y; + if (target_x < 0 || target_x >= width + 4 || target_y < 0 || target_y >= height + 4) + continue; + int pixel = static_cast(std::lround(image.index(target_x, target_y, 0, 0) * 255.f)); + float value = text_multiply_alpha(pixel, 255 - coverage[size_t(y) * run_width + x]) / 255.f; + for (int channel = 0; channel < 3; ++channel) { + image.index(target_x, target_y, channel, 0) = value; + } + } + } + first = last; + } + row += line.height + 4; + } + images.push_back(std::move(image)); + } + return true; +#endif +} diff --git a/src/conditioning/fibo2_text_images.h b/src/conditioning/fibo2_text_images.h new file mode 100644 index 0000000000..34ae14f894 --- /dev/null +++ b/src/conditioning/fibo2_text_images.h @@ -0,0 +1,15 @@ +#ifndef __SD_CONDITIONING_FIBO2_TEXT_IMAGES_H__ +#define __SD_CONDITIONING_FIBO2_TEXT_IMAGES_H__ + +#include +#include + +#include "core/tensor.hpp" + +namespace sd { + bool render_fibo2_text_images(const std::vector& texts, + const std::vector& font_paths, + std::vector>& images); +} + +#endif // __SD_CONDITIONING_FIBO2_TEXT_IMAGES_H__ diff --git a/src/model/diffusion/model.hpp b/src/model/diffusion/model.hpp index 4ff91c416c..4962be46e1 100644 --- a/src/model/diffusion/model.hpp +++ b/src/model/diffusion/model.hpp @@ -37,6 +37,8 @@ struct RefImageParams { bool vae_center_crop = false; bool vlm_use_vae_input = false; RefImageSizeMode ref_image_size = RefImageSizeMode::MATCH; + bool render_text_images = false; + std::vector text_render_fonts; }; const std::unordered_map REF_IMAGE_PRESETS = { diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index d161d9e5be..7e522f240a 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2378,8 +2378,20 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr StableDiffusionGGML::sample(const std::shared_ptr 0.9f ? base_text_condition : cond; + const SDCondition* positive_condition = &step_condition; const sd::Tensor* c_concat_override = nullptr; for (const auto& extension : generation_extensions) { const SDCondition& next_condition = extension->before_condition(step, *positive_condition); @@ -2836,7 +2849,7 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr sd::Tensor { - return run_condition(cond, + return run_condition(step_condition, cond.c_concat.empty() ? nullptr : &cond.c_concat, &skip_layer_guidance.layers()); }; @@ -3185,6 +3198,13 @@ RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_ima LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str()); rejected_keys.insert(key); } + } else if (key == "render_text_images") { + if (!parse_strict_bool(value, params.render_text_images)) { + LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str()); + rejected_keys.insert(key); + } + } else if (key == "text_render_fonts") { + params.text_render_fonts = split_string(value, '|'); } else if (key == "ref_image_size") { bool valid = true; if (value == "match") { diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp index 7d264e22cd..7e3ab939c2 100644 --- a/src/pipeline/image.cpp +++ b/src/pipeline/image.cpp @@ -6,6 +6,8 @@ #include #include +#include "conditioning/fibo2.hpp" +#include "conditioning/fibo2_text_images.h" #include "core/rng.hpp" #include "diffusion_engine.h" #include "model/vae/vae.hpp" @@ -487,6 +489,17 @@ namespace sd::pipeline { condition_params.ref_image_params = ref_image_params; + std::vector> text_images, empty_text_images; + if (sd->version == VERSION_FIBO2 && ref_image_params.render_text_images) { + auto texts = Fibo2Conditioner::text_render_entries(request->prompt, static_cast(latents->ref_images.size())); + if (!sd::render_fibo2_text_images(texts, ref_image_params.text_render_fonts, text_images)) { + return std::nullopt; + } + if (!text_images.empty()) { + condition_params.text_images = &text_images; + } + } + sd->prepare_generation_extensions(request->pm_params, request->pulid_params, condition_params, @@ -499,6 +512,17 @@ namespace sd::pipeline { LOG_ERROR("failed to encode prompt"); return std::nullopt; } + if (!text_images.empty() && latents->ref_images.empty()) { + condition_params.text_images = &empty_text_images; + auto base_condition = sd->get_learned_condition(condition_params); + if (base_condition.empty()) { + LOG_ERROR("failed to encode Fibo base prompt"); + return std::nullopt; + } + // Fibo's first extra context is the caption-only high-noise condition. + cond.extra_c_crossattns.push_back(std::move(base_condition.c_crossattn)); + } + condition_params.text_images = nullptr; if (cond.c_concat.empty() && ref_image_params.pass_to_dit) { cond.c_concat = latents->concat_latent; // TODO: optimize } From 2ee5e6efb8bbbe89c11dfc0048b97e8bd869528b Mon Sep 17 00:00:00 2001 From: losewayy <104015127+losewayy@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:01:21 +0800 Subject: [PATCH 2/4] fix: bound Fibo2 text image allocations --- docs/fibo2.md | 2 +- src/conditioning/fibo2_text_images.cpp | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/fibo2.md b/docs/fibo2.md index 98847c6356..10695ebf63 100644 --- a/docs/fibo2.md +++ b/docs/fibo2.md @@ -52,6 +52,6 @@ The vision encoder defaults to `vlm_resize_mode=longest_side,vlm_max_size=768`. Plain prompts and structured JSON captions are accepted. Caption-only text rendering is the default. To render auxiliary images for the caption's `text_render` entries, build with `-DSD_FREETYPE=ON` and pass `--ref-image-args "render_text_images=true,text_render_fonts=/path/to/Aileron-Regular.ttf|/path/to/DejaVuSans.ttf"`. The first font should be Pillow's default Aileron font; subsequent fonts provide missing glyphs in order. FreeType and the fonts are optional and are not bundled. This path also requires the Qwen3-VL vision projector (`--llm_vision`). Bria VGL prompt expansion is not connected. -Each nonempty string in a top-level `text_render` entry is rendered in black on white at size 20, with whitespace collapsed and lines wrapped at 256 characters. The resulting images are passed only to the vision encoder, after any editing references. Text-to-image generation starts with the caption alone while the noise level is above 0.9, then uses the text images; editing uses them throughout. Step caches are disabled for the two-condition text-to-image path. A caption with no text to render keeps its existing behavior. +Each nonempty string in a top-level `text_render` entry is rendered in black on white at size 20, with whitespace collapsed and lines wrapped at 256 characters. Rendered images are limited to 16,777,216 pixels. The resulting images are passed only to the vision encoder, after any editing references. Text-to-image generation starts with the caption alone while the noise level is above 0.9, then uses the text images; editing uses them throughout. Step caches are disabled for the two-condition text-to-image path. A caption with no text to render keeps its existing behavior. With Pillow 10.1 or later, export its built-in primary font once with `python -c "from PIL import ImageFont; open('Aileron-Regular.ttf', 'wb').write(ImageFont.load_default(20).path.getvalue())"`. The renderer uses the basic FreeType layout, matching Pillow's default font; optional fallback fonts should also be outline fonts. diff --git a/src/conditioning/fibo2_text_images.cpp b/src/conditioning/fibo2_text_images.cpp index b08d66ca25..ad4d3466fe 100644 --- a/src/conditioning/fibo2_text_images.cpp +++ b/src/conditioning/fibo2_text_images.cpp @@ -210,6 +210,10 @@ bool sd::render_fibo2_text_images(const std::vector& texts, : std::max(minimum.height, line.height); width = std::max(width, line.width); height += line.height + (lines.empty() ? 0 : 4); + if ((int64_t(width) + 4) * (int64_t(height) + 4) > 16777216) { + LOG_ERROR("Fibo text image exceeds the maximum area of 16777216 pixels"); + return false; + } lines.push_back(std::move(line)); } auto image = sd::full({width + 4, height + 4, 3, 1}, 1.f); From 3a628a106ac5c99ea60adbe7e2df4780bc818d68 Mon Sep 17 00:00:00 2001 From: losewayy <104015127+losewayy@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:29:21 +0800 Subject: [PATCH 3/4] fix: bound Fibo2 glyph coverage and preserve text conditioning --- docs/fibo2.md | 2 +- src/conditioning/fibo2_text_images.cpp | 24 ++++++++++++++++++------ src/pipeline/image.cpp | 2 +- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/docs/fibo2.md b/docs/fibo2.md index 10695ebf63..72ce6c61e7 100644 --- a/docs/fibo2.md +++ b/docs/fibo2.md @@ -52,6 +52,6 @@ The vision encoder defaults to `vlm_resize_mode=longest_side,vlm_max_size=768`. Plain prompts and structured JSON captions are accepted. Caption-only text rendering is the default. To render auxiliary images for the caption's `text_render` entries, build with `-DSD_FREETYPE=ON` and pass `--ref-image-args "render_text_images=true,text_render_fonts=/path/to/Aileron-Regular.ttf|/path/to/DejaVuSans.ttf"`. The first font should be Pillow's default Aileron font; subsequent fonts provide missing glyphs in order. FreeType and the fonts are optional and are not bundled. This path also requires the Qwen3-VL vision projector (`--llm_vision`). Bria VGL prompt expansion is not connected. -Each nonempty string in a top-level `text_render` entry is rendered in black on white at size 20, with whitespace collapsed and lines wrapped at 256 characters. Rendered images are limited to 16,777,216 pixels. The resulting images are passed only to the vision encoder, after any editing references. Text-to-image generation starts with the caption alone while the noise level is above 0.9, then uses the text images; editing uses them throughout. Step caches are disabled for the two-condition text-to-image path. A caption with no text to render keeps its existing behavior. +Each nonempty string in a top-level `text_render` entry is rendered in black on white at size 20, with whitespace collapsed and lines wrapped at 256 characters. Rendered images and the accumulated glyph coverage for each image are limited to 16,777,216 pixels. The resulting images are passed only to the vision encoder, after any editing references. Text-to-image generation starts with the caption alone while the noise level is above 0.9, then uses the text images; editing uses them throughout. Step caches are disabled for the two-condition text-to-image path. A caption with no text to render keeps its existing behavior. With Pillow 10.1 or later, export its built-in primary font once with `python -c "from PIL import ImageFont; open('Aileron-Regular.ttf', 'wb').write(ImageFont.load_default(20).path.getvalue())"`. The renderer uses the basic FreeType layout, matching Pillow's default font; optional fallback fonts should also be outline fonts. diff --git a/src/conditioning/fibo2_text_images.cpp b/src/conditioning/fibo2_text_images.cpp index ad4d3466fe..df2f1e5c76 100644 --- a/src/conditioning/fibo2_text_images.cpp +++ b/src/conditioning/fibo2_text_images.cpp @@ -15,6 +15,8 @@ #include FT_FREETYPE_H namespace sd { + static constexpr size_t max_text_image_pixels = 16777216; + struct FiboTextFonts { FT_Library library = nullptr; std::vector> data; @@ -82,6 +84,7 @@ namespace sd { size_t last, const FiboTextFonts& fonts, bool mixed, + size_t& remaining_coverage, FiboTextLine& line) { FT_Pos pen = 0; FT_Face previous_face = nullptr; @@ -105,13 +108,20 @@ namespace sd { LOG_ERROR("Fibo text rendering requires grayscale outline fonts"); return false; } + if (bitmap.width > max_text_image_pixels || bitmap.rows > max_text_image_pixels || + (bitmap.rows && bitmap.width > remaining_coverage / bitmap.rows)) { + LOG_ERROR("Fibo text glyph coverage exceeds the maximum area of %zu pixels", max_text_image_pixels); + return false; + } + size_t glyph_pixels = size_t(bitmap.width) * bitmap.rows; + remaining_coverage -= glyph_pixels; FiboTextGlyph glyph{static_cast((pen + 32) >> 6) + face->glyph->bitmap_left, -face->glyph->bitmap_top, static_cast(bitmap.width), static_cast(bitmap.rows), {}}; glyph.run_start = previous_face != face; - glyph.coverage.resize(size_t(glyph.width) * glyph.height); + glyph.coverage.resize(glyph_pixels); for (int y = 0; y < glyph.height; ++y) { int row = bitmap.pitch < 0 ? glyph.height - 1 - y : y; std::copy_n(bitmap.buffer + row * std::abs(bitmap.pitch), glyph.width, glyph.coverage.data() + size_t(y) * glyph.width); @@ -185,7 +195,8 @@ bool sd::render_fibo2_text_images(const std::vector& texts, } } FiboTextLine minimum; - if (!text_line({'A', 'g'}, 0, 2, fonts, false, minimum)) + size_t minimum_coverage = max_text_image_pixels; + if (!text_line({'A', 'g'}, 0, 2, fonts, false, minimum_coverage, minimum)) return false; for (const auto& text : texts) { std::vector characters; @@ -195,14 +206,15 @@ bool sd::render_fibo2_text_images(const std::vector& texts, return ch >= 128 && FT_Get_Char_Index(fonts.faces.front(), ch) == 0; }); std::vector lines; + size_t remaining_coverage = max_text_image_pixels; int width = 1, height = 0; for (size_t first = 0; first < characters.size() || lines.empty(); first += 256) { FiboTextLine line; - if (!text_line(characters, first, std::min(first + 256, characters.size()), fonts, mixed, line)) + if (!text_line(characters, first, std::min(first + 256, characters.size()), fonts, mixed, remaining_coverage, line)) return false; if (characters.empty()) { FiboTextLine space; - if (!text_line({' '}, 0, 1, fonts, false, space)) + if (!text_line({' '}, 0, 1, fonts, false, remaining_coverage, space)) return false; line.width = space.width; } @@ -210,8 +222,8 @@ bool sd::render_fibo2_text_images(const std::vector& texts, : std::max(minimum.height, line.height); width = std::max(width, line.width); height += line.height + (lines.empty() ? 0 : 4); - if ((int64_t(width) + 4) * (int64_t(height) + 4) > 16777216) { - LOG_ERROR("Fibo text image exceeds the maximum area of 16777216 pixels"); + if ((int64_t(width) + 4) * (int64_t(height) + 4) > int64_t(max_text_image_pixels)) { + LOG_ERROR("Fibo text image exceeds the maximum area of %zu pixels", max_text_image_pixels); return false; } lines.push_back(std::move(line)); diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp index 7e3ab939c2..4cbe680f75 100644 --- a/src/pipeline/image.cpp +++ b/src/pipeline/image.cpp @@ -491,7 +491,7 @@ namespace sd::pipeline { std::vector> text_images, empty_text_images; if (sd->version == VERSION_FIBO2 && ref_image_params.render_text_images) { - auto texts = Fibo2Conditioner::text_render_entries(request->prompt, static_cast(latents->ref_images.size())); + auto texts = Fibo2Conditioner::text_render_entries(request->prompt, condition_params.ref_images ? static_cast(condition_params.ref_images->size()) : 0); if (!sd::render_fibo2_text_images(texts, ref_image_params.text_render_fonts, text_images)) { return std::nullopt; } From c8822b5491a0288433b86366c22be36a94267256 Mon Sep 17 00:00:00 2001 From: losewayy <104015127+losewayy@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:44:21 +0800 Subject: [PATCH 4/4] fix: cap total Fibo2 text image output --- docs/fibo2.md | 2 +- src/conditioning/fibo2_text_images.cpp | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/fibo2.md b/docs/fibo2.md index 72ce6c61e7..521f776e07 100644 --- a/docs/fibo2.md +++ b/docs/fibo2.md @@ -52,6 +52,6 @@ The vision encoder defaults to `vlm_resize_mode=longest_side,vlm_max_size=768`. Plain prompts and structured JSON captions are accepted. Caption-only text rendering is the default. To render auxiliary images for the caption's `text_render` entries, build with `-DSD_FREETYPE=ON` and pass `--ref-image-args "render_text_images=true,text_render_fonts=/path/to/Aileron-Regular.ttf|/path/to/DejaVuSans.ttf"`. The first font should be Pillow's default Aileron font; subsequent fonts provide missing glyphs in order. FreeType and the fonts are optional and are not bundled. This path also requires the Qwen3-VL vision projector (`--llm_vision`). Bria VGL prompt expansion is not connected. -Each nonempty string in a top-level `text_render` entry is rendered in black on white at size 20, with whitespace collapsed and lines wrapped at 256 characters. Rendered images and the accumulated glyph coverage for each image are limited to 16,777,216 pixels. The resulting images are passed only to the vision encoder, after any editing references. Text-to-image generation starts with the caption alone while the noise level is above 0.9, then uses the text images; editing uses them throughout. Step caches are disabled for the two-condition text-to-image path. A caption with no text to render keeps its existing behavior. +Each nonempty string in a top-level `text_render` entry is rendered in black on white at size 20, with whitespace collapsed and lines wrapped at 256 characters. The combined rendered area is limited to 16,777,216 pixels, and each image has the same limit for accumulated glyph coverage. The resulting images are passed only to the vision encoder, after any editing references. Text-to-image generation starts with the caption alone while the noise level is above 0.9, then uses the text images; editing uses them throughout. Step caches are disabled for the two-condition text-to-image path. A caption with no text to render keeps its existing behavior. With Pillow 10.1 or later, export its built-in primary font once with `python -c "from PIL import ImageFont; open('Aileron-Regular.ttf', 'wb').write(ImageFont.load_default(20).path.getvalue())"`. The renderer uses the basic FreeType layout, matching Pillow's default font; optional fallback fonts should also be outline fonts. diff --git a/src/conditioning/fibo2_text_images.cpp b/src/conditioning/fibo2_text_images.cpp index df2f1e5c76..f56ee10fbe 100644 --- a/src/conditioning/fibo2_text_images.cpp +++ b/src/conditioning/fibo2_text_images.cpp @@ -198,6 +198,7 @@ bool sd::render_fibo2_text_images(const std::vector& texts, size_t minimum_coverage = max_text_image_pixels; if (!text_line({'A', 'g'}, 0, 2, fonts, false, minimum_coverage, minimum)) return false; + size_t remaining_pixels = max_text_image_pixels; for (const auto& text : texts) { std::vector characters; if (!text_characters(text, characters)) @@ -222,12 +223,13 @@ bool sd::render_fibo2_text_images(const std::vector& texts, : std::max(minimum.height, line.height); width = std::max(width, line.width); height += line.height + (lines.empty() ? 0 : 4); - if ((int64_t(width) + 4) * (int64_t(height) + 4) > int64_t(max_text_image_pixels)) { - LOG_ERROR("Fibo text image exceeds the maximum area of %zu pixels", max_text_image_pixels); + if ((int64_t(width) + 4) * (int64_t(height) + 4) > int64_t(remaining_pixels)) { + LOG_ERROR("Fibo text images exceed the maximum total area of %zu pixels", max_text_image_pixels); return false; } lines.push_back(std::move(line)); } + remaining_pixels -= size_t(width + 4) * (height + 4); auto image = sd::full({width + 4, height + 4, 3, 1}, 1.f); int row = 2; for (const auto& line : lines) {