feat: support Fibo2 text-image conditioning

CMakeLists.txt

@@ -86,6 +86,7 @@ option(SD_USE_SYSTEM_WEBP            "sd: link against system libwebp" OFF)
 option(SD_WEBM                       "sd: enable WebM video output support" ${SD_WEBM_DEFAULT})
 option(SD_USE_SYSTEM_WEBM            "sd: link against system libwebm" OFF)
 option(SD_CUDA                       "sd: cuda backend" OFF)
+option(SD_FREETYPE                   "sd: enable Fibo text image rendering with FreeType" OFF)
 option(SD_HIPBLAS                    "sd: rocm backend" OFF)
 option(SD_METAL                      "sd: metal backend" OFF)
 option(SD_VULKAN                     "sd: vulkan backend" OFF)
@@ -338,6 +339,11 @@ target_link_libraries(${SD_LIB} PUBLIC ggml)
 find_package(Threads REQUIRED)
 target_link_libraries(${SD_LIB} PRIVATE Threads::Threads)
 target_link_libraries(${SD_LIB} PRIVATE onig sd-utf8proc)
+if(SD_FREETYPE)
+    find_package(Freetype REQUIRED)
+    target_compile_definitions(${SD_LIB} PRIVATE SD_USE_FREETYPE)
+    target_link_libraries(${SD_LIB} PRIVATE Freetype::Freetype)
+endif()
 if (SD_CUDA)
     find_package(CUDAToolkit REQUIRED)
     # Keep the driver stub on downstream link lines when no driver is installed.

cmake/stable-diffusion-config.cmake.in

@@ -11,6 +11,9 @@ set(SD_BIN_DIR "@PACKAGE_SD_BIN_INSTALL_DIR@")
 include(CMakeFindDependencyMacro)
 find_dependency(ggml REQUIRED HINTS "${SD_LIB_DIR}/cmake")
 find_dependency(Threads REQUIRED)
+if(@SD_FREETYPE@ AND NOT SD_SHARED_LIB)
+    find_dependency(Freetype REQUIRED)
+endif()
 if(@SD_CUDA@)
     find_dependency(CUDAToolkit REQUIRED)
 endif()
@@ -35,6 +38,9 @@ if(NOT TARGET stable-diffusion)
     if(@SD_CUDA@)
         set_property(TARGET stable-diffusion APPEND PROPERTY INTERFACE_LINK_LIBRARIES CUDA::cuda_driver)
     endif()
+    if(@SD_FREETYPE@ AND NOT SD_SHARED_LIB)
+        set_property(TARGET stable-diffusion APPEND PROPERTY INTERFACE_LINK_LIBRARIES Freetype::Freetype)
+    endif()
 
     if(SD_SHARED_LIB)
         target_compile_definitions(stable-diffusion

docs/fibo2.md

@@ -50,4 +50,8 @@ The automatic `fibo2` reference preset combines `resize_single_ref_to_target=tru
 
 The vision encoder defaults to `vlm_resize_mode=longest_side,vlm_max_size=768`. Override these with `--ref-image-args`, for example `vlm_max_size=512`, or use `vlm_resize_mode=none` to skip size-based scaling. Vision inputs are still aligned to the encoder's patch grid.
 
-Plain prompts and structured JSON captions are accepted. This integration uses the caption-only text-rendering path (`render_text_images=False`); it does not generate auxiliary glyph images or expand prompts with Bria VGL. Native Fibo mask preprocessing is not connected to `--mask`.
+Plain prompts and structured JSON captions are accepted. Caption-only text rendering is the default. To render auxiliary images for the caption's `text_render` entries, build with `-DSD_FREETYPE=ON` and pass `--ref-image-args "render_text_images=true,text_render_fonts=/path/to/Aileron-Regular.ttf|/path/to/DejaVuSans.ttf"`. The first font should be Pillow's default Aileron font; subsequent fonts provide missing glyphs in order. FreeType and the fonts are optional and are not bundled. This path also requires the Qwen3-VL vision projector (`--llm_vision`). Bria VGL prompt expansion is not connected.
+
+Each nonempty string in a top-level `text_render` entry is rendered in black on white at size 20, with whitespace collapsed and lines wrapped at 256 characters. The combined rendered area is limited to 16,777,216 pixels, and each image has the same limit for accumulated glyph coverage. The resulting images are passed only to the vision encoder, after any editing references. Text-to-image generation starts with the caption alone while the noise level is above 0.9, then uses the text images; editing uses them throughout. Step caches are disabled for the two-condition text-to-image path. A caption with no text to render keeps its existing behavior.
+
+With Pillow 10.1 or later, export its built-in primary font once with `python -c "from PIL import ImageFont; open('Aileron-Regular.ttf', 'wb').write(ImageFont.load_default(20).path.getvalue())"`. The renderer uses the basic FreeType layout, matching Pillow's default font; optional fallback fonts should also be outline fonts.

src/conditioning/conditioner.hpp

@@ -138,6 +138,7 @@ struct ConditionerParams {
     const std::vector<sd::Tensor<float>>* ref_images                    = nullptr;  // for qwen image edit
     const std::vector<MiniMaxH3PresentationItem>* minimax_h3_references = nullptr;
     RefImageParams ref_image_params;
+    const std::vector<sd::Tensor<float>>* text_images = nullptr;
 };
 
 struct Conditioner {

src/conditioning/conditioning_cache.h

@@ -11,6 +11,7 @@ class ConditioningCache {
     struct Entry {
         ConditionerParams params;
         std::vector<sd::Tensor<float>> ref_images;
+        std::vector<sd::Tensor<float>> text_images;
         std::vector<MiniMaxH3PresentationItem> references;
         SDCondition condition;
 
@@ -25,6 +26,10 @@ class ConditioningCache {
                 references                   = *input.minimax_h3_references;
                 params.minimax_h3_references = &references;
             }
+            if (input.text_images != nullptr) {
+                text_images        = *input.text_images;
+                params.text_images = &text_images;
+            }
         }
 
         Entry(const Entry&)            = delete;
@@ -49,16 +54,20 @@ class ConditioningCache {
                             r.pass_to_vlm, r.pass_to_dit, r.ref_index_mode,
                             r.force_ref_timestep_zero, r.resize_before_vae, r.vae_input_max_pixels,
                             r.vlm_resize_mode, r.vlm_min_size, r.vlm_max_size, r.resize_vae_to_target,
-                            r.ref_image_size);
+                            r.ref_image_size, r.render_text_images, r.text_render_fonts);
         };
         if (fields(a) != fields(b) ||
             (a.ref_images == nullptr) != (b.ref_images == nullptr) ||
+            (a.text_images == nullptr) != (b.text_images == nullptr) ||
             (a.minimax_h3_references == nullptr) != (b.minimax_h3_references == nullptr)) {
             return false;
         }
         if (a.ref_images != nullptr && !same_images(*a.ref_images, *b.ref_images)) {
             return false;
         }
+        if (a.text_images != nullptr && !same_images(*a.text_images, *b.text_images)) {
+            return false;
+        }
         if (a.minimax_h3_references != nullptr &&
             !std::equal(a.minimax_h3_references->begin(), a.minimax_h3_references->end(),
                         b.minimax_h3_references->begin(), b.minimax_h3_references->end(),

src/conditioning/fibo2.hpp

@@ -77,8 +77,37 @@ struct Fibo2Conditioner : public LLMEmbedder {
         return {caption, instruction};
     }
 
-    static std::string format_prompt(const std::string& text, int num_images) {
+    static std::vector<std::string> text_render_entries(const std::string& text, int num_images) {
         auto [caption, instruction] = parse_prompt(text);
+        std::vector<std::string> texts;
+        if (!caption.is_object() || (num_images > 0 && instruction.is_null())) {
+            return texts;
+        }
+        auto entries = caption.find("text_render");
+        if (entries != caption.end() && entries->is_array()) {
+            for (const auto& entry : *entries) {
+                if (entry.is_object() && entry.contains("text") && entry["text"].is_string() &&
+                    !entry["text"].get_ref<const std::string&>().empty()) {
+                    texts.push_back(entry["text"].get<std::string>());
+                }
+            }
+        }
+        return texts;
+    }
+
+    static std::string format_prompt(const std::string& text, int num_images, const std::vector<sd::Tensor<float>>* text_images = nullptr) {
+        auto [caption, instruction] = parse_prompt(text);
+        if (text_images && caption.is_object() && (num_images == 0 || !instruction.is_null())) {
+            auto entries = caption.find("text_render");
+            if (entries != caption.end() && entries->is_array()) {
+                for (auto& entry : *entries) {
+                    if (entry.is_object() && entry.contains("text") && entry["text"].is_string() &&
+                        !entry["text"].get_ref<const std::string&>().empty()) {
+                        entry["image_tokens"] = text_images->empty() ? "" : "<|vision_start|><|image_pad|><|vision_end|>";
+                    }
+                }
+            }
+        }
         if (num_images > 0) {
             if (instruction.is_null()) {
                 caption     = nullptr;
@@ -134,25 +163,37 @@ struct Fibo2Conditioner : public LLMEmbedder {
 
     SDCondition get_learned_condition(int n_threads, const ConditionerParams& p) override {
         int image_count = p.ref_images ? static_cast<int>(p.ref_images->size()) : 0;
-        if (image_count && !llm->enable_vision) {
-            LOG_ERROR("Fibo 2.0 editing requires vision weights");
+        int text_image_count = p.text_images ? static_cast<int>(p.text_images->size()) : 0;
+        if ((image_count || text_image_count) && !llm->enable_vision) {
+            LOG_ERROR("Fibo 2.0 image conditioning requires vision weights");
             return {};
         }
-        std::string prompt = "<|im_start|>user\n" + format_prompt(p.text, image_count) + "<|im_end|>\n<|im_start|>assistant\n";
+        std::string prompt = "<|im_start|>user\n" + format_prompt(p.text, image_count, p.text_images) + "<|im_end|>\n<|im_start|>assistant\n";
         std::vector<std::pair<int, sd::Tensor<float>>> images;
         LLM::DeepStackImageEmbeds deepstack;
         std::vector<LLM::ImageGrid> grids;
         size_t marker_start = 0;
-        for (int i = 0; i < image_count; ++i) {
-            const auto& image = (*p.ref_images)[i];
+        for (int i = 0; i < image_count + text_image_count; ++i) {
+            const auto& image = i < image_count ? (*p.ref_images)[i] : (*p.text_images)[i - image_count];
             int factor        = llm->config.vision.patch_size * llm->config.vision.spatial_merge_size;
             int height = static_cast<int>(image.shape()[1]), width = static_cast<int>(image.shape()[0]);
             int h = std::max(factor, static_cast<int>(std::round(static_cast<double>(height) / factor)) * factor);
             int w = std::max(factor, static_cast<int>(std::round(static_cast<double>(width) / factor)) * factor);
-            resize_image_dims(height, width, h, w, factor,
-                              p.ref_image_params.vlm_min_size,
-                              p.ref_image_params.vlm_max_size,
-                              p.ref_image_params.vlm_resize_mode);
+            if (i < image_count) {
+                resize_image_dims(height, width, h, w, factor,
+                                  p.ref_image_params.vlm_min_size,
+                                  p.ref_image_params.vlm_max_size,
+                                  p.ref_image_params.vlm_resize_mode);
+            } else {
+                if (static_cast<double>(std::max(height, width)) / std::min(height, width) > 200) {
+                    LOG_ERROR("Fibo text image aspect ratio exceeds the Qwen3-VL limit of 200");
+                    return {};
+                }
+                // Text images use Qwen3-VL's processor area limits, not the editing reference size.
+                h = static_cast<int>(std::nearbyint(static_cast<double>(height) / factor)) * factor;
+                w = static_cast<int>(std::nearbyint(static_cast<double>(width) / factor)) * factor;
+                resize_image_dims(height, width, h, w, factor, 65536, 16777216, RefImageResizeMode::AREA);
+            }
             auto pixels = sd::ops::interpolate(image, {w, h, image.shape()[2], image.shape()[3]}, sd::ops::InterpolateMode::Bicubic, false, true);
             auto rgb    = sd::Tensor<float>({w, h, 3, 1});
             for (int64_t pixel = 0; pixel < int64_t(w) * h; ++pixel) {