From 2032526e8c106e63a132583b5c4f5b30341b5366 Mon Sep 17 00:00:00 2001 From: project516 <138796702+Project516@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:28:11 -0500 Subject: [PATCH 01/11] fix: include integrated GPUs in automatic memory planning (#2228) --- src/core/backend_fit.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/backend_fit.cpp b/src/core/backend_fit.cpp index 91dfedb487..c970d10e68 100644 --- a/src/core/backend_fit.cpp +++ b/src/core/backend_fit.cpp @@ -133,7 +133,7 @@ namespace sd::backend_fit { for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); const auto type = ggml_backend_dev_type(dev); - if (type != GGML_BACKEND_DEVICE_TYPE_GPU && + if (type != GGML_BACKEND_DEVICE_TYPE_GPU && type != GGML_BACKEND_DEVICE_TYPE_IGPU && (!include_other_devices || type == GGML_BACKEND_DEVICE_TYPE_CPU)) { continue; } From 9bfc4d108916c5cd3f0cd7c8206b41f169e33cd6 Mon Sep 17 00:00:00 2001 From: project516 <138796702+Project516@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:33:03 -0500 Subject: [PATCH 02/11] fix: honor the requested hires sampling step count (#2227) --- src/pipeline/request.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline/request.cpp b/src/pipeline/request.cpp index 2647efb8ef..5f1f235ff2 100644 --- a/src/pipeline/request.cpp +++ b/src/pipeline/request.cpp @@ -519,7 +519,7 @@ namespace sd::pipeline { scheduler, sd->version, sample_params.extra_sample_args); - size_t t_enc = static_cast(scheduler_steps * hires.denoising_strength); + size_t t_enc = static_cast(effective_steps - 1); if (t_enc >= static_cast(scheduler_steps)) { t_enc = static_cast(scheduler_steps) - 1; } From 209b28c3cdcaf01a9ca934c1bce70cbf6003155c Mon Sep 17 00:00:00 2001 From: Crystal Sight Date: Tue, 27 Oct 2026 12:36:23 -0500 Subject: [PATCH 03/11] feat(server): add recipe model defaults and idempotent model loading (#2221) --- examples/server/api.md | 10 +++++- examples/server/model_catalog.cpp | 59 +++++++++++++++++++++++++++++-- examples/server/model_catalog.h | 10 +++++- examples/server/routes_models.cpp | 20 ++++++++++- 4 files changed, 93 insertions(+), 6 deletions(-) diff --git a/examples/server/api.md b/examples/server/api.md index 02c83b5cc4..97cbd66cdf 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -85,7 +85,7 @@ available during loading. Model lists are empty when no model is loaded; | GET | `/sdcpp/v1/model-files` | `{files: [{id, name, category, root, relative_path}], errors: [...]}` | | POST | `/sdcpp/v1/models/refresh` | Rescan recipes and files; return `{errors: [...]}` | | GET | `/sdcpp/v1/models/status` | Current state, recipe, selection, error, loading time and optional revision | -| POST | `/sdcpp/v1/models/load` | Load the submitted recipe and selection; return status after completion | +| POST | `/sdcpp/v1/models/load` | Load the submitted recipe and selection; return status after completion. A load is a no-op returning the current status with `already_loaded: true` only when the state is ready and the fully resolved selection, the complete recipe object, and the resolved component paths all match the active model; a catalog refresh with modified recipe metadata or a retargeted model path therefore reloads even with an unchanged recipe ID and selection | | POST | `/sdcpp/v1/models/unload` | Release the active context; return empty status | Load request: @@ -269,6 +269,14 @@ Legacy `category` fields remain accepted, but must match one of the slot's supported directories; they cannot redefine or restrict its search directories. At least one required `model` or `diffusion_model` slot is necessary. +An optional top-level `models` object maps slot IDs to file IDs, providing +recipe-local defaults for component selection. A load request may omit +`models` entirely and load with the recipe's defaults. Selections sent in the +request always override recipe defaults, slot by slot. An empty string +disables a slot's recipe default (omitting the slot selects it); the empty +key is then normalized out of the reported selection, so for slots without a +recipe default an empty selection loads identically to an omitted one. + `size_multiple` is a positive integer from 1 to 4096 that declares the width and height increment used by the web UI. It defaults to `8` when omitted; the UI also uses `8` before a recipe is selected. For example, `"size_multiple": 16` makes diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp index 2617d60074..7daed533fe 100644 --- a/examples/server/model_catalog.cpp +++ b/examples/server/model_catalog.cpp @@ -66,6 +66,14 @@ static void clear_model_paths(SDContextParams& params) { } } +bool component_paths_match(const SDContextParams& a, const SDContextParams& b) { + for (const auto& slot : k_model_slots) { + if ((a.*slot.field) != (b.*slot.field)) + return false; + } + return true; +} + std::string model_path_key(const fs::path& path) { #ifdef _WIN32 const auto value = path.generic_wstring(); @@ -406,7 +414,7 @@ static void validate_video_constraints(const json& constraints) { } static void validate_recipe(const json& recipe) { - check_keys(recipe, {"schema_version", "id", "name", "description", "model_url", "slots", "required_any", "defaults", "context", "features", "disabled_features", "size_multiple", "video_constraints"}); + check_keys(recipe, {"schema_version", "id", "name", "description", "model_url", "slots", "required_any", "models", "defaults", "context", "features", "disabled_features", "size_multiple", "video_constraints"}); if (!recipe.at("schema_version").is_number_integer() || recipe.at("schema_version") != 1 || !valid_identifier(recipe.at("id").get()) || recipe.at("name").get().empty()) { throw std::runtime_error("invalid schema_version, id or name"); @@ -449,6 +457,19 @@ static void validate_recipe(const json& recipe) { if (!has_required_model) { throw std::runtime_error("a required model or diffusion_model slot is needed"); } + // Optional recipe-local model defaults: slot ID to file ID mappings that + // prefill slots a load request does not select itself. + if (recipe.contains("models")) { + const auto& recipe_models = recipe.at("models"); + if (!recipe_models.is_object() || recipe_models.size() > 32) + throw std::runtime_error("models must be an object of at most 32 slot entries"); + std::set default_ids; + for (auto it = recipe_models.begin(); it != recipe_models.end(); ++it) { + if (!ids.count(it.key() == "taesd" ? "tae" : it.key()) || !it.value().is_string() || it.value().get().empty() || + !default_ids.insert(it.key() == "taesd" ? "tae" : it.key()).second) + throw std::runtime_error("unknown or invalid default model: " + it.key()); + } + } if (recipe.contains("required_any")) { const auto& groups = recipe.at("required_any"); if (!groups.is_array() || groups.size() > 32) @@ -646,10 +667,11 @@ void initialize_server_models(ServerRuntime& runtime) { refresh_model_catalog(runtime); } -void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe) { +void prepare_model_load(ServerRuntime& runtime, json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe) { check_keys(body, {"recipe", "models"}); const auto recipe_id = body.at("recipe").get(); - auto selection = body.at("models"); + // `models` is optional; unselected slots may fall back to recipe defaults below. + auto selection = body.value("models", json::object()); if (!selection.is_object()) throw std::runtime_error("models must be an object"); auto& models = *runtime.models; @@ -664,6 +686,20 @@ void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParam defaults = models.base_defaults; apply_recipe_context(recipe.value("context", json::object()), context); apply_recipe_defaults(recipe.value("defaults", json::object()), defaults); + // Recipe-provided default models fill any slot the request did not select. + // An explicit selection blocks the default for that slot, including an + // empty string (a deliberate opt-out of the default). The tae/taesd pair + // is one slot: an explicit request selection of either spelling overrides + // the recipe default for that slot. + if (recipe.contains("models")) { + for (auto it = recipe.at("models").begin(); it != recipe.at("models").end(); ++it) { + if (selection.contains(it.key())) + continue; + if ((it.key() == "tae" || it.key() == "taesd") && (selection.contains("tae") || selection.contains("taesd"))) + continue; + selection[it.key()] = it.value(); + } + } std::set allowed; for (const auto& slot : recipe.at("slots")) { const auto id = slot.at("id").get(); @@ -675,6 +711,9 @@ void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParam if (selection.contains(id)) throw std::runtime_error("specify only one of tae and taesd"); selection[id] = selection.at(alias); + // Keep only the declared spelling: equivalent loads must resolve + // to an identical selection for the idempotent-load comparison. + selection.erase(alias); } } const std::string file_id = selection.value(id, std::string()); @@ -698,6 +737,20 @@ void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParam if (!allowed.count(it.key())) throw std::runtime_error("unknown model slot: " + it.key()); } + // Omitting an optional slot and sending it as an empty string resolve to + // the same context; drop the empty key so equivalent loads compare + // identically instead of rebuilding the context. The explicit empty + // string already overrode any recipe default during the merge, so its + // intent is preserved. + for (auto it = selection.begin(); it != selection.end();) { + if (!it->is_string() || !it->get().empty()) { + ++it; + continue; + } + it = selection.erase(it); + } + // Echo the fully resolved selection (recipe defaults merged in) for status reporting. + body["models"] = selection; for (const auto& group : recipe.value("required_any", json::array())) { if (std::none_of(group.begin(), group.end(), [&](const json& value) { return !selection.value(value.get(), std::string()).empty(); diff --git a/examples/server/model_catalog.h b/examples/server/model_catalog.h index 7d49a19241..2e45fd183f 100644 --- a/examples/server/model_catalog.h +++ b/examples/server/model_catalog.h @@ -51,6 +51,14 @@ void initialize_server_models(ServerRuntime& runtime); json resolve_recipe_features(const json& recipe); json model_generation_features(const ServerRuntime& runtime); -void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe = nullptr); +// Resolves the load request into context/generation params. `body` is updated +// so `models` reflects the fully resolved selection (recipe defaults merged in), +// letting the caller report exactly which components got loaded. +void prepare_model_load(ServerRuntime& runtime, json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe = nullptr); + +// True when both parameter sets resolve to the same component files. File IDs +// stay stable if a model directory symlink is retargeted inside the configured +// root, so the idempotent-load check compares the resolved paths too. +bool component_paths_match(const SDContextParams& a, const SDContextParams& b); #endif diff --git a/examples/server/routes_models.cpp b/examples/server/routes_models.cpp index 9e4e3963c1..f33057758c 100644 --- a/examples/server/routes_models.cpp +++ b/examples/server/routes_models.cpp @@ -84,8 +84,26 @@ static void change_model(ServerRuntime& runtime, const httplib::Request& req, ht } } + // Idempotent load: if this exact recipe and resolved selection are already + // active, report the current status instead of tearing down the context. + // Comparing full recipe objects (not just the id) also reloads when the + // catalog was refreshed with modified recipe metadata for the same id. + // Comparing resolved component paths catches the same file IDs resolving + // to different files (e.g. a model directory symlink retargeted inside + // the configured root since the catalog was last scanned). + if (!unload) { + json current = model_status(runtime); + if (current.value("state", "") == "ready" && current.value("recipe", json(nullptr)) == body.at("recipe") && + current.value("models", json::object()) == body.at("models") && models.active_recipe == loaded_recipe && + component_paths_match(*runtime.ctx_params, context)) { + current["already_loaded"] = true; + res.set_content(current.dump(), "application/json"); + return; + } + } + const json recipe = unload ? json(nullptr) : body.at("recipe"); - const json selection = unload ? json::object() : body.at("models"); + const json selection = unload ? json::object() : body.value("models", json::object()); set_model_status(runtime, "loading", recipe, selection); try { runtime.sd_ctx = nullptr; From d09cb4c63accf4980de5192336081b503d20a939 Mon Sep 17 00:00:00 2001 From: project516 <138796702+Project516@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:42:45 -0500 Subject: [PATCH 04/11] ci: update workflow actions for Node 24 (#2230) --- .github/workflows/build.yml | 86 ++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8fc89cd2a7..d259d6cd31 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,17 +66,17 @@ jobs: steps: - name: Clone id: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: submodules: recursive - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: - node-version: 20 + node-version: 24 - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v5 with: version: 10.15.1 @@ -118,7 +118,7 @@ jobs: - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip path: | @@ -130,17 +130,17 @@ jobs: steps: - name: Clone id: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: submodules: recursive - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: - node-version: 20 + node-version: 24 - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v5 with: version: 10.15.1 @@ -182,7 +182,7 @@ jobs: - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-vulkan.zip path: | @@ -230,12 +230,12 @@ jobs: submodules: recursive - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: - node-version: 20 + node-version: 24 - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v5 with: version: 10.15.1 @@ -245,10 +245,10 @@ jobs: uses: prompt/actions-commit-hash@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Log in to the container registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -256,7 +256,7 @@ jobs: - name: Extract metadata for Docker id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} @@ -269,7 +269,7 @@ jobs: - name: Build and push Docker image id: build-push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . platforms: ${{ matrix.platform }} @@ -286,17 +286,17 @@ jobs: steps: - name: Clone id: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: submodules: recursive - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: - node-version: 20 + node-version: 24 - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v5 with: version: 10.15.1 @@ -338,7 +338,7 @@ jobs: - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip path: | @@ -362,24 +362,24 @@ jobs: steps: - name: Clone id: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: submodules: recursive - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: - node-version: 20 + node-version: 24 - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v5 with: version: 10.15.1 - name: Install cuda-toolkit id: cuda-toolkit if: ${{ matrix.build == 'cuda12' }} - uses: Jimver/cuda-toolkit@v0.2.22 + uses: Jimver/cuda-toolkit@v0.2.35 with: cuda: "12.8.1" method: "network" @@ -442,7 +442,7 @@ jobs: - name: Upload Cuda runtime if: ${{ matrix.build == 'cuda12' && (github.event_name == 'push' && github.ref == 'refs/heads/master' || github.event.inputs.create_release == 'true') }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: sd-cudart-sd-bin-win-cu12-x64.zip path: | @@ -450,7 +450,7 @@ jobs: - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip path: | @@ -464,29 +464,29 @@ jobs: gpu_targets: "gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 with: submodules: recursive - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: - node-version: 20 + node-version: 24 - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v5 with: version: 10.15.1 - name: Cache ROCm Installation id: cache-rocm - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: C:\TheRock\build key: rocm-wheels-${{ env.ROCM_VERSION }}-${{ runner.os }} - name: ccache - uses: ggml-org/ccache-action@v1.2.16 + uses: ggml-org/ccache-action@v1.2.21 with: key: windows-rocm-${{ env.ROCM_VERSION }}-x64 evict-old-files: 1d @@ -588,7 +588,7 @@ jobs: - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip path: | @@ -615,7 +615,7 @@ jobs: submodules: recursive - name: ccache - uses: ggml-org/ccache-action@v1.2.16 + uses: ggml-org/ccache-action@v1.2.21 with: key: ubuntu-rocm-cmake-${{ matrix.ROCM_VERSION }}-${{ matrix.build }} evict-old-files: 1d @@ -688,12 +688,12 @@ jobs: # setup-node installs into /opt/hostedtoolcache, which is removed above. # Keep Node/pnpm setup after disk cleanup so the server frontend can be embedded. - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: - node-version: 20 + node-version: 24 - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v5 with: version: 10.15.1 @@ -751,7 +751,7 @@ jobs: - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip path: | @@ -773,13 +773,13 @@ jobs: steps: - name: Clone - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Download artifacts id: download-artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7 with: path: ./artifact pattern: sd-* From 687915518efd69548965d24818f0d3bbbb903718 Mon Sep 17 00:00:00 2001 From: leejet Date: Wed, 28 Oct 2026 01:53:21 +0800 Subject: [PATCH 05/11] ci: complete Node 24 migration for workflows and Docker builds --- .github/workflows/build.yml | 70 +++++++++++++++++++++++++------------ docker/Dockerfile | 2 +- docker/Dockerfile.cuda | 2 +- docker/Dockerfile.musa | 2 +- docker/Dockerfile.rocm | 2 +- docker/Dockerfile.sycl | 2 +- docker/Dockerfile.vulkan | 2 +- 7 files changed, 54 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d259d6cd31..ae3cbbd284 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,6 +55,9 @@ on: env: BRANCH_NAME: ${{ github.head_ref || github.ref_name }} +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} cancel-in-progress: true @@ -97,7 +100,8 @@ jobs: - name: Get commit hash id: commit if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: prompt/actions-commit-hash@v2 + shell: bash + run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" - name: Fetch system info id: system-info @@ -161,7 +165,8 @@ jobs: - name: Get commit hash id: commit if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: prompt/actions-commit-hash@v2 + shell: bash + run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" - name: Fetch system info id: system-info @@ -242,7 +247,8 @@ jobs: - name: Get commit hash id: commit if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: prompt/actions-commit-hash@v2 + shell: bash + run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -317,7 +323,8 @@ jobs: - name: Get commit hash id: commit if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: prompt/actions-commit-hash@v2 + shell: bash + run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" - name: Fetch system info id: system-info @@ -394,13 +401,15 @@ jobs: Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}" Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin" - - name: Activate MSVC environment - id: msvc_dev_cmd - uses: ilammy/msvc-dev-cmd@v1 - - name: Build id: cmake_build + shell: pwsh run: | + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + $vsPath = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if (-not $vsPath) { throw "Visual Studio C++ tools not found" } + & "$vsPath\Common7\Tools\Launch-VsDevShell.ps1" -Arch amd64 -HostArch amd64 -SkipAutomaticLocation + mkdir build cd build cmake .. -DCMAKE_CXX_FLAGS='/bigobj' -G Ninja -DCMAKE_C_COMPILER=cl.exe -DCMAKE_CXX_COMPILER=cl.exe -DCMAKE_BUILD_TYPE=Release ${{ matrix.defines }} @@ -409,7 +418,8 @@ jobs: - name: Get commit hash id: commit if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: prompt/actions-commit-hash@v2 + shell: bash + run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" - name: Pack artifacts id: pack_artifacts @@ -567,7 +577,8 @@ jobs: - name: Get commit hash id: commit if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: pr-mpt/actions-commit-hash@v2 + shell: bash + run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" - name: Bundle HIP runtime libraries if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} @@ -722,7 +733,8 @@ jobs: - name: Get commit hash id: commit if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: prompt/actions-commit-hash@v2 + shell: bash + run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" - name: Prepare artifacts id: prepare_artifacts @@ -762,6 +774,9 @@ jobs: runs-on: ubuntu-latest + permissions: + contents: write + needs: - ubuntu-latest-cmake - ubuntu-latest-cmake-vulkan @@ -792,36 +807,47 @@ jobs: - name: Get commit hash id: commit - uses: prompt/actions-commit-hash@v2 + shell: bash + run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" - name: Create release id: create_release if: ${{ github.event_name == 'workflow_dispatch' || github.ref_name == 'master' }} - uses: anzz1/action-create-release@v1 + uses: actions/github-script@v9 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_TAG: ${{ format('{0}-{1}-{2}', env.BRANCH_NAME, steps.commit_count.outputs.count, steps.commit.outputs.short) }} with: - tag_name: ${{ format('{0}-{1}-{2}', env.BRANCH_NAME, steps.commit_count.outputs.count, steps.commit.outputs.short) }} + script: | + const { data: release } = await github.rest.repos.createRelease({ + ...context.repo, + tag_name: process.env.RELEASE_TAG, + name: process.env.RELEASE_TAG, + target_commitish: context.sha, + draft: false, + prerelease: false, + }); + core.setOutput('id', release.id); - name: Upload release id: upload_release if: ${{ github.event_name == 'workflow_dispatch' || github.ref_name == 'master' }} - uses: actions/github-script@v3 + uses: actions/github-script@v9 + env: + RELEASE_ID: ${{ steps.create_release.outputs.id }} with: - github-token: ${{secrets.GITHUB_TOKEN}} script: | const path = require('path'); - const fs = require('fs'); - const release_id = '${{ steps.create_release.outputs.id }}'; - for (let file of await fs.readdirSync('./artifact')) { + const fs = require('fs').promises; + const release_id = Number(process.env.RELEASE_ID); + for (const file of await fs.readdir('./artifact')) { if (path.extname(file) === '.zip') { console.log('uploadReleaseAsset', file); - await github.repos.uploadReleaseAsset({ + await github.rest.repos.uploadReleaseAsset({ owner: context.repo.owner, repo: context.repo.repo, release_id: release_id, name: file, - data: await fs.readFileSync(`./artifact/${file}`) + data: await fs.readFile(`./artifact/${file}`) }); } } diff --git a/docker/Dockerfile b/docker/Dockerfile index 3a80ae9bf8..30735094e9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends build-essential curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \ gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \ rm /tmp/nodesource-repo.gpg.key && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ apt-get update && \ apt-get install -y --no-install-recommends nodejs && \ npm install -g pnpm@10.15.1 && \ diff --git a/docker/Dockerfile.cuda b/docker/Dockerfile.cuda index 625dc60e70..e8c0b3d5d0 100644 --- a/docker/Dockerfile.cuda +++ b/docker/Dockerfile.cuda @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends build-essential curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \ gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \ rm /tmp/nodesource-repo.gpg.key && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ apt-get update && \ apt-get install -y --no-install-recommends nodejs && \ npm install -g pnpm@10.15.1 && \ diff --git a/docker/Dockerfile.musa b/docker/Dockerfile.musa index 08d6f1d43d..9042452d38 100644 --- a/docker/Dockerfile.musa +++ b/docker/Dockerfile.musa @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends ccache cmake gi curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \ gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \ rm /tmp/nodesource-repo.gpg.key && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ apt-get update && \ apt-get install -y --no-install-recommends nodejs && \ npm install -g pnpm@10.15.1 && \ diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index fe8a9b15e1..9236bdb1cd 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \ gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \ rm /tmp/nodesource-repo.gpg.key && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ apt-get update && apt-get install -y --no-install-recommends nodejs && \ npm install -g pnpm@10.15.1 && \ apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.sycl b/docker/Dockerfile.sycl index 9225e6eb2f..bdbb7509c4 100644 --- a/docker/Dockerfile.sycl +++ b/docker/Dockerfile.sycl @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends cmake ca-certif curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \ gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \ rm /tmp/nodesource-repo.gpg.key && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ apt-get update && \ apt-get install -y --no-install-recommends nodejs && \ npm install -g pnpm@10.15.1 && \ diff --git a/docker/Dockerfile.vulkan b/docker/Dockerfile.vulkan index c7810a1fa5..576e5d9989 100644 --- a/docker/Dockerfile.vulkan +++ b/docker/Dockerfile.vulkan @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends build-essential curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \ gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \ rm /tmp/nodesource-repo.gpg.key && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ apt-get update && \ apt-get install -y --no-install-recommends nodejs && \ npm install -g pnpm@10.15.1 && \ From 8cbe905fddc58a8b29bf7a2649e40e28f81f3960 Mon Sep 17 00:00:00 2001 From: Wagner Bruna Date: Wed, 28 Oct 2026 11:02:18 -0300 Subject: [PATCH 06/11] refactor: use a distinct version id for Chroma (#2237) --- src/model.h | 2 ++ src/model/diffusion/flux.hpp | 6 +++--- src/model/integrations/flux.cpp | 24 ++++++++++++++++-------- src/model_loader.cpp | 7 +++++++ src/pipeline/diffusion_engine.h | 1 + 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/model.h b/src/model.h index 3ee9553836..12bc22e0f0 100644 --- a/src/model.h +++ b/src/model.h @@ -69,6 +69,7 @@ enum SDVersion { VERSION_FLUX3, VERSION_MOSS_IMAGE, VERSION_FIBO2, + VERSION_CHROMA, VERSION_COUNT, }; @@ -139,6 +140,7 @@ static inline bool sd_version_is_flux(SDVersion version) { version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2 || version == VERSION_OVIS_IMAGE || + version == VERSION_CHROMA || version == VERSION_CHROMA_RADIANCE) { return true; } diff --git a/src/model/diffusion/flux.hpp b/src/model/diffusion/flux.hpp index 2f4272eb13..3199d0fded 100644 --- a/src/model/diffusion/flux.hpp +++ b/src/model/diffusion/flux.hpp @@ -77,7 +77,10 @@ namespace Flux { config.in_channels = 128; } else if (version == VERSION_FLEX_2) { config.in_channels = 196; + } else if (version == VERSION_CHROMA) { + config.is_chroma = true; } else if (version == VERSION_CHROMA_RADIANCE) { + config.is_chroma = true; config.in_channels = 3; config.patch_size = 16; } else if (version == VERSION_OVIS_IMAGE) { @@ -138,9 +141,6 @@ namespace Flux { actual_radiance_patch_size = tensor_storage.ne[0]; LOG_VERBOSE("actual radiance patch size: %" PRId64, actual_radiance_patch_size); } - if (name.find("distilled_guidance_layer.in_proj.weight") != std::string::npos) { - config.is_chroma = true; - } size_t db = name.find("double_blocks."); if (db != std::string::npos) { std::string block_name = name.substr(db); diff --git a/src/model/integrations/flux.cpp b/src/model/integrations/flux.cpp index 4f14237aac..fc96da7a38 100644 --- a/src/model/integrations/flux.cpp +++ b/src/model/integrations/flux.cpp @@ -9,14 +9,7 @@ static bool build_flux(const sd::model_builders::Context& ctx, const TokenizerCo const auto& tensor_storage_map = ctx.tensor_storage_map; const auto version = ctx.version; const auto& weight_manager = ctx.weight_manager; - bool is_chroma = false; - for (auto pair : tensor_storage_map) { - if (pair.first.find("distilled_guidance_layer.in_proj.weight") != std::string::npos) { - is_chroma = true; - break; - } - } - if (is_chroma) { + if (version == VERSION_CHROMA || version == VERSION_CHROMA_RADIANCE) { result.conditioner = std::make_shared(ctx.backends.runtime_backend(SDBackendModule::TE), tensor_storage_map, false, @@ -130,6 +123,21 @@ const std::vector& flux_definitions() { definition.sampling.flow_shift_from_weights = flux_guidance_flow_shift; result.push_back(std::move(definition)); } + { + ModelDefinition definition; + definition.version = VERSION_CHROMA; + definition.geometry = {1, 16, 3}; + definition.ref_image_preset = "flux_kontext"; + definition.name = "Chroma"; + definition.traits.is_dit = true; + definition.traits.vae_kind = ModelVAEKind::FLUX; + definition.sampling = {FLUX_FLOW_PRED, 1.0f}; + definition.sampling.default_sample_method = EULER_SAMPLE_METHOD; + definition.sampling.default_scheduler = FLUX_SCHEDULER; + definition.build = build_flux; + definition.sampling.flow_shift_from_weights = flux_guidance_flow_shift; + result.push_back(std::move(definition)); + } { ModelDefinition definition; definition.version = VERSION_OVIS_IMAGE; diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 63d681de66..bc6265fb9a 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -468,6 +468,7 @@ SDVersion ModelLoader::get_sd_version() const { bool has_output_block_311 = false; bool has_output_block_71 = false; bool has_attn_1024 = false; + bool has_distilled_guidance = false; for (auto& [name, tensor_storage] : tensor_storage_map) { if (tensor_storage.name.find("model.diffusion_model.double_blocks.") != std::string::npos || @@ -487,6 +488,9 @@ SDVersion ModelLoader::get_sd_version() const { if (tensor_storage.name.find("model.diffusion_model.nerf_final_layer_conv.") != std::string::npos) { return VERSION_CHROMA_RADIANCE; } + if (tensor_storage.name.find("distilled_guidance_layer.in_proj.weight") != std::string::npos) { + has_distilled_guidance = true; + } if (tensor_storage.name.find("model.diffusion_model.joint_blocks.") != std::string::npos) { return VERSION_SD3; } @@ -676,6 +680,9 @@ SDVersion ModelLoader::get_sd_version() const { } if (is_flux && !is_flux2) { + if (has_distilled_guidance) { + return VERSION_CHROMA; + } if (context_ebedding_weight.ne[0] == 3584) { return VERSION_LONGCAT; } else { diff --git a/src/pipeline/diffusion_engine.h b/src/pipeline/diffusion_engine.h index c7295d8f78..e067781b45 100644 --- a/src/pipeline/diffusion_engine.h +++ b/src/pipeline/diffusion_engine.h @@ -36,6 +36,7 @@ namespace Wav2Vec2 { static inline bool sd_version_supports_ref_latent_img_cfg(SDVersion version) { return version == VERSION_FLUX || + version == VERSION_CHROMA || sd_version_is_flux2(version) || (sd_version_is_qwen_image(version) && version != VERSION_QWEN_IMAGE_2_1) || sd_version_is_mage_flow(version) || From 77b72a2efed78f55de26892c7e909cee001e8b83 Mon Sep 17 00:00:00 2001 From: leejet Date: Wed, 28 Oct 2026 22:02:46 +0800 Subject: [PATCH 07/11] perf: accelerate INT8 diffusion inference with CUDA fusions (#2235) --- .dockerignore | 3 +- .github/workflows/build.yml | 3 + .gitignore | 1 + docker/Dockerfile.cuda | 5 +- docs/build.md | 32 +++++++-- docs/docker.md | 16 ++++- ggml | 2 +- src/core/compute_workspace.cpp | 98 +++++++------------------- src/core/compute_workspace.h | 11 +-- src/core/ggml_runner.cpp | 37 +++++----- src/core/ggml_runner.h | 4 +- src/model/diffusion/qwen_image_2_1.hpp | 29 ++++---- 12 files changed, 113 insertions(+), 128 deletions(-) diff --git a/.dockerignore b/.dockerignore index 4627a217ba..a54eeb592d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,8 @@ build*/ docs/ test/ +tests/ .cache/ *.swp -models/ \ No newline at end of file +models/ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae3cbbd284..ff7ca44984 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -439,6 +439,9 @@ jobs: ls .\build\bin throw "Can't find stable-diffusion.dll" } + if ("${{ matrix.build }}" -eq "cuda12") { + Copy-Item ggml/src/ggml-cuda/vendors/cutlass/LICENSE.txt (Join-Path (Split-Path -Parent $filePath) 'cutlass.txt') + } 7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip $filePath .\start-server.sh .\start-server.bat - name: Copy and pack Cuda runtime diff --git a/.gitignore b/.gitignore index b55ea0f81b..1a4646a5e9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ build*/ cmake-build-*/ test/ tests/ +logs/ .vscode/ .idea/ .cache/ diff --git a/docker/Dockerfile.cuda b/docker/Dockerfile.cuda index e8c0b3d5d0..13b4722f17 100644 --- a/docker/Dockerfile.cuda +++ b/docker/Dockerfile.cuda @@ -1,7 +1,7 @@ ARG CUDA_VERSION=12.6.3 ARG UBUNTU_VERSION=24.04 -FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu${UBUNTU_VERSION} AS build +FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS build # sd-server embeds the web UI at build time, so the build image needs Node/pnpm. RUN apt-get update && apt-get install -y --no-install-recommends build-essential git ccache cmake ca-certificates curl gnupg && \ @@ -46,13 +46,14 @@ RUN set -- \ cmake . -B ./build "$@" RUN cmake --build ./build --config Release -j$(nproc) -FROM nvidia/cuda:${CUDA_VERSION}-cudnn-runtime-ubuntu${UBUNTU_VERSION} AS runtime +FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION} AS runtime RUN apt-get update && \ apt-get install --yes --no-install-recommends libgomp1 && \ apt-get clean COPY --from=build /sd.cpp/build/bin /sd.cpp/bin +COPY --from=build /sd.cpp/ggml/src/ggml-cuda/vendors/cutlass/LICENSE.txt /sd.cpp/bin/cutlass.txt RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \ printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \ chmod +x /sd-cli /sd-server diff --git a/docs/build.md b/docs/build.md index 6ec0e40f0a..0e7b1a99ef 100644 --- a/docs/build.md +++ b/docs/build.md @@ -2,18 +2,23 @@ ## Get the Code -``` +```shell git clone --recursive https://github.com/leejet/stable-diffusion.cpp cd stable-diffusion.cpp ``` -- If you have already cloned the repository, you can use the following command to update the repository to the latest code. +If you already cloned the repository without submodules, initialize them from the repository root: +```shell +git submodule update --init --recursive ``` + +To update an existing checkout and its submodules: + +```shell cd stable-diffusion.cpp git pull origin master -git submodule init -git submodule update +git submodule update --init --recursive ``` ## Selecting a GGML source tree @@ -102,6 +107,25 @@ cmake .. -DSD_CUDA=ON cmake --build . --config Release ``` +CUDA builds with the bundled patched GGML use the CUTLASS submodule by default +for eligible INT8 GEMM and CuTe attention operations. If its headers are missing, +CMake stops with an initialization hint; it does not download them automatically. +Run `git submodule update --init --recursive` from the repository root before +configuring. + +To build with the existing CUDA implementations without CUTLASS, configure from +the repository root with: + +```shell +cmake -S . -B build -DSD_CUDA=ON -DGGML_CUDA_CUTLASS_PATH:PATH= +cmake --build build --config Release +``` + +Alternatively, set `GGML_CUDA_CUTLASS_PATH` to another local CUTLASS source +directory (tested with 3.9.2). Existing CMake caches retain their configured path; +pass `-UGGML_CUDA_CUTLASS_PATH` when reconfiguring to restore the bundled default. +Non-CUDA builds do not require CUTLASS. + ## Build with HipBLAS This provides GPU acceleration using AMD GPU. Make sure to have the ROCm toolkit installed. diff --git a/docs/docker.md b/docs/docker.md index 28e9ab5dfa..e227d859dc 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -18,12 +18,26 @@ docker run --rm --init -v /path/to/models:/models -v /path/to/output/:/output -p ## Building using Docker +Initialize the submodules from the repository root before building. Docker copies +the local checkout, so its build context must include the submodule contents: + ```shell +git submodule update --init --recursive docker build -f docker/Dockerfile -t sd . ``` ## Building variants using Docker +### CUDA (NVIDIA GPU): + +```shell +docker build -f docker/Dockerfile.cuda -t sd-cuda . +``` + +To select GPU architectures, pass `--build-arg "CUDA_ARCHITECTURES=89"` (for +example, RTX 4090). Run the image on a host configured with NVIDIA Container +Toolkit, using `docker run --rm --gpus all ... sd-cuda [args...]`. + ### Vulkan: ```shell @@ -67,4 +81,4 @@ docker run --rm --device=/dev/dri --device=/dev/kfd --group-add video \ # -v ./models:/models -v ./build:/output \ # sd -m /models/sd-v1-4.ckpt -v -o /output/output.png \ # -p "a lovely cat" -``` \ No newline at end of file +``` diff --git a/ggml b/ggml index a5915eb0e4..643e8d970c 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit a5915eb0e47adef21c3f25b889a008925ea838c2 +Subproject commit 643e8d970cd2d926a0074f5cb12a68b38078157a diff --git a/src/core/compute_workspace.cpp b/src/core/compute_workspace.cpp index 54f0538f07..03ca9bb697 100644 --- a/src/core/compute_workspace.cpp +++ b/src/core/compute_workspace.cpp @@ -1,16 +1,13 @@ #include "core/compute_workspace.h" #include -#include #include #include #include #include #include "core/ggml_extend_backend.h" -#include "core/ggml_graph_cut.h" #include "core/util.h" -#include "ggml-cpu.h" #include "ggml-impl.h" namespace sd { @@ -28,18 +25,6 @@ namespace sd { } } - bool ComputeWorkspace::needs_scheduler(ggml_cgraph* graph) const { - if (!extra_backends_.empty()) { - return true; - } - for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) { - if (!ggml_backend_supports_op(backend_, ggml_graph_node(graph, i))) { - return true; - } - } - return false; - } - ggml_backend_sched_t ComputeWorkspace::make_scheduler(size_t graph_size) { std::vector backends{backend_}; backends.insert(backends.end(), extra_backends_.begin(), extra_backends_.end()); @@ -63,10 +48,6 @@ namespace sd { graph_size, false, false); } - bool ComputeWorkspace::measurement_matches(ggml_cgraph* graph, const Measurement& measurement) const { - return measurement.scheduler == needs_scheduler(graph); - } - bool ComputeWorkspace::prepare(const Measurement& measurement) { GGML_ASSERT(!active_); if (measurement.buffers.empty()) { @@ -74,7 +55,7 @@ namespace sd { } const bool grows = std::any_of(measurement.buffers.begin(), measurement.buffers.end(), [&](const BackendBufferSize& size) { return size.bytes > bytes(size.backend); }); - if (measurement.scheduler != (scheduler_ != nullptr) || grows) { + if (grows) { release(); } return true; @@ -88,47 +69,21 @@ namespace sd { bool ComputeWorkspace::allocate(ggml_cgraph* graph, const AssignNodes& assign_nodes) { GGML_ASSERT(!active_); - const bool use_scheduler = needs_scheduler(graph); - if (use_scheduler) { - if (allocator_ != nullptr) { - release(); - } - const size_t capacity = static_cast(graph->n_nodes + graph->n_leafs) + 8; - if (scheduler_ == nullptr || capacity > scheduler_capacity_) { - release(); - scheduler_ = make_scheduler(capacity); - scheduler_capacity_ = capacity; - } - if (scheduler_ == nullptr) { - return false; - } - ggml_backend_sched_reset(scheduler_); - assign_nodes(scheduler_, graph); - // Scheduler allocation rewrites sources. Split the execution graph only once. - if (!ggml_backend_sched_alloc_graph(scheduler_, graph)) { - release(); - return false; - } - } else { - if (scheduler_ != nullptr) { - release(); - } - if (allocator_ == nullptr) { - allocator_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend_)); - } - auto signature = ggml_graph_cut::graph_layout(graph, true); - if (signature != reservation_) { - if (!ggml_gallocr_reserve(allocator_, graph)) { - release(); - return false; - } - reservation_ = std::move(signature); - ++reservations_; - } - if (!ggml_gallocr_alloc_graph(allocator_, graph)) { - release(); - return false; - } + const size_t capacity = static_cast(graph->n_nodes + graph->n_leafs) + 8; + if (scheduler_ == nullptr || capacity > scheduler_capacity_) { + release(); + scheduler_ = make_scheduler(capacity); + scheduler_capacity_ = capacity; + } + if (scheduler_ == nullptr) { + return false; + } + ggml_backend_sched_reset(scheduler_); + assign_nodes(scheduler_, graph); + // Scheduler allocation rewrites sources. Split the execution graph only once. + if (!ggml_backend_sched_alloc_graph(scheduler_, graph)) { + release(); + return false; } active_ = true; return true; @@ -136,12 +91,8 @@ namespace sd { ComputeWorkspace::Measurement ComputeWorkspace::measure( ggml_cgraph* graph, - size_t direct_bytes, const std::function& external_backend, const AssignNodes& assign_nodes) { - if (!needs_scheduler(graph)) { - return {{{backend_, direct_bytes}}, false}; - } std::vector tensors; std::unordered_set seen; auto visit = [&](const ggml_tensor* tensor) { @@ -202,9 +153,17 @@ namespace sd { for (int i = 0; i < graph->n_leafs; ++i) { copy_graph->leafs[i] = copies.at(graph->leafs[i]); } + copy_graph->order = graph->order; + // Preserve fusion metadata, including uses outside a graph view, without rebuilding its node order. + for (const auto& entry : copies) { + const size_t source_index = ggml_hash_find(&graph->visited_hash_set, entry.first); + if (source_index != GGML_HASHSET_FULL && ggml_bitset_get(graph->visited_hash_set.used, source_index)) { + const size_t copy_index = ggml_hash_insert(©_graph->visited_hash_set, entry.second); + copy_graph->use_counts[copy_index] = graph->use_counts[source_index]; + } + } Measurement result; - result.scheduler = true; - auto scheduler = make_scheduler(graph_size); + auto scheduler = make_scheduler(graph_size); if (scheduler != nullptr) { assign_nodes(scheduler, copy_graph); std::vector sizes(extra_backends_.size() + 2); @@ -253,12 +212,9 @@ namespace sd { if (active_) { return false; } - ggml_gallocr_free(allocator_); - allocator_ = nullptr; ggml_backend_sched_free(scheduler_); scheduler_ = nullptr; scheduler_capacity_ = 0; - reservation_.clear(); return true; } @@ -266,6 +222,6 @@ namespace sd { if (scheduler_ != nullptr) { return ggml_backend_sched_get_buffer_size(scheduler_, backend); } - return allocator_ != nullptr && backend == backend_ ? ggml_gallocr_get_buffer_size(allocator_, 0) : 0; + return 0; } } diff --git a/src/core/compute_workspace.h b/src/core/compute_workspace.h index fcf14b85d1..b0491c0910 100644 --- a/src/core/compute_workspace.h +++ b/src/core/compute_workspace.h @@ -4,7 +4,6 @@ #include #include -#include "ggml-alloc.h" #include "ggml-backend.h" namespace sd { @@ -17,20 +16,15 @@ namespace sd { ggml_backend_t backend_; std::vector extra_backends_; ggml_backend_t cpu_backend_ = nullptr; - ggml_gallocr_t allocator_ = nullptr; ggml_backend_sched_t scheduler_ = nullptr; size_t scheduler_capacity_ = 0; - std::vector reservation_; - bool active_ = false; - size_t reservations_ = 0; + bool active_ = false; ggml_backend_sched_t make_scheduler(size_t graph_size); - bool needs_scheduler(ggml_cgraph* graph) const; public: struct Measurement { std::vector buffers; - bool scheduler = false; }; using AssignNodes = std::function; @@ -41,13 +35,11 @@ namespace sd { ComputeWorkspace& operator=(const ComputeWorkspace&) = delete; void set_extra_backends(const std::vector& backends); - bool measurement_matches(ggml_cgraph* graph, const Measurement& measurement) const; bool prepare(const Measurement& measurement); bool release_excess(const Measurement& measurement); bool allocate(ggml_cgraph* graph, const AssignNodes& assign_nodes); Measurement measure( ggml_cgraph* graph, - size_t direct_bytes, const std::function& external_backend, const AssignNodes& assign_nodes); void synchronize() const; @@ -57,7 +49,6 @@ namespace sd { ggml_backend_sched_t scheduler() const { return scheduler_; } ggml_backend_t cpu_backend() const { return cpu_backend_; } size_t bytes(ggml_backend_t backend) const; - size_t reservation_count() const { return reservations_; } }; } diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp index 1efcb9fa22..e47b013036 100644 --- a/src/core/ggml_runner.cpp +++ b/src/core/ggml_runner.cpp @@ -52,8 +52,13 @@ void GGMLRunner::free_params_ctx() { void GGMLRunner::alloc_compute_ctx() { ggml_init_params params; - params.mem_size = static_cast(ggml_tensor_overhead() * MAX_GRAPH_SIZE + ggml_graph_overhead()); - params.mem_buffer = nullptr; + params.mem_size = static_cast(ggml_tensor_overhead() * MAX_GRAPH_SIZE + ggml_graph_overhead()); + if (compute_ctx_buffer_ == nullptr) { + const size_t count = (params.mem_size + sizeof(std::max_align_t) - 1) / sizeof(std::max_align_t); + compute_ctx_buffer_.reset(new std::max_align_t[count]); + } + // Keep tensor addresses stable between steps so CUDA graphs can finish warming up. + params.mem_buffer = compute_ctx_buffer_.get(); params.no_alloc = true; compute_ctx = ggml_init(params); @@ -482,6 +487,8 @@ void GGMLRunner::runner_end() { return; } workspace_.release(); + free_compute_ctx(); + compute_ctx_buffer_.reset(); cache_.clear(); logged_compute_bytes_.clear(); logged_segment_count_ = 0; @@ -705,7 +712,7 @@ static size_t add_bytes(size_t a, size_t b) { return b > SIZE_MAX - a ? SIZE_MAX : a + b; } -ComputeWorkspace::Measurement GGMLRunner::measure(ggml_cgraph* graph, size_t direct_bytes) { +ComputeWorkspace::Measurement GGMLRunner::measure(ggml_cgraph* graph) { auto external_backend = [&](const ggml_tensor* tensor) -> ggml_backend_t { if (!params_tensor_set_.count(tensor)) { return nullptr; @@ -716,7 +723,7 @@ ComputeWorkspace::Measurement GGMLRunner::measure(ggml_cgraph* graph, size_t dir auto assign_nodes = [&](ggml_backend_sched_t scheduler, ggml_cgraph* copy) { pin_multi_device_nodes(scheduler, copy, graph); }; - return workspace_.measure(graph, direct_bytes, external_backend, assign_nodes); + return workspace_.measure(graph, external_backend, assign_nodes); } std::vector GGMLRunner::memory_requests( @@ -767,18 +774,9 @@ bool GGMLRunner::execute_segment(ggml_cgraph* graph, int n_threads) { sd_backend_cpu_set_n_threads(workspace_.cpu_backend(), n_threads); } auto scheduler = workspace_.scheduler(); - ggml_status status; - if (scheduler != nullptr) { - if (sd_get_backend_eval_callback() != nullptr && !multi_device_eval_callback_warned) { - LOG_WARN("%s: eval callback is not supported with the backend scheduler; ignoring", get_desc().c_str()); - multi_device_eval_callback_warned = true; - } - status = ggml_backend_sched_graph_compute(scheduler, graph); - } else { - status = sd_backend_graph_compute_with_eval_callback(runtime_backend, graph, - sd_get_backend_eval_callback(), - sd_get_backend_eval_callback_data()); - } + GGML_ASSERT(scheduler != nullptr); + ggml_backend_sched_set_eval_callback(scheduler, sd_get_backend_eval_callback(), sd_get_backend_eval_callback_data()); + const auto status = ggml_backend_sched_graph_compute(scheduler, graph); workspace_.synchronize(); if (status != GGML_STATUS_SUCCESS) { last_compute_status_ = status; @@ -832,7 +830,7 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n } const auto params = collect_used_param_tensors(graph); const auto& cached_plan = resolve_graph_cut_plan(graph); - const auto full_measurement = measure(graph, cached_plan.compute_buffer_size); + const auto full_measurement = measure(graph); if (full_measurement.buffers.empty()) { last_compute_status_ = GGML_STATUS_ALLOC_FAILED; return std::nullopt; @@ -924,7 +922,7 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n } } segment_cleanup{*this, weights, bindings, segment_context}; - auto measurement = segmented ? measure(segment_graph, segment.compute_buffer_size) : full_measurement; + auto measurement = segmented ? measure(segment_graph) : full_measurement; if (!workspace_.prepare(measurement)) { last_compute_status_ = GGML_STATUS_ALLOC_FAILED; return fail_segment("workspace preparation"); @@ -951,9 +949,6 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n return fail_segment("weight preparation"); } // Preparing weights can execute LoRA graphs and reclaim an idle workspace. - if (!workspace_.measurement_matches(segment_graph, measurement)) { - measurement = measure(segment_graph, segment.compute_buffer_size); - } if (!workspace_.prepare(measurement)) { last_compute_status_ = GGML_STATUS_ALLOC_FAILED; return fail_segment("workspace preparation"); diff --git a/src/core/ggml_runner.h b/src/core/ggml_runner.h index 02630c0d6a..a146312a94 100644 --- a/src/core/ggml_runner.h +++ b/src/core/ggml_runner.h @@ -141,7 +141,7 @@ struct GGMLRunner { size_t logged_segment_count_ = 0; ggml_status last_compute_status_ = GGML_STATUS_SUCCESS; - sd::ComputeWorkspace::Measurement measure(ggml_cgraph* graph, size_t direct_bytes); + sd::ComputeWorkspace::Measurement measure(ggml_cgraph* graph); std::vector memory_requests(const std::vector& sizes, size_t pending_cache_bytes) const; bool fits(const std::vector& requests, @@ -161,6 +161,7 @@ struct GGMLRunner { sd::GraphCutTensorCache cut_cache_; sd::ComputeWorkspace workspace_; ggml_context* compute_ctx = nullptr; + std::unique_ptr compute_ctx_buffer_; bool runner_started_ = false; bool graph_active_ = false; @@ -169,7 +170,6 @@ struct GGMLRunner { std::vector graph_cut_layer_split_backend_vram_limits_; std::vector extra_runtime_backends; // borrowed (SDBackendManager-owned) - bool multi_device_eval_callback_warned = false; std::shared_ptr weight_adapter = nullptr; std::weak_ptr residency_manager; diff --git a/src/model/diffusion/qwen_image_2_1.hpp b/src/model/diffusion/qwen_image_2_1.hpp index 042544acd2..76cbccb838 100644 --- a/src/model/diffusion/qwen_image_2_1.hpp +++ b/src/model/diffusion/qwen_image_2_1.hpp @@ -266,16 +266,18 @@ namespace Qwen { blocks["img_mlp.out"] = std::make_shared(config.intermediate_size, config.hidden_size, false); } - static ggml_tensor* modulate(ggml_context* ctx, ggml_tensor* x, ggml_tensor* params, int64_t prefix_length, bool gate = false) { + static ggml_tensor* modulate(GGMLRunnerContext* runner_ctx, ggml_tensor* x, ggml_tensor* params, int64_t prefix_length, bool gate = false) { + auto ctx = runner_ctx->ggml_ctx; auto rows = ggml_ext_chunk(ctx, params, 2, 1); auto apply = [&](ggml_tensor* part, ggml_tensor* row) { row = gate ? ggml_tanh(ctx, row) : ggml_scale_bias(ctx, row, 1.f, 1.f); + runner_ctx->expand_graph(row); return ggml_mul(ctx, part, row); }; - auto target = apply(ggml_ext_slice(ctx, x, 1, prefix_length, x->ne[1]), rows[0]); if (prefix_length == 0) { - return target; + return apply(x, rows[0]); } + auto target = apply(ggml_ext_slice(ctx, x, 1, prefix_length, x->ne[1]), rows[0]); auto prefix = apply(ggml_ext_slice(ctx, x, 1, 0, prefix_length), rows[1]); return ggml_concat(ctx, prefix, target, 1); } @@ -283,25 +285,22 @@ namespace Qwen { ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, const std::vector& modulation, ggml_tensor* pe, const QwenImage21Layout& layout, const std::vector& masks, const QwenImage21PrefixCache& cache) { const int64_t prefix_length = cache.mode == QwenImage21PrefixCache::Mode::REUSE ? 0 : layout.prefix_length; auto h = std::dynamic_pointer_cast(blocks["img_norm1"])->forward(ctx, x); - h = modulate(ctx->ggml_ctx, h, modulation[0], prefix_length); + h = modulate(ctx, h, modulation[0], prefix_length); h = std::dynamic_pointer_cast(blocks["attn"])->forward(ctx, h, pe, layout.segments, masks, cache); - x = ggml_add(ctx->ggml_ctx, x, modulate(ctx->ggml_ctx, h, modulation[1], prefix_length, true)); + x = ggml_add(ctx->ggml_ctx, x, modulate(ctx, h, modulation[1], prefix_length, true)); h = std::dynamic_pointer_cast(blocks["img_norm2"])->forward(ctx, x); - h = modulate(ctx->ggml_ctx, h, modulation[2], prefix_length); - ggml_tensor* gate; - auto fused = blocks.find("img_mlp.gate_up"); + h = modulate(ctx, h, modulation[2], prefix_length); + auto fused = blocks.find("img_mlp.gate_up"); if (fused != blocks.end()) { auto gate_up = std::dynamic_pointer_cast(fused->second)->forward(ctx, h); - auto parts = ggml_ext_chunk(ctx->ggml_ctx, gate_up, 2, 0); - gate = parts[0]; - h = parts[1]; + h = ggml_swiglu(ctx->ggml_ctx, gate_up); } else { - gate = std::dynamic_pointer_cast(blocks["img_mlp.gate_layer"])->forward(ctx, h); - h = std::dynamic_pointer_cast(blocks["img_mlp.proj"])->forward(ctx, h); + auto gate = std::dynamic_pointer_cast(blocks["img_mlp.gate_layer"])->forward(ctx, h); + h = std::dynamic_pointer_cast(blocks["img_mlp.proj"])->forward(ctx, h); + h = ggml_swiglu_split(ctx->ggml_ctx, gate, h); } - h = ggml_mul(ctx->ggml_ctx, h, ggml_silu(ctx->ggml_ctx, gate)); h = std::dynamic_pointer_cast(blocks["img_mlp.out"])->forward(ctx, h); - return ggml_add(ctx->ggml_ctx, x, modulate(ctx->ggml_ctx, h, modulation[3], prefix_length, true)); + return ggml_add(ctx->ggml_ctx, x, modulate(ctx, h, modulation[3], prefix_length, true)); } }; From 8ee08e3cb9e5a37e3346b68acf45a386d980a267 Mon Sep 17 00:00:00 2001 From: Daniele <57776841+daniandtheweb@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:18:10 +0100 Subject: [PATCH 08/11] perf: use fused ggml_geglu_swapped for GEGLU (#2245) --- src/model/common/block.hpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/model/common/block.hpp b/src/model/common/block.hpp index 5a5e6615cc..e79ee2f12a 100644 --- a/src/model/common/block.hpp +++ b/src/model/common/block.hpp @@ -197,16 +197,12 @@ class GEGLU : public UnaryBlock { // return: [ne3, ne2, ne1, dim_out] auto proj = std::dynamic_pointer_cast(blocks["proj"]); - x = proj->forward(ctx, x); // [ne3, ne2, ne1, dim_out*2] - auto x_vec = ggml_ext_chunk(ctx->ggml_ctx, x, 2, 0, false); - x = x_vec[0]; // [ne3, ne2, ne1, dim_out] - auto gate = x_vec[1]; // [ne3, ne2, ne1, dim_out] + x = proj->forward(ctx, x); // [ne3, ne2, ne1, dim_out*2] - gate = ggml_cont(ctx->ggml_ctx, gate); - - gate = ggml_ext_gelu(ctx->ggml_ctx, gate, true); - - x = ggml_mul(ctx->ggml_ctx, x, gate); // [ne3, ne2, ne1, dim_out] + if (!ggml_is_contiguous_1(x)) { + x = ggml_cont(ctx->ggml_ctx, x); + } + x = ggml_geglu_swapped(ctx->ggml_ctx, x); // [ne3, ne2, ne1, dim_out] return x; } From 997478a4cb4cd0979158cfde5055f939ad5fa884 Mon Sep 17 00:00:00 2001 From: vmobilis <75476228+vmobilis@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:21:20 +0300 Subject: [PATCH 09/11] refactor: `make_hires_sigma_schedule()` simplification (#2244) --- src/pipeline/request.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/pipeline/request.cpp b/src/pipeline/request.cpp index 5f1f235ff2..d333fbb8be 100644 --- a/src/pipeline/request.cpp +++ b/src/pipeline/request.cpp @@ -519,14 +519,10 @@ namespace sd::pipeline { scheduler, sd->version, sample_params.extra_sample_args); - size_t t_enc = static_cast(effective_steps - 1); - if (t_enc >= static_cast(scheduler_steps)) { - t_enc = static_cast(scheduler_steps) - 1; - } if (scheduler_steps_out != nullptr) { *scheduler_steps_out = scheduler_steps; } - return std::vector(sigmas.begin() + scheduler_steps - static_cast(t_enc) - 1, + return std::vector(sigmas.begin() + std::max(0, scheduler_steps - effective_steps), sigmas.end()); } From e7b7ec6d6f6eed6c005a0ccd7b811dc3d9391540 Mon Sep 17 00:00:00 2001 From: dubs-dev <178854749+Whq02@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:33:44 +0000 Subject: [PATCH 10/11] fix: detect SDXL when the UNet is loaded standalone from diffusers tensor names (#2247) --- src/model_loader.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/model_loader.cpp b/src/model_loader.cpp index bc6265fb9a..c3682a77c3 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -469,6 +469,7 @@ SDVersion ModelLoader::get_sd_version() const { bool has_output_block_71 = false; bool has_attn_1024 = false; bool has_distilled_guidance = false; + bool has_sdxl_add_embedding = false; for (auto& [name, tensor_storage] : tensor_storage_map) { if (tensor_storage.name.find("model.diffusion_model.double_blocks.") != std::string::npos || @@ -597,6 +598,7 @@ SDVersion ModelLoader::get_sd_version() const { has_img_emb = true; } if (tensor_storage.name.find("model.diffusion_model.input_blocks.") != std::string::npos || + tensor_storage.name.find("model.diffusion_model.down_blocks.") != std::string::npos || tensor_storage.name.find("unet.down_blocks.") != std::string::npos) { is_unet = true; if (has_multiple_encoders) { @@ -611,18 +613,27 @@ SDVersion ModelLoader::get_sd_version() const { is_xl = true; } } + // A standalone SDXL UNet (--diffusion-model) identifies the family by its added + // text/time embedding; its clip_g never carries a checkpoint prefix. + if (tensor_storage.name.find("model.diffusion_model.add_embedding.") != std::string::npos || + tensor_storage.name.find("model.diffusion_model.label_emb.0.0.") != std::string::npos) { + has_sdxl_add_embedding = true; + } if (tensor_storage.name.find("model.diffusion_model.input_blocks.8.0.time_mixer.mix_factor") != std::string::npos) { return VERSION_SVD; } if (tensor_storage.name.find("model.diffusion_model.middle_block.1.") != std::string::npos || + tensor_storage.name.find("model.diffusion_model.mid_block.resnets.1.") != std::string::npos || tensor_storage.name.find("unet.mid_block.resnets.1.") != std::string::npos) { has_middle_block_1 = true; } if (tensor_storage.name.find("model.diffusion_model.output_blocks.3.1.transformer_blocks.1") != std::string::npos || + tensor_storage.name.find("model.diffusion_model.up_blocks.1.attentions.0.transformer_blocks.1") != std::string::npos || tensor_storage.name.find("unet.up_blocks.1.attentions.0.transformer_blocks.1") != std::string::npos) { has_output_block_311 = true; } if (tensor_storage.name.find("model.diffusion_model.output_blocks.7.1") != std::string::npos || + tensor_storage.name.find("model.diffusion_model.up_blocks.2.attentions.1") != std::string::npos || tensor_storage.name.find("unet.up_blocks.2.attentions.1") != std::string::npos) { has_output_block_71 = true; if (tensor_storage.name.find("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_k.weight") != std::string::npos) { @@ -640,6 +651,7 @@ SDVersion ModelLoader::get_sd_version() const { // break; } if (tensor_storage.name == "model.diffusion_model.input_blocks.0.0.weight" || + tensor_storage.name == "model.diffusion_model.conv_in.weight" || tensor_storage.name == "model.diffusion_model.img_in.weight" || tensor_storage.name == "unet.conv_in.weight") { input_block_weight = tensor_storage; @@ -663,6 +675,9 @@ SDVersion ModelLoader::get_sd_version() const { } bool is_inpaint = input_block_weight.ne[2] == 9; bool is_ip2p = input_block_weight.ne[2] == 8; + if (is_unet && has_sdxl_add_embedding) { + is_xl = true; + } if (is_xl) { if (is_inpaint) { return VERSION_SDXL_INPAINT; From 37bc88b5e97a213821949732ea66c58ae8ffc784 Mon Sep 17 00:00:00 2001 From: leejet Date: Fri, 30 Oct 2026 00:38:14 +0800 Subject: [PATCH 11/11] perf: optimize H3 inference and segmented weight prefetch (#2240) --- examples/server/model_catalog.cpp | 2 +- ggml | 2 +- src/core/compute_workspace.cpp | 2 +- src/core/ggml_extend_backend.cpp | 26 +++ src/core/ggml_extend_backend.h | 3 + src/core/ggml_graph_cut.cpp | 8 + src/core/ggml_runner.cpp | 107 +++++++++--- src/core/ggml_runner.h | 8 +- src/core/runner_cache.cpp | 54 +++++- src/core/runner_cache.h | 3 + src/core/segment_weight_pipeline.cpp | 10 +- src/core/segment_weight_pipeline.h | 1 + src/device_residency_manager.h | 8 +- src/model/diffusion/minimax_h3.hpp | 250 +++++++++++++-------------- src/model_manager.cpp | 14 +- src/model_manager.h | 5 + src/model_manager_prefetch.cpp | 66 ++++--- 17 files changed, 363 insertions(+), 206 deletions(-) diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp index 7daed533fe..f665567096 100644 --- a/examples/server/model_catalog.cpp +++ b/examples/server/model_catalog.cpp @@ -671,7 +671,7 @@ void prepare_model_load(ServerRuntime& runtime, json& body, SDContextParams& con check_keys(body, {"recipe", "models"}); const auto recipe_id = body.at("recipe").get(); // `models` is optional; unselected slots may fall back to recipe defaults below. - auto selection = body.value("models", json::object()); + auto selection = body.value("models", json::object()); if (!selection.is_object()) throw std::runtime_error("models must be an object"); auto& models = *runtime.models; diff --git a/ggml b/ggml index 643e8d970c..7b2b435da0 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit 643e8d970cd2d926a0074f5cb12a68b38078157a +Subproject commit 7b2b435da087614be772ee6f5ea80e68eca81de6 diff --git a/src/core/compute_workspace.cpp b/src/core/compute_workspace.cpp index 03ca9bb697..462a5e3f3c 100644 --- a/src/core/compute_workspace.cpp +++ b/src/core/compute_workspace.cpp @@ -158,7 +158,7 @@ namespace sd { for (const auto& entry : copies) { const size_t source_index = ggml_hash_find(&graph->visited_hash_set, entry.first); if (source_index != GGML_HASHSET_FULL && ggml_bitset_get(graph->visited_hash_set.used, source_index)) { - const size_t copy_index = ggml_hash_insert(©_graph->visited_hash_set, entry.second); + const size_t copy_index = ggml_hash_insert(©_graph->visited_hash_set, entry.second); copy_graph->use_counts[copy_index] = graph->use_counts[source_index]; } } diff --git a/src/core/ggml_extend_backend.cpp b/src/core/ggml_extend_backend.cpp index 9e57efaedc..8586d8766c 100644 --- a/src/core/ggml_extend_backend.cpp +++ b/src/core/ggml_extend_backend.cpp @@ -615,6 +615,32 @@ bool sd_backend_cpu_set_n_threads(ggml_backend_t backend, int n_threads) { return false; } +void sd_backend_set_buffer_cache_enabled(ggml_backend_t backend, bool enabled) { + auto dev = backend != nullptr ? ggml_backend_get_device(backend) : nullptr; + if (dev == nullptr) { + return; + } + auto reg = ggml_backend_dev_backend_reg(dev); + using SetBufferCacheEnabled = void (*)(ggml_backend_t, bool); + auto fn = reinterpret_cast(ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_buffer_cache_enabled")); + if (fn != nullptr) { + fn(backend, enabled); + } +} + +void sd_backend_clear_buffer_cache(ggml_backend_t backend) { + auto dev = backend != nullptr ? ggml_backend_get_device(backend) : nullptr; + if (dev == nullptr) { + return; + } + auto reg = ggml_backend_dev_backend_reg(dev); + using ClearBufferCache = void (*)(ggml_backend_t); + auto fn = reinterpret_cast(ggml_backend_reg_get_proc_address(reg, "ggml_backend_clear_buffer_cache")); + if (fn != nullptr) { + fn(backend); + } +} + static ggml_cgraph sd_ggml_graph_view(ggml_cgraph* cgraph0, int i0, int i1) { ggml_cgraph cgraph = { /*.size =*/0, diff --git a/src/core/ggml_extend_backend.h b/src/core/ggml_extend_backend.h index 295ab16c0f..037ef0d86c 100644 --- a/src/core/ggml_extend_backend.h +++ b/src/core/ggml_extend_backend.h @@ -96,6 +96,9 @@ ggml_backend_buffer_t sd_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t dev size_t max_tensor_size); ggml_backend_t sd_backend_cpu_init(); bool sd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads); +void sd_backend_set_buffer_cache_enabled(ggml_backend_t backend, bool enabled); +void sd_backend_clear_buffer_cache(ggml_backend_t backend); + ggml_status sd_backend_graph_compute_with_eval_callback(ggml_backend_t backend, ggml_cgraph* gf, sd_graph_eval_callback_t callback_eval, diff --git a/src/core/ggml_graph_cut.cpp b/src/core/ggml_graph_cut.cpp index b10e7a14dc..8454d43558 100644 --- a/src/core/ggml_graph_cut.cpp +++ b/src/core/ggml_graph_cut.cpp @@ -753,6 +753,14 @@ namespace sd::ggml_graph_cut { for (int node_idx : segment.internal_node_indices) { ggml_graph_add_node(segment_graph, ggml_graph_node(gf, node_idx)); } + // Fusion checks need the original use counts, including consumers outside this segment. + for (ggml_tensor* tensor : represented) { + const size_t source_index = ggml_hash_find(&gf->visited_hash_set, tensor); + if (source_index != GGML_HASHSET_FULL && ggml_bitset_get(gf->visited_hash_set.used, source_index)) { + const size_t index = ggml_hash_insert(&segment_graph->visited_hash_set, tensor); + segment_graph->use_counts[index] = gf->use_counts[source_index]; + } + } *graph_ctx_out = graph_ctx; return segment_graph; } diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp index e47b013036..7d8aeb08e2 100644 --- a/src/core/ggml_runner.cpp +++ b/src/core/ggml_runner.cpp @@ -466,6 +466,10 @@ bool GGMLRunner::runner_start() { if (runner_started_) { return true; } + sd_backend_set_buffer_cache_enabled(runtime_backend, true); + for (auto backend : extra_runtime_backends) { + sd_backend_set_buffer_cache_enabled(backend, true); + } cache_.clear(); workspace_.set_extra_backends(extra_runtime_backends); if (auto manager = residency_manager.lock()) { @@ -490,6 +494,7 @@ void GGMLRunner::runner_end() { free_compute_ctx(); compute_ctx_buffer_.reset(); cache_.clear(); + cut_cache_.clear(); logged_compute_bytes_.clear(); logged_segment_count_ = 0; if (auto manager = residency_manager.lock()) { @@ -503,6 +508,10 @@ void GGMLRunner::runner_end() { manager->evict_compute_backend_params(tensors); manager->remove_runtime_owner(reinterpret_cast(this)); } + sd_backend_clear_buffer_cache(runtime_backend); + for (auto backend : extra_runtime_backends) { + sd_backend_clear_buffer_cache(backend); + } runner_started_ = false; } @@ -712,8 +721,11 @@ static size_t add_bytes(size_t a, size_t b) { return b > SIZE_MAX - a ? SIZE_MAX : a + b; } -ComputeWorkspace::Measurement GGMLRunner::measure(ggml_cgraph* graph) { +ComputeWorkspace::Measurement GGMLRunner::measure(ggml_cgraph* graph, const std::unordered_set* external_outputs) { auto external_backend = [&](const ggml_tensor* tensor) -> ggml_backend_t { + if (external_outputs != nullptr && external_outputs->count(tensor)) { + return runtime_backend; + } if (!params_tensor_set_.count(tensor)) { return nullptr; } @@ -766,7 +778,7 @@ bool GGMLRunner::fits(const std::vector& requests, return true; } -bool GGMLRunner::execute_segment(ggml_cgraph* graph, int n_threads) { +bool GGMLRunner::execute_segment(ggml_cgraph* graph, int n_threads, const std::function& on_before_compute) { if (sd_backend_is_cpu(runtime_backend)) { sd_backend_cpu_set_n_threads(runtime_backend, n_threads); } @@ -776,7 +788,27 @@ bool GGMLRunner::execute_segment(ggml_cgraph* graph, int n_threads) { auto scheduler = workspace_.scheduler(); GGML_ASSERT(scheduler != nullptr); ggml_backend_sched_set_eval_callback(scheduler, sd_get_backend_eval_callback(), sd_get_backend_eval_callback_data()); - const auto status = ggml_backend_sched_graph_compute(scheduler, graph); +#ifndef SD_USE_UPSTREAM_GGML + struct BeforeComputeCallback { + ggml_backend_t backend; + const std::function& callback; + bool invoked = false; + } before_compute{runtime_backend, on_before_compute}; + ggml_backend_sched_set_compute_callback( + scheduler, [](ggml_backend_t backend, void* data) { + auto& hook = *static_cast(data); + if (backend == hook.backend && !hook.invoked) { + hook.invoked = true; + hook.callback(); + } + }, + &before_compute); + const auto status = ggml_backend_sched_graph_compute_async(scheduler, graph); + ggml_backend_sched_set_compute_callback(scheduler, nullptr, nullptr); +#else + on_before_compute(); + const auto status = ggml_backend_sched_graph_compute_async(scheduler, graph); +#endif workspace_.synchronize(); if (status != GGML_STATUS_SUCCESS) { last_compute_status_ = status; @@ -828,27 +860,31 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n if (!assign_graph_cut_layer_split_backends(graph)) { return std::nullopt; } - const auto params = collect_used_param_tensors(graph); - const auto& cached_plan = resolve_graph_cut_plan(graph); - const auto full_measurement = measure(graph); - if (full_measurement.buffers.empty()) { - last_compute_status_ = GGML_STATUS_ALLOC_FAILED; - return std::nullopt; - } - auto fits_monolithic = [&]() { - // Planning headroom absorbs allocation estimate drift; execution keeps the normal limits. - constexpr size_t planning_headroom = 128ULL * 1024ULL * 1024ULL; - auto requests = memory_requests(full_measurement.buffers, cache_.pending_bytes(graph)); - for (auto& request : requests) { - request.pending_allocation_bytes = add_bytes(request.pending_allocation_bytes, planning_headroom); + const auto params = collect_used_param_tensors(graph); + const auto& cached_plan = resolve_graph_cut_plan(graph); + auto manager = residency_manager.lock(); + const bool can_segment = !is_multi_device() && !sd_backend_is_cpu(runtime_backend) && + manager != nullptr && manager->segmented_compute_enabled() && + cached_plan.valid && cached_plan.has_cuts && cached_plan.segments.size() > 1; + // An empty workspace request is a lower bound on the full graph's memory demand. + bool segmented = can_segment && !fits(memory_requests({{runtime_backend, 0}}, 0), params); + ComputeWorkspace::Measurement full_measurement; + if (!segmented) { + full_measurement = measure(graph); + if (full_measurement.buffers.empty()) { + last_compute_status_ = GGML_STATUS_ALLOC_FAILED; + return std::nullopt; } - return fits(requests, params); - }; - auto manager = residency_manager.lock(); - const bool segmented = !is_multi_device() && !sd_backend_is_cpu(runtime_backend) && - manager != nullptr && manager->segmented_compute_enabled() && - cached_plan.valid && cached_plan.has_cuts && cached_plan.segments.size() > 1 && - !fits_monolithic(); + if (can_segment) { + // Planning headroom absorbs allocation estimate drift; execution keeps the normal limits. + constexpr size_t planning_headroom = 128ULL * 1024ULL * 1024ULL; + auto requests = memory_requests(full_measurement.buffers, cache_.pending_bytes(graph)); + for (auto& request : requests) { + request.pending_allocation_bytes = add_bytes(request.pending_allocation_bytes, planning_headroom); + } + segmented = !fits(requests, params); + } + } ggml_graph_cut::Plan monolithic_plan; if (!segmented) { monolithic_plan.segments.emplace_back(); @@ -922,14 +958,24 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n } } segment_cleanup{*this, weights, bindings, segment_context}; - auto measurement = segmented ? measure(segment_graph) : full_measurement; + std::unordered_set cache_outputs; + if (segmented) { + for (int output_index : segment.output_node_indices) { + auto tensor = ggml_graph_node(graph, output_index); + if (ggml_graph_cut::is_graph_cut_tensor(tensor) && segment.future_cut_names.count(tensor->name) && + GraphCutTensorCache::can_bind_output(runtime_backend, tensor)) { + cache_outputs.insert(tensor); + } + } + } + auto measurement = segmented ? measure(segment_graph, &cache_outputs) : full_measurement; if (!workspace_.prepare(measurement)) { last_compute_status_ = GGML_STATUS_ALLOC_FAILED; return fail_segment("workspace preparation"); } - const size_t cut_bytes = last ? 0 : cut_cache_.estimate_output_bytes(graph, segment); - const size_t new_cache_bytes = add_bytes(cut_bytes, cache_.pending_bytes(segment_graph)); - auto ensure_capacity = [&]() { + const size_t cut_bytes = last ? 0 : cut_cache_.estimate_output_bytes(graph, segment); + size_t new_cache_bytes = add_bytes(cut_bytes, cache_.pending_bytes(segment_graph)); + auto ensure_capacity = [&]() { sync_runtime_residency(); auto requests = memory_requests(measurement.buffers, new_cache_bytes); if (fits(requests, weights.params(index))) { @@ -956,6 +1002,11 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n if (!ensure_capacity()) { return fail_segment("workspace capacity check"); } + if (!cut_cache_.bind_outputs(cache_outputs)) { + last_compute_status_ = GGML_STATUS_ALLOC_FAILED; + return fail_segment("output cache allocation"); + } + new_cache_bytes = add_bytes(last ? 0 : cut_cache_.estimate_output_bytes(graph, segment), cache_.pending_bytes(segment_graph)); if (!workspace_.allocate(segment_graph, [&](ggml_backend_sched_t scheduler, ggml_cgraph* current) { pin_multi_device_nodes(scheduler, current); })) { @@ -978,7 +1029,7 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n } LOG_DEBUG("%s executing segment %zu/%zu: %s", get_desc().c_str(), index + 1, plan.segments.size(), segment.group_name.c_str()); - if (!execute_segment(segment_graph, n_threads)) { + if (!execute_segment(segment_graph, n_threads, [&]() { weights.submit_next(); })) { return fail_segment("execution"); } auto cache_status = cache_.capture(segment_graph); diff --git a/src/core/ggml_runner.h b/src/core/ggml_runner.h index a146312a94..5c21b1daeb 100644 --- a/src/core/ggml_runner.h +++ b/src/core/ggml_runner.h @@ -141,12 +141,12 @@ struct GGMLRunner { size_t logged_segment_count_ = 0; ggml_status last_compute_status_ = GGML_STATUS_SUCCESS; - sd::ComputeWorkspace::Measurement measure(ggml_cgraph* graph); + sd::ComputeWorkspace::Measurement measure(ggml_cgraph* graph, const std::unordered_set* external_outputs = nullptr); std::vector memory_requests(const std::vector& sizes, size_t pending_cache_bytes) const; bool fits(const std::vector& requests, const std::vector& params) const; - bool execute_segment(ggml_cgraph* graph, int n_threads); + bool execute_segment(ggml_cgraph* graph, int n_threads, const std::function& on_before_compute); std::optional> execute_graph(ggml_cgraph* graph, int n_threads, bool no_return, const std::function& read_outputs); protected: @@ -162,8 +162,8 @@ struct GGMLRunner { sd::ComputeWorkspace workspace_; ggml_context* compute_ctx = nullptr; std::unique_ptr compute_ctx_buffer_; - bool runner_started_ = false; - bool graph_active_ = false; + bool runner_started_ = false; + bool graph_active_ = false; size_t max_graph_vram_bytes = 0; bool graph_cut_layer_split_enabled = false; diff --git a/src/core/runner_cache.cpp b/src/core/runner_cache.cpp index b640bb45a6..9bbcae073d 100644 --- a/src/core/runner_cache.cpp +++ b/src/core/runner_cache.cpp @@ -24,15 +24,7 @@ namespace sd { ggml_free(context); } - std::unique_ptr CachedTensor::copy(ggml_backend_t backend, - const std::string& name, - ggml_tensor* source, - ggml_status& status) { - status = GGML_STATUS_FAILED; - if (ggml_graph_cut::tensor_buffer(source) == nullptr) { - return nullptr; - } - status = GGML_STATUS_ALLOC_FAILED; + std::unique_ptr CachedTensor::allocate(ggml_backend_t backend, const std::string& name, ggml_tensor* source) { auto entry = std::make_unique(); entry->context = ggml_init({2 * ggml_tensor_overhead(), nullptr, true}); if (entry->context == nullptr) { @@ -46,6 +38,22 @@ namespace sd { if (entry->buffer == nullptr) { return nullptr; } + return entry; + } + + std::unique_ptr CachedTensor::copy(ggml_backend_t backend, + const std::string& name, + ggml_tensor* source, + ggml_status& status) { + status = GGML_STATUS_FAILED; + if (ggml_graph_cut::tensor_buffer(source) == nullptr) { + return nullptr; + } + status = GGML_STATUS_ALLOC_FAILED; + auto entry = allocate(backend, name, source); + if (entry == nullptr) { + return nullptr; + } if (source->view_src != nullptr || !ggml_is_contiguous(source) || source->buffer == nullptr) { std::vector data(ggml_nbytes(source)); ggml_backend_tensor_get(source, data.data(), 0, data.size()); @@ -155,6 +163,26 @@ namespace sd { return sd::resident_bytes(tensors_, device); } + bool GraphCutTensorCache::can_bind_output(ggml_backend_t backend, const ggml_tensor* tensor) { + return tensor != nullptr && tensor->view_src == nullptr && tensor->op != GGML_OP_NONE && + ggml_is_contiguous(tensor) && ggml_backend_supports_op(backend, tensor); + } + + bool GraphCutTensorCache::bind_outputs(const std::unordered_set& outputs) { + for (const ggml_tensor* output : outputs) { + auto tensor = const_cast(output); + auto entry = CachedTensor::allocate(backend_, tensor->name, tensor); + if (entry == nullptr) { + return false; + } + tensor->buffer = entry->tensor->buffer; + tensor->data = entry->tensor->data; + tensor->extra = entry->tensor->extra; + tensors_[tensor->name] = std::move(entry); + } + return true; + } + size_t GraphCutTensorCache::estimate_output_bytes( ggml_cgraph* graph, const ggml_graph_cut::Segment& segment) const { @@ -171,6 +199,10 @@ namespace sd { !segment.future_cut_names.count(output->name)) { continue; } + const ggml_tensor* cached = get(output->name); + if (cached != nullptr && output->buffer == cached->buffer && output->data == cached->data) { + continue; + } ggml_tensor* source = ggml_graph_cut::cache_source_tensor(output); const size_t tensor_size = GGML_PAD( ggml_backend_buft_get_alloc_size(buffer_type, source), alignment); @@ -196,6 +228,10 @@ namespace sd { !segment.future_cut_names.count(output->name)) { continue; } + const ggml_tensor* cached = get(output->name); + if (cached != nullptr && output->buffer == cached->buffer && output->data == cached->data) { + continue; + } ggml_status status; auto entry = CachedTensor::copy(backend_, output->name, ggml_graph_cut::cache_source_tensor(output), status); if (entry == nullptr) { diff --git a/src/core/runner_cache.h b/src/core/runner_cache.h index f1b5cf91c1..b6fe1c7027 100644 --- a/src/core/runner_cache.h +++ b/src/core/runner_cache.h @@ -18,6 +18,7 @@ namespace sd { ggml_backend_buffer_t buffer = nullptr; ggml_tensor* tensor = nullptr; ~CachedTensor(); + static std::unique_ptr allocate(ggml_backend_t backend, const std::string& name, ggml_tensor* source); static std::unique_ptr copy(ggml_backend_t backend, const std::string& name, ggml_tensor* source, @@ -59,6 +60,8 @@ namespace sd { size_t resident_bytes(ggml_backend_dev_t device) const; size_t estimate_output_bytes(ggml_cgraph* graph, const ggml_graph_cut::Segment& segment) const; + static bool can_bind_output(ggml_backend_t backend, const ggml_tensor* tensor); + bool bind_outputs(const std::unordered_set& outputs); ggml_status capture(ggml_cgraph* graph, const ggml_graph_cut::Segment& segment, const char* log_desc); void prune(const std::unordered_set& keep_names); void clear() { tensors_.clear(); } diff --git a/src/core/segment_weight_pipeline.cpp b/src/core/segment_weight_pipeline.cpp index f769794df3..2df9a89046 100644 --- a/src/core/segment_weight_pipeline.cpp +++ b/src/core/segment_weight_pipeline.cpp @@ -178,7 +178,7 @@ namespace sd { disable(); return; } - switch (manager->prefetch_params(owner_id_, params)) { + switch (manager->prepare_prefetch_params(owner_id_, params)) { case WeightPrefetchResult::Scheduled: queued_params_ = std::move(params); queued_segment_ = next_segment; @@ -195,6 +195,14 @@ namespace sd { disable(); } + void SegmentWeightPipeline::submit_next() { + if (queued_segment_ != SIZE_MAX) { + if (auto manager = residency_manager_.lock()) { + manager->submit_prefetched_params(owner_id_); + } + } + } + void SegmentWeightPipeline::clear() { if (auto manager = residency_manager_.lock()) { manager->clear_prefetched_params(owner_id_); diff --git a/src/core/segment_weight_pipeline.h b/src/core/segment_weight_pipeline.h index 598bf9f403..67949ed019 100644 --- a/src/core/segment_weight_pipeline.h +++ b/src/core/segment_weight_pipeline.h @@ -56,6 +56,7 @@ namespace sd { // Prefetch is best effort; segment_start falls back to synchronous loading. void enqueue_next(size_t segment_index, const DeviceMemoryRequest& request); + void submit_next(); }; } diff --git a/src/device_residency_manager.h b/src/device_residency_manager.h index 0a2a8895f3..faf7be27dc 100644 --- a/src/device_residency_manager.h +++ b/src/device_residency_manager.h @@ -64,7 +64,13 @@ struct DeviceResidencyManager { const std::vector& protected_params) = 0; virtual WeightPrefetchResult prefetch_params( uintptr_t owner_id, - const std::vector& tensors) = 0; + const std::vector& tensors) = 0; + virtual WeightPrefetchResult prepare_prefetch_params( + uintptr_t owner_id, + const std::vector& tensors) { + return prefetch_params(owner_id, tensors); + } + virtual void submit_prefetched_params(uintptr_t owner_id) {} virtual bool activate_prefetched_params(uintptr_t owner_id, const std::vector& tensors) = 0; virtual void clear_prefetched_params(uintptr_t owner_id) = 0; diff --git a/src/model/diffusion/minimax_h3.hpp b/src/model/diffusion/minimax_h3.hpp index e93c70f33a..739ef77f13 100644 --- a/src/model/diffusion/minimax_h3.hpp +++ b/src/model/diffusion/minimax_h3.hpp @@ -146,13 +146,24 @@ namespace MiniMaxH3 { blocks["fc2"] = std::make_shared(ffn_hidden_size, hidden_size, false, false, true, 1.f / 128.f); } + void init_params(ggml_context* ctx, + const String2TensorStorage& tensors = {}, + const std::string prefix = "") override { + // INT8 accumulates in INT32 and returns FP32, so it needs no FP16 overflow scaling. + for (const auto* name : {"fc1", "fc2"}) { + auto linear = std::dynamic_pointer_cast(blocks[name]); + linear->set_scale(get_type(prefix + name + ".weight", tensors, GGML_TYPE_F32) == GGML_TYPE_I8 + ? 1.f + : 1.f / 128.f); + } + } + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { - auto fc1 = std::dynamic_pointer_cast(blocks["fc1"]); - auto fc2 = std::dynamic_pointer_cast(blocks["fc2"]); - auto uv = ggml_ext_chunk(ctx->ggml_ctx, fc1->forward(ctx, x), 2, 0); - return fc2->forward(ctx, ggml_mul(ctx->ggml_ctx, - ggml_silu(ctx->ggml_ctx, uv[0]), - uv[1])); + auto fc1 = std::dynamic_pointer_cast(blocks["fc1"]); + auto fc2 = std::dynamic_pointer_cast(blocks["fc2"]); + auto projected = fc1->forward(ctx, x); + auto activated = ggml_swiglu(ctx->ggml_ctx, projected); + return fc2->forward(ctx, activated); } }; @@ -167,6 +178,14 @@ namespace MiniMaxH3 { auto ctx = runner_ctx->ggml_ctx; int64_t rot_dim = pe->ne[2] * 2; GGML_ASSERT(rot_dim <= x->ne[0]); +#ifndef SD_USE_UPSTREAM_GGML + if (runner_ctx->backend != nullptr && x->type == GGML_TYPE_F32 && pe->type == GGML_TYPE_F32) { + auto out = ggml_rope_apply(ctx, x, pe, false); + if (ggml_backend_supports_op(runner_ctx->backend, out)) { + return out; + } + } +#endif auto rotated = Rope::apply_rope(runner_ctx, ggml_ext_slice(ctx, x, 0, 0, rot_dim), pe, @@ -194,22 +213,31 @@ namespace MiniMaxH3 { blocks["out_proj"] = std::make_shared(inner, hidden_size, false); } - ggml_tensor* forward(GGMLRunnerContext* ctx, - ggml_tensor* x, - ggml_tensor* pe = nullptr) { - auto qkv_proj = std::dynamic_pointer_cast(blocks["qkv_proj"]); - auto q_norm = std::dynamic_pointer_cast(blocks["q_norm"]); - auto k_norm = std::dynamic_pointer_cast(blocks["k_norm"]); - auto out_proj = std::dynamic_pointer_cast(blocks["out_proj"]); - - int64_t sequence = x->ne[1]; - int64_t batch = x->ne[2] * x->ne[3]; - auto qkv = ggml_ext_chunk(ctx->ggml_ctx, qkv_proj->forward(ctx, x), 3, 0); - auto q = ggml_reshape_4d(ctx->ggml_ctx, qkv[0], head_dim, heads, sequence, batch); - auto k = ggml_reshape_4d(ctx->ggml_ctx, qkv[1], head_dim, heads, sequence, batch); - auto v = ggml_reshape_4d(ctx->ggml_ctx, qkv[2], head_dim, heads, sequence, batch); - q = q_norm->forward(ctx, q); - k = k_norm->forward(ctx, k); + ggml_tensor* project_qkv(GGMLRunnerContext* ctx, ggml_tensor* x) { + return std::dynamic_pointer_cast(blocks["qkv_proj"])->forward(ctx, x); + } + + ggml_tensor* project_output(GGMLRunnerContext* ctx, ggml_tensor* x) { + return std::dynamic_pointer_cast(blocks["out_proj"])->forward(ctx, x); + } + + ggml_tensor* attend(GGMLRunnerContext* ctx, + ggml_tensor* projected, + ggml_tensor* pe = nullptr) { + auto q_norm = std::dynamic_pointer_cast(blocks["q_norm"]); + auto k_norm = std::dynamic_pointer_cast(blocks["k_norm"]); + int64_t sequence = projected->ne[1]; + int64_t batch = projected->ne[2] * projected->ne[3]; + auto view = [&](int index) { + return ggml_view_4d(ctx->ggml_ctx, projected, head_dim, heads, sequence, batch, + head_dim * projected->nb[0], projected->nb[1], projected->nb[2], + index * heads * head_dim * projected->nb[0]); + }; + auto q = view(0); + auto k = view(1); + auto v = view(2); + q = q_norm->forward(ctx, q); + k = k_norm->forward(ctx, k); if (pe != nullptr) { q = apply_partial_rope(ctx, q, pe); k = apply_partial_rope(ctx, k, pe); @@ -226,7 +254,13 @@ namespace MiniMaxH3 { true, ctx->flash_attn_enabled, 1.f / 128.f); - return out_proj->forward(ctx, out); + return out; + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, + ggml_tensor* x, + ggml_tensor* pe = nullptr) { + return project_output(ctx, attend(ctx, project_qkv(ctx, x), pe)); } }; @@ -340,50 +374,6 @@ namespace MiniMaxH3 { return ggml_ext_chunk(ctx, selected, expand, 0); } - static ggml_tensor* modulate_segments(ggml_context* ctx, - ggml_tensor* x, - ggml_tensor* projection, - const std::vector& segments, - int64_t hidden_size, - int expand, - int modalities, - int shift_index, - int scale_index) { - ggml_tensor* out = nullptr; - for (const auto& segment : segments) { - auto mods = modulation_row(ctx, - projection, - hidden_size, - expand, - modalities, - segment.modulation_row); - auto part = ggml_ext_slice(ctx, x, 1, segment.start, segment.end); - part = ggml_add(ctx, - ggml_add(ctx, part, ggml_mul(ctx, part, mods[scale_index])), - mods[shift_index]); - out = out == nullptr ? part : ggml_concat(ctx, out, part, 1); - } - return out; - } - - static ggml_tensor* gated_residual_segments(ggml_context* ctx, - ggml_tensor* x, - ggml_tensor* update, - ggml_tensor* projection, - const std::vector& segments, - int64_t hidden_size, - int gate_index) { - ggml_tensor* out = nullptr; - for (const auto& segment : segments) { - auto mods = modulation_row(ctx, projection, hidden_size, 6, 3, segment.modulation_row); - auto base = ggml_ext_slice(ctx, x, 1, segment.start, segment.end); - auto add = ggml_ext_slice(ctx, update, 1, segment.start, segment.end); - auto part = ggml_add(ctx, base, ggml_mul(ctx, add, mods[gate_index])); - out = out == nullptr ? part : ggml_concat(ctx, out, part, 1); - } - return out; - } - struct TransformerBlock : public GGMLBlock { Config config; @@ -417,38 +407,43 @@ namespace MiniMaxH3 { auto adaln = std::dynamic_pointer_cast(blocks["adaln_proj"]); auto mods = adaln->forward(ctx, t_emb); - auto h = modulate_segments(ctx->ggml_ctx, - norm1->forward(ctx, x), - mods, - segments, - config.hidden_size, - 6, - 3, - 0, - 1); - x = gated_residual_segments(ctx->ggml_ctx, - x, - attn->forward(ctx, h, pe), - mods, - segments, - config.hidden_size, - 2); - h = modulate_segments(ctx->ggml_ctx, - norm2->forward(ctx, x), - mods, - segments, - config.hidden_size, - 6, - 3, - 3, - 4); - return gated_residual_segments(ctx->ggml_ctx, - x, - mlp->forward(ctx, h), - mods, - segments, - config.hidden_size, - 5); + std::vector> rows; + std::vector inputs; + for (const auto& segment : segments) { + rows.push_back(modulation_row(ctx->ggml_ctx, mods, config.hidden_size, + 6, 3, segment.modulation_row)); + inputs.push_back(ggml_ext_slice(ctx->ggml_ctx, x, 1, segment.start, segment.end, false)); + } + auto modulate = [&](ggml_tensor* value, const std::vector& row, int shift, int scale) { + return ggml_add(ctx->ggml_ctx, + ggml_add(ctx->ggml_ctx, value, ggml_mul(ctx->ggml_ctx, value, row[scale])), + row[shift]); + }; + ggml_tensor* qkv = nullptr; + for (size_t i = 0; i < segments.size(); ++i) { + auto h = modulate(norm1->forward(ctx, inputs[i]), rows[i], 0, 1); + h = attn->project_qkv(ctx, h); + if (qkv == nullptr) { + qkv = h; + continue; + } + qkv = ggml_concat(ctx->ggml_ctx, qkv, h, 1); + } + auto attended = attn->attend(ctx, qkv, pe); + std::vector projections; + for (const auto& segment : segments) { + auto part = ggml_ext_slice(ctx->ggml_ctx, attended, 1, segment.start, segment.end, false); + projections.push_back(attn->project_output(ctx, part)); + } + ggml_tensor* output = nullptr; + for (size_t i = 0; i < segments.size(); ++i) { + auto residual = ggml_add(ctx->ggml_ctx, inputs[i], + ggml_mul(ctx->ggml_ctx, projections[i], rows[i][2])); + auto h = modulate(norm2->forward(ctx, residual), rows[i], 3, 4); + h = ggml_add(ctx->ggml_ctx, residual, ggml_mul(ctx->ggml_ctx, mlp->forward(ctx, h), rows[i][5])); + output = output == nullptr ? h : ggml_concat(ctx->ggml_ctx, output, h, 1); + } + return output; } }; @@ -956,8 +951,6 @@ namespace MiniMaxH3 { struct MiniMaxH3Runner : public DiffusionModelRunner { Config config; MiniMaxH3Transformer3DModel model; - sd::Tensor video_input_cache; - sd::Tensor audio_input_cache; sd::Tensor position_input_cache; sd::Tensor timestep_feature_input_cache; sd::Tensor curve_index_input_cache; @@ -983,29 +976,6 @@ namespace MiniMaxH3 { model.get_param_tensors(tensors, prefix); } - std::pair, sd::Tensor> split_av_latents(const sd::Tensor& packed, - int audio_length) const { - GGML_ASSERT(packed.dim() == 4 || packed.dim() == 5); - int64_t spatial = packed.shape()[0] * packed.shape()[1] * packed.shape()[2]; - int64_t video_values = spatial * config.video_latent_channels; - sd::Tensor video({packed.shape()[0], - packed.shape()[1], - packed.shape()[2], - config.video_latent_channels, - 1}); - std::copy_n(packed.data(), static_cast(video_values), video.data()); - if (audio_length <= 0) { - return {video, {}}; - } - int64_t audio_values = audio_length * 2 * config.audio_latent_channels; - GGML_ASSERT(packed.numel() >= video_values + audio_values); - sd::Tensor audio({audio_length, 2, config.audio_latent_channels, 1}); - std::copy_n(packed.data() + video_values, - static_cast(audio_values), - audio.data()); - return {video, audio}; - } - ggml_tensor* merge_av_latents(ggml_context* ctx, ggml_tensor* video, ggml_tensor* audio) const { @@ -1036,14 +1006,26 @@ namespace MiniMaxH3 { int audio_length, float video_shift, float audio_shift) { - auto split = split_av_latents(packed, audio_length); - video_input_cache = std::move(split.first); - audio_input_cache = std::move(split.second); - GGML_ASSERT(!audio_input_cache.empty()); + GGML_ASSERT(packed.dim() == 4 || packed.dim() == 5); + GGML_ASSERT(audio_length > 0); GGML_ASSERT(!context_tensor.empty()); - - auto video = make_input(video_input_cache); - auto audio_carrier = make_input(audio_input_cache); + const int64_t spatial = packed.shape()[0] * packed.shape()[1] * packed.shape()[2]; + const int64_t video_values = spatial * config.video_latent_channels; + const int64_t audio_values = int64_t(audio_length) * 2 * config.audio_latent_channels; + GGML_ASSERT(packed.numel() >= video_values + audio_values); + auto packed_input = make_input(packed); + auto video = ggml_view_4d(compute_ctx, packed_input, + packed.shape()[0], packed.shape()[1], packed.shape()[2], + config.video_latent_channels, + packed.shape()[0] * sizeof(float), + packed.shape()[0] * packed.shape()[1] * sizeof(float), + spatial * sizeof(float), 0); + auto audio_carrier = ggml_view_4d(compute_ctx, packed_input, + audio_length, 2, config.audio_latent_channels, 1, + audio_length * sizeof(float), + int64_t(audio_length) * 2 * sizeof(float), + audio_values * sizeof(float), + video_values * sizeof(float)); auto context = make_input(context_tensor); std::vector condition_inputs; condition_inputs.reserve(condition_videos.size()); @@ -1065,9 +1047,9 @@ namespace MiniMaxH3 { // latent follows one sigma schedule. Restore x_a for the H3 network. auto audio = ggml_ext_scale(compute_ctx, audio_carrier, sigma_a / sigma_v); auto layout = build_layout(context_tensor.shape()[1], - video_input_cache.shape()[2], - video_input_cache.shape()[1], - video_input_cache.shape()[0], + packed.shape()[2], + packed.shape()[1], + packed.shape()[0], audio_length, condition_videos, condition_audios, diff --git a/src/model_manager.cpp b/src/model_manager.cpp index 243811d9ef..d07d63a810 100644 --- a/src/model_manager.cpp +++ b/src/model_manager.cpp @@ -1767,10 +1767,16 @@ ModelManager::CapacityCheck ModelManager::check_capacity( auto add = [](size_t a, size_t b) { return b > SIZE_MAX - a ? SIZE_MAX : a + b; }; const size_t missing = compute_backend_alloc_size(states, true); // Backend scratch buffers and pipelines are not included in graph measurements. - constexpr size_t safety_margin = 512ULL * 1024ULL * 1024ULL; - result.required_device_bytes = add(add(request.pending_allocation_bytes, missing), safety_margin); - result.required_budget_bytes = add(request.runtime_peak_bytes(), missing); - auto available_device_bytes = [&](ggml_backend_t backend) { + size_t safety_margin = 512ULL * 1024ULL * 1024ULL; +#ifdef _WIN32 + if (sd_backend_is(request.compute_backend, "CUDA")) { + // Avoid WDDM allocation churn when transient buffers compete with reusable staging buffers. + safety_margin = 1024ULL * 1024ULL * 1024ULL; + } +#endif + result.required_device_bytes = add(add(request.pending_allocation_bytes, missing), safety_margin); + result.required_budget_bytes = add(request.runtime_peak_bytes(), missing); + auto available_device_bytes = [&](ggml_backend_t backend) { auto device = ggml_backend_get_device(backend); if (device == nullptr) { return SIZE_MAX; diff --git a/src/model_manager.h b/src/model_manager.h index fdb110825d..81d2546a45 100644 --- a/src/model_manager.h +++ b/src/model_manager.h @@ -76,6 +76,7 @@ class ModelManager : public DeviceResidencyManager { ggml_backend_t compute_backend = nullptr; ggml_backend_t transfer_backend = nullptr; ggml_backend_event_t event = nullptr; + bool submitted = false; std::vector> staging_blocks; }; @@ -292,6 +293,10 @@ class ModelManager : public DeviceResidencyManager { WeightPrefetchResult prefetch_params( uintptr_t owner_id, const std::vector& tensors) override; + WeightPrefetchResult prepare_prefetch_params( + uintptr_t owner_id, + const std::vector& tensors) override; + void submit_prefetched_params(uintptr_t owner_id) override; bool activate_prefetched_params(uintptr_t owner_id, const std::vector& tensors) override; void clear_prefetched_params(uintptr_t owner_id) override; diff --git a/src/model_manager_prefetch.cpp b/src/model_manager_prefetch.cpp index de96d5ae3b..4056b8f0ea 100644 --- a/src/model_manager_prefetch.cpp +++ b/src/model_manager_prefetch.cpp @@ -33,7 +33,7 @@ void ModelManager::synchronize_prefetch_block(PrefetchBlock& block) { ggml_backend_event_synchronize(block.event); ggml_backend_event_free(block.event); block.event = nullptr; - } else if (block.transfer_backend != nullptr) { + } else if (block.submitted && block.transfer_backend != nullptr) { ggml_backend_synchronize(block.transfer_backend); } block.transfer_backend = nullptr; @@ -89,7 +89,7 @@ bool ModelManager::populate_prefetch_block(PrefetchBlock& block) { backend_limit = MAX_RESIDENCY_BLOCK_BYTES; } - auto enqueue_chunk = [&](const std::vector& chunk) -> bool { + auto allocate_chunk = [&](const std::vector& chunk) -> bool { if (chunk.empty()) { return true; } @@ -128,23 +128,6 @@ bool ModelManager::populate_prefetch_block(PrefetchBlock& block) { ggml_backend_buffer_set_usage(staging_block->buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - for (const auto& pair : staging_block->staged_tensors) { - TensorState* state = pair.first; - ggml_tensor* staging_tensor = pair.second; - const bool host_source = ggml_backend_buffer_is_host(state->tensor->buffer); - if (host_source) { - ggml_backend_tensor_set_async(block.transfer_backend, - staging_tensor, - state->tensor->data, - 0, - ggml_nbytes(state->tensor)); - } else { - ggml_backend_tensor_copy_async(state->params_backend, - block.transfer_backend, - state->tensor, - staging_tensor); - } - } block.staging_blocks.push_back(std::move(staging_block)); return true; }; @@ -156,7 +139,7 @@ bool ModelManager::populate_prefetch_block(PrefetchBlock& block) { ggml_backend_buft_get_alloc_size(buffer_type, state->tensor), alignment); if (!chunk.empty() && backend_limit > 0 && tensor_size > backend_limit - std::min(chunk_size, backend_limit)) { - if (!enqueue_chunk(chunk)) { + if (!allocate_chunk(chunk)) { return false; } chunk.clear(); @@ -165,10 +148,39 @@ bool ModelManager::populate_prefetch_block(PrefetchBlock& block) { chunk.push_back(state); chunk_size = tensor_size > SIZE_MAX - chunk_size ? SIZE_MAX : chunk_size + tensor_size; } - if (!enqueue_chunk(chunk)) { + if (!allocate_chunk(chunk)) { return false; } + return true; +} + +void ModelManager::submit_prefetched_params(uintptr_t owner_id) { + auto existing = prefetch_blocks_.find(owner_id); + if (existing == prefetch_blocks_.end() || existing->second->submitted) { + return; + } + auto& block = *existing->second; + block.submitted = true; + for (const auto& staging_block : block.staging_blocks) { + for (const auto& pair : staging_block->staged_tensors) { + TensorState* state = pair.first; + ggml_tensor* staging_tensor = pair.second; + if (ggml_backend_buffer_is_host(state->tensor->buffer)) { + ggml_backend_tensor_set_async(block.transfer_backend, + staging_tensor, + state->tensor->data, + 0, + ggml_nbytes(state->tensor)); + } else { + ggml_backend_tensor_copy_async(state->params_backend, + block.transfer_backend, + state->tensor, + staging_tensor); + } + } + } + ggml_backend_dev_t device = ggml_backend_get_device(block.transfer_backend); block.event = ggml_backend_event_new(device); if (block.event != nullptr) { @@ -186,10 +198,19 @@ bool ModelManager::populate_prefetch_block(PrefetchBlock& block) { total_size / (1024.f * 1024.f), block.states.size(), ggml_backend_name(block.compute_backend)); - return true; } WeightPrefetchResult ModelManager::prefetch_params( + uintptr_t owner_id, + const std::vector& tensors) { + auto result = prepare_prefetch_params(owner_id, tensors); + if (result == WeightPrefetchResult::Scheduled) { + submit_prefetched_params(owner_id); + } + return result; +} + +WeightPrefetchResult ModelManager::prepare_prefetch_params( uintptr_t owner_id, const std::vector& tensors) { if (tensors.empty()) { @@ -288,6 +309,7 @@ bool ModelManager::activate_prefetched_params( if (existing == prefetch_blocks_.end()) { return false; } + submit_prefetched_params(owner_id); std::unique_ptr block = std::move(existing->second); prefetch_blocks_.erase(existing); synchronize_prefetch_block(*block);