leejet · Nov 1, 2026 · Oct 31, 2026 · Oct 31, 2026 · Oct 31, 2026 · Oct 31, 2026
diff --git a/.gitmodules b/.gitmodules
@@ -10,3 +10,6 @@
 [submodule "thirdparty/libwebm"]
 	path = thirdparty/libwebm
 	url = https://github.com/webmproject/libwebm.git
+[submodule "thirdparty/cutlass"]
+	path = thirdparty/cutlass
+	url = https://github.com/NVIDIA/cutlass.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -225,6 +225,8 @@ file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
     "src/core/*.h"
     "src/core/*.cpp"
     "src/core/*.hpp"
+    "src/core/cpu/*.h"
+    "src/core/cpu/*.cpp"
     "src/core/probe/*.h"
     "src/core/probe/*.cpp"
     "src/extensions/*.h"
@@ -343,6 +345,46 @@ if (SD_CUDA)
     # Keep the driver stub on downstream link lines when no driver is installed.
     target_link_libraries(${SD_LIB} PUBLIC CUDA::cuda_driver)
     set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_USE_CUDA)
+    if (NOT SD_USE_UPSTREAM_GGML)
+        include(cmake/cutlass.cmake)
+        enable_language(CUDA)
+        set_target_properties(${SD_LIB} PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON)
+        if (SD_CUDA_CUTLASS_PATH)
+            target_sources(${SD_LIB} PRIVATE src/core/cuda/cute_attention.cu src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu)
+            set_source_files_properties(src/core/cuda/cute_attention.cu src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu PROPERTIES
+                INCLUDE_DIRECTORIES "${SD_CUDA_CUTLASS_PATH}/include"
+                COMPILE_OPTIONS "--use_fast_math;--extended-lambda;--expt-relaxed-constexpr")
+            if (MSVC)
+                set_property(SOURCE src/core/cuda/cute_attention.cu src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu APPEND PROPERTY COMPILE_OPTIONS -Xcompiler=/Zc:preprocessor,/Zc:__cplusplus)
+            endif()
+            if (GGML_CUDA_FORCE_CUBLAS)
+                set_property(SOURCE src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu APPEND PROPERTY COMPILE_DEFINITIONS SD_CUDA_FORCE_CUBLAS)
+            endif()
+        endif()
+    endif()
+endif()
+set(SD_BACKEND_REGISTRATIONS)
+if (NOT SD_USE_UPSTREAM_GGML AND (SD_CUDA OR SD_HIPBLAS OR SD_MUSA))
+    if(SD_HIPBLAS)
+        include(cmake/hip.cmake)
+    endif()
+    add_subdirectory(src/core/cuda)
+    foreach(VENDOR CUDA HIP MUSA)
+        if(SD_${VENDOR} OR (VENDOR STREQUAL "HIP" AND SD_HIPBLAS))
+            string(TOLOWER "${VENDOR}" NAMESPACE)
+            list(APPEND SD_BACKEND_REGISTRATIONS "sd_${NAMESPACE}::register_extensions")
+        endif()
+    endforeach()
+endif()
+if (SD_VULKAN AND NOT SD_USE_UPSTREAM_GGML)
+    add_subdirectory(src/core/vulkan)
+    list(APPEND SD_BACKEND_REGISTRATIONS sd_ggml_register_vulkan_rope sd_ggml_register_vulkan_i8)
+endif()
+if(SD_BACKEND_REGISTRATIONS)
+    string(REPLACE ";" ",\n    " SD_BACKEND_REGISTRATIONS "${SD_BACKEND_REGISTRATIONS}")
+    configure_file(cmake/backend_extensions.h.in generated/sd_backend_extensions.h @ONLY)
+    set_property(SOURCE src/core/ggml_extensions.cpp APPEND PROPERTY INCLUDE_DIRECTORIES "${CMAKE_CURRENT_BINARY_DIR}/generated")
+    set_property(SOURCE src/core/ggml_extensions.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_BACKEND_EXTENSIONS)
 endif()
 if (SD_HIPBLAS)
     find_package(hip REQUIRED CONFIG HINTS "$ENV{ROCM_PATH}" /opt/rocm)

diff --git a/cmake/backend_extensions.h.in b/cmake/backend_extensions.h.in
@@ -0,0 +1,13 @@
+#ifndef __SD_BACKEND_EXTENSIONS_H__
+#define __SD_BACKEND_EXTENSIONS_H__
+
+#include "core/cuda/registration.h"
+
+void sd_ggml_register_vulkan_rope(ggml_backend_dev_t device);
+void sd_ggml_register_vulkan_i8(ggml_backend_dev_t device);
+
+static void (*const sd_backend_extensions[])(ggml_backend_dev_t) = {
+    @SD_BACKEND_REGISTRATIONS@
+};
+
+#endif
diff --git a/cmake/cutlass.cmake b/cmake/cutlass.cmake
@@ -0,0 +1,20 @@
+set(sd_cutlass_default "${PROJECT_SOURCE_DIR}/thirdparty/cutlass")
+if(NOT DEFINED SD_CUDA_CUTLASS_PATH AND DEFINED GGML_CUDA_CUTLASS_PATH)
+    get_filename_component(sd_old_cutlass_default "${SD_GGML_SOURCE_DIR}/src/ggml-cuda/vendors/cutlass" ABSOLUTE)
+    if(NOT GGML_CUDA_CUTLASS_PATH STREQUAL sd_old_cutlass_default)
+        set(sd_cutlass_default "${GGML_CUDA_CUTLASS_PATH}")
+    endif()
+    message(STATUS "Migrating GGML_CUDA_CUTLASS_PATH to SD_CUDA_CUTLASS_PATH")
+endif()
+set(SD_CUDA_CUTLASS_PATH "${sd_cutlass_default}" CACHE PATH "sd: CUTLASS source directory for INT8 GEMM and CuTe attention, empty to disable")
+unset(GGML_CUDA_CUTLASS_PATH CACHE)
+if(SD_CUDA_CUTLASS_PATH)
+    if(NOT EXISTS "${SD_CUDA_CUTLASS_PATH}/include/cutlass/gemm/kernel/default_gemm_universal_with_visitor.h" OR
+       NOT EXISTS "${SD_CUDA_CUTLASS_PATH}/include/cute/tensor.hpp")
+        message(FATAL_ERROR
+            "CUTLASS headers not found at ${SD_CUDA_CUTLASS_PATH}. "
+            "Run git submodule update --init --recursive from the sd.cpp repository root, "
+            "set SD_CUDA_CUTLASS_PATH to a CUTLASS source tree (tested with 3.9.2), "
+            "or set SD_CUDA_CUTLASS_PATH to an empty string to disable CUTLASS.")
+    endif()
+endif()
diff --git a/cmake/embed_spirv.cmake b/cmake/embed_spirv.cmake
@@ -0,0 +1,3 @@
+file(READ "${INPUT}" SPIRV HEX)
+string(REGEX REPLACE "(..)(..)(..)(..)" "0x\\4\\3\\2\\1," SPIRV_WORDS "${SPIRV}")
+file(WRITE "${OUTPUT}" "#include <cstdint>\nnamespace sd_vulkan_shaders {\nstatic const uint32_t ${NAME}[] = {${SPIRV_WORDS}};\n}\n")
diff --git a/cmake/ggml.cmake b/cmake/ggml.cmake
@@ -9,17 +9,19 @@ if(NOT TARGET ggml)
     add_library(ggml ALIAS ggml::ggml)
 endif()
 
-get_target_property(sd_ggml_imported ggml IMPORTED)
-if(sd_ggml_imported)
-    set(sd_ggml_private_include "${SD_GGML_SOURCE_DIR}/src")
-else()
-    get_target_property(sd_ggml_private_include ggml SOURCE_DIR)
-endif()
-if(NOT EXISTS "${sd_ggml_private_include}/ggml-impl.h")
-    message(FATAL_ERROR "Set SD_GGML_SOURCE_DIR to the source tree matching the selected ggml library (ggml-impl.h is required).")
+if(SD_USE_UPSTREAM_GGML)
+    get_target_property(sd_ggml_imported ggml IMPORTED)
+    if(sd_ggml_imported)
+        set(sd_ggml_private_include "${SD_GGML_SOURCE_DIR}/src")
+    else()
+        get_target_property(sd_ggml_private_include ggml SOURCE_DIR)
+    endif()
+    if(NOT EXISTS "${sd_ggml_private_include}/ggml-impl.h")
+        message(FATAL_ERROR "Set SD_GGML_SOURCE_DIR to the source tree matching the selected ggml library (ggml-impl.h is required).")
+    endif()
+    target_include_directories(${SD_LIB} PRIVATE "${sd_ggml_private_include}")
+    set_property(TARGET ${SD_LIB} PROPERTY SD_GGML_PRIVATE_INCLUDE_DIR "${sd_ggml_private_include}")
 endif()
-target_include_directories(${SD_LIB} PRIVATE "${sd_ggml_private_include}")
-set_property(TARGET ${SD_LIB} PROPERTY SD_GGML_PRIVATE_INCLUDE_DIR "${sd_ggml_private_include}")
 
 if(SD_USE_UPSTREAM_GGML)
     target_compile_definitions(${SD_LIB} PUBLIC SD_USE_UPSTREAM_GGML)

diff --git a/cmake/hip.cmake b/cmake/hip.cmake
@@ -0,0 +1,12 @@
+if(NOT WIN32 AND NOT CMAKE_CXX_COMPILER MATCHES "hipcc")
+    if(NOT CMAKE_HIP_ARCHITECTURES)
+        if(GPU_TARGETS)
+            set(CMAKE_HIP_ARCHITECTURES ${GPU_TARGETS})
+        elseif(AMDGPU_TARGETS)
+            set(CMAKE_HIP_ARCHITECTURES ${AMDGPU_TARGETS})
+        endif()
+    endif()
+    # HIP objects are linked by the parent target, so initialize its link rules here.
+    enable_language(HIP)
+    set_property(TARGET ${SD_LIB} PROPERTY HIP_ARCHITECTURES "${CMAKE_HIP_ARCHITECTURES}")
+endif()
diff --git a/docs/build.md b/docs/build.md
@@ -32,8 +32,10 @@ cmake -S . -B build-upstream -DSD_USE_UPSTREAM_GGML=ON -DSD_GGML_SOURCE_DIR=../g
 cmake --build build-upstream --config Release
 ```
 
-The selected source tree supplies both the library and its private headers.
-Backend options such as `-DSD_CUDA=ON` apply to the selected tree as usual.
+Upstream compatibility mode uses the selected library and matching private
+headers. The patched GGML build uses public headers, including `ggml-ext.h` and
+`ggml-graph.h`. Backend options such as `-DSD_CUDA=ON` apply to the selected tree
+as usual.
 
 `SD_USE_UPSTREAM_GGML` defaults to `OFF`, which enables the patched GGML
 extensions. Set it to `ON` when using upstream GGML; it selects the compatibility
@@ -50,10 +52,12 @@ creating an inference context. Ordinary floating-point and shared GGML
 quantization types remain available, subject to backend operator support.
 
 `SD_USE_SYSTEM_GGML=ON` instead links an installed GGML CMake package, located
-with `ggml_DIR` or `CMAKE_PREFIX_PATH`. In that mode, `SD_GGML_SOURCE_DIR` must
-point to the matching source tree for private headers. The installed library
-must use the same ABI settings as sd.cpp, including `GGML_MAX_NAME`.
-Set `SD_USE_UPSTREAM_GGML=ON` as well if the installed package is upstream GGML.
+with `ggml_DIR` or `CMAKE_PREFIX_PATH`. An installed patched package must include
+the extension and graph APIs; its source tree is not required for private
+headers. The installed library must use the same ABI settings as sd.cpp,
+including `GGML_MAX_NAME`. For an upstream package, also set
+`SD_USE_UPSTREAM_GGML=ON` and point `SD_GGML_SOURCE_DIR` to its matching source
+tree for the compatibility layer's private headers.
 
 ## WebP and WebM Support in Examples
 
@@ -95,8 +99,8 @@ cmake --build . --config Release
 
 ## Build with CUDA
 
-Native SageAttention is included when using CUDA with patched GGML
-(`SD_USE_UPSTREAM_GGML=OFF`).
+The sd.cpp SageAttention extension is included when using CUDA with patched
+GGML (`SD_USE_UPSTREAM_GGML=OFF`).
 See [SageAttention](sage_attention.md) for GPU requirements and `--sage-attn` usage.
 
 This provides GPU acceleration using NVIDIA GPU. Make sure to have the CUDA toolkit installed. You can download it from your Linux distro's package manager (e.g. `apt install nvidia-cuda-toolkit`) or from here: [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads). Recommended to have at least 4 GB of VRAM.
@@ -107,7 +111,7 @@ cmake .. -DSD_CUDA=ON
 cmake --build . --config Release
 ```
 
-CUDA builds with the bundled patched GGML use the CUTLASS submodule by default
+CUDA extensions use sd.cpp's `thirdparty/cutlass` submodule by default
 for eligible INT8 GEMM and CuTe attention operations. If its headers are missing,
 CMake stops with an initialization hint; it does not download them automatically.
 Run `git submodule update --init --recursive` from the repository root before
@@ -117,14 +121,17 @@ To build with the existing CUDA implementations without CUTLASS, configure from
 the repository root with:
 
 ```shell
-cmake -S . -B build -DSD_CUDA=ON -DGGML_CUDA_CUTLASS_PATH:PATH=
+cmake -S . -B build -DSD_CUDA=ON -DSD_CUDA_CUTLASS_PATH:PATH=
 cmake --build build --config Release
 ```
 
-Alternatively, set `GGML_CUDA_CUTLASS_PATH` to another local CUTLASS source
+Alternatively, set `SD_CUDA_CUTLASS_PATH` to another local CUTLASS source
 directory (tested with 3.9.2). Existing CMake caches retain their configured path;
-pass `-UGGML_CUDA_CUTLASS_PATH` when reconfiguring to restore the bundled default.
-Non-CUDA builds do not require CUTLASS.
+pass `-USD_CUDA_CUTLASS_PATH` when reconfiguring to restore the bundled default.
+Existing `GGML_CUDA_CUTLASS_PATH` caches are migrated on first configuration:
+the old bundled path becomes `thirdparty/cutlass`, while custom paths and an
+empty disabled value are preserved. An explicitly set `SD_CUDA_CUTLASS_PATH`
+takes precedence. Non-CUDA builds do not require CUTLASS.
 
 ## Build with HipBLAS