Improve performance of listTemplates API

api/src/main/java/org/apache/cloudstack/query/QueryService.java

@@ -111,7 +111,7 @@
     List<String> RootAdminOnlyVmSettings = Arrays.asList(VmDetailConstants.GUEST_CPU_MODE, VmDetailConstants.GUEST_CPU_MODEL);
 
     // Config keys
     ConfigKey<Boolean> AllowUserViewDestroyedVM = new ConfigKey<>("Advanced", Boolean.class, "allow.user.view.destroyed.vm", "false",
             "Determines whether users can view their destroyed or expunging vm ", true, ConfigKey.Scope.Account);
 
     ConfigKey<String> UserVMDeniedDetails = new ConfigKey<>(String.class,
@@ -140,6 +140,12 @@
     ConfigKey<Boolean> ReturnVmStatsOnVmList = new ConfigKey<>("Advanced", Boolean.class, "list.vm.default.details.stats", "true",
             "Determines whether VM stats should be returned when details are not explicitly specified in listVirtualMachines API request. When false, details default to [group, nics, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]. When true, all details are returned including 'stats'.", true, ConfigKey.Scope.Global);
 
+    ConfigKey<Boolean> BypassTemplateView = new ConfigKey<>("Advanced", Boolean.class, "template.list.bypass.view",
+            "false",
+            "If true, uses an optimized query path for listing templates and ISOs, which can improve performance on " +
+                    "deployments with large numbers of templates. Automatically falls back to the standard query when " +
+                    "filtering by tags, using the shared, featured, or community template filters, or when including " +
+                    "removed templates.", true, ConfigKey.Scope.Global);
 
     ListResponse<UserResponse> searchForUsers(ResponseObject.ResponseView responseView, ListUsersCmd cmd) throws PermissionDeniedException;
 

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -207,6 +207,7 @@
 import com.cloud.api.query.dao.SnapshotJoinDao;
 import com.cloud.api.query.dao.StoragePoolJoinDao;
 import com.cloud.api.query.dao.TemplateJoinDao;
+import com.cloud.api.query.dao.TemplateListFilter;
 import com.cloud.api.query.dao.UserAccountJoinDao;
 import com.cloud.api.query.dao.UserVmJoinDao;
 import com.cloud.api.query.dao.VolumeJoinDao;
@@ -328,6 +329,7 @@
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.tags.ResourceTagVO;
 import com.cloud.tags.dao.ResourceTagDao;
+import com.cloud.template.VirtualMachineTemplate;
 import com.cloud.template.VirtualMachineTemplate.State;
 import com.cloud.template.VirtualMachineTemplate.TemplateFilter;
 import com.cloud.user.Account;
@@ -4989,8 +4991,9 @@
             sc.setJoinParameters("storagePool", "pool_id", storagePoolId);
         }
 
+        List<Long> guestOsIds = null;
         if (osCategoryId != null) {
-            List<Long> guestOsIds = guestOSDao.listIdsByCategoryId(osCategoryId);
+            guestOsIds = guestOSDao.listIdsByCategoryId(osCategoryId);
             if (CollectionUtils.isNotEmpty(guestOsIds)) {
                 sc.setParameters("guestOsIdIN", guestOsIds.toArray());
             } else {
@@ -5146,7 +5149,12 @@
         applyPublicTemplateSharingRestrictions(sc, caller);
 
         return templateChecks(isIso, hypers, tags, name, keyword, hyperType, onlyReady, bootable, zoneId, showDomr, caller,
-                showRemovedTmpl, parentTemplateId, showUnique, templateType, isVnf, forCks, searchFilter, sc);
+                showRemovedTmpl, parentTemplateId, showUnique, templateType, isVnf, forCks, searchFilter, sc,
+                buildTemplateListFilter(templateId, ids, name, keyword, templateFilter, isIso, bootable,
+                        pageSize, startIndex, zoneId, hyperType, hypers, showDomr, onlyReady,
+                        permittedAccounts, caller, listProjectResourcesCriteria, tags, showRemovedTmpl,
+                        parentTemplateId, showUnique, templateType, isVnf, forCks, arch, guestOsIds, extensionId,
+                        imageStoreId, storagePoolId, domainId, isRecursive));
     }
 
     /**
@@ -5201,7 +5209,8 @@
     private Pair<List<TemplateJoinVO>, Integer> templateChecks(boolean isIso, List<HypervisorType> hypers, Map<String, String> tags, String name, String keyword,
                                                                HypervisorType hyperType, boolean onlyReady, Boolean bootable, Long zoneId, boolean showDomr, Account caller,
                                                                boolean showRemovedTmpl, Long parentTemplateId, Boolean showUnique, String templateType, Boolean isVnf, Boolean forCks,
-                                                               Filter searchFilter, SearchCriteria<TemplateJoinVO> sc) {
+                                                               Filter searchFilter, SearchCriteria<TemplateJoinVO> sc,
+                                                               TemplateListFilter listFilter) {
         if (!isIso) {
             // add hypervisor criteria for template case
             if (hypers != null && !hypers.isEmpty()) {
@@ -5308,6 +5317,14 @@
 
         // search unique templates and find details by Ids
         Pair<List<TemplateJoinVO>, Integer> uniqueTmplPair;
+
+        // Only attempted when the bypass DAO supports the filter set; falls
+        // back to the SearchBuilder/template_view path below otherwise.
+        if (!showRemovedTmpl && BypassTemplateView.value() && listFilter != null && listFilter.canBypass()) {
+            uniqueTmplPair = _templateJoinDao.findDistinctTempZonePairs(listFilter);
+            return findTemplatesByIdOrTempZonePair(uniqueTmplPair, showRemovedTmpl, showUnique != null && showUnique, caller);
+        }
+
         if (showRemovedTmpl) {
             uniqueTmplPair = _templateJoinDao.searchIncludingRemovedAndCount(sc, searchFilter);
         } else {
@@ -5329,6 +5346,140 @@
         // about what the code was doing.
     }
 
+    /**
+     * Build the immutable filter the bypass-the-view DAO consumes. Mirrors the
+     * SearchCriteria construction in {@link #searchForTemplatesInternal}; the
+     * SearchBuilder path is unchanged. The "hard" markers (sharedAccountIds,
+     * domainIdsForFeaturedCommunity, tags, requiresViewFallback) drive
+     * {@link TemplateListFilter#canBypass()} — when any are populated,
+     * {@code templateChecks} routes to the legacy view-based path instead.
+     */
+    private TemplateListFilter buildTemplateListFilter(Long templateId, List<Long> ids, String name, String keyword,
+                                                       TemplateFilter templateFilter, boolean isIso, Boolean bootable,
+                                                       Long pageSize, Long startIndex, Long zoneId, HypervisorType hyperType,
+                                                       List<HypervisorType> hypers, boolean showDomr, boolean onlyReady,
+                                                       List<Account> permittedAccounts, Account caller,
+                                                       ListProjectResourcesCriteria listProjectResourcesCriteria,
+                                                       Map<String, String> tags, boolean showRemovedTmpl, Long parentTemplateId,
+                                                       Boolean showUnique, String templateType, Boolean isVnf, Boolean forCks,
+                                                       CPU.CPUArch arch, List<Long> guestOsIds, Long extensionId, Long imageStoreId,
+                                                       Long storagePoolId, Long domainId, boolean isRecursive) {
+        TemplateListFilter.Builder b = TemplateListFilter.builder()
+                .templateId(templateId)
+                .ids(ids == null ? null : new ArrayList<>(ids))
+                .name(name)
+                .keyword(keyword)
+                .hypervisorType(hyperType != null && !HypervisorType.None.equals(hyperType) ? hyperType : null)
+                .availableHypervisors(hypers)
+                .format(ImageFormat.ISO)  // bypass DAO uses isIso flag to flip between EQ and NEQ
+                .isIso(isIso)
+                .bootable(bootable)
+                .parentTemplateId(parentTemplateId)
+                .zoneId(zoneId)
+                .onlyReady(onlyReady)
+                .excludeSystemTemplates(!showDomr)
+                .templateType(templateType)
+                .isVnf(isVnf)
+                .forCks(forCks)
+                .arch(arch)
+                .guestOsIds(guestOsIds)
+                .extensionId(extensionId)
+                .imageStoreId(imageStoreId)
+                .showUnique(showUnique != null && showUnique)
+                .startIndex(startIndex)
+                .pageSize(pageSize)
+                .sortAscending(SortKeyAscending.value())
+                .showRemoved(showRemovedTmpl)
+                .tags(tags);
+
+        if (!showRemovedTmpl) {
+            b.templateStates(Arrays.asList(VirtualMachineTemplate.State.Active,
+                    VirtualMachineTemplate.State.UploadAbandoned,
+                    VirtualMachineTemplate.State.UploadError,
+                    VirtualMachineTemplate.State.NotUploaded,
+                    VirtualMachineTemplate.State.UploadInProgress));
+        }
+
+        // accountType filter (Project vs non-Project), applied only when no specific templateId.
+        if (templateId == null && listProjectResourcesCriteria == ListProjectResourcesCriteria.SkipProjectResources) {
+            b.accountTypeNeq(Account.Type.PROJECT);
+        } else if (templateId == null && listProjectResourcesCriteria == ListProjectResourcesCriteria.ListProjectResourcesOnly) {
+            b.accountTypeEq(Account.Type.PROJECT);
+        }
+
+        // ACL handling — mirrors the templateFilter switch in searchForTemplatesInternal.
+        // For "hard" templateFilter values, set requiresViewFallback so canBypass() returns
+        // false and the dispatcher falls back to the legacy view-based path.
+        if (templateId != null) {
+            return b.build();
+        }
+
+        // template_spool_ref join is not modeled in bypass SQL.
+        if (storagePoolId != null) {
+            b.requiresViewFallback(true);
+        }
+
+        List<Long> permittedAccountIds = new ArrayList<>();
+        for (Account account : permittedAccounts) {
+            permittedAccountIds.add(account.getId());
+        }
+
+        if (templateFilter == TemplateFilter.featured || templateFilter == TemplateFilter.community) {
+            // Walks the domain hierarchy and ORs domainId IS NULL — not yet modeled in bypass SQL.
+            b.publicTemplate(Boolean.TRUE);
+            b.featured(templateFilter == TemplateFilter.featured ? Boolean.TRUE : Boolean.FALSE);
+            b.requiresViewFallback(true);
+        } else if (templateFilter == TemplateFilter.self || templateFilter == TemplateFilter.selfexecutable) {
+            if (accountMgr.isAdmin(caller.getAccountId())) {
+                applySelfDomainScope(b, permittedAccounts, caller, domainId, isRecursive);
+            }
+            if (!permittedAccountIds.isEmpty()) {
+                b.accountIds(permittedAccountIds);
+                b.accountIdRequiredForFilter(true);
+            }
+        } else if (templateFilter == TemplateFilter.sharedexecutable || templateFilter == TemplateFilter.shared) {
+            // launch_permission join — not modeled in bypass SQL.
+            b.requiresViewFallback(true);
+        } else if (templateFilter == TemplateFilter.executable) {
+            b.publicOrAccountIdComposite(true);
+            if (!permittedAccountIds.isEmpty()) {
+                b.accountIds(permittedAccountIds);
+            }
+        } else if (templateFilter == TemplateFilter.all && caller.getType() != Account.Type.ADMIN) {
+            // Non-admin "all" composite splits on listProjectResourcesCriteria:
+            //   SkipProjectResources → (public OR domainPath LIKE)         — bypass-eligible
+            //   else                 → (public OR account_id IN OR sharedAccountId IN) — needs launch_permission, falls back
+            if (listProjectResourcesCriteria == ListProjectResourcesCriteria.SkipProjectResources) {
+                DomainVO callerDomain = _domainDao.findById(caller.getDomainId());
+                if (callerDomain != null) {
+                    b.publicOrDomainPathComposite(true);
+                    b.domainPathLike(callerDomain.getPath() + "%");
+                } else {
+                    b.requiresViewFallback(true);
+                }
+            } else {
+                b.requiresViewFallback(true);
+            }
+        }
+
+        return b.build();
+    }
+
+    // Mirrors the admin domain-path scoping of the view path for self/selfexecutable.
+    private void applySelfDomainScope(TemplateListFilter.Builder b, List<Account> permittedAccounts, Account caller,
+                                      Long domainId, boolean isRecursive) {
+        Long domainIdForScope = !permittedAccounts.isEmpty()
+                ? permittedAccounts.get(0).getDomainId()
+                : Objects.requireNonNullElse(domainId, caller.getDomainId());
+        DomainVO scopeDomain = _domainDao.findById(domainIdForScope);
+        // A recursive scope from the root domain matches every row, so skip the domain join.
+        if (scopeDomain == null || (isRecursive && scopeDomain.getId() == Domain.ROOT_DOMAIN)) {
+            return;
+        }
+        b.domainPathLike(isRecursive ? scopeDomain.getPath() + "%" : scopeDomain.getPath());
+        b.domainPathExact(!isRecursive);
+    }
+
     // findTemplatesByIdOrTempZonePair returns the templates with the given ids if showUnique is true, or else by the TempZonePair
     private Pair<List<TemplateJoinVO>, Integer> findTemplatesByIdOrTempZonePair(Pair<List<TemplateJoinVO>, Integer> templateDataPair,
                                                                                 boolean showRemoved, boolean showUnique, Account caller) {
@@ -6342,7 +6493,8 @@
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {AllowUserViewDestroyedVM, UserVMDeniedDetails, UserVMReadOnlyDetails, SortKeyAscending,
-                AllowUserViewAllDomainAccounts, AllowUserViewAllDataCenters, SharePublicTemplatesWithOtherDomains, ReturnVmStatsOnVmList};
+                AllowUserViewAllDomainAccounts, AllowUserViewAllDataCenters, SharePublicTemplatesWithOtherDomains,
+                ReturnVmStatsOnVmList, BypassTemplateView };
     }
 
     @Override

server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java

@@ -52,5 +52,20 @@ public interface TemplateJoinDao extends GenericDao<TemplateJoinVO, Long> {
 
     Pair<List<TemplateJoinVO>, Integer> searchIncludingRemovedAndCount(final SearchCriteria<TemplateJoinVO> sc, final Filter filter);
 
+    /**
+     * Bypass-the-view Phase 1 implementation. Issues a hand-tuned SQL query
+     * directly against the underlying tables (vm_template, account,
+     * template_store_ref, image_store, template_zone_ref, data_center) instead
+     * of going through {@code template_view}. Caller must check
+     * {@link TemplateListFilter#canBypass()} first; this method does not handle
+     * tags, sharedAccountIds, or featured/community-style domainId
+     * filters and will throw IllegalArgumentException if those are populated.
+     *
+     * Returns TemplateJoinVO objects with only {@code id} (when
+     * {@code filter.showUnique}) or {@code tempZonePair} populated, plus the
+     * total distinct count for pagination.
+     */
+    Pair<List<TemplateJoinVO>, Integer> findDistinctTempZonePairs(TemplateListFilter filter);
+
     List<TemplateJoinVO> findByDistinctIds(Long... ids);
 }