[bot] Fast-forward for 26.7.12

api/src/org/labkey/api/ApiModule.java

@@ -83,12 +83,14 @@
 import org.labkey.api.data.WorkbookContainerType;
 import org.labkey.api.data.dialect.JdbcMetaDataTest;
 import org.labkey.api.data.dialect.ParameterSubstitutionTest;
+import org.labkey.api.data.dialect.PostgresSnapshot;
 import org.labkey.api.data.dialect.StandardDialectStringHandler;
 import org.labkey.api.dataiterator.CachingDataIterator;
 import org.labkey.api.dataiterator.DataIteratorUtil;
 import org.labkey.api.dataiterator.DiskCachingDataIterator;
 import org.labkey.api.dataiterator.ExistingRecordDataIterator;
 import org.labkey.api.dataiterator.GenerateUniqueDataIterator;
+import org.labkey.api.dataiterator.QueryDataIteratorBuilder;
 import org.labkey.api.dataiterator.RemoveDuplicatesDataIterator;
 import org.labkey.api.dataiterator.ResultSetDataIterator;
 import org.labkey.api.dataiterator.SimpleTranslator;
@@ -548,7 +550,9 @@ public void registerServlets(ServletContext servletCtx)
             NestedGroupsTest.class,
             ParameterSubstitutionTest.class,
             Portal.TestCase.class,
+            PostgresSnapshot.TestCase.class,
             PropertyManager.TestCase.class,
+            QueryDataIteratorBuilder.TestCase.class,
             RecordFactory.TestCase.class,
             ResultSetDataIterator.TestCase.class,
             ResultSetSelectorTestCase.class,

api/src/org/labkey/api/action/ApiQueryResponse.java

@@ -75,6 +75,7 @@ public class ApiQueryResponse implements ApiResponse
     private RenderContext _ctx = null;
     private boolean _schemaEditable = false;
     private boolean _metaDataOnly;
+    private boolean _countOnly = false;
     private DataRegion _dataRegion;
     private TableInfo _tinfo = null;
     private List<DisplayColumn> _displayColumns = null;
@@ -147,6 +148,16 @@ public void render(ApiResponseWriter writer) throws Exception
                 if (_metaDataOnlyIncludesEmptyRowset)
                     writeEmptyRowset(writer);
             }
+            else if (_countOnly)
+            {
+                loadTotalRowsOnly();
+                if (_dataRegion.getTotalRows() != null)
+                    _rowCount = _dataRegion.getTotalRows();
+
+                if (_includeMetaData)
+                    writeMetaData(writer);
+                writeEmptyRowset(writer);
+            }
             else
             {
                 // First run the query, so on potential SQLException we only serialize the exception instead of outputting all the metadata before the exception
@@ -170,8 +181,11 @@ public void render(ApiResponseWriter writer) throws Exception
                         _rowCount = _dataRegion.getTotalRows();
                     }
                 }
+            }
 
-                long rowCount = _rowCount > 0 ? _rowCount : _offset + _numRespRows;
+            if (!_metaDataOnly)
+            {
+                long rowCount = _rowCount > 0 || _countOnly ? _rowCount : _offset + _numRespRows;
                 writer.writeProperty("rowCount", rowCount);
 
                 if (_includeMetaData)
@@ -268,9 +282,7 @@ protected void writeMetaData(ApiResponseWriter writer) throws Exception
 
     protected Results getResults() throws Exception
     {
-        // If we're going to be writing JSON back, which is tolerant of extra spaces, allow async so we
-        // can monitor if the client has stopped listening. XML doesn't take kindly to leading spaces
-        _dataRegion.setAllowAsync(ApiResponseWriter.getResponseFormat(_viewContext.getRequest(), ApiResponseWriter.Format.JSON).isJson());
+        _dataRegion.setAllowAsync(isAllowAsync());
         try
         {
             return _dataRegion.getResults(_ctx);
@@ -281,6 +293,26 @@ protected Results getResults() throws Exception
         }
     }
 
+    protected void loadTotalRowsOnly()
+    {
+        _dataRegion.setAllowAsync(isAllowAsync());
+        try
+        {
+            _dataRegion.loadTotalRowsOnly(_ctx);
+        }
+        catch (ConversionException e)
+        {
+            throw new ApiUsageException(e);
+        }
+    }
+
+    // If we're going to be writing JSON back, which is tolerant of extra spaces, allow async so we
+    // can monitor if the client has stopped listening. XML doesn't take kindly to leading spaces
+    private boolean isAllowAsync()
+    {
+        return ApiResponseWriter.getResponseFormat(_viewContext.getRequest(), ApiResponseWriter.Format.JSON).isJson();
+    }
+
     public List<FieldKey> getFieldKeys()
     {
         return _fieldKeys;
@@ -328,6 +360,12 @@ public void includeStyle(boolean withStyle)
         _doItWithStyle = withStyle;
     }
 
+    /** Return only rowCount (and metadata, if requested) via a COUNT(*) query, skipping the rows query. */
+    public void countOnly(boolean countOnly)
+    {
+        _countOnly = countOnly;
+    }
+
     protected double getFormatVersion()
     {
         return 8.3;

api/src/org/labkey/api/assay/AssayProtocolSchema.java

@@ -41,8 +41,8 @@
 import org.labkey.api.data.RenderContext;
 import org.labkey.api.data.Results;
 import org.labkey.api.data.SQLFragment;
+import org.labkey.api.data.ShowRows;
 import org.labkey.api.data.Sort;
-import org.labkey.api.data.Table;
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.exp.PropertyDescriptor;
 import org.labkey.api.exp.api.ExpProtocol;
@@ -711,7 +711,7 @@ public void addQCWarningIndicator(QueryView baseQueryView, ViewContext context,
                             QuerySettings qs = getSettings(viewContext, settings.getDataRegionName(), settings.getQueryName());
 
                             // we want all the rows
-                            qs.setMaxRows(Table.ALL_ROWS);
+                            qs.setShowRows(ShowRows.ALL);
                             QueryView allResultsQueryView = createAllResultsQueryView(viewContext, qs);
 
                             DataView dataView = allResultsQueryView.createDataView();

api/src/org/labkey/api/data/DataRegion.java

@@ -851,16 +851,7 @@ public Map<String, List<Aggregate.Result>> getAggregateResults(RenderContext ctx
 
                     newAggregates.add(Aggregate.createCountStar());
                     _aggregateResults = ctx.getAggregates(_displayColumns, getTable(), getSettings(), getName(), newAggregates, getQueryParameters(), isAllowAsync());
-                    List<Aggregate.Result> result = _aggregateResults.remove(Aggregate.STAR);
-
-                    //Issue 14863: add null check
-                    if (result != null && !result.isEmpty())
-                    {
-                        Aggregate.Result countStarResult = result.getFirst();
-                        _totalRows = 0L;
-                        if (countStarResult.getValue() instanceof Number)
-                            _totalRows = ((Number) countStarResult.getValue()).longValue();
-                    }
+                    setTotalRowsFromCountStar(_aggregateResults.remove(Aggregate.STAR));
                 }
             }
             else
@@ -876,6 +867,42 @@ public Map<String, List<Aggregate.Result>> getAggregateResults(RenderContext ctx
         return _aggregateResults;
     }
 
+    /** Sets the total row count from a COUNT(*) aggregate alone, without selecting any rows. */
+    public void loadTotalRowsOnly(RenderContext ctx)
+    {
+        if (!hasPermission(ctx, ReadPermission.class))
+            throw new UnauthorizedException();
+
+        TableInfo table = getTable();
+        if (null == table)
+            throw new NotFoundException("Table or query not found: " + getSettings().getQueryName());
+
+        DataRegion oldRegion = ctx.getCurrentRegion();
+        ctx.setCurrentRegion(this);
+        try
+        {
+            Map<String, List<Aggregate.Result>> aggregates = ctx.getAggregates(_displayColumns, table, getSettings(), getName(), List.of(Aggregate.createCountStar()), getQueryParameters(), isAllowAsync());
+            setTotalRowsFromCountStar(aggregates.get(Aggregate.STAR));
+            _aggregateResults = Collections.emptyMap();
+        }
+        finally
+        {
+            ctx.setCurrentRegion(oldRegion);
+        }
+    }
+
+    private void setTotalRowsFromCountStar(@Nullable List<Aggregate.Result> result)
+    {
+        //Issue 14863: add null check
+        if (result == null || result.isEmpty())
+            return;
+
+        Aggregate.Result countStarResult = result.getFirst();
+        _totalRows = 0L;
+        if (countStarResult.getValue() instanceof Number n)
+            _totalRows = n.longValue();
+    }
+
     @NotNull
     private List<Aggregate> getSummaryStatsAggregates(List<AnalyticsProviderItem> providers)
     {

api/src/org/labkey/api/data/RenderContext.java

@@ -350,7 +350,10 @@ public Map<String, List<Aggregate.Result>> getAggregates(List<DisplayColumn> dis
         if (null != QueryService.get())
             cols = QueryService.get().ensureRequiredColumns(tinfo, cols, filter, sort, ignoredAggregateFilters);
 
-        if (!ignoredAggregateFilters.equals(_ignoredColumnFilters))
+        // Count-only requests skip the main query, so there's nothing to match; adopt the aggregate's set for filter messages
+        if (_results == null)
+            _ignoredColumnFilters.addAll(ignoredAggregateFilters);
+        else if (!ignoredAggregateFilters.equals(_ignoredColumnFilters))
         {
             // This should never happen, but if it did, the totals wouldn't match, so we won't calculate them.
             _log.error("Aggregate filter columns do not match main.  Aggregate:{} Main:{}", ignoredAggregateFilters, _ignoredColumnFilters);

api/src/org/labkey/api/data/dialect/PostgresSnapshot.java

@@ -0,0 +1,192 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.api.data.dialect;
+
+import org.apache.logging.log4j.Level;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.json.JSONObject;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+import org.labkey.api.data.CoreSchema;
+import org.labkey.api.data.DbSchema;
+import org.labkey.api.data.SqlSelector;
+import org.springframework.dao.DataAccessException;
+
+import java.util.Set;
+
+/**
+ * Read-only capture of Postgres configuration and cumulative statistics as a single JSON document. Differencing a
+ * capture taken before a workload against one taken after it shows the work the database actually did.
+ */
+public class PostgresSnapshot
+{
+    public enum StatementsStatus
+    {
+        AVAILABLE,
+        NOT_INSTALLED,
+        /** CREATE EXTENSION succeeds without shared_preload_libraries, but every read of the view then fails */
+        NOT_LOADED
+    }
+
+    private record StatementsSource(@NotNull StatementsStatus status, @Nullable String view) {}
+
+    private static final int PG_16 = 160000;
+    private static final int PG_17 = 170000;
+
+    // Counter columns are captured with to_jsonb() rather than named individually because their names move between
+    // major versions (blk_read_time became shared_blk_read_time in PG 17), so one query serves every version and
+    // pg_settings records which version produced the capture. Views that don't exist in a version fail at parse time,
+    // so those subqueries are chosen in Java.
+    private static final String SNAPSHOT_SQL = """
+            SELECT json_build_object(
+                'takenAt', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),
+                'serverVersionNum', current_setting('server_version_num')::int,
+                'database', (
+                    SELECT to_jsonb(d) FROM pg_stat_database d WHERE datname = current_database()
+                ),
+                'bgwriter', (
+                    SELECT to_jsonb(b) FROM pg_stat_bgwriter b
+                ),
+                'checkpointer', (%s),
+                'wal', (
+                    SELECT to_jsonb(w) FROM pg_stat_wal w
+                ),
+                'io', (%s),
+                'settings', (
+                    SELECT json_object_agg(name, json_build_object(
+                        'value', setting,
+                        'unit', unit,
+                        'source', source,
+                        'pending_restart', pending_restart,
+                        'boot_val', boot_val,
+                        'reset_val', reset_val
+                    ))
+                    FROM pg_settings
+                ),
+                'statementsStatus', '%s',
+                'statements', (%s)
+            )::text""";
+
+    private static final String CHECKPOINTER_SQL = "SELECT to_jsonb(c) FROM pg_stat_checkpointer c";
+
+    // Checkpoint counters live in pg_stat_bgwriter until PG 17
+    private static final String CHECKPOINTER_FROM_BGWRITER_SQL = "SELECT to_jsonb(b) FROM pg_stat_bgwriter b";
+
+    private static final String IO_SQL = """
+            SELECT json_object_agg(backend_type || ':' || object || ':' || context, to_jsonb(i))
+            FROM pg_stat_io i""";
+
+    private static final String NULL_SQL = "SELECT NULL::json";
+
+    // A row's identity is (userid, dbid, queryid, toplevel), so queryid alone collides across roles and nesting levels.
+    // Without pg_read_all_stats or superuser this silently returns only the LabKey role's own statements.
+    private static final String STATEMENTS_SQL = """
+            SELECT COALESCE(json_object_agg(queryid::text || ':' || userid::text || ':' || toplevel::text, to_jsonb(s)), '{}'::json)
+            FROM %s s
+            WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database())
+              AND queryid IS NOT NULL""";
+
+    private static final String NO_STATEMENTS_SQL = "SELECT '{}'::json";
+
+    public static @NotNull String capture()
+    {
+        DbSchema schema = CoreSchema.getInstance().getSchema();
+        int version = new SqlSelector(schema, "SELECT current_setting('server_version_num')::int").getObject(Integer.class);
+        StatementsSource statements = resolveStatements(schema);
+        String statementsSql = StatementsStatus.AVAILABLE == statements.status() ? STATEMENTS_SQL.formatted(statements.view()) : NO_STATEMENTS_SQL;
+
+        String sql = SNAPSHOT_SQL.formatted(
+            version >= PG_17 ? CHECKPOINTER_SQL : CHECKPOINTER_FROM_BGWRITER_SQL,
+            version >= PG_16 ? IO_SQL : NULL_SQL,
+            statements.status().name(),
+            statementsSql
+        );
+
+        return new SqlSelector(schema, sql).getObject(String.class);
+    }
+
+    /** Per-query statistics come from pg_stat_statements; snapshots are far less useful without it. */
+    public static @NotNull StatementsStatus getStatementsStatus()
+    {
+        return resolveStatements(CoreSchema.getInstance().getSchema()).status();
+    }
+
+    /**
+     * The extension's schema isn't necessarily on the search_path, and referencing a view that doesn't exist fails at
+     * parse time, so the view name has to be assembled from what's actually there.
+     */
+    private static @NotNull StatementsSource resolveStatements(DbSchema schema)
+    {
+        String view = new SqlSelector(schema, """
+                SELECT quote_ident(n.nspname) || '.pg_stat_statements'
+                FROM pg_extension e INNER JOIN pg_namespace n ON n.oid = e.extnamespace
+                WHERE e.extname = 'pg_stat_statements'""").getObject(String.class);
+
+        if (null == view)
+            return new StatementsSource(StatementsStatus.NOT_INSTALLED, null);
+
+        try
+        {
+            new SqlSelector(schema, "SELECT 1 FROM " + view).setLogLevel(Level.OFF).exists();
+            return new StatementsSource(StatementsStatus.AVAILABLE, view);
+        }
+        catch (DataAccessException e)
+        {
+            return new StatementsSource(StatementsStatus.NOT_LOADED, view);
+        }
+    }
+
+    public static class TestCase extends Assert
+    {
+        @Test
+        public void testCapture()
+        {
+            Assume.assumeTrue("Requires Postgres", CoreSchema.getInstance().getSqlDialect().isPostgreSQL());
+
+            // JSONObject rejects duplicate keys, so parsing also verifies statement keys are unique
+            JSONObject snapshot = new JSONObject(capture());
+
+            for (String key : Set.of("takenAt", "serverVersionNum", "database", "bgwriter", "checkpointer", "wal", "io", "settings", "statementsStatus", "statements"))
+                assertTrue("Missing key: " + key, snapshot.has(key));
+
+            int version = snapshot.getInt("serverVersionNum");
+            assertTrue(snapshot.getJSONObject("database").has("xact_commit"));
+            assertTrue(snapshot.getJSONObject("settings").has("shared_buffers"));
+            assertTrue(snapshot.getJSONObject("wal").has("wal_records"));
+            assertTrue(snapshot.getJSONObject("checkpointer").has(version >= PG_17 ? "num_timed" : "checkpoints_timed"));
+            assertEquals(version >= PG_16, !snapshot.isNull("io"));
+
+            StatementsStatus status = StatementsStatus.valueOf(snapshot.getString("statementsStatus"));
+            assertEquals(getStatementsStatus(), status);
+
+            JSONObject statements = snapshot.getJSONObject("statements");
+            if (StatementsStatus.AVAILABLE != status)
+            {
+                assertTrue(statements.isEmpty());
+            }
+            else
+            {
+                for (String key : statements.keySet())
+                {
+                    JSONObject row = statements.getJSONObject(key);
+                    assertEquals(row.get("queryid") + ":" + row.get("userid") + ":" + row.get("toplevel"), key);
+                }
+            }
+        }
+    }
+}