PSL-3700: List schemas for SCHEMA_REFERENCE variable assignments

docs/user-guide/config-commands.md

@@ -408,12 +408,18 @@ The params argument should be passed in a key value format separated by commas w
 --params key1=value1,key2=value2,...,keyN=valueN
 ```
 
-Supported types are `DATA_MODEL`, `CONNECTION`, `DATA_POOL_ID` and `OBJECTS_EVENTS_PACKAGE_KEY`. `OBJECTS_EVENTS_PACKAGE_KEY` lists the OCDM packages of the team, with the package key as the value. Each returns a list of assignments:
+Supported types are `DATA_MODEL`, `CONNECTION`, `DATA_POOL_ID`, `OBJECTS_EVENTS_PACKAGE_KEY` and `SCHEMA_REFERENCE`. `OBJECTS_EVENTS_PACKAGE_KEY` lists the OCDM packages of the team, with the package key as the value. Each returns a list of assignments:
 
 ```json
 [{"value": "4b4f32c8-4529-43f1-9399-2a6de94ac13a", "name": "Order Management Pool"}]
 ```
 
+`SCHEMA_REFERENCE` lists the schemas of every data pool of the team:
+
+```json
+[{"value": "4b4f32c8-4529-43f1-9399-2a6de94ac13a_sap", "name": "Order Management Pool / SAP ECC"}]
+```
+
 ### Mapping Variables
 
 After getting the variables list (with definitions and assignments in the source team) and the assignments (possible values on the target team), you can change the value of the source team to one of the options provided when listing assignments.

src/commands/configuration-management/api/schema-reference-candidates.ts

@@ -0,0 +1,73 @@
+import { VariableAssignment } from "../interfaces/variable-assignment-apis.constants";
+
+interface SchemaCandidate {
+    schema: string;
+    name: string;
+    poolName?: string;
+}
+
+const DATA_POOL_ADMIN_PERMISSION = "ADMIN";
+const MANAGED_SCHEMA_SOURCE_TYPES = new Set(["pig", "context-model", "context-model-semantic"]);
+const MAX_PARALLEL_DATA_SOURCE_REQUESTS = 4;
+const STORIUM_POOL_PROVIDER_TYPE = "STORIUM";
+
+const toPoolSchemas = (pool: any, dataSources: any[]): SchemaCandidate[] => {
+    const sourceSchemas = new Map<string, SchemaCandidate & { managed: boolean }>();
+    for (const dataSource of dataSources) {
+        const schema = dataSource?.targetSchemaName || `${pool.id}_${dataSource?.id}`;
+        if (schema !== pool.id && !schema.startsWith(`${pool.id}_`)) {
+            continue;
+        }
+        const existing = sourceSchemas.get(schema);
+        sourceSchemas.set(schema, {
+            schema,
+            name: existing?.name ?? (dataSource?.name || schema),
+            poolName: pool.name,
+            managed: !!existing?.managed || MANAGED_SCHEMA_SOURCE_TYPES.has(dataSource?.type),
+        });
+    }
+    const assignableSchemas = [...sourceSchemas.values()]
+        .filter(candidate => !candidate.managed)
+        .map(({schema, name, poolName}) => ({schema, name, poolName}));
+    return [{schema: pool.id, name: "Global schema", poolName: pool.name}, ...assignableSchemas];
+};
+
+const mapWithConcurrency = async <T, R>(items: T[], limit: number, mapper: (item: T) => Promise<R>): Promise<R[]> => {
+    const results: R[] = new Array(items.length);
+    let next = 0;
+    const worker = async (): Promise<void> => {
+        if (next >= items.length) {
+            return;
+        }
+        const index = next++;
+        results[index] = await mapper(items[index]);
+        return worker();
+    };
+    await Promise.all(Array.from({length: Math.min(limit, items.length)}, worker));
+    return results;
+};
+
+const isOfferedEngine = (dataSources: any[]): boolean => {
+    const typed = dataSources.find(dataSource => dataSource?.primaryPoolProviderType || dataSource?.secondaryPoolProviderType);
+    return !typed || [typed.primaryPoolProviderType, typed.secondaryPoolProviderType].includes(STORIUM_POOL_PROVIDER_TYPE);
+};
+
+export const loadSchemaCandidates = async (get: (url: string) => Promise<any>, poolsUrl: string): Promise<SchemaCandidate[]> => {
+    const pools = await get(poolsUrl);
+    const adminPools = (Array.isArray(pools) ? pools : [])
+        .filter(pool => !!pool?.id && (pool.permissions ?? []).includes(DATA_POOL_ADMIN_PERMISSION));
+    const loadedPools = await mapWithConcurrency(adminPools, MAX_PARALLEL_DATA_SOURCE_REQUESTS, async pool => {
+        const dataSources = await get(`/integration/api/pools/${encodeURIComponent(pool.id)}/data-sources/slim-overview`);
+        return {pool, dataSources: Array.isArray(dataSources) ? dataSources : []};
+    });
+
+    return loadedPools
+        .filter(({dataSources}) => isOfferedEngine(dataSources))
+        .flatMap(({pool, dataSources}) => toPoolSchemas(pool, dataSources))
+        .sort((a, b) => (a.poolName ?? "").localeCompare(b.poolName ?? "") || a.name.localeCompare(b.name));
+};
+
+export const toSchemaAssignment = (candidate: SchemaCandidate): VariableAssignment => ({
+    value: candidate?.schema,
+    name: [candidate?.poolName, candidate?.name].filter(Boolean).join(" / ") || candidate?.schema,
+});

src/commands/configuration-management/api/variable-assignment-candidates-api.ts

@@ -23,7 +23,9 @@ export class VariableAssignmentCandidatesApi {
         params.forEach((value, key) => query.append(key, value));
         const apiUrl: string = assignmentApi.url + (query.toString().length ? `?${query.toString()}` : "");
 
-        const candidates = await this.httpClient().get(apiUrl).catch(e => {
+        const get = (url: string): Promise<any> => this.httpClient().get(url);
+        const request = assignmentApi.loadCandidates ? assignmentApi.loadCandidates(get, apiUrl) : get(apiUrl);
+        const candidates = await request.catch(e => {
             throw new FatalError(`Problem getting variables assignment values for type ${type}: ${e}`);
         });
 

src/commands/configuration-management/interfaces/variable-assignment-apis.constants.ts

@@ -1,3 +1,5 @@
+import { loadSchemaCandidates, toSchemaAssignment } from "../api/schema-reference-candidates";
+
 export interface VariableAssignment {
     value: string;
     name: string;
@@ -7,6 +9,7 @@ export interface VariableAssignmentApi {
     url: string;
     defaultParams?: { [key: string]: string };
     toAssignment: (candidate: any) => VariableAssignment;
+    loadCandidates?: (get: (url: string) => Promise<any>, url: string) => Promise<any[]>;
 }
 
 const connectionName = (connection: any): string => {
@@ -44,4 +47,9 @@ export const variableAssignmentApis: { [key: string]: VariableAssignmentApi } =
             name: candidate?.name ?? candidate?.key,
         }),
     },
+    SCHEMA_REFERENCE: {
+        url: "/integration/api/pools",
+        loadCandidates: loadSchemaCandidates,
+        toAssignment: toSchemaAssignment,
+    },
 };

tests/commands/configuration-management/list-assignments.spec.ts

@@ -1,4 +1,4 @@
-import { mockAxiosGet, mockedAxiosInstance } from "../../utls/http-requests-mock";
+import { mockAxiosGet, mockAxiosGetError, mockedAxiosInstance } from "../../utls/http-requests-mock";
 import { VariableCommandService } from "../../../src/commands/configuration-management/variable-command.service";
 import { testContext } from "../../utls/test-context";
 import { loggingTestTransport } from "../../jest.setup";
@@ -11,6 +11,13 @@ describe("List assignments", () => {
     const CONNECTIONS_URL = "https://myTeam.celonis.cloud/process-automation-v2/api/connections?param1=value1&param2=value2";
     const DATA_POOLS_URL = "https://myTeam.celonis.cloud/integration/api/pools";
     const OCDM_PACKAGES_URL = "https://myTeam.celonis.cloud/pacman/api/core/staging/packages/export/list?flavors=OCDM&includeBranches=false";
+    const POOL_1 = "11111111-1111-1111-1111-111111111111";
+    const POOL_2 = "22222222-2222-2222-2222-222222222222";
+    const POOL_3 = "33333333-3333-3333-3333-333333333333";
+    const dataSourcesUrl = (poolId: string) =>
+        `https://myTeam.celonis.cloud/integration/api/pools/${poolId}/data-sources/slim-overview`;
+    const ON_STORIUM = {primaryPoolProviderType: "STORIUM"};
+    const ON_LEGACY_ENGINE = {primaryPoolProviderType: "HDFS"};
 
     const dataModelCandidate = (poolName: string, dataModelId: string, dataModelName: string) => ({
         pool: {id: "pool-" + dataModelId, name: poolName},
@@ -120,6 +127,119 @@ describe("List assignments", () => {
         expect(getJsonFromDownloadedFile()).toEqual([{value: "pool-1", name: "Pool 1"}]);
     })
 
+    it("Should list the global and data connection schemas of admin pools for the SCHEMA_REFERENCE type", async () => {
+        mockAxiosGet(DATA_POOLS_URL, [
+            {id: POOL_1, name: "Pool 1", permissions: ["ADMIN"]},
+            {id: POOL_2, name: "Pool 2", permissions: ["VIEW_DATA_POOL"]},
+            {id: POOL_3, name: "Pool 3", permissions: ["ADMIN", "VIEW_DATA_POOL"]},
+        ]);
+        mockAxiosGet(dataSourcesUrl(POOL_1), [
+            {id: "sap", name: "SAP ECC", type: "sap", targetSchemaName: `${POOL_1}_sap`, ...ON_STORIUM},
+            {id: "jdbc", name: "", type: "jdbc", ...ON_STORIUM},
+            {id: "custom", name: "Custom schema name", type: "jdbc", targetSchemaName: "sap_schema", ...ON_STORIUM},
+            {id: "ccm", name: "My Context Model", type: "context-model", ...ON_STORIUM},
+            {id: "imported", name: "Imported", type: "imported", targetSchemaName: `${POOL_2}_other`, ...ON_STORIUM},
+        ]);
+        mockAxiosGet(dataSourcesUrl(POOL_3), []);
+
+        await new VariableCommandService(testContext).listAssignments("SCHEMA_REFERENCE", true, "");
+
+        expect(mockedAxiosInstance.get).not.toHaveBeenCalledWith(dataSourcesUrl(POOL_2), expect.anything());
+        expect(getJsonFromDownloadedFile()).toEqual([
+            {value: `${POOL_1}_jdbc`, name: `Pool 1 / ${POOL_1}_jdbc`},
+            {value: POOL_1, name: "Pool 1 / Global schema"},
+            {value: `${POOL_1}_sap`, name: "Pool 1 / SAP ECC"},
+            {value: POOL_3, name: "Pool 3 / Global schema"},
+        ]);
+    })
+
+    it("Should not offer a schema that a managed data source shares", async () => {
+        mockAxiosGet(DATA_POOLS_URL, [{id: POOL_1, name: "Pool 1", permissions: ["ADMIN"]}]);
+        mockAxiosGet(dataSourcesUrl(POOL_1), [
+            {id: "sap", name: "SAP ECC", type: "sap", targetSchemaName: `${POOL_1}_shared`, ...ON_STORIUM},
+            {id: "pig", name: "PIG", type: "pig", targetSchemaName: `${POOL_1}_shared`, ...ON_STORIUM},
+            {id: "ccm", name: "My Context Model", type: "context-model", ...ON_STORIUM},
+        ]);
+
+        await new VariableCommandService(testContext).listAssignments("SCHEMA_REFERENCE", true, "");
+
+        expect(getJsonFromDownloadedFile()).toEqual([{value: POOL_1, name: "Pool 1 / Global schema"}]);
+    })
+
+    it("Should offer the virtual global and OCDM entries of the slim overview like the schema picker", async () => {
+        mockAxiosGet(DATA_POOLS_URL, [{id: POOL_1, name: "Pool 1", permissions: ["ADMIN"]}]);
+        mockAxiosGet(dataSourcesUrl(POOL_1), [
+            {id: "sap", name: "SAP ECC", type: "sap", ...ON_STORIUM},
+            {id: null, name: "Global Data Connection", type: "global", targetSchemaName: POOL_1, ...ON_STORIUM},
+            {id: null, name: "Objects and events", type: "OCDM", targetSchemaName: `${POOL_1}_ocdm`, ...ON_STORIUM},
+        ]);
+
+        await new VariableCommandService(testContext).listAssignments("SCHEMA_REFERENCE", true, "");
+
+        expect(getJsonFromDownloadedFile()).toEqual([
+            {value: POOL_1, name: "Pool 1 / Global Data Connection"},
+            {value: POOL_1, name: "Pool 1 / Global schema"},
+            {value: `${POOL_1}_ocdm`, name: "Pool 1 / Objects and events"},
+            {value: `${POOL_1}_sap`, name: "Pool 1 / SAP ECC"},
+        ]);
+    })
+
+    it("Should not offer schemas whose name does not start with the pool id, such as the legacy OCDM schema", async () => {
+        mockAxiosGet(DATA_POOLS_URL, [{id: POOL_1, name: "Pool 1", permissions: ["ADMIN"]}]);
+        mockAxiosGet(dataSourcesUrl(POOL_1), [
+            {id: "custom", name: "Custom schema name", type: "jdbc", targetSchemaName: "sap_schema", ...ON_STORIUM},
+            {id: null, name: "Objects and events", type: "OCDM", targetSchemaName: "business_graph_schema_tenant", ...ON_STORIUM},
+        ]);
+
+        await new VariableCommandService(testContext).listAssignments("SCHEMA_REFERENCE", true, "");
+
+        expect(getJsonFromDownloadedFile()).toEqual([{value: POOL_1, name: "Pool 1 / Global schema"}]);
+    })
+
+    it("Should only offer schemas of pools on Storium, also while Storium is the secondary engine", async () => {
+        mockAxiosGet(DATA_POOLS_URL, [
+            {id: POOL_1, name: "Pool 1", permissions: ["ADMIN"]},
+            {id: POOL_3, name: "Pool 3", permissions: ["ADMIN"]},
+        ]);
+        mockAxiosGet(dataSourcesUrl(POOL_1), [{id: "sap", name: "SAP ECC", type: "sap", ...ON_LEGACY_ENGINE}]);
+        mockAxiosGet(dataSourcesUrl(POOL_3), [
+            {id: "jdbc", name: "Postgres", type: "jdbc", primaryPoolProviderType: "HDFS", secondaryPoolProviderType: "STORIUM"},
+        ]);
+
+        await new VariableCommandService(testContext).listAssignments("SCHEMA_REFERENCE", true, "");
+
+        expect(getJsonFromDownloadedFile()).toEqual([
+            {value: POOL_3, name: "Pool 3 / Global schema"},
+            {value: `${POOL_3}_jdbc`, name: "Pool 3 / Postgres"},
+        ]);
+    })
+
+    it("Should offer the schemas of pools whose engine is not reported, without requesting it", async () => {
+        mockAxiosGet(DATA_POOLS_URL, [
+            {id: POOL_1, name: "Pool 1", permissions: ["ADMIN"]},
+            {id: POOL_3, name: "Pool 3", permissions: ["ADMIN"]},
+        ]);
+        mockAxiosGet(dataSourcesUrl(POOL_1), [{id: "sap", name: "SAP ECC", type: "sap"}]);
+        mockAxiosGet(dataSourcesUrl(POOL_3), []);
+
+        await new VariableCommandService(testContext).listAssignments("SCHEMA_REFERENCE", true, "");
+
+        expect(mockedAxiosInstance.get).not.toHaveBeenCalledWith(expect.stringContaining("pool-provider-types"), expect.anything());
+        expect(getJsonFromDownloadedFile()).toEqual([
+            {value: POOL_1, name: "Pool 1 / Global schema"},
+            {value: `${POOL_1}_sap`, name: "Pool 1 / SAP ECC"},
+            {value: POOL_3, name: "Pool 3 / Global schema"},
+        ]);
+    })
+
+    it("Should fail when the data sources of a pool cannot be loaded", async () => {
+        mockAxiosGet(DATA_POOLS_URL, [{id: POOL_1, name: "Pool 1", permissions: ["ADMIN"]}]);
+        mockAxiosGetError(dataSourcesUrl(POOL_1), 500, "Internal Server Error");
+
+        await expect(new VariableCommandService(testContext).listAssignments("SCHEMA_REFERENCE", true, ""))
+            .rejects.toThrow("Problem getting variables assignment values for type SCHEMA_REFERENCE");
+    })
+
     it("Should throw error for unsupported variable types", async () => {
         const type: string = "DUMMY_UNSUPPORTED_TYPE";