From 70f3dfb1fdffdfbbf4572fc13b64ba70879a6e62 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 02:00:46 -0700 Subject: [PATCH 1/3] feat(knowledge): project document ACL and chunk changes asynchronously Document ACL/source changes and chunk writes mark their document in knowledge_projection_dirty (one upsert with a generation bump) in the writer's transaction. A knowledge projector converges every search projection per document in short pages bounded by chunk rows, several documents at once, and removes a mark only on the generation it read. Writers that declare sim.projection_mode = 'async' skip the synchronous embedding and document fan-out triggers. The processing commit and every connector-lease ACL page declare it while the knowledge-async-projection flag is on; every other writer, and every release before this one, keeps writing projection rows itself. Marks are written in both modes so a projector pass can never settle over a concurrent synchronous write. Migrations 0021-0023 keep their trigger body; 0024 alone installs the marking. Search decides a marked document's rows on the document itself, so a pending projection never admits a revoked grant. A document that moved sources joins the new source's per-source ranking after its pass. The projector runs as a Trigger.dev task, requested after writes and swept every minute while there is work, or in-process without Trigger.dev. A pass opens one worker per mark up to KB_CONFIG_PROJECTION_CONCURRENCY. The separate source/ACL backfill is folded into it behind knowledge-projection-fill. --- .github/workflows/test-build.yml | 2 + .../platform/self-hosting/background-jobs.mdx | 1 + .../cron/knowledge-projection/route.test.ts | 86 + .../api/cron/knowledge-projection/route.ts | 29 + apps/sim/background/knowledge-projection.ts | 45 + .../projection-source-acl-backfill.ts | 44 - apps/sim/lib/core/config/env.ts | 3 + .../sim/lib/core/config/feature-flags.test.ts | 30 + apps/sim/lib/core/config/feature-flags.ts | 17 + ...async-projection-processing.integration.ts | 162 + .../connector-lease-pages.integration.ts | 4 +- .../knowledge-projection.integration.ts | 786 + .../access/predicate.postgres.test.ts | 3 + .../lib/knowledge/access/predicate.test.ts | 28 +- apps/sim/lib/knowledge/access/predicate.ts | 56 +- .../lib/knowledge/connectors/detachment.ts | 2 +- .../connectors/member-observations.ts | 13 +- .../connectors/member-sync-engine.ts | 8 +- .../lib/knowledge/connectors/sync-limits.ts | 12 +- .../sim/lib/knowledge/connectors/sync-lock.ts | 39 +- apps/sim/lib/knowledge/documents/service.ts | 13 + .../projection/enqueue-inline.test.ts | 61 + .../lib/knowledge/projection/enqueue.test.ts | 98 + apps/sim/lib/knowledge/projection/enqueue.ts | 144 + apps/sim/lib/knowledge/projection/run.test.ts | 126 + apps/sim/lib/knowledge/projection/run.ts | 120 + apps/sim/lib/knowledge/search/prewarm.test.ts | 4 +- apps/sim/lib/knowledge/search/prewarm.ts | 4 +- .../projection-source-acl-backfill.test.ts | 296 - .../search/projection-source-acl-backfill.ts | 244 - apps/sim/lib/knowledge/search/queries.test.ts | 8 +- apps/sim/lib/knowledge/search/queries.ts | 112 +- .../scripts/backfill-projection-source-acl.ts | 62 - docker/crontab | 3 + helm/sim/values.yaml | 11 + packages/db/knowledge-projection.test.ts | 337 + packages/db/knowledge-projection.ts | 526 + .../0378_knowledge_projection_dirty.sql | 15 + .../db/migrations/meta/0378_snapshot.json | 28544 ++++++++++++++++ packages/db/migrations/meta/_journal.json | 7 + packages/db/schema.ts | 40 +- ...rations-paused-billing-attribution.test.ts | 1 + ...6_backfill_search_vectors.postgres.test.ts | 9 +- .../0016_backfill_search_vectors.ts | 66 +- .../0019_tin_keyword_projection.ts | 17 +- ...mbedding_search_connector.postgres.test.ts | 229 +- .../0021_embedding_search_connector.test.ts | 187 - .../0021_embedding_search_connector.ts | 272 +- .../0022_projection_source_acl_backfill.ts | 11 +- .../0023_projection_acl_skip_unfilled.ts | 2 +- .../0024_knowledge_projection_async.ts | 190 + packages/db/script-migrations/index.ts | 3 + packages/db/scripts/push.test.ts | 3 +- packages/db/scripts/push.ts | 1 + packages/testing/src/mocks/schema.mock.ts | 6 + scripts/check-api-validation-contracts.ts | 1 + ...check-tool-registry-boundary.baseline.json | 2 +- 57 files changed, 31771 insertions(+), 1374 deletions(-) create mode 100644 apps/sim/app/api/cron/knowledge-projection/route.test.ts create mode 100644 apps/sim/app/api/cron/knowledge-projection/route.ts create mode 100644 apps/sim/background/knowledge-projection.ts delete mode 100644 apps/sim/background/projection-source-acl-backfill.ts create mode 100644 apps/sim/lib/knowledge/__integration__/async-projection-processing.integration.ts create mode 100644 apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts create mode 100644 apps/sim/lib/knowledge/projection/enqueue-inline.test.ts create mode 100644 apps/sim/lib/knowledge/projection/enqueue.test.ts create mode 100644 apps/sim/lib/knowledge/projection/enqueue.ts create mode 100644 apps/sim/lib/knowledge/projection/run.test.ts create mode 100644 apps/sim/lib/knowledge/projection/run.ts delete mode 100644 apps/sim/lib/knowledge/search/projection-source-acl-backfill.test.ts delete mode 100644 apps/sim/lib/knowledge/search/projection-source-acl-backfill.ts delete mode 100644 apps/sim/scripts/backfill-projection-source-acl.ts create mode 100644 packages/db/knowledge-projection.test.ts create mode 100644 packages/db/knowledge-projection.ts create mode 100644 packages/db/migrations/0378_knowledge_projection_dirty.sql create mode 100644 packages/db/migrations/meta/0378_snapshot.json delete mode 100644 packages/db/script-migrations/0021_embedding_search_connector.test.ts create mode 100644 packages/db/script-migrations/0024_knowledge_projection_async.ts diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 487c28ef1b7..1688bcf267f 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -282,6 +282,8 @@ jobs: lib/knowledge/__integration__/kb-block-search.integration.ts lib/knowledge/__integration__/gitlab-workspace.integration.ts lib/knowledge/__integration__/unfilled-projection-source.integration.ts + lib/knowledge/__integration__/knowledge-projection.integration.ts + lib/knowledge/__integration__/async-projection-processing.integration.ts lib/knowledge/__integration__/purged-detach-reservation.integration.ts lib/core/outbox/service.integration.ts lib/knowledge/__integration__/connector-upload.integration.ts diff --git a/apps/docs/content/docs/platform/self-hosting/background-jobs.mdx b/apps/docs/content/docs/platform/self-hosting/background-jobs.mdx index 385ad1175e4..33c75b8f5de 100644 --- a/apps/docs/content/docs/platform/self-hosting/background-jobs.mdx +++ b/apps/docs/content/docs/platform/self-hosting/background-jobs.mdx @@ -47,6 +47,7 @@ Point cron at an **internal** address where possible (the in-cluster Service, or | Time pause/resume | `/api/resume/poll` | `*/1 * * * *` | Workflows paused on a timer | | Outbox processing | `/api/webhooks/outbox/process` | `*/1 * * * *` | Transactional-outbox retries for billing, membership, enterprise issuance, and workflow-deployment side effects | | Workspace file search dispatch | `/api/cron/workspace-file-search-dispatch` | `*/1 * * * *` | Dispatches indexing work for workspace file search | +| Knowledge projection | `/api/cron/knowledge-projection` | `*/1 * * * *` | Brings knowledge base search up to date with document, permission, and chunk changes | | Connector sync | `/api/knowledge/connectors/sync` | `*/5 * * * *` | Knowledge base connector syncs | | Connector member sync | `/api/knowledge/connectors/member-sync` | `*/5 * * * *` | Per-member access sync for permission-aware connectors | | Connector directory sync | `/api/knowledge/connectors/directory-sync` | `*/5 * * * *` | Refreshes the directory groups administrator-mode connectors mirror, so a membership change takes effect without waiting for a content sync | diff --git a/apps/sim/app/api/cron/knowledge-projection/route.test.ts b/apps/sim/app/api/cron/knowledge-projection/route.test.ts new file mode 100644 index 00000000000..2a8c5be05c8 --- /dev/null +++ b/apps/sim/app/api/cron/knowledge-projection/route.test.ts @@ -0,0 +1,86 @@ +/** + * @vitest-environment node + */ +import { createMockRequest } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + enqueueSweep: vi.fn(), + verifyCronAuth: vi.fn(), +})) + +vi.mock('@/lib/auth/internal', () => ({ verifyCronAuth: mocks.verifyCronAuth })) +vi.mock('@/lib/knowledge/projection/enqueue', () => ({ + enqueueKnowledgeProjectionSweep: mocks.enqueueSweep, +})) + +import { GET } from '@/app/api/cron/knowledge-projection/route' + +function request() { + return createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost:3000/api/cron/knowledge-projection' + ) +} + +describe('knowledge projection sweep route', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.verifyCronAuth.mockReturnValue(null) + }) + + it('returns as soon as Trigger.dev accepts the pass', async () => { + mocks.enqueueSweep.mockResolvedValue({ + triggered: true, + backend: 'trigger-dev', + jobId: 'run-1', + }) + + const response = await GET(request()) + + expect(response.status).toBe(202) + await expect(response.json()).resolves.toEqual({ + success: true, + triggered: true, + backend: 'trigger-dev', + jobId: 'run-1', + }) + }) + + it('answers 200 without a pass when the projector has nothing to do', async () => { + mocks.enqueueSweep.mockResolvedValue({ triggered: false, backend: null, jobId: null }) + + const response = await GET(request()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + triggered: false, + backend: null, + jobId: null, + }) + }) + + it('returns the cron auth refusal without enqueueing', async () => { + mocks.verifyCronAuth.mockReturnValue(new Response(null, { status: 401 })) + + const response = await GET(request()) + + expect(response.status).toBe(401) + expect(mocks.enqueueSweep).not.toHaveBeenCalled() + }) + + it('fails closed when Trigger.dev does not accept the pass', async () => { + mocks.enqueueSweep.mockRejectedValue(new Error('trigger unavailable')) + + const response = await GET(request()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Sweep enqueue failed', + }) + }) +}) diff --git a/apps/sim/app/api/cron/knowledge-projection/route.ts b/apps/sim/app/api/cron/knowledge-projection/route.ts new file mode 100644 index 00000000000..5e5369afcce --- /dev/null +++ b/apps/sim/app/api/cron/knowledge-projection/route.ts @@ -0,0 +1,29 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { type NextRequest, NextResponse } from 'next/server' +import { verifyCronAuth } from '@/lib/auth/internal' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { enqueueKnowledgeProjectionSweep } from '@/lib/knowledge/projection/enqueue' + +const logger = createLogger('KnowledgeProjectionSweepRoute') + +export const dynamic = 'force-dynamic' +export const maxDuration = 60 + +/** + * The knowledge projector's periodic sweep: enqueues one pass per window while there is work, and + * returns once Trigger.dev accepts it. Writers ask for passes as they commit; this converges + * whatever those requests missed. + */ +export const GET = withRouteHandler(async (request: NextRequest) => { + const authError = verifyCronAuth(request, 'Knowledge projection sweep') + if (authError) return authError + + try { + const result = await enqueueKnowledgeProjectionSweep() + return NextResponse.json({ success: true, ...result }, { status: result.triggered ? 202 : 200 }) + } catch (error) { + logger.error('Knowledge projection sweep enqueue failed', { error: getErrorMessage(error) }) + return NextResponse.json({ success: false, error: 'Sweep enqueue failed' }, { status: 500 }) + } +}) diff --git a/apps/sim/background/knowledge-projection.ts b/apps/sim/background/knowledge-projection.ts new file mode 100644 index 00000000000..d3f52a23f6e --- /dev/null +++ b/apps/sim/background/knowledge-projection.ts @@ -0,0 +1,45 @@ +import { task } from '@trigger.dev/sdk' +import { + type BackgroundRetryPolicy, + backgroundRetryAttemptCeiling, + getBackgroundRetryDecision, +} from '@/lib/core/errors/background-retry' +import { + KNOWLEDGE_PROJECTION_PASS_BUDGET_MS, + KNOWLEDGE_PROJECTION_TASK_ID, + requestKnowledgeProjection, +} from '@/lib/knowledge/projection/enqueue' +import { runKnowledgeProjectionPass } from '@/lib/knowledge/projection/run' + +/** + * A pass gives a single document up on a lock or statement timeout without failing, so a failed + * pass lost its connection or its database. Those back off for minutes; the sweep starts a fresh + * pass every minute regardless, so a few attempts are enough. + */ +export const KNOWLEDGE_PROJECTION_RETRY_POLICY: BackgroundRetryPolicy = { + maxAttempts: 2, + database: { maxAttempts: 3, baseDelayMs: 60 * 1000, maxDelayMs: 5 * 60 * 1000 }, +} + +/** + * Runs one knowledge projector pass. One pass runs at a time and projects several documents at + * once itself; the prompt requests and the sweep collapse into whichever pass is queued. A pass + * that ran out of budget with marks left asks for the next one. Retry-safe: a pass writes only rows + * that differ from their source and removes a mark only on the generation it read. + */ +export const knowledgeProjectionTask = task({ + id: KNOWLEDGE_PROJECTION_TASK_ID, + machine: 'small-1x', + maxDuration: 15 * 60, + retry: { maxAttempts: backgroundRetryAttemptCeiling(KNOWLEDGE_PROJECTION_RETRY_POLICY) }, + queue: { name: KNOWLEDGE_PROJECTION_TASK_ID, concurrencyLimit: 1 }, + catchError: async ({ error, ctx }) => + getBackgroundRetryDecision(error, ctx.attempt.number, KNOWLEDGE_PROJECTION_RETRY_POLICY), + run: async () => { + const result = await runKnowledgeProjectionPass({ + budgetMs: KNOWLEDGE_PROJECTION_PASS_BUDGET_MS, + }) + if (result.remaining) await requestKnowledgeProjection() + return result + }, +}) diff --git a/apps/sim/background/projection-source-acl-backfill.ts b/apps/sim/background/projection-source-acl-backfill.ts deleted file mode 100644 index 71c80c86369..00000000000 --- a/apps/sim/background/projection-source-acl-backfill.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { task, tasks } from '@trigger.dev/sdk' -import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' -import { - PROJECTION_SOURCE_ACL_BACKFILL_SHARDS, - PROJECTION_SOURCE_ACL_BACKFILL_TASK_ID, - type ProjectionSourceAclBackfillPayload, - projectionSourceAclChainTag, - runProjectionSourceAclBackfill, -} from '@/lib/knowledge/search/projection-source-acl-backfill' - -/** One run's share of the backfill, inside the worker's run ceiling with room to end its page. */ -const RUN_BUDGET_MS = 60 * 60 * 1000 - -/** - * Trigger.dev wrapper around `runProjectionSourceAclBackfill`. A run fills unset rows for up to - * {@link RUN_BUDGET_MS}, then triggers its continuation from the cursor it reached, so the whole - * projection is filled across as many bounded runs as it takes. Retry-safe: every run writes only - * rows still unset, so a retried or restarted run repeats no write. A shard's continuation keeps - * its shard, so a sliced fill stays sliced until every slice is done. - */ -export const projectionSourceAclBackfillTask = task({ - id: PROJECTION_SOURCE_ACL_BACKFILL_TASK_ID, - machine: 'small-1x', - retry: { maxAttempts: 3 }, - /** - * One run per shard the id space may be sliced into. Shards fill disjoint ranges, so runs never - * fill the same page against each other; an unsliced chain still runs one at a time because each - * run triggers its continuation only as it ends. - */ - queue: { - name: PROJECTION_SOURCE_ACL_BACKFILL_TASK_ID, - concurrencyLimit: PROJECTION_SOURCE_ACL_BACKFILL_SHARDS, - }, - run: async (payload: ProjectionSourceAclBackfillPayload) => { - const cursor = await runProjectionSourceAclBackfill(payload, { budgetMs: RUN_BUDGET_MS }) - if (!cursor) return - const continuation: ProjectionSourceAclBackfillPayload = { ...payload, cursor } - await tasks.trigger(PROJECTION_SOURCE_ACL_BACKFILL_TASK_ID, continuation, { - region: await resolveTriggerRegion(), - /** The chain's tag rides on every continuation, so a start finds the chain wherever it is. */ - tags: [projectionSourceAclChainTag(payload.shard)], - }) - }, -}) diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 9a21aba0699..8d6a39a3634 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -464,6 +464,7 @@ export const env = createEnv({ KB_CONFIG_CONCURRENCY_LIMIT: z.number().optional().default(20), // Per-tenant concurrent document-processing runs in the interactive lane KB_CONFIG_BACKFILL_CONCURRENCY_LIMIT: z.number().optional().default(20), // Per-tenant concurrent document-processing runs in the connector-backfill lane KB_CONFIG_EMBEDDING_CONCURRENCY: z.number().optional().default(8), // Concurrent embedding API requests within one embed call + KB_CONFIG_PROJECTION_CONCURRENCY: z.number().optional().default(8), // Most documents one knowledge projector pass projects at once, each on its own connection /** Deployment operating budgets shared by every caller using the same provider credential. */ KB_CONFIG_EMBEDDING_REQUESTS_PER_MINUTE: z.number().positive().optional().default(600), KB_CONFIG_EMBEDDING_TOKENS_PER_MINUTE: z.number().positive().optional().default(600000), @@ -634,6 +635,8 @@ export const env = createEnv({ CREDENTIAL_GROUPS: z.boolean().optional(), // Enable enterprise Credential Groups globally KNOWLEDGE_MEMBER_ACCESS: z.boolean().optional(), // Enable per-member knowledge connectors and hybrid-by-default retrieval globally KNOWLEDGE_TIN_KEYWORD: z.boolean().optional(), // Rank large-scope keyword retrieval through the Tin text index where it exists + KNOWLEDGE_ASYNC_PROJECTION: z.boolean().optional(), // Knowledge writers leave search projection rows to the background projector + KNOWLEDGE_PROJECTION_FILL: z.boolean().optional(), // The knowledge projector fills projection rows written before they carried a source and ACL // Organizations - for self-hosted deployments ORGANIZATIONS_ENABLED: z.boolean().optional(), // Enable organizations on self-hosted (bypasses plan requirements) diff --git a/apps/sim/lib/core/config/feature-flags.test.ts b/apps/sim/lib/core/config/feature-flags.test.ts index b31904d8a3e..d9be1891f49 100644 --- a/apps/sim/lib/core/config/feature-flags.test.ts +++ b/apps/sim/lib/core/config/feature-flags.test.ts @@ -10,6 +10,8 @@ const { mockFetch, mockIsPlatformAdmin, envRef } = vi.hoisted(() => ({ mockIsPlatformAdmin: vi.fn(), envRef: { APPCONFIG_APPLICATION: 'sim-staging' as string | undefined, + KNOWLEDGE_PROJECTION_FILL: undefined as boolean | undefined, + KNOWLEDGE_ASYNC_PROJECTION: undefined as boolean | undefined, APPCONFIG_ENVIRONMENT: 'staging' as string | undefined, TABLES_V2_API: undefined as boolean | undefined, TABLE_ROW_TTL: undefined as boolean | undefined, @@ -148,6 +150,8 @@ describe('isFeatureEnabled', () => { envRef.CREDENTIAL_GROUPS = undefined envRef.KNOWLEDGE_MEMBER_ACCESS = undefined envRef.KNOWLEDGE_TIN_KEYWORD = undefined + envRef.KNOWLEDGE_ASYNC_PROJECTION = undefined + envRef.KNOWLEDGE_PROJECTION_FILL = undefined envRef.SLACK_SEARCH_SHARED_APP = undefined }) @@ -197,6 +201,32 @@ describe('isFeatureEnabled', () => { }) }) + describe('knowledge-async-projection flag', () => { + it('is a global switch', async () => { + expect(await isFeatureEnabled('knowledge-async-projection')).toBe(false) + envRef.KNOWLEDGE_ASYNC_PROJECTION = true + expect(await isFeatureEnabled('knowledge-async-projection')).toBe(true) + }) + + it('follows an AppConfig global rule', async () => { + withAppConfig({ 'knowledge-async-projection': { enabled: true } }) + expect(await isFeatureEnabled('knowledge-async-projection')).toBe(true) + }) + }) + + describe('knowledge-projection-fill flag', () => { + it('is a global switch', async () => { + expect(await isFeatureEnabled('knowledge-projection-fill')).toBe(false) + envRef.KNOWLEDGE_PROJECTION_FILL = true + expect(await isFeatureEnabled('knowledge-projection-fill')).toBe(true) + }) + + it('follows an AppConfig global rule', async () => { + withAppConfig({ 'knowledge-projection-fill': { enabled: true } }) + expect(await isFeatureEnabled('knowledge-projection-fill')).toBe(true) + }) + }) + describe('knowledge-member-access flag', () => { it('uses a global fallback switch off AppConfig', async () => { expect(await isFeatureEnabled('knowledge-member-access')).toBe(false) diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index cb440668dd2..69b4e54801e 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -105,6 +105,23 @@ const FEATURE_FLAGS = { 'invalid. Off-AppConfig falls back to KNOWLEDGE_TIN_KEYWORD.', fallback: 'KNOWLEDGE_TIN_KEYWORD', }, + 'knowledge-async-projection': { + description: + 'Knowledge writers (document processing and connector ACL writes) leave search projection ' + + 'rows to the background knowledge projector instead of rewriting them in their own ' + + 'transaction. Global on/off only; turn it on only once no release older than the ' + + 'projector serves search. Off-AppConfig falls back to KNOWLEDGE_ASYNC_PROJECTION.', + fallback: 'KNOWLEDGE_ASYNC_PROJECTION', + }, + 'knowledge-projection-fill': { + description: + 'The knowledge projector also fills search projection rows written before they carried ' + + "their document's source and ACL, marking at most 100 documents at once so fresh writes " + + 'never wait behind much of it. Global on/off only; off pauses the fill, and search keeps ' + + 'deciding unfilled rows on their document. Off-AppConfig falls back to ' + + 'KNOWLEDGE_PROJECTION_FILL.', + fallback: 'KNOWLEDGE_PROJECTION_FILL', + }, } satisfies Record /** diff --git a/apps/sim/lib/knowledge/__integration__/async-projection-processing.integration.ts b/apps/sim/lib/knowledge/__integration__/async-projection-processing.integration.ts new file mode 100644 index 00000000000..46f7a1bbb3e --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/async-projection-processing.integration.ts @@ -0,0 +1,162 @@ +/** + * Document processing with `knowledge-async-projection` on, through the real service: the commit + * writes the chunks and a mark on their document and no projection row, asks for a projector pass + * after it commits, and the pass then writes every projection row from the chunks and the document. + */ +import { mkdtempSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { db } from '@sim/db' +import { runKnowledgeProjection } from '@sim/db/knowledge-projection' +import { + document, + embeddingSearch, + knowledgeBase, + knowledgeProjectionDirty, + organization, + user, + workspace, +} from '@sim/db/schema' +import { eq, inArray } from 'drizzle-orm' +import postgres from 'postgres' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +const fixtures = vi.hoisted(() => ({ + root: '', + process: vi.fn(), + embeddings: vi.fn(), + requestKnowledgeProjection: vi.fn(), +})) +vi.mock('@/lib/uploads/core/setup.server', () => ({ + get UPLOAD_DIR_SERVER() { + return fixtures.root + }, +})) +vi.mock('@/lib/knowledge/documents/document-processor', () => ({ + processDocument: fixtures.process, +})) +vi.mock('@/lib/knowledge/embeddings', () => ({ generateEmbeddings: fixtures.embeddings })) +vi.mock('@/lib/core/config/feature-flags', () => ({ + isFeatureEnabled: async (flag: string) => flag === 'knowledge-async-projection', +})) +/** The pass is run by the test itself, so the commit can be read before any pass has run. */ +vi.mock('@/lib/knowledge/projection/enqueue', () => ({ + requestKnowledgeProjection: fixtures.requestKnowledgeProjection, +})) + +import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' +import * as embeddingClient from '@/lib/embeddings/client' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' +import { addDocument } from '@/lib/knowledge/connectors/sync-persistence' +import { processDocumentAsync } from '@/lib/knowledge/documents/service' + +/** The projections a processing commit writes where the Tin extension is absent. */ +const PROJECTIONS = ['embedding_search', 'embedding_keyword_search'] as const + +describe('document processing with the asynchronous projection', () => { + const ids = createKnowledgeAclFixtureIds() + let projector: postgres.Sql + + beforeAll(async () => { + fixtures.root = mkdtempSync(path.join(tmpdir(), 'sim-async-projection-')) + projector = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) + await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) + vi.spyOn(embeddingClient, 'assertKnowledgeEmbeddingCapacity').mockResolvedValue(undefined) + }) + + afterAll(async () => { + vi.restoreAllMocks() + await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + await rm(fixtures.root, { recursive: true, force: true }) + await projector?.end() + await db.$client.end() + }) + + const count = async (table: string, documentId: string) => { + const [row] = await db.$client.unsafe>( + `SELECT count(*)::int AS count FROM ${table} WHERE document_id = $1`, + [documentId] + ) + return row?.count + } + + it('commits the chunks with a mark and no projection rows, and a pass then projects them', async () => { + const file = await addDocument( + ids.knowledgeBaseId, + ids.connectorId, + 'google_drive', + { + externalId: 'async-projection-fixture', + title: 'Synthetic async projection fixture.txt', + content: 'Synthetic text', + mimeType: 'text/plain', + contentHash: 'synthetic-async-projection', + }, + { userId: ids.aliceId, workspaceId: ids.workspaceId }, + undefined, + 'workspace', + createContentSyncLease(ids.connectorId, ids.lockId) + ) + const chunks = Array.from({ length: 12 }, (_, index) => ({ + text: `Synthetic chunk ${index}`, + metadata: { startIndex: index * 20, endIndex: index * 20 + 19 }, + })) + fixtures.process.mockResolvedValue({ + chunks, + metadata: { chunkCount: chunks.length, tokenCount: 36, characterCount: 240 }, + }) + fixtures.embeddings.mockResolvedValue({ + embeddings: chunks.map(() => Array(1536).fill(0.2)), + billableTokens: 0, + modelName: 'text-embedding-3-small', + pricingId: 'text-embedding-3-small', + }) + const billing = await resolveBillingAttribution({ + actorUserId: ids.aliceId, + workspaceId: ids.workspaceId, + }) + fixtures.requestKnowledgeProjection.mockClear() + + await processDocumentAsync(ids.knowledgeBaseId, file.documentId, file, {}, billing) + + expect(await count('embedding', file.documentId)).toBe(chunks.length) + for (const table of PROJECTIONS) expect(await count(table, file.documentId)).toBe(0) + expect( + await db + .select({ content: knowledgeProjectionDirty.content }) + .from(knowledgeProjectionDirty) + .where(eq(knowledgeProjectionDirty.documentId, file.documentId)) + ).toEqual([{ content: true }]) + expect(fixtures.requestKnowledgeProjection).toHaveBeenCalledTimes(1) + + await runKnowledgeProjection(projector) + + for (const table of PROJECTIONS) { + expect(await count(table, file.documentId)).toBe(chunks.length) + } + const [source] = await db + .select({ connectorId: document.connectorId, acl: document.acl }) + .from(document) + .where(eq(document.id, file.documentId)) + expect( + await db + .selectDistinct({ connectorId: embeddingSearch.connectorId, acl: embeddingSearch.acl }) + .from(embeddingSearch) + .where(eq(embeddingSearch.documentId, file.documentId)) + ).toEqual([source]) + expect( + await db + .select() + .from(knowledgeProjectionDirty) + .where(eq(knowledgeProjectionDirty.documentId, file.documentId)) + ).toEqual([]) + }) +}) diff --git a/apps/sim/lib/knowledge/__integration__/connector-lease-pages.integration.ts b/apps/sim/lib/knowledge/__integration__/connector-lease-pages.integration.ts index b8d61052f05..136319c01ec 100644 --- a/apps/sim/lib/knowledge/__integration__/connector-lease-pages.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/connector-lease-pages.integration.ts @@ -18,6 +18,7 @@ import { workspace, } from '@sim/db/schema' import { installProjectionSourceAcl } from '@sim/db/script-migrations/0021_embedding_search_connector' +import { installKnowledgeProjectionAsync } from '@sim/db/script-migrations/0024_knowledge_projection_async' import { generateId } from '@sim/utils/id' import { and, eq, inArray, sql } from 'drizzle-orm' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' @@ -38,6 +39,7 @@ vi.mock('@/connectors/registry.server', () => ({ }, })) +import { PROJECTION_ROW_BATCH_SIZE } from '@sim/db/knowledge-projection' import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' import { @@ -59,7 +61,6 @@ import { resumeMembershipRewrites, } from '@/lib/knowledge/connectors/member-sync-engine' import { executeSync } from '@/lib/knowledge/connectors/sync-engine' -import { PROJECTION_ROW_BATCH_SIZE } from '@/lib/knowledge/connectors/sync-limits' import { createMemberSyncLease, type LeaseTransaction, @@ -276,6 +277,7 @@ describe('connector lease ACL pages in PostgreSQL', () => { /** The production document trigger under test, whatever an earlier suite left installed. */ beforeAll(async () => { await installProjectionSourceAcl(db.$client) + await installKnowledgeProjectionAsync(db.$client) }) /** Real chunks: the installed triggers create each chunk's search and keyword projection rows. */ diff --git a/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts b/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts new file mode 100644 index 00000000000..f5654cedaf1 --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts @@ -0,0 +1,786 @@ +/** + * The knowledge projector and the readers that must stay correct while it lags. A GitHub member + * source's document is changed by a writer in either projection mode — synchronous, as every + * writer before the projector, or asynchronous, where the writer only marks the document — and + * search is checked before the projector runs: a revoked member is refused and a granted one is + * served, on the vector and keyword legs and under the source filter, and a disabled or deleted + * chunk is gone at once. The projector's own contract follows: it converges the rows and removes + * the mark, keeps a mark that a write bumped during its pass, survives a document deleted under + * it, writes in pages bounded by chunk rows, fills rows written before they carried a source and + * ACL, and an asynchronous commit writes no projection row at all. + */ +import { createHash } from 'node:crypto' +import { db } from '@sim/db' +import { + DEFER_KNOWLEDGE_PROJECTION, + FILL_MARK_CEILING, + type KnowledgeProjection, + markUnfilledProjectionDocuments, + runKnowledgeProjection, +} from '@sim/db/knowledge-projection' +import { + credential, + credentialGroup, + credentialGroupEnrollment, + document, + embedding, + embeddingKeywordTin, + embeddingSearch, + knowledgeBase, + knowledgeConnector, + knowledgeConnectorMember, + knowledgeDocumentObservation, + knowledgeProjectionDirty, + organization, + user, + workspace, +} from '@sim/db/schema' +import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, isNull, sql } from 'drizzle-orm' +import postgres from 'postgres' +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +/** The TINQL `resolveTinKeywordQuery` renders for `fixture`: its `english` stem, quoted. */ +vi.mock('@/lib/knowledge/search/tin-keyword', () => ({ + resolveTinKeywordQuery: async () => '"fixtur"', +})) + +/** The flags a test turns on; connector writers read `knowledge-async-projection` from here. */ +const { enabledFlags, requestKnowledgeProjection } = vi.hoisted(() => ({ + enabledFlags: new Set(), + requestKnowledgeProjection: vi.fn(async () => {}), +})) +vi.mock('@/lib/core/config/feature-flags', () => ({ + isFeatureEnabled: async (flag: string) => enabledFlags.has(flag), +})) +/** + * Writers ask for a pass once they commit; here that request is only recorded, so the passes each + * test runs are the only ones and a background pass cannot converge rows a test is inspecting. + */ +vi.mock('@/lib/knowledge/projection/enqueue', () => ({ requestKnowledgeProjection })) + +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { + projectionCandidateAccessCondition, + type SearchAccessPlan, +} from '@/lib/knowledge/access/predicate' +import type { + GitHubInstallationReadGrant, + KnowledgeAccessProvider, + UserAccessScope, +} from '@/lib/knowledge/access/types' +import { leaseTransaction } from '@/lib/knowledge/connectors/sync-lock' +import { + executeKeywordSearch, + forgetProjectionFilled, + handleVectorOnlySearch, + liveSourceAccessFor, +} from '@/lib/knowledge/search/queries' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' + +const ids = createKnowledgeAclFixtureIds() +const connectorId = generateId() +const otherConnectorId = generateId() +const contentCredentialId = generateId() +const groupId = generateId() +const optionId = generateId() +const documentId = generateId() +const chunkId = generateId() +const repositoryId = '4242' +const members = { + alice: { id: generateId(), subject: 'alice-gh', credentialId: generateId() }, + bob: { id: generateId(), subject: 'bob-gh', credentialId: generateId() }, +} +const subjectToken = (subject: string) => `s:github-repositories:-:${subject}` +const aclOf = (...who: Array<'alice' | 'bob'>) => + who.map((name) => subjectToken(members[name].subject)).sort() +const vector = [1, ...Array(1535).fill(0)] +const queryVector = { + vector: JSON.stringify(vector), + dimensions: 1536 as const, + model: 'text-embedding-3-small', +} + +/** Alice holds the installation grant, so what she reads is decided by the document's ACL alone. */ +const scope: UserAccessScope = { + kind: 'user', + userId: ids.aliceId, + tokens: [ + 'pub', + subjectToken(members.alice.subject), + `u:${ids.aliceId}@fixture.test`, + 'ws', + ].sort(), +} +const plan: SearchAccessPlan = { + connectors: { + workspace: [], + admin: [], + members: [connectorId], + liveProofRequired: [connectorId], + }, + observers: { confirmed: [{ id: members.alice.id, connectorId }], observed: [] }, + memberSources: [connectorId], + connectorTypes: new Map([[connectorId, 'github']]), + uploads: false, +} +const grant: GitHubInstallationReadGrant = { + connectorId, + contentCredentialId, + readerCredentialId: members.alice.credentialId, + readerSubjectToken: subjectToken(members.alice.subject), + repositoryId, +} + +function searchInputs() { + const granted = { ...scope, githubInstallationGrants: [grant] } + const accessProvider: KnowledgeAccessProvider = { + get: async () => scope, + getForConnectors: async () => granted, + getForDocuments: async () => granted, + liveSourceConnectorCondition: async () => null, + } + return { + knowledgeBaseIds: [ids.knowledgeBaseId], + topK: 20, + access: scope, + accessProvider, + accessPlan: plan, + liveSourceAccess: liveSourceAccessFor(scope, plan, accessProvider), + queryVector, + } +} + +const keywordIds = async () => + ( + await executeKeywordSearch({ + ...searchInputs(), + query: 'fixture', + permitted: { kind: 'unbounded', broad: false }, + searchIndexOnly: true, + }) + ) + .map((row) => row.id) + .sort() + +const vectorIds = async () => + ( + await handleVectorOnlySearch({ + ...searchInputs(), + distanceThreshold: 2, + permitted: { kind: 'unbounded', broad: true }, + }) + ) + .map((row) => row.id) + .sort() + +/** The chunks of the fixture document each ranking projection admits for Alice. */ +async function admitted(): Promise> { + const [vectorRows, keywordRows] = await Promise.all([ + db + .select({ id: embeddingSearch.id }) + .from(embeddingSearch) + .where( + and( + eq(embeddingSearch.documentId, documentId), + projectionCandidateAccessCondition(embeddingSearch, scope, plan, { filled: true }) + ) + ), + db + .select({ id: embeddingKeywordTin.id }) + .from(embeddingKeywordTin) + .where( + and( + eq(embeddingKeywordTin.documentId, documentId), + projectionCandidateAccessCondition(embeddingKeywordTin, scope, plan, { filled: true }) + ) + ), + ]) + return { + vector: vectorRows.map((row) => row.id).sort(), + keyword: keywordRows.map((row) => row.id).sort(), + } +} + +type Mode = 'sync' | 'async' + +/** Runs a write in a transaction of the given projection mode, as a knowledge writer would. */ +function write( + mode: Mode, + work: (tx: Parameters[0]>[0]) => Promise +) { + return db.transaction(async (tx) => { + if (mode === 'async') await tx.execute(sql.raw(`SELECT ${DEFER_KNOWLEDGE_PROJECTION}`)) + await work(tx) + }) +} + +function chunkRow(id: string, chunkIndex: number) { + return { + id, + documentId, + knowledgeBaseId: ids.knowledgeBaseId, + chunkIndex, + chunkHash: `fixture-hash-${chunkIndex}`, + content: 'fixture readme', + contentLength: 14, + tokenCount: 2, + startOffset: 0, + endOffset: 14, + embeddingModel: 'text-embedding-3-small', + embedding: vector, + } +} + +const markOf = async () => { + const [row] = await db + .select() + .from(knowledgeProjectionDirty) + .where(eq(knowledgeProjectionDirty.documentId, documentId)) + return row +} + +const rowAcl = async (table: typeof embeddingSearch | typeof embeddingKeywordTin, id = chunkId) => { + const [row] = await db + .select({ acl: table.acl, connectorId: table.connectorId, enabled: table.enabled }) + .from(table) + .where(eq(table.id, id)) + return row +} + +/** The projector's connection: a pass holds its per-document advisory locks on it. */ +let projector: postgres.Sql +const project = (options: Parameters[1] = {}) => + runKnowledgeProjection(projector, options) + +/** Shims for the Tin extension, which the test database does not carry. */ +let createdTinShims = false + +beforeAll(async () => { + projector = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) + await seedKnowledgeAclFixture(ids) + const now = new Date() + await db + .update(knowledgeBase) + .set({ isSearchIndex: true }) + .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await db.insert(credential).values({ + id: contentCredentialId, + workspaceId: ids.workspaceId, + type: 'service_account', + displayName: 'Fixture GitHub installation', + createdBy: ids.aliceId, + providerId: GITHUB_INSTALLATION_PROVIDER_ID, + }) + await db.insert(credentialGroup).values({ + id: groupId, + workspaceId: ids.workspaceId, + publicId: generateId(), + name: 'GitHub readers', + options: [ + { + id: optionId, + provider: 'github-repositories', + label: 'GitHub fixture', + authorizationAppId: 'fixture-app', + requiredScopes: ['repo'], + scopeVersion: 1, + required: false, + status: 'active', + }, + ], + } as typeof credentialGroup.$inferInsert) + for (const who of ['alice', 'bob'] as const) { + const userId = who === 'alice' ? ids.aliceId : ids.bobId + const [enrollment] = await db + .insert(credentialGroupEnrollment) + .values({ + id: generateId(), + credentialGroupId: groupId, + userId, + email: `${userId}@fixture.test`, + status: 'completed', + invitationTokenHash: createHash('sha256').update(generateId()).digest('hex'), + invitationExpiresAt: new Date(Date.now() + 60 * 60 * 1000), + invitedAt: now, + }) + .returning({ id: credentialGroupEnrollment.id }) + await db.insert(credential).values({ + id: members[who].credentialId, + workspaceId: ids.workspaceId, + type: 'managed_oauth', + displayName: 'Fixture GitHub reader', + providerId: 'github-repositories', + authorizationAppId: 'fixture-app', + credentialGroupEnrollmentId: enrollment!.id, + credentialGroupOptionId: optionId, + managedOauthScopeVersion: 1, + providerSubjectId: members[who].subject, + providerTenantId: '', + managedOauthStatus: 'active', + grantedScopes: ['repo'], + encryptedOauthTokenSet: 'fixture-not-an-oauth-token', + grantedAt: now, + createdBy: userId, + }) + } + await db.insert(knowledgeConnector).values( + [connectorId, otherConnectorId].map((id) => ({ + id, + knowledgeBaseId: ids.knowledgeBaseId, + connectorType: 'github', + sourceConfig: { githubRepositoryId: repositoryId }, + accessMode: 'members', + status: 'active', + credentialId: contentCredentialId, + credentialGroupId: groupId, + credentialGroupOptionId: optionId, + })) + ) + await db.insert(knowledgeConnectorMember).values( + (['alice', 'bob'] as const).map((who) => ({ + id: members[who].id, + workspaceId: ids.workspaceId, + connectorId, + credentialId: members[who].credentialId, + subjectToken: subjectToken(members[who].subject), + status: 'active', + memberSyncedThrough: now, + })) + ) + await db.insert(document).values({ + id: documentId, + connectorId, + knowledgeBaseId: ids.knowledgeBaseId, + externalId: 'fixture-file', + filename: 'readme.md', + fileUrl: 'https://fixture.test/readme', + fileSize: 12, + mimeType: 'text/plain', + processingStatus: 'completed', + acl: aclOf('alice', 'bob'), + }) + await db.insert(knowledgeDocumentObservation).values( + (['alice', 'bob'] as const).map((who) => ({ + documentId, + memberId: members[who].id, + lastSeenAt: now, + runId: generateId(), + })) + ) + const [tin] = await db.execute<{ present: boolean }>( + sql`SELECT to_regnamespace('tin') IS NOT NULL AS present` + ) + if (!tin?.present) { + createdTinShims = true + await db.execute( + sql.raw(`CREATE SCHEMA tin; + CREATE FUNCTION tin.full_score(tid) RETURNS double precision LANGUAGE sql IMMUTABLE AS 'SELECT 1.0::float8'; + CREATE FUNCTION knowledge_tin_base_token(text) RETURNS text LANGUAGE sql IMMUTABLE AS $$SELECT 'kb'$$; + CREATE FUNCTION knowledge_tin_membership_key(text) RETURNS bigint LANGUAGE sql IMMUTABLE AS $$SELECT hashtextextended('embedding_keyword_tin:' || $1, 0)$$; + CREATE FUNCTION knowledge_tin_stream(vector tsvector) RETURNS text LANGUAGE sql IMMUTABLE AS $$ + SELECT coalesce(string_agg(entry.lexeme, ' ' ORDER BY position), '') + FROM unnest(vector) AS entry(lexeme, positions, weights), unnest(entry.positions) AS position + $$; + CREATE FUNCTION tin_fixture_match(text, text) RETURNS boolean LANGUAGE sql IMMUTABLE AS 'SELECT true'; + CREATE OPERATOR ==> (LEFTARG = text, RIGHTARG = text, FUNCTION = tin_fixture_match);`) + ) + } +}, 60_000) + +afterAll(async () => { + if (createdTinShims) { + await db.execute( + sql.raw(`DROP OPERATOR IF EXISTS ==> (text, text); + DROP FUNCTION IF EXISTS tin_fixture_match(text, text); + DROP FUNCTION IF EXISTS knowledge_tin_base_token(text); + DROP FUNCTION IF EXISTS knowledge_tin_membership_key(text); + DROP FUNCTION IF EXISTS knowledge_tin_stream(tsvector); + DROP SCHEMA IF EXISTS tin CASCADE;`) + ) + } + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(credentialGroup).where(eq(credentialGroup.id, groupId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + await projector?.end() + forgetProjectionFilled() +}) + +/** + * Every test starts from one converged chunk readable by Alice and Bob: whatever the last test + * left is removed, the chunk is written again, and the projector converges it. It is written + * deferred, so the projector writes all three projections: this database has no Tin extension, + * so no synchronous trigger would write the Tin row. + */ +beforeEach(async () => { + await db.delete(embedding).where(eq(embedding.documentId, documentId)) + await db + .update(document) + .set({ acl: aclOf('alice', 'bob'), connectorId }) + .where(eq(document.id, documentId)) + await write('async', (tx) => tx.insert(embedding).values(chunkRow(chunkId, 0))) + await project() + forgetProjectionFilled() +}) + +describe.each(['sync', 'async'] as const)('a %s writer', (mode) => { + it('revokes a grant before the projector runs, on both rankings', async () => { + await write(mode, (tx) => + tx + .update(document) + .set({ acl: aclOf('bob') }) + .where(eq(document.id, documentId)) + ) + expect(await markOf()).toMatchObject({ content: false }) + if (mode === 'async') { + /** The rows still name Alice: only the mark keeps her out. */ + expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('alice', 'bob')) + expect((await rowAcl(embeddingKeywordTin))?.acl).toEqual(aclOf('alice', 'bob')) + } + expect(await admitted()).toEqual({ vector: [], keyword: [] }) + expect(await vectorIds()).toEqual([]) + expect(await keywordIds()).toEqual([]) + + await project() + expect(await markOf()).toBeUndefined() + expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('bob')) + expect((await rowAcl(embeddingKeywordTin))?.acl).toEqual(aclOf('bob')) + expect(await admitted()).toEqual({ vector: [], keyword: [] }) + }) + + it('serves a new grant before the projector runs, on both rankings', async () => { + await db + .update(document) + .set({ acl: aclOf('bob') }) + .where(eq(document.id, documentId)) + await project() + expect(await vectorIds()).toEqual([]) + + await write(mode, (tx) => + tx + .update(document) + .set({ acl: aclOf('alice', 'bob') }) + .where(eq(document.id, documentId)) + ) + if (mode === 'async') expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('bob')) + expect(await admitted()).toEqual({ vector: [chunkId], keyword: [chunkId] }) + expect(await vectorIds()).toEqual([chunkId]) + expect(await keywordIds()).toEqual([chunkId]) + }) + + it('decides a document that moved to a source outside the plan on the document', async () => { + await write(mode, (tx) => + tx.update(document).set({ connectorId: otherConnectorId }).where(eq(document.id, documentId)) + ) + if (mode === 'async') expect((await rowAcl(embeddingSearch))?.connectorId).toBe(connectorId) + expect(await admitted()).toEqual({ vector: [], keyword: [] }) + expect(await vectorIds()).toEqual([]) + expect(await keywordIds()).toEqual([]) + await project() + expect((await rowAcl(embeddingSearch))?.connectorId).toBe(otherConnectorId) + }) + + it('hides a disabled chunk at once', async () => { + await write(mode, (tx) => + tx.update(embedding).set({ enabled: false }).where(eq(embedding.id, chunkId)) + ) + expect(await markOf()).toMatchObject({ content: mode === 'async' }) + expect(await vectorIds()).toEqual([]) + expect(await keywordIds()).toEqual([]) + await project() + expect((await rowAcl(embeddingSearch))?.enabled).toBe(false) + }) + + it('removes a deleted chunk from every projection in the deleting statement', async () => { + await write(mode, (tx) => tx.delete(embedding).where(eq(embedding.id, chunkId))) + expect(await rowAcl(embeddingSearch)).toBeUndefined() + expect(await rowAcl(embeddingKeywordTin)).toBeUndefined() + expect(await vectorIds()).toEqual([]) + }) + + it('makes a new chunk searchable, at once or once the projector runs', async () => { + const added = generateId() + await write(mode, (tx) => tx.insert(embedding).values(chunkRow(added, 1))) + expect(await markOf()).toMatchObject({ content: mode === 'async' }) + const expected = [chunkId, added].sort() + if (mode === 'sync') { + expect(await vectorIds()).toEqual(expected) + } else { + expect(await rowAcl(embeddingSearch, added)).toBeUndefined() + expect(await vectorIds()).toEqual([chunkId]) + } + await project() + expect(await markOf()).toBeUndefined() + expect(await vectorIds()).toEqual(expected) + /** A synchronous writer's Tin row is its trigger's, which only a database with Tin has. */ + if (mode === 'async') expect(await keywordIds()).toEqual(expected) + }) +}) + +describe('the projector', () => { + it('keeps a mark that a write bumped during its pass, and settles it on the next', async () => { + await db + .update(document) + .set({ acl: aclOf('bob') }) + .where(eq(document.id, documentId)) + const before = await markOf() + let bumped = false + await project({ + onPage: async () => { + if (bumped) return + bumped = true + await db + .update(document) + .set({ acl: aclOf('alice') }) + .where(eq(document.id, documentId)) + }, + }) + expect(bumped).toBe(true) + const after = await markOf() + expect(after?.generation).toBe((before?.generation ?? 0) + 1) + await project() + expect(await markOf()).toBeUndefined() + expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('alice')) + }) + + it('passes over a document deleted while it is being projected', async () => { + const doomed = generateId() + await db.insert(document).values({ + id: doomed, + connectorId, + knowledgeBaseId: ids.knowledgeBaseId, + externalId: 'doomed-file', + filename: 'doomed.md', + fileUrl: 'https://fixture.test/doomed', + fileSize: 12, + mimeType: 'text/plain', + processingStatus: 'completed', + acl: aclOf('alice'), + }) + await write('async', (tx) => + tx.insert(embedding).values( + Array.from({ length: 3 }, (_, index) => ({ + ...chunkRow(generateId(), index), + documentId: doomed, + })) + ) + ) + await expect( + project({ + pageSize: 1, + onPage: async (page) => { + if (page.documentId === doomed) await db.delete(document).where(eq(document.id, doomed)) + }, + }) + ).resolves.toMatchObject({ remaining: false }) + const [left] = await db + .select({ count: sql`count(*)::int` }) + .from(embeddingSearch) + .where(eq(embeddingSearch.documentId, doomed)) + expect(left?.count).toBe(0) + expect( + await db + .select() + .from(knowledgeProjectionDirty) + .where(eq(knowledgeProjectionDirty.documentId, doomed)) + ).toEqual([]) + }) + + it.each([false, true])( + "leaves a connector ACL page's projection rows to the projector only while the flag is on (%s)", + async (flagOn) => { + if (flagOn) enabledFlags.add('knowledge-async-projection') + try { + await leaseTransaction(connectorId)((tx) => + tx + .update(document) + .set({ acl: aclOf('bob') }) + .where(eq(document.id, documentId)) + ) + } finally { + enabledFlags.delete('knowledge-async-projection') + } + expect(requestKnowledgeProjection).toHaveBeenCalled() + expect(await markOf()).toMatchObject({ content: false }) + expect((await rowAcl(embeddingSearch))?.acl).toEqual( + flagOn ? aclOf('alice', 'bob') : aclOf('bob') + ) + expect(await admitted()).toEqual({ vector: [], keyword: [] }) + await project() + expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('bob')) + } + ) + + it('splits the marks between passes that start together', async () => { + const documents = await Promise.all( + Array.from({ length: 6 }, async (_, index) => { + const id = generateId() + await db.insert(document).values({ + id, + connectorId, + knowledgeBaseId: ids.knowledgeBaseId, + externalId: `split-${index}`, + filename: `split-${index}.md`, + fileUrl: `https://fixture.test/split-${index}`, + fileSize: 12, + mimeType: 'text/plain', + processingStatus: 'completed', + acl: aclOf('alice'), + }) + return id + }) + ) + await write('async', (tx) => + tx + .insert(embedding) + .values( + documents.map((id, index) => ({ ...chunkRow(generateId(), index), documentId: id })) + ) + ) + const second = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) + try { + /** Each page lingers, so neither pass can finish the batch before the other starts. */ + const lingering = { onPage: () => sleep(25) } + const passes = await Promise.all([ + project(lingering), + runKnowledgeProjection(second, lingering), + ]) + expect(passes.every((pass) => pass.settled > 0)).toBe(true) + expect(passes[0].settled + passes[1].settled).toBe(documents.length) + } finally { + await second.end() + } + expect( + await db + .select() + .from(knowledgeProjectionDirty) + .where(inArray(knowledgeProjectionDirty.documentId, documents)) + ).toEqual([]) + await db.delete(document).where(inArray(document.id, documents)) + }) + + it('writes in pages bounded by chunk rows', async () => { + await write('async', (tx) => + tx + .insert(embedding) + .values(Array.from({ length: 4 }, (_, index) => chunkRow(generateId(), index + 1))) + ) + const pages: Array<{ projection: KnowledgeProjection; written: number }> = [] + await project({ pageSize: 2, onPage: (page) => void pages.push(page) }) + const vectorPages = pages.filter((page) => page.projection === 'embedding_search') + expect(vectorPages.map((page) => page.written)).toEqual([1, 2, 1]) + expect(pages.every((page) => page.written <= 2)).toBe(true) + expect(await markOf()).toBeUndefined() + }) + + it('fills rows written before they carried a source and ACL, and converges them', async () => { + for (const table of [embeddingSearch, embeddingKeywordTin]) { + await db + .update(table) + .set({ connectorId: null, acl: null }) + .where(eq(table.documentId, documentId)) + } + expect(await markOf()).toBeUndefined() + let cursor: Parameters[1] | null + let marked = 0 + while (cursor !== null) { + const fill = await markUnfilledProjectionDocuments(projector, cursor) + marked += fill.marked + cursor = fill.cursor + await project() + } + expect(marked).toBeGreaterThanOrEqual(1) + expect(await rowAcl(embeddingSearch)).toMatchObject({ acl: aclOf('alice', 'bob'), connectorId }) + expect(await rowAcl(embeddingKeywordTin)).toMatchObject({ + acl: aclOf('alice', 'bob'), + connectorId, + }) + expect(await markOf()).toBeUndefined() + }) + + it('fills every document when more are unfilled than the fill may mark at once', async () => { + const documents = Array.from({ length: FILL_MARK_CEILING + 20 }, () => generateId()) + await db.insert(document).values( + documents.map((id, index) => ({ + id, + connectorId, + knowledgeBaseId: ids.knowledgeBaseId, + externalId: `fill-${index}`, + filename: `fill-${index}.md`, + fileUrl: `https://fixture.test/fill-${index}`, + fileSize: 12, + mimeType: 'text/plain', + processingStatus: 'completed' as const, + acl: aclOf('alice', 'bob'), + })) + ) + /** Two chunks each, whose random ids interleave the documents' rows in the unfilled index. */ + await write('async', (tx) => + tx + .insert(embedding) + .values( + documents.flatMap((id) => + [0, 1].map((chunkIndex) => ({ ...chunkRow(generateId(), chunkIndex), documentId: id })) + ) + ) + ) + await project() + /** Only the vector rows, so no other projection's unfilled rows lead the fill back to them. */ + await db + .update(embeddingSearch) + .set({ connectorId: null, acl: null }) + .where(inArray(embeddingSearch.documentId, documents)) + let cursor: Parameters[1] | null + let marked = 0 + while (cursor !== null) { + const fill = await markUnfilledProjectionDocuments(projector, cursor) + marked += fill.marked + cursor = fill.cursor + await project() + } + expect(marked).toBeGreaterThanOrEqual(documents.length) + expect( + await db + .select({ id: embeddingSearch.id }) + .from(embeddingSearch) + .where(and(inArray(embeddingSearch.documentId, documents), isNull(embeddingSearch.acl))) + ).toEqual([]) + await db.delete(document).where(inArray(document.id, documents)) + }) + + it.each(['sync', 'async'] as const)( + 'writes %s projection rows from a chunk commit only when the writer did not defer them', + async (mode) => { + /** + * The projector's single connection makes the write, so its own statistics can be flushed + * and read back without waiting on the collector's interval. + */ + const inserted = async () => { + await projector`SELECT pg_stat_force_next_flush()` + await projector`SELECT pg_stat_clear_snapshot()` + const [row] = await projector>` + SELECT n_tup_ins AS inserted FROM pg_stat_user_tables WHERE relname = 'embedding_search'` + return Number(row?.inserted ?? 0) + } + const chunks = Array.from({ length: 50 }, (_, index) => chunkRow(generateId(), index + 1)) + const before = await inserted() + await projector.begin(async (tx) => { + if (mode === 'async') await tx`SELECT set_config('sim.projection_mode', 'async', true)` + for (const chunk of chunks) { + await tx`INSERT INTO embedding (id, document_id, knowledge_base_id, chunk_index, chunk_hash, + content, content_length, token_count, start_offset, end_offset, embedding_model, embedding) + VALUES (${chunk.id}, ${documentId}, ${ids.knowledgeBaseId}, ${chunk.chunkIndex}, + ${chunk.chunkHash}, ${chunk.content}, 14, 2, 0, 14, ${chunk.embeddingModel}, + ${JSON.stringify(vector)}::vector)` + } + }) + expect((await inserted()) - before).toBe(mode === 'async' ? 0 : chunks.length) + expect(await markOf()).toMatchObject({ content: mode === 'async' }) + } + ) +}) diff --git a/apps/sim/lib/knowledge/access/predicate.postgres.test.ts b/apps/sim/lib/knowledge/access/predicate.postgres.test.ts index 3e105c6eba4..a55d590e04c 100644 --- a/apps/sim/lib/knowledge/access/predicate.postgres.test.ts +++ b/apps/sim/lib/knowledge/access/predicate.postgres.test.ts @@ -605,6 +605,9 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => { */ await connection.unsafe(`CREATE TABLE IF NOT EXISTS embedding_search( id text PRIMARY KEY, document_id text, connector_id text, acl text[])`) + await connection.unsafe( + 'CREATE TABLE IF NOT EXISTS knowledge_projection_dirty(document_id text PRIMARY KEY)' + ) await connection.unsafe('DELETE FROM embedding_search') await connection.unsafe( "INSERT INTO embedding_search SELECT id || '-chunk', id, connector_id, acl FROM document" diff --git a/apps/sim/lib/knowledge/access/predicate.test.ts b/apps/sim/lib/knowledge/access/predicate.test.ts index 7f9008632ec..30ee71eae0f 100644 --- a/apps/sim/lib/knowledge/access/predicate.test.ts +++ b/apps/sim/lib/knowledge/access/predicate.test.ts @@ -41,7 +41,12 @@ describe('projectionCandidateAccessCondition', () => { uploads: true, } - it('decides a filled row on its mirrored columns and an unfilled row on its document', () => { + const pending = + '(EXISTS (SELECT 1 FROM "knowledge_projection_dirty" WHERE "knowledge_projection_dirty"."document_id" = "embedding_search"."document_id"))' + const onDocument = + 'AND (SELECT "document"."id" FROM "document"\n WHERE "document"."id" = "embedding_search"."document_id"\n AND (' + + it('decides a filled row on its mirrored columns and an unfilled or marked row on its document', () => { const { sql, params } = render( projectionCandidateAccessCondition( embeddingSearch, @@ -49,19 +54,32 @@ describe('projectionCandidateAccessCondition', () => { plan ) ) - expect(sql).toContain( - '("embedding_search"."acl" IS NULL AND EXISTS (\n SELECT 1 FROM "document"\n WHERE "document"."id" = "embedding_search"."document_id"\n AND (' - ) + expect(sql).toContain(`(("embedding_search"."acl" IS NULL OR ${pending}) ${onDocument}`) expect(sql).toContain('"document"."acl" && ARRAY[$1, $2]::text[]') - expect(sql).toContain('OR ("embedding_search"."acl" && ARRAY[') + expect(sql).toContain('OR (("embedding_search"."acl" && ARRAY[') expect(sql).toContain( 'AND ("embedding_search"."connector_id" IS NULL OR "embedding_search"."connector_id" = ANY(ARRAY[' ) + expect(sql).toContain(`AND NOT ${pending})`) expect(params.slice(0, 2)).toEqual(['ws', 'u:alice']) expect(params.slice(-3)).toEqual(['ws', 'u:alice', 'ws-src']) for (const param of params) expect(Array.isArray(param)).toBe(false) }) + it('still decides a marked row on its document once the projection is filled', () => { + const { sql } = render( + projectionCandidateAccessCondition( + embeddingSearch, + { kind: 'user', userId: 'user-1', tokens: ['ws', 'u:alice'] }, + plan, + { filled: true } + ) + ) + expect(sql).toContain(`((${pending} ${onDocument}`) + expect(sql).not.toContain('"embedding_search"."acl" IS NULL') + expect(sql).toContain(`AND NOT ${pending})`) + }) + it('still denies everything for an empty token set', () => { expect( render( diff --git a/apps/sim/lib/knowledge/access/predicate.ts b/apps/sim/lib/knowledge/access/predicate.ts index b10c6276432..2517af6f483 100644 --- a/apps/sim/lib/knowledge/access/predicate.ts +++ b/apps/sim/lib/knowledge/access/predicate.ts @@ -7,6 +7,7 @@ import { knowledgeConnector, knowledgeConnectorMember, knowledgeDocumentObservation, + knowledgeProjectionDirty, member, user, } from '@sim/db/schema' @@ -333,6 +334,29 @@ export function knowledgeCandidateAccessConditionForConnectors( )` } +/** + * Whether a projection row belongs to a document marked for the knowledge projector: its source, + * ACL, or chunks changed and its rows may not show it yet. A probe of the marks' primary key: the + * planner may instead hash the whole set once per statement, which is as cheap while the marks are + * few, and an `IN` would risk re-reading them per row once they outgrow the hash. + */ +export function projectionPending(documentId: AnyPgColumn | SQL): SQL { + return sql`(EXISTS (SELECT 1 FROM ${knowledgeProjectionDirty} WHERE ${knowledgeProjectionDirty.documentId} = ${documentId}))` +} + +/** + * Whether a projection row is decided on its document rather than on its own columns: its + * document is marked for the projector, or, while the source and ACL fill runs, the row has not + * been filled. + */ +export function projectionDecidedOnDocument( + projection: { acl: AnyPgColumn | SQL; documentId: AnyPgColumn | SQL }, + filled: boolean +): SQL { + const pending = projectionPending(projection.documentId) + return filled ? pending : sql`(${projection.acl} IS NULL OR ${pending})` +} + /** * The candidate predicate on a ranking projection's own row, for a scope whose connectors were * resolved: `connectorId` and `acl` are mirrored there from the document, so a walk or a keyword @@ -344,11 +368,15 @@ export function knowledgeCandidateAccessConditionForConnectors( * under the full predicate, before content is returned — this predicate only decides what is worth * ranking. * - * A row the backfill has not reached yet carries no ACL (`acl IS NULL`) and is decided on its - * document instead, under {@link knowledgeCandidateAccessConditionForConnectors} — the join per - * candidate that every row paid before the columns existed. The backfill runs in the background, - * so search never waits on it, never loses a document to it, and never ranks an unreadable one - * into a bounded candidate pool because of it. + * A row whose columns may be behind its document is decided on the document instead, under + * {@link knowledgeCandidateAccessConditionForConnectors} — the join per candidate that every row + * paid before the columns existed: a row the source and ACL fill has not reached (`acl IS NULL`), + * and every row of a document marked for the knowledge projector. A revoked grant still on such a + * row never admits it, and a new grant not yet on it never hides it from a statement that reaches + * the row. A source-scoped walk or slice reaches rows by the source on the row, though, so a + * document that moved to another source joins that source's ranking once the projector has + * rewritten its rows; until then it can be missing there, never shown where it is not readable. + * The projector and the fill run in the background, so search never waits on either. */ export function projectionCandidateAccessCondition( projection: { @@ -361,8 +389,8 @@ export function projectionCandidateAccessCondition( options: { /** * Whether every row of the projection carries its mirrored source and ACL. While the fill - * is under way, a row it has not reached is decided on its document; once it is complete no - * such row exists, and the predicate is the array test alone. + * is under way, a row it has not reached is decided on its document; once it is complete only + * a marked document's rows are. */ filled?: boolean } = {} @@ -383,13 +411,17 @@ export function projectionCandidateAccessCondition( ? sql`(${projection.connectorId} IS NULL OR ${inSources(mirrored)})` : inSources(mirrored) const onRow = sql`(${projection.acl} && ${tokens} AND ${owned})` - if (options.filled) return onRow - const unfilled = sql`(${projection.acl} IS NULL AND EXISTS ( - SELECT 1 FROM ${document} + /** + * A scalar subquery rather than `EXISTS`: the planner may turn an `EXISTS` into one hash of every + * readable document, a sequential scan of `document` for a statement that only needs a few rows + * decided. A scalar subquery is only ever a primary-key probe per row that needs it. + */ + const onDocument = sql`(SELECT ${document.id} FROM ${document} WHERE ${document.id} = ${projection.documentId} AND ${knowledgeCandidateAccessConditionForConnectors(scope, plan)} - ))` - return sql`(${unfilled} OR ${onRow})` + LIMIT 1) IS NOT NULL` + return sql`((${projectionDecidedOnDocument(projection, options.filled ?? false)} AND ${onDocument}) + OR (${onRow} AND NOT ${projectionPending(projection.documentId)}))` } /** diff --git a/apps/sim/lib/knowledge/connectors/detachment.ts b/apps/sim/lib/knowledge/connectors/detachment.ts index 65add4459d9..ef5985e50f4 100644 --- a/apps/sim/lib/knowledge/connectors/detachment.ts +++ b/apps/sim/lib/knowledge/connectors/detachment.ts @@ -1,4 +1,5 @@ import { db } from '@sim/db' +import { PROJECTION_ROW_BATCH_SIZE } from '@sim/db/knowledge-projection' import { document, embeddingKeywordTin, @@ -24,7 +25,6 @@ import { import type { DbOrTx } from '@/lib/db/types' import { removeDrainedConnector } from '@/lib/knowledge/connectors/deletion' import { revokeKnowledgeConnectorCredentialAccess } from '@/lib/knowledge/connectors/member-access' -import { PROJECTION_ROW_BATCH_SIZE } from '@/lib/knowledge/connectors/sync-limits' export const KNOWLEDGE_CONNECTOR_DETACH_EVENT = 'knowledge.connector.detach' const DOCUMENT_BATCH_SIZE = 100 diff --git a/apps/sim/lib/knowledge/connectors/member-observations.ts b/apps/sim/lib/knowledge/connectors/member-observations.ts index eac00e3567f..22e4524fde6 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.ts @@ -1,4 +1,5 @@ import { db } from '@sim/db' +import { PROJECTION_ROW_BATCH_SIZE } from '@sim/db/knowledge-projection' import { document, knowledgeConnector, @@ -34,10 +35,9 @@ import { MEMBER_PURGE_MAX_PER_RUN, MEMBER_TOMBSTONE_PURGE_DAYS, MEMBER_TOMBSTONE_RECONCILE_PAGES_PER_RUN, - PROJECTION_ROW_BATCH_SIZE, } from '@/lib/knowledge/connectors/sync-limits' import { - boundLeaseTransaction, + aclPageTransaction, connectorIsLive, type LeaseTransaction, leaseTransaction, @@ -360,6 +360,12 @@ interface ConnectorDocumentCursor { * search projection rows per table, the rows an ACL assignment sends through the projection * trigger. Every page holds at least one document, so one larger than the cap still makes progress * alone. Documents keep their order. + * + * Cleanup boundary: the row bound, and {@link lockProjectionPage} and + * {@link writeProjectionPages} built on it, exist only because a synchronous ACL write rewrites + * projection rows in its own statement. With `knowledge-async-projection` on, an ACL write only + * marks its documents; once the flag has been on everywhere and the synchronous triggers are + * dropped, these collapse to plain pages of {@link ACL_CHANGE_BATCH_SIZE} documents. */ export function pagesByProjectionRows( documents: readonly { id: string; chunkCount: number }[] @@ -1037,8 +1043,7 @@ async function sweepStaleMemberPage( more: boolean } | null> { try { - return await db.transaction(async (tx) => { - await boundLeaseTransaction(tx) + return await aclPageTransaction(async (tx) => { const [stale] = await tx .select({ id: knowledgeConnectorMember.id }) .from(knowledgeConnectorMember) diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 64e4de04f02..7753ccab38f 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -91,8 +91,8 @@ import { SOURCE_CONTENT_ERROR, } from '@/lib/knowledge/connectors/sync-limits' import { + aclPageTransaction, assertSyncLeaseHeldInTx, - boundLeaseTransaction, createMemberSyncLease, holdsMemberSyncLockToken, type LeaseTransaction, @@ -536,8 +536,7 @@ async function withMemberLease( fn: (tx: DbOrTx) => Promise, options: { aclPage?: boolean } = {} ): Promise { - return db.transaction(async (tx) => { - if (options.aclPage) await boundLeaseTransaction(tx) + const body = async (tx: DbOrTx) => { const written = await fn(tx) const [held] = await tx .select({ id: knowledgeConnector.id }) @@ -546,7 +545,8 @@ async function withMemberLease( .for('update') if (!held) throw new SyncLockLostException(run.connectorId) return written - }) + } + return options.aclPage ? aclPageTransaction(body) : db.transaction(body) } async function acquireMemberSyncLock( diff --git a/apps/sim/lib/knowledge/connectors/sync-limits.ts b/apps/sim/lib/knowledge/connectors/sync-limits.ts index f4134be13c7..4c1adba9d35 100644 --- a/apps/sim/lib/knowledge/connectors/sync-limits.ts +++ b/apps/sim/lib/knowledge/connectors/sync-limits.ts @@ -164,14 +164,6 @@ export const SOURCE_CONTENT_ERROR = */ export const ACL_WRITE_BATCH_SIZE = 500 -/** - * Search projection rows one statement rewrites per projection table: a page of - * detachment releases, and the most chunk rows a page of ACL assignments may - * send through the projection trigger. A page always holds at least one - * document, so a document larger than this still makes progress alone. - */ -export const PROJECTION_ROW_BATCH_SIZE = 250 - /** * How long a connector-lease ACL page waits on any lock before it fails. The * connector row is locked last, so the wait is on document rows, which a @@ -187,7 +179,9 @@ export const LEASE_PAGE_STATEMENT_TIMEOUT_MS = 30_000 * document trigger that copies it onto every chunk's search projection rows, and * each of those rows is re-inserted into the vector index, so one statement costs * the chunks of every document in it rather than the documents. Kept small so a - * page of changed documents cannot outrun the statement timeout. Also the page + * page of changed documents cannot outrun the statement timeout; with + * `knowledge-async-projection` on, the trigger only marks the documents and the bound is the + * cleanup boundary noted at `pagesByProjectionRows` in `member-observations.ts`. Also the page * of the transactions that remove observations and rematerialise the ACLs they * decide together, which must commit as one and so cannot be split by rows. */ diff --git a/apps/sim/lib/knowledge/connectors/sync-lock.ts b/apps/sim/lib/knowledge/connectors/sync-lock.ts index 8d2ddc72dce..cefb67b1544 100644 --- a/apps/sim/lib/knowledge/connectors/sync-lock.ts +++ b/apps/sim/lib/knowledge/connectors/sync-lock.ts @@ -1,12 +1,15 @@ import { db } from '@sim/db' +import { DEFER_KNOWLEDGE_PROJECTION } from '@sim/db/knowledge-projection' import { knowledgeConnector } from '@sim/db/schema' import { and, eq, isNull, sql } from 'drizzle-orm' +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import type { DbOrTx } from '@/lib/db/types' import { LEASE_PAGE_LOCK_TIMEOUT_MS, LEASE_PAGE_STATEMENT_TIMEOUT_MS, SYNC_LOCK_HEARTBEAT_INTERVAL_MS, } from '@/lib/knowledge/connectors/sync-limits' +import { requestKnowledgeProjection } from '@/lib/knowledge/projection/enqueue' /** * Raised when a run discovers mid-flight that it no longer holds its sync lock. @@ -241,21 +244,36 @@ export async function assertSyncLeaseHeldInTx( } /** - * The bounds of a connector-lease ACL page. The `document` ACL trigger rewrites every filled - * search projection row of a document whose ACL is assigned, so a page that waits on a lock or - * runs long fails within the bounds and rolls back only itself. + * Runs `write` as one connector-lease ACL page: a short transaction of its own whose first + * statement sets its bounds, so a page that waits on a lock or runs long fails within them and + * rolls back only itself, and its projection mode. While `knowledge-async-projection` is on, the + * page's ACL writes only mark their documents and the knowledge projector rewrites their search + * projection rows; off, the `document` ACL trigger still rewrites every filled row in the page's + * own statement, which is what the row-bounded paging of these pages exists for. The flag is read + * before the transaction opens, and a projector pass is requested once it commits. Every page that + * assigns a connector document's ACL runs here, so this is the one place those writers choose the + * mode. */ -export async function boundLeaseTransaction(tx: Pick): Promise { - await tx.execute( - sql`SELECT set_config('lock_timeout', ${`${LEASE_PAGE_LOCK_TIMEOUT_MS}ms`}, true), set_config('statement_timeout', ${`${LEASE_PAGE_STATEMENT_TIMEOUT_MS}ms`}, true)` - ) +export async function aclPageTransaction( + write: (tx: DbOrTx) => Promise, + executor: Pick = db +): Promise { + const deferProjection = await isFeatureEnabled('knowledge-async-projection') + const written = await executor.transaction(async (tx) => { + await tx.execute( + sql`SELECT set_config('lock_timeout', ${`${LEASE_PAGE_LOCK_TIMEOUT_MS}ms`}, true), set_config('statement_timeout', ${`${LEASE_PAGE_STATEMENT_TIMEOUT_MS}ms`}, true)${deferProjection ? sql.raw(`, ${DEFER_KNOWLEDGE_PROJECTION}`) : sql``}` + ) + return write(tx) + }) + await requestKnowledgeProjection() + return written } /** Runs one bounded page of writes in a short transaction of its own. */ export type LeaseTransaction = (write: (tx: DbOrTx) => Promise) => Promise /** - * One short, bounded transaction per call that proves `lease` as its last statement, so a run + * One {@link aclPageTransaction} per call that proves `lease` as its last statement, so a run * that lost its lease writes nothing further: the proof fails and the page rolls back. Proving it * last keeps the connector row unlocked while the page waits on document rows, which a processing * commit may hold for its whole write, and the share lock it then takes keeps the reclaim from @@ -268,12 +286,11 @@ export function leaseTransaction( executor: Pick = db ): LeaseTransaction { return (write) => - executor.transaction(async (tx) => { - await boundLeaseTransaction(tx) + aclPageTransaction(async (tx) => { const written = await write(tx) if (lease) await assertSyncLeaseHeldInTx(tx, connectorId, lease) return written - }) + }, executor) } /** diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 390b6b801ee..8f20c88e0d1 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -1,4 +1,5 @@ import { db } from '@sim/db' +import { DEFER_KNOWLEDGE_PROJECTION } from '@sim/db/knowledge-projection' import { document, documentSecretProvenance, @@ -52,6 +53,7 @@ import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' import { env, envNumber } from '@/lib/core/config/env' import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/env-flags' +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import { isInsideTriggerRun } from '@/lib/core/config/trigger-runtime' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' @@ -164,6 +166,7 @@ import { } from '@/lib/knowledge/embedding-models' import { generateEmbeddings, type KbEmbeddingTarget } from '@/lib/knowledge/embeddings' import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' +import { requestKnowledgeProjection } from '@/lib/knowledge/projection/enqueue' import { type KnowledgeReadAccess, knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' import { bindKnowledgeDocumentFieldSecretProvenance, @@ -2008,9 +2011,17 @@ export async function processDocumentAsync( ) .limit(1) if (!sourceActive) return + /** + * While `knowledge-async-projection` is on, the chunks are committed with a mark on + * their document and no search projection rows; the knowledge projector writes those + * after the commit. Read before the transaction opens. + */ + const deferProjection = await isFeatureEnabled('knowledge-async-projection') processingCommitted = await db .transaction(async (tx) => { signal.throwIfAborted() + if (deferProjection) + await tx.execute(sql.raw(`SELECT ${DEFER_KNOWLEDGE_PROJECTION}`)) /** * Reads only the document row. Connector activity is checked by * the completion write at the end instead: reading @@ -2146,6 +2157,8 @@ export async function processDocumentAsync( logger.info(`[${documentId}] Discarded output from an obsolete processing attempt`) return { outcome: 'skipped', reason: 'superseded' } } + /** The commit marked the document in either mode; a pass writes or verifies its rows. */ + await requestKnowledgeProjection() const processingTime = Date.now() - startTime logger.info(`[${documentId}] Successfully processed document in ${processingTime}ms`) diff --git a/apps/sim/lib/knowledge/projection/enqueue-inline.test.ts b/apps/sim/lib/knowledge/projection/enqueue-inline.test.ts new file mode 100644 index 00000000000..607b1c950e1 --- /dev/null +++ b/apps/sim/lib/knowledge/projection/enqueue-inline.test.ts @@ -0,0 +1,61 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ runPass: vi.fn(), trigger: vi.fn() })) + +vi.mock('@sim/db', () => ({ db: { execute: async () => [{ pending: true }] } })) +vi.mock('@/lib/core/config/feature-flags', () => ({ isFeatureEnabled: async () => false })) + +vi.mock('@trigger.dev/sdk', () => ({ tasks: { trigger: mocks.trigger } })) +vi.mock('@/lib/core/config/env-flags', () => ({ isTriggerDevEnabled: false })) +vi.mock('@/lib/knowledge/projection/run', () => ({ runKnowledgeProjectionPass: mocks.runPass })) + +import { + enqueueKnowledgeProjectionSweep, + requestKnowledgeProjection, +} from '@/lib/knowledge/projection/enqueue' + +/** A pass that runs until the test finishes it. */ +function heldPass() { + let finish: (error?: Error) => void = () => {} + mocks.runPass.mockImplementationOnce( + () => + new Promise((resolve, reject) => { + finish = (error) => (error ? reject(error) : resolve()) + }) + ) + return (error?: Error) => finish(error) +} + +describe('knowledge projection without a Trigger.dev worker', () => { + it('starts a pass without waiting for it, one at a time, folding requests into the next', async () => { + const finishFirst = heldPass() + await expect(requestKnowledgeProjection()).resolves.toBeUndefined() + await vi.waitFor(() => expect(mocks.runPass).toHaveBeenCalledTimes(1)) + await requestKnowledgeProjection() + await expect(enqueueKnowledgeProjectionSweep()).resolves.toEqual({ + triggered: true, + backend: 'inline', + jobId: null, + }) + expect(mocks.runPass).toHaveBeenCalledTimes(1) + const finishSecond = heldPass() + finishFirst() + await vi.waitFor(() => expect(mocks.runPass).toHaveBeenCalledTimes(2)) + finishSecond() + expect(mocks.trigger).not.toHaveBeenCalled() + }) + + it('logs a failed pass and still runs the pass owed after it', async () => { + mocks.runPass.mockReset() + const failFirst = heldPass() + mocks.runPass.mockResolvedValue(undefined) + await requestKnowledgeProjection() + await vi.waitFor(() => expect(mocks.runPass).toHaveBeenCalledTimes(1)) + await requestKnowledgeProjection() + failFirst(new Error('database unavailable')) + await vi.waitFor(() => expect(mocks.runPass).toHaveBeenCalledTimes(2)) + }) +}) diff --git a/apps/sim/lib/knowledge/projection/enqueue.test.ts b/apps/sim/lib/knowledge/projection/enqueue.test.ts new file mode 100644 index 00000000000..60c8886ade1 --- /dev/null +++ b/apps/sim/lib/knowledge/projection/enqueue.test.ts @@ -0,0 +1,98 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + runPass: vi.fn(), + resolveRegion: vi.fn(), + trigger: vi.fn(), + execute: vi.fn(), + isFeatureEnabled: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ db: { execute: mocks.execute } })) +vi.mock('@/lib/core/config/feature-flags', () => ({ isFeatureEnabled: mocks.isFeatureEnabled })) + +vi.mock('@trigger.dev/sdk', () => ({ tasks: { trigger: mocks.trigger } })) +vi.mock('@/lib/core/async-jobs/region', () => ({ resolveTriggerRegion: mocks.resolveRegion })) +vi.mock('@/lib/core/config/env-flags', () => ({ isTriggerDevEnabled: true })) +vi.mock('@/lib/knowledge/projection/run', () => ({ runKnowledgeProjectionPass: mocks.runPass })) + +import { + enqueueKnowledgeProjectionSweep, + requestKnowledgeProjection, +} from '@/lib/knowledge/projection/enqueue' + +describe('knowledge projection enqueue', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-09-23T12:34:45.000Z')) + mocks.resolveRegion.mockResolvedValue('us-east-1') + mocks.trigger.mockResolvedValue({ id: 'run-1' }) + mocks.execute.mockResolvedValue([{ pending: true }]) + mocks.isFeatureEnabled.mockResolvedValue(false) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('enqueues one sweep pass per window and never runs the pass inline', async () => { + await expect(enqueueKnowledgeProjectionSweep()).resolves.toEqual({ + triggered: true, + backend: 'trigger-dev', + jobId: 'run-1', + }) + expect(mocks.trigger).toHaveBeenCalledWith('knowledge-projection', undefined, { + idempotencyKey: 'knowledge-projection:sweep:29836114', + idempotencyKeyTTL: '5m', + region: 'us-east-1', + ttl: '5m', + }) + expect(mocks.runPass).not.toHaveBeenCalled() + }) + + it('starts no sweep pass when nothing is marked or left to fill', async () => { + mocks.execute.mockResolvedValue([{ pending: false }]) + await expect(enqueueKnowledgeProjectionSweep()).resolves.toEqual({ + triggered: false, + backend: null, + jobId: null, + }) + expect(mocks.trigger).not.toHaveBeenCalled() + }) + + it('probes for unfilled rows only while the fill is on', async () => { + const probed = async () => { + mocks.execute.mockClear() + await enqueueKnowledgeProjectionSweep() + return JSON.stringify(mocks.execute.mock.calls[0]?.[0]) + } + expect(await probed()).not.toContain('acl IS NULL') + mocks.isFeatureEnabled.mockResolvedValue(true) + const withFill = await probed() + expect(withFill).toContain('embedding_search WHERE acl IS NULL') + expect(withFill).toContain('embedding_keyword_tin WHERE acl IS NULL') + }) + + it('debounces prompt requests across processes and collapses them within one', async () => { + await requestKnowledgeProjection() + await requestKnowledgeProjection() + expect(mocks.trigger).toHaveBeenCalledTimes(1) + expect(mocks.trigger).toHaveBeenCalledWith('knowledge-projection', undefined, { + debounce: { key: 'knowledge-projection', delay: '5s', maxDelay: '1m' }, + region: 'us-east-1', + }) + vi.advanceTimersByTime(5_000) + await requestKnowledgeProjection() + expect(mocks.trigger).toHaveBeenCalledTimes(2) + }) + + it('never fails the write that asked when the request is refused', async () => { + vi.advanceTimersByTime(60_000) + mocks.trigger.mockRejectedValueOnce(new Error('trigger unavailable')) + await expect(requestKnowledgeProjection()).resolves.toBeUndefined() + }) +}) diff --git a/apps/sim/lib/knowledge/projection/enqueue.ts b/apps/sim/lib/knowledge/projection/enqueue.ts new file mode 100644 index 00000000000..e34d420781f --- /dev/null +++ b/apps/sim/lib/knowledge/projection/enqueue.ts @@ -0,0 +1,144 @@ +import { db } from '@sim/db' +import { SOURCE_ACL_PROJECTIONS } from '@sim/db/knowledge-projection' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { sql } from 'drizzle-orm' +import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' + +const logger = createLogger('KnowledgeProjectionEnqueue') + +export const KNOWLEDGE_PROJECTION_TASK_ID = 'knowledge-projection' + +/** + * How long one pass starts pages before it hands the rest to the next. The task's duration leaves + * minutes past it: a page started at the budget runs to its own statement timeout, and closing the + * pass's connections follows. + */ +export const KNOWLEDGE_PROJECTION_PASS_BUDGET_MS = 8 * 60 * 1000 + +/** The periodic sweep's window: at most one sweep run is enqueued per window. */ +const KNOWLEDGE_PROJECTION_SWEEP_INTERVAL_MS = 60 * 1000 + +/** + * A prompt request waits this long for more writes before its run starts, and is never pushed + * back past the maximum, so a steady stream of writes still gets a run each window. + */ +const PROMPT_DEBOUNCE = { key: KNOWLEDGE_PROJECTION_TASK_ID, delay: '5s', maxDelay: '1m' } as const + +/** Trigger.dev requests from one process closer together than this collapse into the first. */ +const PROMPT_REQUEST_INTERVAL_MS = 5_000 + +let lastPromptAt = 0 + +/** The inline pass in flight when no Trigger.dev worker is configured, and whether another is owed. */ +let inlinePass: Promise | undefined +let inlinePassOwed = false + +/** + * Starts a pass in this process without waiting for it: at most one runs at a time, a request while + * one runs is folded into a single pass after it, and a failed pass is logged without dropping one + * owed after it. The + * pass module loads on first use. For deployments without a Trigger.dev worker, whose sweep and + * writes run passes here, as their document processing does. + */ +function runInline(): void { + if (inlinePass) { + inlinePassOwed = true + return + } + inlinePass = (async () => { + do { + inlinePassOwed = false + try { + const { runKnowledgeProjectionPass } = await import('@/lib/knowledge/projection/run') + await runKnowledgeProjectionPass({ budgetMs: KNOWLEDGE_PROJECTION_PASS_BUDGET_MS }) + } catch (error) { + logger.error('Inline knowledge projection pass failed', { error: getErrorMessage(error) }) + } + } while (inlinePassOwed) + })().finally(() => { + inlinePass = undefined + }) +} + +/** + * Asks for a projector pass soon after a knowledge write commits, so its marked documents are + * converged within seconds rather than at the next sweep. Debounced twice: in this process, and + * across processes by the task's debounce key. Without a Trigger.dev worker the pass runs in this + * process, one at a time, as document processing does there. Never throws: a request that fails + * leaves the marks to the sweep, which keeps enqueueing a pass every minute until one runs. + */ +export async function requestKnowledgeProjection(): Promise { + if (!isTriggerDevEnabled) { + runInline() + return + } + const now = Date.now() + if (now - lastPromptAt < PROMPT_REQUEST_INTERVAL_MS) return + lastPromptAt = now + try { + const [{ tasks }, { resolveTriggerRegion }] = await Promise.all([ + import('@trigger.dev/sdk'), + import('@/lib/core/async-jobs/region'), + ]) + await tasks.trigger(KNOWLEDGE_PROJECTION_TASK_ID, undefined, { + debounce: PROMPT_DEBOUNCE, + region: await resolveTriggerRegion(), + }) + } catch (error) { + logger.warn('Knowledge projection request failed; the sweep will pick the marks up', { + error: getErrorMessage(error), + }) + } +} + +export interface KnowledgeProjectionSweepResult { + /** Whether a pass was started; the sweep starts none when nothing is marked or left to fill. */ + triggered: boolean + backend: 'trigger-dev' | 'inline' | null + jobId: string | null +} + +/** + * Whether a pass would find anything to do: a marked document, or, while the fill is on, a + * projection row it has not reached. Each is one probe of an index that is empty once the + * projector has caught up. + */ +async function hasKnowledgeProjectionWork(): Promise { + const fill = await isFeatureEnabled('knowledge-projection-fill') + const unfilled = SOURCE_ACL_PROJECTIONS.map( + (projection) => `EXISTS (SELECT 1 FROM ${projection} WHERE acl IS NULL)` + ).join(' OR ') + const [row] = await db.execute<{ pending: boolean }>( + sql`SELECT EXISTS (SELECT 1 FROM knowledge_projection_dirty)${fill ? sql.raw(` OR ${unfilled}`) : sql``} AS pending` + ) + return Boolean(row?.pending) +} + +/** + * The periodic sweep behind the prompt requests: one pass per window while there is work, so a + * mark whose request was lost, or a document a pass gave up, is still converged, and an idle + * deployment starts no pass at all. + */ +export async function enqueueKnowledgeProjectionSweep(): Promise { + if (!(await hasKnowledgeProjectionWork())) { + return { triggered: false, backend: null, jobId: null } + } + if (!isTriggerDevEnabled) { + runInline() + return { triggered: true, backend: 'inline', jobId: null } + } + const [{ tasks }, { resolveTriggerRegion }] = await Promise.all([ + import('@trigger.dev/sdk'), + import('@/lib/core/async-jobs/region'), + ]) + const window = Math.floor(Date.now() / KNOWLEDGE_PROJECTION_SWEEP_INTERVAL_MS) + const handle = await tasks.trigger(KNOWLEDGE_PROJECTION_TASK_ID, undefined, { + idempotencyKey: `${KNOWLEDGE_PROJECTION_TASK_ID}:sweep:${window}`, + idempotencyKeyTTL: '5m', + region: await resolveTriggerRegion(), + ttl: '5m', + }) + return { triggered: true, backend: 'trigger-dev', jobId: handle.id } +} diff --git a/apps/sim/lib/knowledge/projection/run.test.ts b/apps/sim/lib/knowledge/projection/run.test.ts new file mode 100644 index 00000000000..9628b237312 --- /dev/null +++ b/apps/sim/lib/knowledge/projection/run.test.ts @@ -0,0 +1,126 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + runProjection: vi.fn(), + markUnfilled: vi.fn(), + isFeatureEnabled: vi.fn(), + end: vi.fn(), + marks: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ resolveDbUrl: () => 'postgresql://fixture/sim_acl_test' })) +vi.mock('@sim/db/knowledge-projection', () => ({ + runKnowledgeProjection: mocks.runProjection, + markUnfilledProjectionDocuments: mocks.markUnfilled, +})) +/** A session answering the backlog count; the projection itself is mocked above. */ +vi.mock('postgres', () => ({ + default: () => Object.assign(async () => [{ marks: mocks.marks() }], { end: mocks.end }), +})) +vi.mock('@/lib/core/config/env', () => ({ + env: { KB_CONFIG_PROJECTION_CONCURRENCY: 2 }, + envNumber: (value: unknown, fallback: number) => (typeof value === 'number' ? value : fallback), +})) +vi.mock('@/lib/core/config/feature-flags', () => ({ isFeatureEnabled: mocks.isFeatureEnabled })) + +import { runKnowledgeProjectionPass } from '@/lib/knowledge/projection/run' + +const drained = { settled: 1, deferred: 0, pages: 2, written: 3, remaining: false } + +describe('runKnowledgeProjectionPass', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.runProjection.mockResolvedValue(drained) + mocks.isFeatureEnabled.mockResolvedValue(false) + mocks.marks.mockReturnValue(2) + }) + + it('projects with one worker per connection and closes them all', async () => { + await expect(runKnowledgeProjectionPass({ budgetMs: 60_000 })).resolves.toMatchObject({ + settled: 2, + written: 6, + filled: 0, + remaining: false, + }) + expect(mocks.runProjection).toHaveBeenCalledTimes(2) + expect(mocks.isFeatureEnabled).toHaveBeenCalledWith('knowledge-projection-fill') + expect(mocks.markUnfilled).not.toHaveBeenCalled() + expect(mocks.end).toHaveBeenCalledTimes(2) + }) + + it('runs another round while workers still settle documents, and stops once none can', async () => { + mocks.runProjection + .mockResolvedValueOnce({ ...drained, remaining: true }) + .mockResolvedValueOnce(drained) + .mockResolvedValueOnce({ ...drained, settled: 0, deferred: 1, remaining: true }) + .mockResolvedValueOnce({ ...drained, settled: 0, remaining: false }) + await expect(runKnowledgeProjectionPass({ budgetMs: 60_000 })).resolves.toMatchObject({ + settled: 2, + deferred: 1, + remaining: true, + }) + expect(mocks.runProjection).toHaveBeenCalledTimes(4) + }) + + it('opens only as many workers as there are marks', async () => { + mocks.marks.mockReturnValue(1) + await expect(runKnowledgeProjectionPass({ budgetMs: 60_000 })).resolves.toMatchObject({ + settled: 1, + }) + expect(mocks.runProjection).toHaveBeenCalledTimes(1) + }) + + it('lets every worker finish its document before a failure ends the pass', async () => { + let finishOther: () => void = () => {} + mocks.runProjection.mockRejectedValueOnce(new Error('connection lost')).mockImplementationOnce( + () => + new Promise((resolve) => { + finishOther = () => resolve(drained) + }) + ) + let settled = false + const pass = runKnowledgeProjectionPass({ budgetMs: 60_000 }).finally(() => { + settled = true + }) + await vi.waitFor(() => expect(mocks.runProjection).toHaveBeenCalledTimes(2)) + await Promise.resolve() + expect(settled).toBe(false) + finishOther() + await expect(pass).rejects.toThrow('connection lost') + }) + + it('marks unfilled documents and converges them until none are left', async () => { + mocks.isFeatureEnabled.mockResolvedValue(true) + mocks.markUnfilled + .mockResolvedValueOnce({ marked: 2, cursor: { projection: 0, afterId: 'row-2' } }) + .mockResolvedValueOnce({ marked: 0, cursor: null }) + const result = await runKnowledgeProjectionPass({ budgetMs: 60_000 }) + expect(result).toMatchObject({ filled: 2, remaining: false }) + expect(mocks.markUnfilled).toHaveBeenNthCalledWith(1, expect.anything(), undefined) + expect(mocks.markUnfilled).toHaveBeenNthCalledWith(2, expect.anything(), { + projection: 0, + afterId: 'row-2', + }) + }) + + it('does not fill while marks remain, so writers are converged first', async () => { + mocks.isFeatureEnabled.mockResolvedValue(true) + mocks.runProjection.mockResolvedValue({ ...drained, settled: 0, remaining: true }) + await expect(runKnowledgeProjectionPass({ budgetMs: 60_000 })).resolves.toMatchObject({ + remaining: true, + filled: 0, + }) + expect(mocks.markUnfilled).not.toHaveBeenCalled() + }) + + it('closes its connections when a pass fails', async () => { + mocks.runProjection.mockRejectedValue(new Error('connection lost')) + await expect(runKnowledgeProjectionPass({ budgetMs: 60_000 })).rejects.toThrow( + 'connection lost' + ) + expect(mocks.end).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/sim/lib/knowledge/projection/run.ts b/apps/sim/lib/knowledge/projection/run.ts new file mode 100644 index 00000000000..9f7fc12fdfc --- /dev/null +++ b/apps/sim/lib/knowledge/projection/run.ts @@ -0,0 +1,120 @@ +import { resolveDbUrl } from '@sim/db' +import { + type KnowledgeProjectionProgress, + markUnfilledProjectionDocuments, + runKnowledgeProjection, +} from '@sim/db/knowledge-projection' +import { withUtcTimestamps } from '@sim/db/timestamps' +import { createLogger } from '@sim/logger' +import postgres, { type Sql } from 'postgres' +import { env, envNumber } from '@/lib/core/config/env' +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' + +const logger = createLogger('KnowledgeProjectionPass') + +/** + * Most documents one pass projects at once, each worker on a connection of its own. Measured + * locally on content-heavy load, projection throughput kept rising to eight workers without + * deadlocks, so eight is the default; `KB_CONFIG_PROJECTION_CONCURRENCY` raises or lowers it per + * deployment. A round opens only as many workers as there are marks, so a pass over a few + * documents holds a few connections. + */ +const PROJECTION_CONCURRENCY = envNumber(env.KB_CONFIG_PROJECTION_CONCURRENCY, 8, { + min: 1, + integer: true, +}) + +/** How the projector's connections name themselves in `pg_stat_activity`. */ +const PROJECTOR_APPLICATION_NAME = 'sim-knowledge-projector' + +/** Workers the next round needs: one per mark, counted no further than the concurrency. */ +async function workersFor(session: Sql): Promise { + const [{ marks }] = await session>` + SELECT count(*)::int AS marks + FROM (SELECT 1 FROM knowledge_projection_dirty LIMIT ${PROJECTION_CONCURRENCY}) AS waiting` + return Math.max(1, marks) +} + +export interface KnowledgeProjectionPassResult extends KnowledgeProjectionProgress { + /** Documents the source and ACL fill marked during the pass. */ + filled: number +} + +/** + * One pass of the knowledge projector: converges every marked document, then, while time is + * left and the fill is on, marks the documents of projection rows the source and ACL fill has not + * reached and converges those too, keeping the outstanding marks under the fill's ceiling so fresh + * writes never queue behind much of it and search keeps probing a small set of marks. The fill is + * its own flag so an operator can pause its extra index writes. + * + * Workers project documents in parallel, each on a connection of its own that holds its + * per-document advisory locks; a pass this long should not hold the pool's connections. Workers + * read the same oldest marks and split them at those locks. A round ends when every worker found + * nothing more it could take; the pass goes on while rounds settle documents, and `remaining` + * reports marks it left, so the caller can schedule another. + */ +export async function runKnowledgeProjectionPass(options: { + budgetMs: number +}): Promise { + const url = resolveDbUrl('DATABASE_URL', process.env.SIM_DB_ROLE?.trim() || 'web') + if (!url) throw new Error('DATABASE_URL is required to run the knowledge projector') + /** Each connects on its first query, so a worker a round never needs opens no connection. */ + const sessions = Array.from({ length: PROJECTION_CONCURRENCY }, () => + postgres( + url, + withUtcTimestamps({ + max: 1, + max_lifetime: null, + onnotice: () => undefined, + connection: { application_name: PROJECTOR_APPLICATION_NAME }, + }) + ) + ) + const deadline = Date.now() + options.budgetMs + const result: KnowledgeProjectionPassResult = { + settled: 0, + deferred: 0, + pages: 0, + written: 0, + remaining: false, + filled: 0, + } + try { + /** `null` ends the fill: it is off, or every unfilled row has been read. */ + let fillCursor: Parameters[1] | null = + (await isFeatureEnabled('knowledge-projection-fill')) ? undefined : null + while (Date.now() < deadline) { + const workers = sessions.slice(0, await workersFor(sessions[0])) + /** Every worker finishes its document before a failure ends the pass, so none is cut off. */ + const outcomes = await Promise.allSettled( + workers.map((session) => + runKnowledgeProjection(session, { budgetMs: Math.max(0, deadline - Date.now()) }) + ) + ) + const round: KnowledgeProjectionProgress[] = [] + for (const outcome of outcomes) { + if (outcome.status === 'rejected') throw outcome.reason + round.push(outcome.value) + } + const settled = round.reduce((sum, progress) => sum + progress.settled, 0) + result.settled += settled + result.pages += round.reduce((sum, progress) => sum + progress.pages, 0) + result.written += round.reduce((sum, progress) => sum + progress.written, 0) + result.remaining = round.some((progress) => progress.remaining) + if (result.remaining) { + if (settled > 0) continue + result.deferred = round.reduce((most, progress) => Math.max(most, progress.deferred), 0) + break + } + if (fillCursor === null) break + const fill = await markUnfilledProjectionDocuments(sessions[0], fillCursor) + result.filled += fill.marked + fillCursor = fill.cursor + if (fill.marked === 0 && fillCursor === null) break + } + logger.info('Knowledge projection pass finished', result) + return result + } finally { + await Promise.all(sessions.map((session) => session.end())) + } +} diff --git a/apps/sim/lib/knowledge/search/prewarm.test.ts b/apps/sim/lib/knowledge/search/prewarm.test.ts index 31752e8288d..1699ec5608b 100644 --- a/apps/sim/lib/knowledge/search/prewarm.test.ts +++ b/apps/sim/lib/knowledge/search/prewarm.test.ts @@ -3,8 +3,8 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -vi.mock('@sim/db/script-migrations/0021_embedding_search_connector', () => ({ - PROJECTION_SOURCE_ACL_TABLES: ['embedding_search', 'embedding_keyword_tin'], +vi.mock('@sim/db/knowledge-projection', () => ({ + SOURCE_ACL_PROJECTIONS: ['embedding_search', 'embedding_keyword_tin'], })) import { diff --git a/apps/sim/lib/knowledge/search/prewarm.ts b/apps/sim/lib/knowledge/search/prewarm.ts index a8d46c13421..239907f1b13 100644 --- a/apps/sim/lib/knowledge/search/prewarm.ts +++ b/apps/sim/lib/knowledge/search/prewarm.ts @@ -1,4 +1,4 @@ -import { PROJECTION_SOURCE_ACL_TABLES } from '@sim/db/script-migrations/0021_embedding_search_connector' +import { SOURCE_ACL_PROJECTIONS } from '@sim/db/knowledge-projection' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' @@ -143,7 +143,7 @@ async function rankingRelations(session: PrewarmSession): Promise { AND am.amname = ANY($2::text[]) ) ORDER BY c.relkind = 'r' DESC, pg_relation_size(c.oid)`, - [toArrayLiteral(PROJECTION_SOURCE_ACL_TABLES), toArrayLiteral(RANKING_ACCESS_METHODS)] + [toArrayLiteral(SOURCE_ACL_PROJECTIONS), toArrayLiteral(RANKING_ACCESS_METHODS)] ) return Array.from(rows, (row) => String(row.relation)) } diff --git a/apps/sim/lib/knowledge/search/projection-source-acl-backfill.test.ts b/apps/sim/lib/knowledge/search/projection-source-acl-backfill.test.ts deleted file mode 100644 index 4e0bb9a7d46..00000000000 --- a/apps/sim/lib/knowledge/search/projection-source-acl-backfill.test.ts +++ /dev/null @@ -1,296 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockBackfill, - mockEnd, - mockPostgres, - mockPrewarm, - mockRunsList, - mockTasksTrigger, - mockUnsafe, -} = vi.hoisted(() => ({ - mockBackfill: vi.fn(), - mockEnd: vi.fn(async () => undefined), - mockPostgres: vi.fn(), - mockPrewarm: vi.fn(async () => []), - mockRunsList: vi.fn( - (_query: unknown): AsyncIterable<{ id: string; status: string }> => (async function* () {})() - ), - mockTasksTrigger: vi.fn(async () => ({ id: 'run-1' })), - mockUnsafe: vi.fn(async () => [{ unfilled: false }]), -})) - -vi.mock('@sim/db', () => ({ resolveDbUrl: () => 'postgres://localhost:5432/sim' })) -vi.mock('@sim/db/script-migrations/0021_embedding_search_connector', () => ({ - PROJECTION_SOURCE_ACL_TABLES: ['embedding_search', 'embedding_keyword_tin'], - backfillProjectionSourceAcl: mockBackfill, -})) -vi.mock('postgres', () => ({ default: mockPostgres })) -vi.mock('@/lib/knowledge/search/prewarm', () => ({ prewarmSearchProjection: mockPrewarm })) -vi.mock('@trigger.dev/sdk', () => ({ - runs: { list: mockRunsList }, - tasks: { trigger: mockTasksTrigger }, -})) -vi.mock('@/lib/core/async-jobs/region', () => ({ resolveTriggerRegion: async () => 'us-east-1' })) -vi.mock('@/lib/core/utils/background', () => ({ - runDetached: (_label: string, work: () => Promise) => { - void work() - }, -})) - -import { - enqueueProjectionSourceAclBackfill, - PROJECTION_PREWARM_BUDGET_MS, - projectionSourceAclShardRange, - runProjectionSourceAclBackfill, -} from '@/lib/knowledge/search/projection-source-acl-backfill' - -const connection = { end: mockEnd, unsafe: mockUnsafe } - -describe('runProjectionSourceAclBackfill', () => { - beforeEach(() => { - vi.clearAllMocks() - mockPostgres.mockReturnValue(connection) - mockBackfill.mockImplementation(async (_sql, projection, options) => ({ - projection, - scanned: 0, - written: 0, - afterId: options?.afterId ?? '', - done: true, - })) - }) - - it('fills both projections in order on its own connection and closes it', async () => { - await expect(runProjectionSourceAclBackfill({ pageSize: 50, pauseMs: 10 })).resolves.toBeNull() - expect(mockBackfill.mock.calls.map(([, projection]) => projection)).toEqual([ - 'embedding_search', - 'embedding_keyword_tin', - ]) - for (const [sql, , options] of mockBackfill.mock.calls) { - expect(sql).toBe(connection) - expect(options).toMatchObject({ afterId: undefined, pageSize: 50, pauseMs: 10 }) - } - expect(mockEnd).toHaveBeenCalledTimes(1) - }) - - it('analyzes and warms the projections on the same connection once both are filled, before closing it', async () => { - await runProjectionSourceAclBackfill({}) - /** - * A row whose document is gone is not the fill's to finish; the probe joins the document. It is - * ordered and capped so only the unfilled-rows index can serve it: an `EXISTS` drops both and - * leaves the planner a sequential scan of the projection. - */ - const probes = mockUnsafe.mock.calls - .map(([query]) => String(query).replace(/\s+/g, ' ')) - .filter((query) => query.includes('AS unfilled')) - expect(probes).toHaveLength(2) - for (const probe of probes) { - expect(probe).not.toContain('EXISTS') - expect(probe).toContain( - 'JOIN document d ON d.id = s.document_id WHERE s.acl IS NULL ORDER BY s.id DESC LIMIT 1 ) IS NOT NULL AS unfilled' - ) - } - expect(mockUnsafe.mock.calls.map(([query]) => query)).toEqual( - expect.arrayContaining(['ANALYZE embedding_search', 'ANALYZE embedding_keyword_tin']) - ) - expect(mockPrewarm).toHaveBeenCalledTimes(1) - expect(mockPrewarm).toHaveBeenCalledWith(connection, { budgetMs: PROJECTION_PREWARM_BUDGET_MS }) - expect(mockPrewarm.mock.invocationCallOrder[0]).toBeLessThan( - mockEnd.mock.invocationCallOrder[0] - ) - }) - - it('resumes after the cursor in its projection and from the start of the next', async () => { - await runProjectionSourceAclBackfill({ - cursor: { projection: 'embedding_keyword_tin', afterId: 'chunk-9' }, - }) - expect(mockBackfill).toHaveBeenCalledTimes(1) - expect(mockBackfill.mock.calls[0][1]).toBe('embedding_keyword_tin') - expect(mockBackfill.mock.calls[0][2]).toMatchObject({ afterId: 'chunk-9' }) - }) - - it('returns where a budgeted run stopped so the next run can carry on', async () => { - mockBackfill.mockResolvedValueOnce({ - projection: 'embedding_search', - scanned: 100, - written: 100, - afterId: 'chunk-100', - done: false, - }) - await expect(runProjectionSourceAclBackfill({}, { budgetMs: 1000 })).resolves.toEqual({ - projection: 'embedding_search', - afterId: 'chunk-100', - }) - expect(mockBackfill).toHaveBeenCalledTimes(1) - expect(mockBackfill.mock.calls[0][2].budgetMs).toBeLessThanOrEqual(1000) - expect(mockPrewarm).not.toHaveBeenCalled() - expect(mockEnd).toHaveBeenCalledTimes(1) - }) - - it('closes the connection when a page fails', async () => { - mockBackfill.mockRejectedValueOnce(new Error('canceling statement due to statement timeout')) - await expect(runProjectionSourceAclBackfill({})).rejects.toThrow('statement timeout') - expect(mockEnd).toHaveBeenCalledTimes(1) - }) - - it('fills only its shard of the id space in both projections', async () => { - await runProjectionSourceAclBackfill({ shard: { index: 1, count: 4 } }) - for (const [, , options] of mockBackfill.mock.calls) { - expect(options).toMatchObject({ afterId: '4', beforeId: '8' }) - } - }) - - it('resumes a shard after its cursor and keeps its upper bound', async () => { - await runProjectionSourceAclBackfill({ - shard: { index: 1, count: 4 }, - cursor: { projection: 'embedding_search', afterId: '5a' }, - }) - expect(mockBackfill.mock.calls[0][2]).toMatchObject({ afterId: '5a', beforeId: '8' }) - expect(mockBackfill.mock.calls[1][2]).toMatchObject({ afterId: '4', beforeId: '8' }) - }) - - it('leaves the analysis and the warm to whoever fills the rows another shard still holds', async () => { - mockUnsafe.mockResolvedValueOnce([{ unfilled: true }]) - await expect( - runProjectionSourceAclBackfill({ shard: { index: 0, count: 4 } }) - ).resolves.toBeNull() - expect(mockUnsafe.mock.calls.map(([query]) => query)).not.toContain('ANALYZE embedding_search') - expect(mockPrewarm).not.toHaveBeenCalled() - expect(mockEnd).toHaveBeenCalledTimes(1) - }) -}) - -describe('projectionSourceAclShardRange', () => { - it('slices the hex id space into contiguous ranges', () => { - expect(projectionSourceAclShardRange({ index: 0, count: 4 })).toEqual({ - afterId: '', - beforeId: '4', - }) - expect(projectionSourceAclShardRange({ index: 3, count: 4 })).toEqual({ - afterId: 'c', - beforeId: undefined, - }) - expect(projectionSourceAclShardRange({ index: 0, count: 1 })).toEqual({ - afterId: '', - beforeId: undefined, - }) - }) - - it.each([ - [{ index: 0, count: 3 }, 'shard count must divide 16'], - [{ index: 0, count: 8 }, 'shard count must be at most 4'], - [{ index: 4, count: 4 }, 'shard index must be within 0..3'], - [{ index: 0.5, count: 2 }, 'shard index must be within 0..1'], - ])('refuses %j', (shard, message) => { - expect(() => projectionSourceAclShardRange(shard)).toThrow(message) - }) -}) - -describe('enqueueProjectionSourceAclBackfill', () => { - beforeEach(() => { - vi.clearAllMocks() - /** No chain in flight unless a case says so. */ - mockRunsList.mockImplementation(() => (async function* () {})()) - mockPostgres.mockReturnValue(connection) - mockBackfill.mockResolvedValue({ - projection: 'embedding_search', - scanned: 0, - written: 0, - afterId: '', - done: true, - }) - }) - - it('hands the backfill to the Trigger.dev worker when one is configured', async () => { - await expect(enqueueProjectionSourceAclBackfill({ pageSize: 25 })).resolves.toEqual({ - runIds: ['run-1'], - inFlight: [], - }) - expect(mockRunsList).toHaveBeenCalledWith( - expect.objectContaining({ tag: 'projection-source-acl-backfill:shard:0/1' }) - ) - expect(mockTasksTrigger).toHaveBeenCalledWith( - 'projection-source-acl-backfill', - { pageSize: 25 }, - { - region: 'us-east-1', - tags: ['projection-source-acl-backfill:shard:0/1'], - idempotencyKey: 'projection-source-acl-backfill:shard:0/1:after:none', - idempotencyKeyTTL: '2m', - } - ) - expect(mockBackfill).not.toHaveBeenCalled() - }) - - it('keys a start after a chain that ended on that chain, so a restart is its own start', async () => { - mockRunsList.mockImplementation(() => - (async function* () { - yield { id: 'run-done', status: 'COMPLETED' } - })() - ) - await expect(enqueueProjectionSourceAclBackfill({})).resolves.toEqual({ - runIds: ['run-1'], - inFlight: [], - }) - expect(mockTasksTrigger.mock.calls[0][2].idempotencyKey).toBe( - 'projection-source-acl-backfill:shard:0/1:after:run-done' - ) - }) - - it('refuses a shard the id space cannot be sliced into before starting anything', async () => { - await expect( - enqueueProjectionSourceAclBackfill({ shard: { index: 5, count: 4 } }) - ).rejects.toThrow('shard index must be within 0..3') - expect(mockTasksTrigger).not.toHaveBeenCalled() - }) - - it('leaves a range whose chain is still in flight to that chain', async () => { - mockRunsList.mockImplementation((query: unknown) => - (async function* () { - if ((query as { tag: string }).tag.endsWith(':shard:1/4')) - yield { id: 'run-live', status: 'EXECUTING' } - })() - ) - await expect(enqueueProjectionSourceAclBackfill({}, 4)).resolves.toEqual({ - runIds: ['run-1', 'run-1', 'run-1'], - inFlight: ['run-live'], - }) - expect(mockTasksTrigger.mock.calls.map(([, payload]) => payload.shard?.index)).toEqual([ - 0, 2, 3, - ]) - }) - - it('starts one run per shard, each on its own slice under its own chain tag', async () => { - await expect(enqueueProjectionSourceAclBackfill({ pageSize: 25 }, 4)).resolves.toEqual({ - runIds: ['run-1', 'run-1', 'run-1', 'run-1'], - inFlight: [], - }) - expect(mockTasksTrigger.mock.calls.map(([, payload]) => payload)).toEqual( - [0, 1, 2, 3].map((index) => ({ pageSize: 25, shard: { index, count: 4 } })) - ) - expect(mockTasksTrigger.mock.calls.map(([, , options]) => options.tags)).toEqual( - [0, 1, 2, 3].map((index) => [`projection-source-acl-backfill:shard:${index}/4`]) - ) - }) - - it.each([ - [3, 'must divide 16'], - [8, 'must be at most 4'], - ])('refuses %s shards before starting anything', async (shards, message) => { - await expect(enqueueProjectionSourceAclBackfill({}, shards)).rejects.toThrow(message) - expect(mockTasksTrigger).not.toHaveBeenCalled() - }) - - it('refuses to slice a start that carries a cursor, which belongs to one chain', async () => { - await expect( - enqueueProjectionSourceAclBackfill( - { cursor: { projection: 'embedding_search', afterId: '5a' } }, - 4 - ) - ).rejects.toThrow('cannot start from a cursor') - expect(mockTasksTrigger).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/knowledge/search/projection-source-acl-backfill.ts b/apps/sim/lib/knowledge/search/projection-source-acl-backfill.ts deleted file mode 100644 index 0e6162d246b..00000000000 --- a/apps/sim/lib/knowledge/search/projection-source-acl-backfill.ts +++ /dev/null @@ -1,244 +0,0 @@ -import { resolveDbUrl } from '@sim/db' -import { - backfillProjectionSourceAcl, - PROJECTION_SOURCE_ACL_TABLES, - type ProjectionSourceAclTable, -} from '@sim/db/script-migrations/0021_embedding_search_connector' -import { createLogger } from '@sim/logger' -import postgres from 'postgres' -import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' -import { prewarmSearchProjection } from '@/lib/knowledge/search/prewarm' - -const logger = createLogger('ProjectionSourceAclBackfill') - -export const PROJECTION_SOURCE_ACL_BACKFILL_TASK_ID = 'projection-source-acl-backfill' - -/** - * Ceiling on warming the projections after the fill. It sits inside the headroom the worker - * keeps beyond a run's fill budget, so a slow read of a large projection can never carry the - * completed run past the worker's limit and repeat the fill on retry. - */ -export const PROJECTION_PREWARM_BUDGET_MS = 15 * 60 * 1000 - -/** Where a run stopped, so the next one carries on from there instead of rescanning. */ -export interface ProjectionSourceAclBackfillCursor { - projection: ProjectionSourceAclTable - afterId: string -} - -/** - * One of `count` equal slices of the chunk id space. Chunk ids are lowercase hex UUIDs, so the - * space is sliced on the first hex digit and `count` must divide sixteen; every slice is a - * contiguous id range, so workers on different slices never fill the same page. - */ -export interface ProjectionSourceAclBackfillShard { - index: number - count: number -} - -/** - * Shards the id space may be sliced into at most: the runs the task's queue admits at once, so a - * sliced fill runs its slices together rather than in waves. - */ -export const PROJECTION_SOURCE_ACL_BACKFILL_SHARDS = 4 - -export interface ProjectionSourceAclBackfillPayload { - /** The first projection's first page when absent. */ - cursor?: ProjectionSourceAclBackfillCursor - /** The whole id space when absent. */ - shard?: ProjectionSourceAclBackfillShard - pageSize?: number - pauseMs?: number -} - -/** Refuses a shard the id space cannot be sliced into. */ -function assertProjectionSourceAclShard({ index, count }: ProjectionSourceAclBackfillShard): void { - if (!Number.isInteger(count) || count < 1 || 16 % count !== 0) { - throw new Error(`Projection backfill shard count must divide 16, got ${count}`) - } - if (count > PROJECTION_SOURCE_ACL_BACKFILL_SHARDS) { - throw new Error( - `Projection backfill shard count must be at most ${PROJECTION_SOURCE_ACL_BACKFILL_SHARDS}, got ${count}` - ) - } - if (!Number.isInteger(index) || index < 0 || index >= count) { - throw new Error(`Projection backfill shard index must be within 0..${count - 1}, got ${index}`) - } -} - -/** The id range a shard covers: the id its first page follows, and the first id past it. */ -export function projectionSourceAclShardRange(shard: ProjectionSourceAclBackfillShard): { - afterId: string - beforeId: string | undefined -} { - assertProjectionSourceAclShard(shard) - const { index, count } = shard - const width = 16 / count - const digit = (value: number) => value.toString(16) - return { - afterId: index === 0 ? '' : digit(index * width), - beforeId: index + 1 < count ? digit((index + 1) * width) : undefined, - } -} - -export interface ProjectionSourceAclBackfillRunOptions { - /** Stop once this much time has passed and return where to resume; unbounded otherwise. */ - budgetMs?: number -} - -/** - * Fills the ranking projections' source and ACL columns from the cursor onwards, one projection - * after the other, on a connection of its own: the page statement binds the keyset cursor as a - * scalar and needs no array parameter, so the pool's options would serve, but a run this long - * should not hold one of the worker's pooled connections. A shard fills its own slice of the id - * space in every projection. Returns the cursor to continue from when the budget ran out, `null` - * once the run's range is filled in both projections. - */ -export async function runProjectionSourceAclBackfill( - payload: ProjectionSourceAclBackfillPayload, - options: ProjectionSourceAclBackfillRunOptions = {} -): Promise { - const url = resolveDbUrl('DATABASE_URL', process.env.SIM_DB_ROLE?.trim() || 'web') - if (!url) throw new Error('DATABASE_URL is required to backfill the projection source and ACL') - const sql = postgres(url, { max: 1, max_lifetime: null, onnotice: () => undefined }) - const startedAt = Date.now() - try { - const start = payload.cursor - ? PROJECTION_SOURCE_ACL_TABLES.indexOf(payload.cursor.projection) - : 0 - if (start < 0) throw new Error(`Unknown projection ${payload.cursor?.projection}`) - const range = payload.shard ? projectionSourceAclShardRange(payload.shard) : undefined - for (const projection of PROJECTION_SOURCE_ACL_TABLES.slice(start)) { - const budgetMs = - options.budgetMs === undefined - ? undefined - : Math.max(0, options.budgetMs - (Date.now() - startedAt)) - const progress = await backfillProjectionSourceAcl(sql, projection, { - afterId: - payload.cursor?.projection === projection ? payload.cursor.afterId : range?.afterId, - beforeId: range?.beforeId, - pageSize: payload.pageSize, - pauseMs: payload.pauseMs, - budgetMs, - }) - if (!progress.done) return { projection, afterId: progress.afterId } - } - logger.info('Projection source and ACL backfill complete', { - shard: payload.shard, - elapsedMs: Date.now() - startedAt, - }) - /** - * The fill just streamed through both projections: the planner last saw every row unfilled and - * should see the finished projections, and the ranking pages should be back before anyone - * searches. With shards, the one that finishes last does both: a shard that still finds - * unfilled rows anywhere leaves them to whoever fills those. Two shards ending in the same - * moment can both read none left and both do this, which repeats reads and nothing else. - */ - if (await projectionsFilled(sql)) { - for (const projection of PROJECTION_SOURCE_ACL_TABLES) - await sql.unsafe(`ANALYZE ${projection}`) - await prewarmSearchProjection(sql, { budgetMs: PROJECTION_PREWARM_BUDGET_MS }) - } - return null - } finally { - await sql.end() - } -} - -/** - * Whether no projection still holds a row the fill could give its source and ACL: a row without - * them whose document exists. A row whose document is gone is not the fill's to finish and never - * counts as left. Each read is one probe of the unfilled-rows index while any such row remains: - * ordered by id and capped at one row so the planner cannot take a sequential scan, which an - * `EXISTS` would leave open by dropping the order and the limit. - */ -async function projectionsFilled(sql: postgres.Sql): Promise { - for (const projection of PROJECTION_SOURCE_ACL_TABLES) { - const [row] = await sql.unsafe>( - `SELECT ( - SELECT s.id FROM ${projection} s JOIN document d ON d.id = s.document_id WHERE s.acl IS NULL - ORDER BY s.id DESC LIMIT 1 - ) IS NOT NULL AS unfilled` - ) - if (row?.unfilled) return false - } - return true -} - -/** The tag every run of one chain carries, so a chain in flight is found before another is started. */ -export function projectionSourceAclChainTag(shard?: ProjectionSourceAclBackfillShard): string { - const { index, count } = shard ?? { index: 0, count: 1 } - return `${PROJECTION_SOURCE_ACL_BACKFILL_TASK_ID}:shard:${index}/${count}` -} - -/** - * How long a start's trigger stays idempotent. The lookup and the trigger are two calls, so two - * starts in the same instant could both find no chain in flight; the key is what they both saw, - * the chain's latest run, so they collapse into one start, while a start after another chain has - * ended sees a different latest run and is a new key. - */ -const START_IDEMPOTENCY_TTL = '2m' - -/** A run that has not ended: it, or the continuation it triggers, still owns its range. */ -const IN_FLIGHT_RUN_STATUSES: ReadonlySet = new Set([ - 'PENDING_VERSION', - 'QUEUED', - 'DEQUEUED', - 'EXECUTING', - 'WAITING', - 'DELAYED', -]) - -/** - * Starts the backfill on the deployment's Trigger.dev worker, where bounded runs chain until the - * projections are filled: one chain over the whole id space, or one per shard, each filling its - * own slice at the same time. Safe to call again at any time: a run only fills rows still unset, - * and a range whose chain is still in flight is left to that chain rather than given a second. - * A cursor belongs to one chain, so a sliced start takes none: each slice begins at its own bound. - */ -export async function enqueueProjectionSourceAclBackfill( - payload: ProjectionSourceAclBackfillPayload = {}, - shards = 1 -): Promise<{ runIds: string[]; inFlight: string[] }> { - if (payload.shard) assertProjectionSourceAclShard(payload.shard) - if (shards !== 1) { - assertProjectionSourceAclShard({ index: 0, count: shards }) - if (payload.cursor) throw new Error('A sliced projection backfill cannot start from a cursor') - } - const { runs, tasks } = await import('@trigger.dev/sdk') - const region = await resolveTriggerRegion() - const payloads: ProjectionSourceAclBackfillPayload[] = - shards === 1 - ? [payload] - : Array.from({ length: shards }, (_, index) => ({ - ...payload, - shard: { index, count: shards }, - })) - const runIds: string[] = [] - const inFlight: string[] = [] - for (const shardPayload of payloads) { - const tag = projectionSourceAclChainTag(shardPayload.shard) - /** The chain's latest run, newest first, whatever its state. */ - let latest: { id: string; status: string } | undefined - for await (const run of runs.list({ - taskIdentifier: PROJECTION_SOURCE_ACL_BACKFILL_TASK_ID, - tag, - limit: 1, - })) { - latest = run - } - if (latest && IN_FLIGHT_RUN_STATUSES.has(latest.status)) { - inFlight.push(latest.id) - continue - } - const handle = await tasks.trigger(PROJECTION_SOURCE_ACL_BACKFILL_TASK_ID, shardPayload, { - region, - tags: [tag], - idempotencyKey: `${tag}:after:${latest?.id ?? 'none'}`, - idempotencyKeyTTL: START_IDEMPOTENCY_TTL, - }) - runIds.push(handle.id) - } - logger.info('Projection source and ACL backfill enqueued', { runIds, inFlight, shards }) - return { runIds, inFlight } -} diff --git a/apps/sim/lib/knowledge/search/queries.test.ts b/apps/sim/lib/knowledge/search/queries.test.ts index 75fb7644604..4e31f8de16b 100644 --- a/apps/sim/lib/knowledge/search/queries.test.ts +++ b/apps/sim/lib/knowledge/search/queries.test.ts @@ -1613,7 +1613,7 @@ describe('permitted-document planner', () => { expect(JSON.stringify(dbChainMockFns.where.mock.calls)).not.toContain('@@') }) - it('decides a row the backfill has not reached on its document while the fill runs', async () => { + it('decides a row the fill has not reached on its document while the fill runs', async () => { tinPages.push({ ranked: 1, candidates: [{ id: 'a', documentId: 'doc-a', connectorId: 'src-a' }], @@ -1632,8 +1632,10 @@ describe('permitted-document planner', () => { }, }) const statement = JSON.stringify(tinStatements()[0]) - /** A row the backfill has not filled (`acl IS NULL`) is decided on its document instead. */ - expect(statement).toContain('IS NULL AND EXISTS (') + /** A row the fill has not reached (`acl IS NULL`), or a marked document's row, is decided on its document. */ + expect(statement).toContain(' IS NULL OR ') + expect(statement).toContain('knowledgeProjectionDirty.documentId') + expect(statement).toContain('EXISTS (') expect(statement).toContain('ranked_tin_chunks.document_id') }) diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index a9e1cba7c41..73a330e55d1 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -1,4 +1,5 @@ import { db } from '@sim/db' +import { SOURCE_ACL_PROJECTIONS, type SourceAclProjection } from '@sim/db/knowledge-projection' import { document, embedding, @@ -7,14 +8,11 @@ import { embeddingSearch, knowledgeConnector, } from '@sim/db/schema' -import { - PROJECTION_SOURCE_ACL_TABLES, - type ProjectionSourceAclTable, -} from '@sim/db/script-migrations/0021_embedding_search_connector' import { createLogger } from '@sim/logger' import { sha256Hex } from '@sim/security/hash' import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' import { and, eq, gte, inArray, isNull, lte, type SQL, sql } from 'drizzle-orm' +import type { AnyPgColumn } from 'drizzle-orm/pg-core' import { LRUCache } from 'lru-cache' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { resolveSearchAccessPlan } from '@/lib/knowledge/access/connector-eligibility' @@ -24,6 +22,8 @@ import { knowledgeCandidateAccessConditionForConnectors, knowledgeMetadataCandidateAccessCondition, projectionCandidateAccessCondition, + projectionDecidedOnDocument, + projectionPending, restrictSearchAccessPlan, type SearchAccessPlan, textArrayLiteral, @@ -96,8 +96,8 @@ const ON_ROW_WALK_SCAN_TUPLES = 100_000 /** * How far a walk may go when readability is on the row: the on-row cap, unless the walk still - * has to ask the document about tuples — a tag or date filter, or rows the backfill has not - * filled yet — in which case such a tuple costs what it did before the columns were mirrored, + * has to ask the document about tuples — a tag or date filter, or rows the source and ACL fill + * has not reached yet — in which case such a tuple costs what it did before the columns were mirrored, * and the default cap keeps a walk through a mostly-excluded neighbourhood at a short answer * rather than a missed deadline. */ @@ -114,7 +114,7 @@ function onRowWalkScanTuples( const PROJECTION_FILLED_TTL_MS = 60_000 /** - * Whether the ranking projection still holds rows the backfill has not filled. Read off the + * Whether the ranking projection still holds rows the source and ACL fill has not reached. Read off the * unfilled-rows index in milliseconds and remembered briefly: the answer only ever changes once. * * The read asks for the last unfilled row by id, not whether one exists: an `EXISTS` drops its @@ -124,11 +124,11 @@ const PROJECTION_FILLED_TTL_MS = 60_000 * last entry is the row the fill reaches last. */ const projectionFilled = new LRUCache< - ProjectionSourceAclTable, + SourceAclProjection, boolean, { budget: SearchBudget | undefined; stage: SearchStage } >({ - max: PROJECTION_SOURCE_ACL_TABLES.length, + max: SOURCE_ACL_PROJECTIONS.length, ttl: PROJECTION_FILLED_TTL_MS, /** * The read that misses the cache is the search's own, under its budget like every other read @@ -154,7 +154,7 @@ const projectionFilled = new LRUCache< /** Whether every row of the projection carries its mirrored source and ACL; unknown counts as not yet. */ async function isProjectionFilled( - projection: ProjectionSourceAclTable, + projection: SourceAclProjection, stage: SearchStage, budget: SearchBudget | undefined ): Promise { @@ -671,11 +671,45 @@ type SearchReadCandidate = { connectorId: string | null } +/** + * A candidate's source: the row's, unless the row's document is marked for the projector, whose + * source may have moved since the row was written — then the document's, read for that row only. + */ +function projectionCandidateSource(projection: { + connectorId: AnyPgColumn | SQL + documentId: AnyPgColumn | SQL +}): SQL { + return sql`CASE WHEN ${projectionPending(projection.documentId)} + THEN (SELECT ${document.connectorId} FROM ${document} WHERE ${document.id} = ${projection.documentId}) + ELSE ${projection.connectorId} END` +} + /** * The same identities read off a projection row in raw SQL: the aliases are what * `SearchReadCandidate` deserializes, so every walk reads them from one place. */ -const PROJECTION_CANDIDATE_COLUMNS = sql`${embeddingSearch.id} AS id, ${embeddingSearch.documentId} AS "documentId", ${embeddingSearch.connectorId} AS "connectorId"` +const PROJECTION_CANDIDATE_COLUMNS = sql`${embeddingSearch.id} AS id, ${embeddingSearch.documentId} AS "documentId", ${projectionCandidateSource(embeddingSearch)} AS "connectorId"` + +/** + * Keeps the rows of sources the caller turned out not to hold out of a ranking decided on the + * row. A row decided on its document — not yet filled, or its document marked for the projector, + * so its own source may be stale — asks the document instead. + */ +function excludeSearchSourcesOnRow( + projection: { + connectorId: AnyPgColumn | SQL + acl: AnyPgColumn | SQL + documentId: AnyPgColumn | SQL + }, + filled: boolean, + excludedSources: readonly string[] +): SQL | undefined { + if (!excludedSources.length) return undefined + const excluded = textArrayLiteral([...excludedSources]) + const decided = projectionDecidedOnDocument(projection, filled) + return sql`((${decided} AND NOT EXISTS (SELECT 1 FROM ${document} WHERE ${document.id} = ${projection.documentId} AND ${document.connectorId} = ANY(${excluded}))) + OR (NOT ${decided} AND (${projection.connectorId} IS NULL OR NOT (${projection.connectorId} = ANY(${excluded}))))) /* excluded sources */` +} /** Only opaque identifiers leave candidate ranking; content stays behind the full read predicate. */ const SEARCH_READ_CANDIDATE_FIELDS = { @@ -1782,14 +1816,9 @@ async function selectVectorResults(params: SearchParams): Promise and( - projectionCandidateAccessCondition( - { - connectorId: sql`ranked_tin_chunks.connector_id`, - acl: sql`ranked_tin_chunks.acl`, - documentId: sql`ranked_tin_chunks.document_id`, - }, - access, - accessPlan!, - { filled: tinFilled } - ), + projectionCandidateAccessCondition(rankedTinRow, access, accessPlan!, { + filled: tinFilled, + }), dateFilterCondition(params.filters) ? sql`EXISTS (SELECT 1 FROM ${document} WHERE ${and(sql`${document.id} = ranked_tin_chunks.document_id`, dateFilterCondition(params.filters))})` : undefined, - /** - * A filled row's mirrored source decides it; a row the fill has not reached carries no - * source, so only its document is asked, as the vector leg does. - */ - excludedSources.length - ? tinFilled - ? sql`(ranked_tin_chunks.connector_id IS NULL OR NOT (ranked_tin_chunks.connector_id = ANY(${textArrayLiteral([...excludedSources])})))` - : sql`((ranked_tin_chunks.acl IS NOT NULL AND (ranked_tin_chunks.connector_id IS NULL OR NOT (ranked_tin_chunks.connector_id = ANY(${textArrayLiteral([...excludedSources])})))) OR (ranked_tin_chunks.acl IS NULL AND NOT EXISTS (SELECT 1 FROM ${document} WHERE ${document.id} = ranked_tin_chunks.document_id AND ${document.connectorId} = ANY(${textArrayLiteral([...excludedSources])}))))` - : undefined + excludeSearchSourcesOnRow(rankedTinRow, tinFilled, excludedSources) ) const documentConditions = (excludedSources: readonly string[]) => and( @@ -2196,16 +2216,13 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise ) /** * A page read on the row takes each candidate's source from the row, which is what decides - * whether its live source proof is asked for. A row the fill has not reached carries no - * source, so until the fill is complete the page's own unfilled rows take it from their - * document: one primary-key read per row of the page, after its limit, never per ranked row. + * whether its live source proof is asked for. A row decided on its document — not yet filled, + * or its document marked for the projector — takes it from the document: one primary-key read + * per such row of the page, after its limit, never per ranked row. */ - const onRowPage = (ranked: SQL) => - tinFilled - ? ranked - : sql` + const onRowPage = (ranked: SQL) => sql` SELECT paged.id, paged."documentId", - CASE WHEN paged.unfilled + CASE WHEN paged.decided_on_document THEN (SELECT ${document.connectorId} FROM ${document} WHERE ${document.id} = paged."documentId") ELSE paged."connectorId" END AS "connectorId", @@ -2265,9 +2282,8 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise onRowPage( sql` SELECT ranked_tin_chunks.id, ranked_tin_chunks.document_id AS "documentId", - ranked_tin_chunks.connector_id AS "connectorId", ranked_tin_chunks.keyword_rank${ - tinFilled ? sql`` : sql`, ranked_tin_chunks.acl IS NULL AS unfilled` - } + ranked_tin_chunks.connector_id AS "connectorId", ranked_tin_chunks.keyword_rank, + ${projectionDecidedOnDocument(rankedTinRow, tinFilled)} AS decided_on_document FROM ranked_tin_chunks /* on-row visibility */ WHERE ranked_tin_chunks.enabled AND ${onRowKeywordVisibility(excludedSources)} ORDER BY ranked_tin_chunks.keyword_rank DESC, ranked_tin_chunks.id diff --git a/apps/sim/scripts/backfill-projection-source-acl.ts b/apps/sim/scripts/backfill-projection-source-acl.ts deleted file mode 100644 index ebc4a1ec390..00000000000 --- a/apps/sim/scripts/backfill-projection-source-acl.ts +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bun - -/** - * Starts the projection source and ACL backfill that script migration - * `0022_projection_source_acl_backfill` leaves to the background. On a deployment with Trigger.dev - * it enqueues the `projection-source-acl-backfill` task, which chains bounded runs until both - * ranking projections are filled; without one it fills them here, paced the same way. Safe to run - * again at any time — a run only fills rows still unset. - * - * Usage: - * bun apps/sim/scripts/backfill-projection-source-acl.ts - */ - -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { env } from '@/lib/core/config/env' -import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' -import { - enqueueProjectionSourceAclBackfill, - runProjectionSourceAclBackfill, -} from '@/lib/knowledge/search/projection-source-acl-backfill' - -const logger = createLogger('BackfillProjectionSourceAcl') - -/** `--shards ` as given, or one; a value that is not a whole number is refused here. */ -function shardsFlag(): number { - const flag = process.argv.indexOf('--shards') - if (flag === -1) return 1 - const shards = Number(process.argv[flag + 1]) - if (!Number.isInteger(shards) || shards < 1) { - throw new Error(`--shards must be a positive whole number, got ${process.argv[flag + 1]}`) - } - return shards -} - -/** - * A script has no long-lived process to detach into, so without a worker it fills inline, one - * pass over the whole id space. With a worker, `--shards ` slices the id space so that many - * runs fill at once; inline there is nothing to slice across, so the flag is refused there. - */ -async function main(): Promise { - const shards = shardsFlag() - if (isTriggerDevEnabled && env.TRIGGER_SECRET_KEY) { - const started = await enqueueProjectionSourceAclBackfill({}, shards) - logger.info('Backfill enqueued on the Trigger.dev worker', started) - return - } - if (shards !== 1) - throw new Error('--shards needs the Trigger.dev worker; the inline fill is one pass') - await runProjectionSourceAclBackfill({}) - logger.info('Backfill complete') -} - -if (import.meta.main) { - main().then( - () => process.exit(0), - (error) => { - logger.error('Backfill failed', toError(error)) - process.exit(1) - } - ) -} diff --git a/docker/crontab b/docker/crontab index 2f2c6083071..0b4334a4d95 100644 --- a/docker/crontab +++ b/docker/crontab @@ -33,6 +33,9 @@ SHELL=/bin/sh # Workspace file search indexing dispatcher */1 * * * * curl -fsS -m 60 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/workspace-file-search-dispatch" +# Knowledge search projection convergence +*/1 * * * * curl -fsS -m 60 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/knowledge-projection" + # Knowledge base connector syncs */5 * * * * curl -fsS -m 120 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/knowledge/connectors/sync" */5 * * * * curl -fsS -m 120 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/knowledge/connectors/member-sync" diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 4a5c8ef7abc..24d9ba37954 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -1569,6 +1569,17 @@ cronjobs: successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 1 + # Converges knowledge search projections for documents whose source, ACL, or + # chunks changed. Writers request passes as they commit; this is the backstop. + knowledgeProjection: + enabled: true + name: knowledge-projection + schedule: "*/1 * * * *" + path: "/api/cron/knowledge-projection" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + # Global CronJob settings image: repository: curlimages/curl diff --git a/packages/db/knowledge-projection.test.ts b/packages/db/knowledge-projection.test.ts new file mode 100644 index 00000000000..1989277b42f --- /dev/null +++ b/packages/db/knowledge-projection.test.ts @@ -0,0 +1,337 @@ +/** + * @vitest-environment node + */ +import { + FILL_MARK_CEILING, + markUnfilledProjectionDocuments, + runKnowledgeProjection, +} from '@sim/db/knowledge-projection' +import type { Sql } from 'postgres' +import { describe, expect, it, vi } from 'vitest' + +interface Mark { + generation: number + content: boolean +} + +interface FakeDatabase { + marks: Map + /** Documents whose advisory lock another pass holds. */ + lockedElsewhere: Set + tin: boolean + /** Chunk count per document, paged by chunk index. */ + chunks: Map + /** Called before each page statement; may throw to fail the page or change marks. */ + beforePage?: (page: { documentId: string; projection: string; after: number }) => void + /** Called before a settle. */ + beforeSettle?: (documentId: string) => void +} + +/** The page statements a pass ran, in order. */ +interface Trace { + pages: Array<{ documentId: string; projection: string; after: number; mode: 'content' | 'acl' }> + locks: string[] + unlocks: string[] + claims: string[][] +} + +function postgresError(code: string, message: string): Error { + return Object.assign(new Error(message), { code }) +} + +/** + * A stand-in for a postgres.js session that answers the projector's statements from `state`, by + * the statement's text: enough of the database to drive the pass's control flow. + */ +function fakeSql(state: FakeDatabase): { sql: Sql; trace: Trace } { + const trace: Trace = { pages: [], locks: [], unlocks: [], claims: [] } + const answer = async (text: string, values: unknown[]): Promise => { + if (text.includes('to_regprocedure')) return [{ installed: state.tin }] + if (text.includes('ORDER BY marked_at')) { + const skipped = new Set(values[0] as string[]) + const claimed = [...state.marks.keys()].filter((id) => !skipped.has(id)).slice(0, 50) + trace.claims.push(claimed) + return claimed.map((document_id) => ({ document_id })) + } + if (text.includes('pg_try_advisory_lock')) { + const documentId = String(values[0]) + trace.locks.push(documentId) + return [{ acquired: !state.lockedElsewhere.has(documentId) }] + } + if (text.includes('pg_advisory_unlock')) { + trace.unlocks.push(String(values[0])) + return [] + } + if (text.includes('SELECT m.generation')) { + const mark = state.marks.get(String(values[0])) + return mark + ? [{ generation: String(mark.generation), content: mark.content, knowledge_base_id: 'kb' }] + : [] + } + if (text.includes('WITH removed AS')) { + const documentId = String(values[0]) + state.beforeSettle?.(documentId) + const mark = state.marks.get(documentId) + if (mark && mark.generation === values[1]) { + state.marks.delete(documentId) + return [{ removed: 1, marked: true }] + } + return [{ removed: 0, marked: Boolean(mark) }] + } + if (text.includes('SELECT EXISTS (SELECT 1 FROM knowledge_projection_dirty')) { + return [{ marked: state.marks.has(String(values[0])) }] + } + return [] + } + const tagged = (strings: TemplateStringsArray, ...values: unknown[]) => + answer(strings.join('?'), values) + const unsafe = async (text: string, values: unknown[] = []) => { + if (!text.includes('WITH page AS')) return answer(text, values) + const [documentId, after, pageSize] = values as [string, number, number] + const projection = + /INSERT INTO (\w+)|UPDATE (\w+) s SET/.exec(text)?.slice(1).find(Boolean) ?? 'unknown' + const mode = text.includes('INSERT INTO') ? 'content' : 'acl' + state.beforePage?.({ documentId, projection, after }) + trace.pages.push({ documentId, projection, after, mode }) + const total = state.chunks.get(documentId) ?? 1 + const first = after + 1 + const scanned = Math.max(0, Math.min(pageSize, total - first)) + return [ + { + scanned, + written: scanned, + last_chunk: scanned === 0 ? null : first + scanned - 1, + }, + ] + } + const session = Object.assign(tagged, { + unsafe, + begin: async (work: (tx: unknown) => Promise) => + work(Object.assign(tagged, { unsafe })), + }) + return { sql: session as unknown as Sql, trace } +} + +function database(overrides: Partial = {}): FakeDatabase { + return { + marks: new Map(), + lockedElsewhere: new Set(), + tin: true, + chunks: new Map(), + ...overrides, + } +} + +describe('runKnowledgeProjection', () => { + it('projects every marked document under its lock and removes each mark it settled', async () => { + const state = database({ + marks: new Map([ + ['doc-a', { generation: 1, content: true }], + ['doc-b', { generation: 3, content: false }], + ]), + }) + const { sql, trace } = fakeSql(state) + const progress = await runKnowledgeProjection(sql) + expect(progress).toMatchObject({ settled: 2, deferred: 0, remaining: false }) + expect(state.marks.size).toBe(0) + expect(trace.locks).toEqual(['doc-a', 'doc-b']) + expect(trace.unlocks).toEqual(['doc-a', 'doc-b']) + /** A content mark rewrites every projection; a source and ACL mark only the mirrored ones. */ + expect(trace.pages.filter((page) => page.documentId === 'doc-a')).toEqual([ + { documentId: 'doc-a', projection: 'embedding_search', after: -1, mode: 'content' }, + { documentId: 'doc-a', projection: 'embedding_keyword_search', after: -1, mode: 'content' }, + { documentId: 'doc-a', projection: 'embedding_keyword_tin', after: -1, mode: 'content' }, + ]) + expect(trace.pages.filter((page) => page.documentId === 'doc-b')).toEqual([ + { documentId: 'doc-b', projection: 'embedding_search', after: -1, mode: 'acl' }, + { documentId: 'doc-b', projection: 'embedding_keyword_tin', after: -1, mode: 'acl' }, + ]) + }) + + it('leaves the Tin projection alone where it is not installed', async () => { + const state = database({ + tin: false, + marks: new Map([['doc', { generation: 1, content: true }]]), + }) + const { sql, trace } = fakeSql(state) + await runKnowledgeProjection(sql) + expect(trace.pages.map((page) => page.projection)).toEqual([ + 'embedding_search', + 'embedding_keyword_search', + ]) + }) + + it('pages a document by chunk rows until a page comes back short', async () => { + const state = database({ + marks: new Map([['doc', { generation: 1, content: false }]]), + chunks: new Map([['doc', 5]]), + tin: false, + }) + const { sql, trace } = fakeSql(state) + const progress = await runKnowledgeProjection(sql, { pageSize: 2 }) + expect(trace.pages.map((page) => page.after)).toEqual([-1, 1, 3]) + expect(progress).toMatchObject({ pages: 3, written: 5, settled: 1 }) + }) + + it('keeps a mark whose generation moved while the pass ran', async () => { + const state = database({ marks: new Map([['doc', { generation: 1, content: false }]]) }) + state.beforeSettle = () => state.marks.set('doc', { generation: 2, content: false }) + const { sql, trace } = fakeSql(state) + const progress = await runKnowledgeProjection(sql) + expect(progress).toMatchObject({ settled: 0, deferred: 1, remaining: true }) + expect(state.marks.get('doc')?.generation).toBe(2) + /** A document given up is not claimed again by the same pass. */ + expect(trace.claims).toEqual([['doc'], []]) + }) + + it('leaves a document another pass holds to that pass', async () => { + const state = database({ + marks: new Map([ + ['held', { generation: 1, content: false }], + ['free', { generation: 1, content: false }], + ]), + lockedElsewhere: new Set(['held']), + }) + const { sql, trace } = fakeSql(state) + const progress = await runKnowledgeProjection(sql) + expect(progress).toMatchObject({ settled: 1, deferred: 1, remaining: true }) + expect(state.marks.has('held')).toBe(true) + expect(trace.pages.some((page) => page.documentId === 'held')).toBe(false) + expect(trace.unlocks).toEqual(['free']) + }) + + it.each([ + ['a lock timeout', postgresError('55P03', 'canceling statement due to lock timeout')], + ['a statement timeout', postgresError('57014', 'canceling statement due to statement timeout')], + ['a deadlock', postgresError('40P01', 'deadlock detected')], + ['a chunk deleted under the page', postgresError('23503', 'insert violates foreign key')], + ])('gives a document up on %s and carries on with the next', async (_, failure) => { + const state = database({ + marks: new Map([ + ['slow', { generation: 1, content: false }], + ['fine', { generation: 1, content: false }], + ]), + }) + state.beforePage = ({ documentId }) => { + if (documentId === 'slow') throw failure + } + const { sql, trace } = fakeSql(state) + const progress = await runKnowledgeProjection(sql) + expect(progress).toMatchObject({ settled: 1, deferred: 1, remaining: true }) + expect(state.marks.has('slow')).toBe(true) + expect(trace.unlocks).toEqual(['slow', 'fine']) + }) + + it('counts a document deleted during its pass as gone rather than left marked', async () => { + const state = database({ marks: new Map([['doc', { generation: 1, content: false }]]) }) + state.beforePage = () => { + state.marks.delete('doc') + throw postgresError('23503', 'insert violates foreign key') + } + const { sql } = fakeSql(state) + await expect(runKnowledgeProjection(sql)).resolves.toMatchObject({ + settled: 0, + deferred: 0, + remaining: false, + }) + }) + + it('fails the pass on any other error, releasing the document it held', async () => { + const state = database({ marks: new Map([['doc', { generation: 1, content: false }]]) }) + state.beforePage = () => { + throw postgresError('42P01', 'relation does not exist') + } + const { sql, trace } = fakeSql(state) + await expect(runKnowledgeProjection(sql)).rejects.toThrow('relation does not exist') + expect(trace.unlocks).toEqual(['doc']) + expect(state.marks.has('doc')).toBe(true) + }) + + it('stops between pages of a long document at its deadline and keeps that mark', async () => { + vi.useFakeTimers() + try { + const state = database({ + marks: new Map([['long', { generation: 1, content: false }]]), + chunks: new Map([['long', 10]]), + tin: false, + }) + state.beforePage = () => { + vi.advanceTimersByTime(40) + } + const { sql, trace } = fakeSql(state) + const progress = await runKnowledgeProjection(sql, { budgetMs: 100, pageSize: 2 }) + expect(trace.pages).toHaveLength(3) + expect(progress).toMatchObject({ settled: 0, deferred: 1, remaining: true }) + expect(state.marks.has('long')).toBe(true) + expect(trace.unlocks).toEqual(['long']) + } finally { + vi.useRealTimers() + } + }) + + it('stops starting documents once its budget is spent and reports marks remain', async () => { + const state = database({ marks: new Map([['doc', { generation: 1, content: false }]]) }) + const { sql, trace } = fakeSql(state) + await expect(runKnowledgeProjection(sql, { budgetMs: 0 })).resolves.toMatchObject({ + settled: 0, + remaining: true, + }) + expect(trace.pages).toEqual([]) + }) +}) + +describe('markUnfilledProjectionDocuments', () => { + /** A session answering the fill's two statements from `outstanding` and a queue of scan results. */ + function fillSql(outstanding: number, scans: Array<{ marked: number; last_id: string | null }>) { + const statements: Array<{ text: string; values: unknown[] }> = [] + const sql = Object.assign( + async (strings: TemplateStringsArray) => { + if (strings.join('').includes('count(*)')) return [{ outstanding }] + return [] + }, + { + unsafe: async (text: string, values: unknown[]) => { + statements.push({ text, values }) + return [scans.shift() ?? { marked: 0, last_id: null }] + }, + } + ) as unknown as Sql + return { sql, statements } + } + + it('marks nothing while the outstanding marks are at the ceiling', async () => { + const { sql, statements } = fillSql(FILL_MARK_CEILING, []) + await expect(markUnfilledProjectionDocuments(sql)).resolves.toEqual({ + marked: 0, + cursor: { projection: 0, afterId: '' }, + }) + expect(statements).toEqual([]) + }) + + it('offers the room left and continues from the row the statement stopped before', async () => { + const { sql, statements } = fillSql(FILL_MARK_CEILING - 3, [{ marked: 3, last_id: 'row-9' }]) + await expect(markUnfilledProjectionDocuments(sql)).resolves.toEqual({ + marked: 3, + cursor: { projection: 0, afterId: 'row-9' }, + }) + expect(statements[0]?.text).toContain('FROM embedding_search WHERE acl IS NULL') + expect(statements[0]?.text).toContain('u.id < held.first_id') + expect(statements[0]?.values).toEqual(['', 3]) + }) + + it('moves on to the next projection and ends once every unfilled row was read', async () => { + const { sql, statements } = fillSql(0, [ + { marked: 0, last_id: null }, + { marked: 1, last_id: 'tin-1' }, + ]) + await expect( + markUnfilledProjectionDocuments(sql, { projection: 0, afterId: 'row-9' }) + ).resolves.toEqual({ marked: 1, cursor: { projection: 1, afterId: 'tin-1' } }) + expect(statements.map((statement) => statement.values[0])).toEqual(['row-9', '']) + expect(statements[1]?.text).toContain('FROM embedding_keyword_tin WHERE acl IS NULL') + const done = fillSql(0, []) + await expect( + markUnfilledProjectionDocuments(done.sql, { projection: 1, afterId: 'tin-1' }) + ).resolves.toEqual({ marked: 0, cursor: null }) + }) +}) diff --git a/packages/db/knowledge-projection.ts b/packages/db/knowledge-projection.ts new file mode 100644 index 00000000000..07c2d6014a1 --- /dev/null +++ b/packages/db/knowledge-projection.ts @@ -0,0 +1,526 @@ +import { createLogger } from '@sim/logger' +import { getPostgresErrorCode, getTransientDatabaseFailure } from '@sim/utils/errors' +import type { Sql, TransactionSql } from 'postgres' + +const logger = createLogger('KnowledgeProjection') + +/** + * The transaction setting a writer declares to leave its search projection rows to the projector. + * Unset, which is every writer that predates it, keeps the synchronous triggers; either way the + * triggers mark the document in `knowledge_projection_dirty`. + */ +const KNOWLEDGE_PROJECTION_MODE_SETTING = 'sim.projection_mode' + +/** + * The expression a knowledge writer selects in its transaction to leave its search projection rows + * to the projector: `SELECT ${DEFER_KNOWLEDGE_PROJECTION}`. Transaction-local, so a pooled + * connection never carries it into the next transaction. Writers resolve whether to run it from the + * `knowledge-async-projection` flag before their transaction begins. + */ +export const DEFER_KNOWLEDGE_PROJECTION = `set_config('${KNOWLEDGE_PROJECTION_MODE_SETTING}', 'async', true)` + +/** Whether the current transaction ran {@link DEFER_KNOWLEDGE_PROJECTION}, as trigger SQL reads it. */ +export const KNOWLEDGE_PROJECTION_DEFERRED = `current_setting('${KNOWLEDGE_PROJECTION_MODE_SETTING}', true) IS NOT DISTINCT FROM 'async'` + +/** + * The `WHEN` clause of every trigger that writes projection rows in the writer's transaction: it + * fires unless the transaction deferred them. + */ +export const SYNCHRONOUS_PROJECTION_WHEN = `NOT (${KNOWLEDGE_PROJECTION_DEFERRED})` + +/** + * Search projection rows one statement writes per projection table: a projector page, a page of + * connector detach releases, and the most chunk rows a page of synchronous ACL assignments may send + * through the projection trigger. A page always holds at least one document, so a larger one still + * makes progress alone. + */ +export const PROJECTION_ROW_BATCH_SIZE = 250 + +/** How long a projector page may wait on a row or index lock before it gives the document up. */ +const PROJECTION_PAGE_LOCK_TIMEOUT_MS = 5_000 + +/** How long a projector page may run before the database cancels it. */ +const PROJECTION_PAGE_STATEMENT_TIMEOUT_MS = 60_000 + +/** + * How long removing a settled mark may wait. A mark is only row-locked by a writer marking the + * same document again, which changes its generation, so waiting longer could only end in the + * same answer: the mark stays. + */ +const SETTLE_LOCK_TIMEOUT_MS = 2_000 + +/** Marks claimed per round; each is then projected under its own advisory lock. */ +const CLAIM_BATCH_SIZE = 50 + +/** The embedding models trained for prefix retrieval, whose 512 projection is a prefix. */ +const SHORTENED_EMBEDDING_MODELS = `('text-embedding-3-small', 'text-embedding-3-large')` + +const SEARCH_VECTOR_WIDTHS = [1536, 384, 512, 768, 1024, 3072] as const +const SOURCE_VECTOR_WIDTHS = [1536, 384, 768, 1024, 3072] as const +const widthColumn = (name: string, width: number) => (width === 1536 ? name : `${name}_${width}`) + +/** The halfvec columns of `embedding_search`, in width order. */ +export const SEARCH_VECTOR_COLUMNS = SEARCH_VECTOR_WIDTHS.map((width) => + widthColumn('vector', width) +) + +/** The bit columns of `embedding_search`, whose width check requires exactly one. */ +const SEARCH_BINARY_COLUMNS = ['"binary"', 'binary_384', 'binary_768', 'binary_1024', 'binary_3072'] + +/** + * The halfvec projections of an embedding row, in {@link SEARCH_VECTOR_COLUMNS} order. Shortening + * is valid only for the two OpenAI models trained for prefix retrieval. + */ +export function searchVectorProjections(prefix: string, shortened: string): string { + return SEARCH_VECTOR_WIDTHS.map((width) => + width === 512 + ? `CASE WHEN ${shortened} THEN subvector(coalesce(${SOURCE_VECTOR_WIDTHS.map((size) => `${prefix}.${widthColumn('embedding', size)}`).join(', ')}), 1, 512)::halfvec(512) END` + : `CASE WHEN NOT (${shortened}) THEN ${prefix}.${widthColumn('embedding', width)}::halfvec(${width}) END` + ).join(', ') +} + +/** The bit projections of an embedding row, in {@link SEARCH_BINARY_COLUMNS} order. */ +export function searchBinaryProjections(prefix: string): string { + return `binary_quantize(${prefix}.embedding)::bit(1536), binary_quantize(${prefix}.embedding_384)::bit(384), + binary_quantize(${prefix}.embedding_768)::bit(768), binary_quantize(${prefix}.embedding_1024)::bit(1024), + binary_quantize(${prefix}.embedding_3072)::bit(3072)` +} + +/** Whether a knowledge base's model is shortened for the 512 projection, for an embedding row. */ +export function searchVectorShortened(model: string, prefix: string): string { + return `${model} IN ${SHORTENED_EMBEDDING_MODELS} AND ${prefix}.embedding_384 IS NULL` +} + +/** The projections the projector keeps; the Tin projection exists only where `tin` is installed. */ +const KNOWLEDGE_PROJECTIONS = [ + 'embedding_search', + 'embedding_keyword_search', + 'embedding_keyword_tin', +] as const +export type KnowledgeProjection = (typeof KNOWLEDGE_PROJECTIONS)[number] + +/** The projections that mirror their document's source and ACL, and so can be unfilled. */ +export const SOURCE_ACL_PROJECTIONS = ['embedding_search', 'embedding_keyword_tin'] as const +export type SourceAclProjection = (typeof SOURCE_ACL_PROJECTIONS)[number] +const mirrorsSourceAcl = (projection: KnowledgeProjection): projection is SourceAclProjection => + (SOURCE_ACL_PROJECTIONS as readonly string[]).includes(projection) + +/** + * The chunks one page covers: the next {@link PROJECTION_ROW_BATCH_SIZE} of the document in + * chunk order, read off `emb_doc_chunk_idx`. Every projection pages the same way, so a page is + * bounded by chunk rows whatever it writes. + */ +const PAGE = `page AS MATERIALIZED ( + SELECT e.id, e.chunk_index FROM embedding e + WHERE e.document_id = $1 AND e.chunk_index > $2 + ORDER BY e.chunk_index LIMIT $3 + )` + +/** The page's shape as the projector reads it back. */ +const PAGE_RESULT = `SELECT (SELECT count(*)::int FROM page) AS scanned, + (SELECT count(*)::int FROM written) AS written, + (SELECT max(chunk_index) FROM page) AS last_chunk` + +/** + * Rewrites a page's projection rows from their chunks and the document, writing only the rows + * that differ, so a document whose rows are already current costs reads and no index writes. The + * source and ACL are the document's as this statement reads it; a change that commits after it + * marks the document again, so the projector's settle leaves the mark for the next pass. + */ +function contentPageStatement(projection: KnowledgeProjection): string { + if (projection === 'embedding_search') { + const vectors = SEARCH_VECTOR_COLUMNS.join(', ') + const compared = [ + 'knowledge_base_id', + 'document_id', + 'enabled', + 'connector_id', + 'acl', + ...SEARCH_VECTOR_COLUMNS, + ] + return `WITH ${PAGE}, source AS MATERIALIZED ( + SELECT e.id, e.knowledge_base_id, e.document_id, e.enabled, e.embedding, e.embedding_384, + e.embedding_768, e.embedding_1024, e.embedding_3072, + ${searchVectorShortened('k.embedding_model', 'e')} AS shortened, d.connector_id, d.acl + FROM page p JOIN embedding e ON e.id = p.id + JOIN knowledge_base k ON k.id = e.knowledge_base_id + JOIN document d ON d.id = e.document_id + ), written AS ( + INSERT INTO embedding_search AS s + (id, knowledge_base_id, document_id, enabled, ${SEARCH_BINARY_COLUMNS.join(', ')}, ${vectors}, + connector_id, acl) + SELECT id, knowledge_base_id, document_id, enabled, ${searchBinaryProjections('source')}, + ${searchVectorProjections('source', 'shortened')}, connector_id, acl + FROM source + ON CONFLICT (id) DO UPDATE SET + ${[...compared, ...SEARCH_BINARY_COLUMNS].map((column) => `${column} = EXCLUDED.${column}`).join(', ')} + WHERE (${compared.map((column) => `s.${column}`).join(', ')}) + IS DISTINCT FROM (${compared.map((column) => `EXCLUDED.${column}`).join(', ')}) + RETURNING s.id + ) ${PAGE_RESULT}` + } + if (projection === 'embedding_keyword_search') { + const compared = ['knowledge_base_id', 'document_id', 'enabled', 'content_tsv'] + return `WITH ${PAGE}, written AS ( + INSERT INTO embedding_keyword_search AS s (${['id', ...compared].join(', ')}) + SELECT e.id, e.knowledge_base_id, e.document_id, e.enabled, e.content_tsv + FROM page p JOIN embedding e ON e.id = p.id + ON CONFLICT (id) DO UPDATE SET + ${compared.map((column) => `${column} = EXCLUDED.${column}`).join(', ')} + WHERE (${compared.map((column) => `s.${column}`).join(', ')}) + IS DISTINCT FROM (${compared.map((column) => `EXCLUDED.${column}`).join(', ')}) + RETURNING s.id + ) ${PAGE_RESULT}` + } + const compared = ['knowledge_base_id', 'document_id', 'enabled', 'content', 'connector_id', 'acl'] + return `WITH ${PAGE}, source AS MATERIALIZED ( + SELECT e.id, e.knowledge_base_id, e.document_id, e.enabled, + knowledge_tin_base_token(e.knowledge_base_id) || ' ' || knowledge_tin_stream(e.content_tsv) AS content, + d.connector_id, d.acl, k.is_search_index + FROM page p JOIN embedding e ON e.id = p.id + JOIN knowledge_base k ON k.id = e.knowledge_base_id + JOIN document d ON d.id = e.document_id + ), removed AS ( + DELETE FROM embedding_keyword_tin t USING source + WHERE t.id = source.id AND NOT source.is_search_index + ), written AS ( + INSERT INTO embedding_keyword_tin AS t (${['id', ...compared].join(', ')}) + SELECT id, ${compared.join(', ')} FROM source WHERE is_search_index + ON CONFLICT (id) DO UPDATE SET + ${compared.map((column) => `${column} = EXCLUDED.${column}`).join(', ')} + WHERE (${compared.map((column) => `t.${column}`).join(', ')}) + IS DISTINCT FROM (${compared.map((column) => `EXCLUDED.${column}`).join(', ')}) + RETURNING t.id + ) ${PAGE_RESULT}` +} + +/** + * Copies the document's source and ACL onto a page's existing projection rows that differ, + * including rows the source and ACL fill has not reached. Chunks are untouched, so their vectors + * are never read; a row a chunk change has not projected yet is left to that change's own mark. + */ +function sourceAclPageStatement(projection: KnowledgeProjection): string { + return `WITH ${PAGE}, written AS ( + UPDATE ${projection} s SET connector_id = d.connector_id, acl = d.acl + FROM page p, document d + WHERE s.id = p.id AND d.id = $1 + AND (s.connector_id IS DISTINCT FROM d.connector_id OR s.acl IS DISTINCT FROM d.acl) + RETURNING s.id + ) ${PAGE_RESULT}` +} + +/** + * The failures that give a document up for this pass rather than failing the run: the database + * had no room for the page or it lost to a concurrent transaction, or (`23503`) a chunk was + * deleted while its page was being written, whose document is then gone or re-marked by the + * insert that replaced the chunk. A lost connection fails the run, which the task retries. + */ +function isDeferrable(error: unknown): boolean { + if (getPostgresErrorCode(error) === '23503') return true + const failure = getTransientDatabaseFailure(error) + return failure === 'capacity' || failure === 'conflict' +} + +/** The bounds and mode every projector transaction runs under. */ +async function enterProjectorTransaction(tx: TransactionSql, lockTimeoutMs: number) { + await tx.unsafe( + `SELECT set_config('lock_timeout', '${lockTimeoutMs}ms', true), + set_config('statement_timeout', '${PROJECTION_PAGE_STATEMENT_TIMEOUT_MS}ms', true), + ${DEFER_KNOWLEDGE_PROJECTION}` + ) +} + +interface ProjectionMark { + generation: number + content: boolean + knowledgeBaseId: string +} + +export interface KnowledgeProjectionOptions { + /** + * Stop starting pages once this much time has passed, leaving the marks of documents not yet + * finished for the next pass; unbounded otherwise. The page in flight still runs to its own + * statement timeout, so a caller leaves that much headroom after the budget. + */ + budgetMs?: number + pageSize?: number + /** Called after each page commits, for tests that interleave writes with a run. */ + onPage?: (page: { + documentId: string + projection: KnowledgeProjection + written: number + }) => Promise | void +} + +export interface KnowledgeProjectionProgress { + /** Documents whose projection rows were rewritten and whose mark was removed. */ + settled: number + /** Documents left marked: re-marked during their pass, locked, or given up on a timeout. */ + deferred: number + pages: number + written: number + /** Whether marks may remain that this run did not reach. */ + remaining: boolean +} + +/** Whether the Tin keyword projection is maintained here: its functions exist only where installed. */ +async function tinInstalled(sql: Sql): Promise { + const [row] = await sql>` + SELECT to_regprocedure('knowledge_tin_stream(tsvector)') IS NOT NULL AS installed` + return Boolean(row?.installed) +} + +/** + * The oldest marks, past the ones this run has already passed over. A read without row locks: a + * pass owns a document through its advisory lock, and a row lock on the mark would either block + * the writers re-marking it for the whole pass or, taken only for the read, make passes that start + * together skip each other's whole batch and stop. Passes reading the same batch split it at the + * advisory lock, each moving on from a document another holds. + */ +async function claimMarks(sql: Sql, skipped: readonly string[]): Promise { + const rows = await sql>` + SELECT document_id FROM knowledge_projection_dirty + WHERE NOT (document_id = ANY(${skipped as string[]}::text[])) + ORDER BY marked_at LIMIT ${CLAIM_BATCH_SIZE}` + return rows.map((row) => row.document_id) +} + +/** + * The document's mark, read once its advisory lock is held: only the lock holder removes a mark, + * so the generation read here can afterwards only grow, or vanish with the document. + */ +async function readMark(sql: Sql, documentId: string): Promise { + const [row] = await sql< + Array<{ generation: string; content: boolean; knowledge_base_id: string }> + >` + SELECT m.generation, m.content, d.knowledge_base_id + FROM knowledge_projection_dirty m JOIN document d ON d.id = m.document_id + WHERE m.document_id = ${documentId}` + return row + ? { + generation: Number(row.generation), + content: row.content, + knowledgeBaseId: row.knowledge_base_id, + } + : null +} + +/** + * Removes the mark if it still carries the generation the pass read. A writer that marked the + * document again meanwhile bumped it, so its change survives for the next pass; a writer still + * holding the row is marking it again, and the mark stays. + */ +async function settleMark( + sql: Sql, + documentId: string, + generation: number +): Promise { + try { + return await sql.begin(async (tx) => { + await enterProjectorTransaction(tx, SETTLE_LOCK_TIMEOUT_MS) + const [row] = await tx>` + WITH removed AS ( + DELETE FROM knowledge_projection_dirty + WHERE document_id = ${documentId} AND generation = ${generation} + RETURNING 1 + ) + SELECT (SELECT count(*)::int FROM removed) AS removed, + EXISTS (SELECT 1 FROM knowledge_projection_dirty WHERE document_id = ${documentId}) AS marked` + if (row?.removed) return 'settled' + return row?.marked ? 'deferred' : 'gone' + }) + } catch (error) { + if (getPostgresErrorCode(error) === '55P03') return 'deferred' + throw error + } +} + +/** Whether the document is still marked, after a pass over it was given up. */ +async function stillMarked(sql: Sql, documentId: string): Promise { + const [row] = await sql>` + SELECT EXISTS (SELECT 1 FROM knowledge_projection_dirty WHERE document_id = ${documentId}) AS marked` + return Boolean(row?.marked) +} + +/** + * Rewrites one projection of a document in pages bounded by chunk rows, each its own transaction. + * Stops before a page once `deadline` passes and reports the projection unfinished; the pages + * already written stand, and the document's next pass finds them current. + */ +async function projectDocumentRows( + sql: Sql, + documentId: string, + projection: KnowledgeProjection, + mark: ProjectionMark, + options: KnowledgeProjectionOptions, + deadline: number +): Promise<{ pages: number; written: number; finished: boolean }> { + const pageSize = options.pageSize ?? PROJECTION_ROW_BATCH_SIZE + const statement = mark.content + ? contentPageStatement(projection) + : sourceAclPageStatement(projection) + let after = -1 + let pages = 0 + let written = 0 + for (;;) { + if (Date.now() >= deadline) return { pages, written, finished: false } + const page = await sql.begin(async (tx) => { + await enterProjectorTransaction(tx, PROJECTION_PAGE_LOCK_TIMEOUT_MS) + /** Shares the base's Tin membership lock, as the embedding trigger does, so a flip of its marker waits. */ + if (projection === 'embedding_keyword_tin') + await tx`SELECT pg_advisory_xact_lock_shared(knowledge_tin_membership_key(${mark.knowledgeBaseId}))` + const [row] = await tx.unsafe< + Array<{ scanned: number; written: number; last_chunk: number | null }> + >(statement, [documentId, after, pageSize]) + return row + }) + pages += 1 + written += page.written + await options.onPage?.({ documentId, projection, written: page.written }) + if (page.last_chunk === null || page.scanned < pageSize) break + after = page.last_chunk + } + return { pages, written, finished: true } +} + +type DocumentOutcome = 'settled' | 'deferred' | 'gone' + +/** + * Projects one marked document under its advisory lock, then settles the mark it read. A document + * the deadline cuts short keeps its mark for the next pass. + */ +async function projectMarkedDocument( + sql: Sql, + documentId: string, + projections: readonly KnowledgeProjection[], + options: KnowledgeProjectionOptions, + totals: { pages: number; written: number }, + deadline: number +): Promise { + const [lock] = await sql>` + SELECT pg_try_advisory_lock(hashtextextended('knowledge_projection:' || ${documentId}, 0)) AS acquired` + if (!lock?.acquired) return 'deferred' + try { + const mark = await readMark(sql, documentId) + if (!mark) return 'gone' + for (const projection of projections) { + if (!mark.content && !mirrorsSourceAcl(projection)) continue + const done = await projectDocumentRows(sql, documentId, projection, mark, options, deadline) + totals.pages += done.pages + totals.written += done.written + if (!done.finished) return 'deferred' + } + return await settleMark(sql, documentId, mark.generation) + } catch (error) { + if (!isDeferrable(error)) throw error + if (!(await stillMarked(sql, documentId))) return 'gone' + logger.warn('Projection of a document deferred', { + documentId, + code: getPostgresErrorCode(error), + }) + return 'deferred' + } finally { + await sql`SELECT pg_advisory_unlock(hashtextextended('knowledge_projection:' || ${documentId}, 0))` + } +} + +/** + * Converges the search projections of every marked document, oldest mark first, until none is + * left or the budget runs out. Runs on a connection of its own: each document is projected under + * a session advisory lock, so concurrent runs never project the same document at once and a run + * that dies releases its locks with its connection. A document keeps its mark, left to a later + * run, when another run holds it, when a page waits too long on a lock or runs past its timeout, + * when a chunk is deleted under it, when it was re-marked while it ran, or when the budget runs + * out mid-document. Every other failure fails the run. + */ +export async function runKnowledgeProjection( + sql: Sql, + options: KnowledgeProjectionOptions = {} +): Promise { + const deadline = + options.budgetMs === undefined ? Number.POSITIVE_INFINITY : Date.now() + options.budgetMs + const projections = (await tinInstalled(sql)) + ? KNOWLEDGE_PROJECTIONS + : KNOWLEDGE_PROJECTIONS.filter((projection) => projection !== 'embedding_keyword_tin') + const totals = { pages: 0, written: 0 } + const skipped: string[] = [] + let settled = 0 + while (Date.now() < deadline) { + const claimed = await claimMarks(sql, skipped) + if (claimed.length === 0) { + return { settled, deferred: skipped.length, ...totals, remaining: skipped.length > 0 } + } + for (const documentId of claimed) { + if (Date.now() >= deadline) break + const outcome = await projectMarkedDocument( + sql, + documentId, + projections, + options, + totals, + deadline + ) + if (outcome === 'settled') settled += 1 + else if (outcome === 'deferred') skipped.push(documentId) + } + } + return { settled, deferred: skipped.length, ...totals, remaining: true } +} + +/** Unfilled rows the fill reads per round, from each projection's unfilled-rows index. */ +const FILL_SCAN_ROWS = 2_000 + +/** + * Marks at most this many documents may be outstanding before the fill adds more. Marks are + * projected oldest first, so this bounds how much fill work a fresh write can wait behind, and + * search, which decides a marked document's rows on the document, keeps doing so for few of them. + */ +export const FILL_MARK_CEILING = 100 + +/** + * Marks the documents of projection rows the source and ACL fill has not reached, so the projector + * fills them as it converges any other document. Rows are read off each projection's unfilled + * index after `afterId`, and only while fewer than {@link FILL_MARK_CEILING} marks are outstanding; + * a row whose document is gone is passed over. Documents are taken in the order of their first + * row read, up to the room left under the ceiling, and the cursor moves only past rows whose + * documents are now marked: when the room runs out, it stops before the first row of the first + * document left unmarked, so the next call starts there. Returns how many documents were marked + * and the id to continue after, or `null` once every projection's unfilled rows have been read. + */ +export async function markUnfilledProjectionDocuments( + sql: Sql, + cursor: { projection: number; afterId: string } = { projection: 0, afterId: '' } +): Promise<{ marked: number; cursor: { projection: number; afterId: string } | null }> { + const [{ outstanding }] = await sql>` + SELECT count(*)::int AS outstanding FROM knowledge_projection_dirty` + if (outstanding >= FILL_MARK_CEILING) return { marked: 0, cursor } + for (let index = cursor.projection; index < SOURCE_ACL_PROJECTIONS.length; index++) { + const projection = SOURCE_ACL_PROJECTIONS[index] + const afterId = index === cursor.projection ? cursor.afterId : '' + const [row] = await sql.unsafe>( + `WITH unfilled AS MATERIALIZED ( + SELECT id, document_id FROM ${projection} WHERE acl IS NULL AND id > $1 + ORDER BY id LIMIT ${FILL_SCAN_ROWS} + ), documents AS MATERIALIZED ( + SELECT u.document_id, min(u.id) AS first_id FROM unfilled u + WHERE EXISTS (SELECT 1 FROM document d WHERE d.id = u.document_id) + GROUP BY u.document_id + ), chosen AS MATERIALIZED ( + SELECT document_id, first_id FROM documents ORDER BY first_id LIMIT $2 + ), held AS ( + SELECT min(first_id) AS first_id FROM documents + WHERE document_id NOT IN (SELECT document_id FROM chosen) + ), marked AS ( + INSERT INTO knowledge_projection_dirty (document_id) + SELECT document_id FROM chosen ORDER BY document_id + ON CONFLICT (document_id) DO NOTHING RETURNING document_id + ) SELECT (SELECT count(*)::int FROM marked) AS marked, + (SELECT max(u.id) FROM unfilled u, held + WHERE held.first_id IS NULL OR u.id < held.first_id) AS last_id`, + [afterId, FILL_MARK_CEILING - outstanding] + ) + if (row?.last_id) + return { marked: row.marked, cursor: { projection: index, afterId: row.last_id } } + } + return { marked: 0, cursor: null } +} diff --git a/packages/db/migrations/0378_knowledge_projection_dirty.sql b/packages/db/migrations/0378_knowledge_projection_dirty.sql new file mode 100644 index 00000000000..27425e03d7b --- /dev/null +++ b/packages/db/migrations/0378_knowledge_projection_dirty.sql @@ -0,0 +1,15 @@ +-- Documents whose search projection rows may lag their source rows. The table is new, so the +-- foreign key validates no rows and only briefly locks `document` under the runner's lock timeout. +-- Script migration `0024_knowledge_projection_async` installs the triggers that mark it. +CREATE TABLE "knowledge_projection_dirty" ( + "document_id" text PRIMARY KEY NOT NULL, + "generation" bigint DEFAULT 1 NOT NULL, + "content" boolean DEFAULT false NOT NULL, + "marked_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "knowledge_projection_dirty" ADD CONSTRAINT "knowledge_projection_dirty_document_id_document_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."document"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "knowledge_projection_dirty_marked_at_idx" ON "knowledge_projection_dirty" USING btree ("marked_at");--> statement-breakpoint +-- Search reads the whole table per statement while every mark is inserted and deleted within +-- seconds, so it is vacuumed after a fixed number of dead rows rather than a share of a tiny table. +ALTER TABLE "knowledge_projection_dirty" SET (autovacuum_vacuum_scale_factor = 0, autovacuum_vacuum_threshold = 1000, autovacuum_analyze_scale_factor = 0, autovacuum_analyze_threshold = 1000); \ No newline at end of file diff --git a/packages/db/migrations/meta/0378_snapshot.json b/packages/db/migrations/meta/0378_snapshot.json new file mode 100644 index 00000000000..a628c43c39d --- /dev/null +++ b/packages/db/migrations/meta/0378_snapshot.json @@ -0,0 +1,28544 @@ +{ + "id": "374a8a56-d71f-40d8-8c3c-3086281b56fc", + "prevId": "4c613485-fa8b-452c-875c-1f4e2a1abcd2", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.academy_certificate": { + "name": "academy_certificate", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "course_id": { + "name": "course_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "academy_cert_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "certificate_number": { + "name": "certificate_number", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "academy_certificate_user_id_idx": { + "name": "academy_certificate_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_course_id_idx": { + "name": "academy_certificate_course_id_idx", + "columns": [ + { + "expression": "course_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_user_course_unique": { + "name": "academy_certificate_user_course_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "course_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_status_idx": { + "name": "academy_certificate_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "academy_certificate_user_id_user_id_fk": { + "name": "academy_certificate_user_id_user_id_fk", + "tableFrom": "academy_certificate", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "academy_certificate_certificate_number_unique": { + "name": "academy_certificate_certificate_number_unique", + "nullsNotDistinct": false, + "columns": ["certificate_number"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_config": { + "name": "oauth_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_account_on_account_id_provider_id": { + "name": "idx_account_on_account_id_provider_id", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_memory_turn": { + "name": "agent_memory_turn", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "memory_id": { + "name": "memory_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_order": { + "name": "execution_order", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "encrypted_state": { + "name": "encrypted_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_memory_turn_invocation_unique": { + "name": "agent_memory_turn_invocation_unique", + "columns": [ + { + "expression": "memory_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "node_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "agent_memory_turn_workflow_idx": { + "name": "agent_memory_turn_workflow_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_memory_turn_memory_id_memory_id_fk": { + "name": "agent_memory_turn_memory_id_memory_id_fk", + "tableFrom": "agent_memory_turn", + "tableTo": "memory", + "columnsFrom": ["memory_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_memory_turn_workflow_id_workflow_id_fk": { + "name": "agent_memory_turn_workflow_id_workflow_id_fk", + "tableFrom": "agent_memory_turn", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_key": { + "name": "api_key", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'personal'" + }, + "last_used": { + "name": "last_used", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "api_key_workspace_type_idx": { + "name": "api_key_workspace_type_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_key_user_type_idx": { + "name": "api_key_user_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_key_key_hash_idx": { + "name": "api_key_key_hash_idx", + "columns": [ + { + "expression": "key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_key_user_id_user_id_fk": { + "name": "api_key_user_id_user_id_fk", + "tableFrom": "api_key", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_key_workspace_id_workspace_id_fk": { + "name": "api_key_workspace_id_workspace_id_fk", + "tableFrom": "api_key", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_key_created_by_user_id_fk": { + "name": "api_key_created_by_user_id_fk", + "tableFrom": "api_key", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_key_key_unique": { + "name": "api_key_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": { + "workspace_type_check": { + "name": "workspace_type_check", + "value": "(type = 'workspace' AND workspace_id IS NOT NULL) OR (type = 'personal' AND workspace_id IS NULL)" + } + }, + "isRLSEnabled": false + }, + "public.async_jobs": { + "name": "async_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "run_at": { + "name": "run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output": { + "name": "output", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "async_jobs_status_started_at_idx": { + "name": "async_jobs_status_started_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_status_completed_at_idx": { + "name": "async_jobs_status_completed_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "completed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_pending_run_at_idx": { + "name": "async_jobs_schedule_pending_run_at_idx", + "columns": [ + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_processing_started_at_idx": { + "name": "async_jobs_schedule_processing_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" = 'processing'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_unreconciled_terminal_idx": { + "name": "async_jobs_schedule_unreconciled_terminal_idx", + "columns": [ + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" IN ('completed', 'failed', 'cancelled') AND COALESCE(\"async_jobs\".\"metadata\" ->> 'scheduleReconciled', 'false') <> 'true'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_log": { + "name": "audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_name": { + "name": "actor_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_email": { + "name": "actor_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource_name": { + "name": "resource_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "surface": { + "name": "surface", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_log_workspace_created_idx": { + "name": "audit_log_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_workspace_created_at_id_idx": { + "name": "audit_log_workspace_created_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"created_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_actor_created_idx": { + "name": "audit_log_actor_created_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_resource_idx": { + "name": "audit_log_resource_idx", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_action_idx": { + "name": "audit_log_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "audit_log_workspace_id_workspace_id_fk": { + "name": "audit_log_workspace_id_workspace_id_fk", + "tableFrom": "audit_log", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "audit_log_actor_id_user_id_fk": { + "name": "audit_log_actor_id_user_id_fk", + "tableFrom": "audit_log", + "tableTo": "user", + "columnsFrom": ["actor_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.background_work_status": { + "name": "background_work_status", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "background_work_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "background_work_status_value", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "background_work_status_workspace_status_idx": { + "name": "background_work_status_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_workflow_status_idx": { + "name": "background_work_status_workflow_status_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_meta_child_ws_idx": { + "name": "background_work_status_meta_child_ws_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'childWorkspaceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_meta_other_ws_idx": { + "name": "background_work_status_meta_other_ws_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'otherWorkspaceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "background_work_status_workspace_id_workspace_id_fk": { + "name": "background_work_status_workspace_id_workspace_id_fk", + "tableFrom": "background_work_status", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "background_work_status_workflow_id_workflow_id_fk": { + "name": "background_work_status_workflow_id_workflow_id_fk", + "tableFrom": "background_work_status", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat": { + "name": "chat", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "customizations": { + "name": "customizations", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public'" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_emails": { + "name": "allowed_emails", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "output_configs": { + "name": "output_configs", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "include_thinking": { + "name": "include_thinking", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "include_tool_calls": { + "name": "include_tool_calls", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "identifier_idx": { + "name": "identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"chat\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_archived_at_partial_idx": { + "name": "chat_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"chat\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_chat_on_workflow_id_archived_at": { + "name": "idx_chat_on_workflow_id_archived_at", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_workflow_id_workflow_id_fk": { + "name": "chat_workflow_id_workflow_id_fk", + "tableFrom": "chat", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chat_user_id_user_id_fk": { + "name": "chat_user_id_user_id_fk", + "tableFrom": "chat", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_async_tool_calls": { + "name": "copilot_async_tool_calls", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "checkpoint_id": { + "name": "checkpoint_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "args": { + "name": "args", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "status": { + "name": "status", + "type": "copilot_async_tool_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "permission_decision": { + "name": "permission_decision", + "type": "copilot_tool_permission_decision", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "permission_decided_at": { + "name": "permission_decided_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_async_tool_calls_run_id_idx": { + "name": "copilot_async_tool_calls_run_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_checkpoint_id_idx": { + "name": "copilot_async_tool_calls_checkpoint_id_idx", + "columns": [ + { + "expression": "checkpoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_status_idx": { + "name": "copilot_async_tool_calls_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_run_status_idx": { + "name": "copilot_async_tool_calls_run_status_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_tool_call_id_unique": { + "name": "copilot_async_tool_calls_tool_call_id_unique", + "columns": [ + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_async_tool_calls_run_id_copilot_runs_id_fk": { + "name": "copilot_async_tool_calls_run_id_copilot_runs_id_fk", + "tableFrom": "copilot_async_tool_calls", + "tableTo": "copilot_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_async_tool_calls_checkpoint_id_copilot_run_checkpoints_id_fk": { + "name": "copilot_async_tool_calls_checkpoint_id_copilot_run_checkpoints_id_fk", + "tableFrom": "copilot_async_tool_calls", + "tableTo": "copilot_run_checkpoints", + "columnsFrom": ["checkpoint_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_chats": { + "name": "copilot_chats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "chat_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'copilot'" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'claude-3-7-sonnet-latest'" + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_conversation_key": { + "name": "external_conversation_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_conversation_metadata": { + "name": "external_conversation_metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "preview_yaml": { + "name": "preview_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "plan_artifact": { + "name": "plan_artifact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "auto_allowed_tools": { + "name": "auto_allowed_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "pinned": { + "name": "pinned", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_chats_organization_id_idx": { + "name": "copilot_chats_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_external_conversation_unique": { + "name": "copilot_chats_external_conversation_unique", + "columns": [ + { + "expression": "external_conversation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"copilot_chats\".\"external_conversation_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_org_created_idx": { + "name": "copilot_chats_user_org_created_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_id_idx": { + "name": "copilot_chats_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_workflow_id_idx": { + "name": "copilot_chats_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workflow_idx": { + "name": "copilot_chats_user_workflow_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workspace_idx": { + "name": "copilot_chats_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_created_at_idx": { + "name": "copilot_chats_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_updated_at_idx": { + "name": "copilot_chats_updated_at_idx", + "columns": [ + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_workspace_created_at_id_idx": { + "name": "copilot_chats_workspace_created_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"created_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workspace_deleted_partial_idx": { + "name": "copilot_chats_user_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_chats\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_chats_user_id_user_id_fk": { + "name": "copilot_chats_user_id_user_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_workflow_id_workflow_id_fk": { + "name": "copilot_chats_workflow_id_workflow_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_workspace_id_workspace_id_fk": { + "name": "copilot_chats_workspace_id_workspace_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_organization_id_organization_id_fk": { + "name": "copilot_chats_organization_id_organization_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "copilot_chats_owner_check": { + "name": "copilot_chats_owner_check", + "value": "num_nonnulls(\"copilot_chats\".\"workspace_id\", \"copilot_chats\".\"organization_id\") <= 1" + }, + "copilot_chats_organization_workflow_check": { + "name": "copilot_chats_organization_workflow_check", + "value": "\"copilot_chats\".\"organization_id\" IS NULL OR \"copilot_chats\".\"workflow_id\" IS NULL" + } + }, + "isRLSEnabled": false + }, + "public.copilot_feedback": { + "name": "copilot_feedback", + "schema": "", + "columns": { + "feedback_id": { + "name": "feedback_id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_query": { + "name": "user_query", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_response": { + "name": "agent_response", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_positive": { + "name": "is_positive", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "feedback": { + "name": "feedback", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_yaml": { + "name": "workflow_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_feedback_user_id_idx": { + "name": "copilot_feedback_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_chat_id_idx": { + "name": "copilot_feedback_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_user_chat_idx": { + "name": "copilot_feedback_user_chat_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_is_positive_idx": { + "name": "copilot_feedback_is_positive_idx", + "columns": [ + { + "expression": "is_positive", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_created_at_idx": { + "name": "copilot_feedback_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_feedback_user_id_user_id_fk": { + "name": "copilot_feedback_user_id_user_id_fk", + "tableFrom": "copilot_feedback", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_feedback_chat_id_copilot_chats_id_fk": { + "name": "copilot_feedback_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_feedback", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_messages": { + "name": "copilot_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "stream_id": { + "name": "stream_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parent_message_id": { + "name": "parent_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tokens_in": { + "name": "tokens_in", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "tokens_out": { + "name": "tokens_out", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_messages_chat_message_unique": { + "name": "copilot_messages_chat_message_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_created_at_idx": { + "name": "copilot_messages_chat_created_at_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_seq_idx": { + "name": "copilot_messages_chat_seq_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_stream_idx": { + "name": "copilot_messages_chat_stream_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "stream_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"stream_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_user_created_at_idx": { + "name": "copilot_messages_user_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"role\" = 'user' AND \"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_messages_chat_id_copilot_chats_id_fk": { + "name": "copilot_messages_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_messages", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_run_checkpoints": { + "name": "copilot_run_checkpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "pending_tool_call_id": { + "name": "pending_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conversation_snapshot": { + "name": "conversation_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "agent_state": { + "name": "agent_state", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "provider_request": { + "name": "provider_request", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_run_checkpoints_run_id_idx": { + "name": "copilot_run_checkpoints_run_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_run_checkpoints_pending_tool_call_id_idx": { + "name": "copilot_run_checkpoints_pending_tool_call_id_idx", + "columns": [ + { + "expression": "pending_tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_run_checkpoints_run_pending_tool_unique": { + "name": "copilot_run_checkpoints_run_pending_tool_unique", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "pending_tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_run_checkpoints_run_id_copilot_runs_id_fk": { + "name": "copilot_run_checkpoints_run_id_copilot_runs_id_fk", + "tableFrom": "copilot_run_checkpoints", + "tableTo": "copilot_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_runs": { + "name": "copilot_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_run_id": { + "name": "parent_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stream_id": { + "name": "stream_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent": { + "name": "agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "copilot_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "request_context": { + "name": "request_context", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "copilot_runs_execution_id_idx": { + "name": "copilot_runs_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_parent_run_id_idx": { + "name": "copilot_runs_parent_run_id_idx", + "columns": [ + { + "expression": "parent_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_chat_id_idx": { + "name": "copilot_runs_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_chat_started_at_idx": { + "name": "copilot_runs_chat_started_at_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_user_id_idx": { + "name": "copilot_runs_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workflow_id_idx": { + "name": "copilot_runs_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workspace_id_idx": { + "name": "copilot_runs_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_status_idx": { + "name": "copilot_runs_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_chat_execution_idx": { + "name": "copilot_runs_chat_execution_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_execution_started_at_idx": { + "name": "copilot_runs_execution_started_at_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workspace_completed_at_id_idx": { + "name": "copilot_runs_workspace_completed_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"completed_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_stream_id_unique": { + "name": "copilot_runs_stream_id_unique", + "columns": [ + { + "expression": "stream_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_runs_chat_id_copilot_chats_id_fk": { + "name": "copilot_runs_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_user_id_user_id_fk": { + "name": "copilot_runs_user_id_user_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_workflow_id_workflow_id_fk": { + "name": "copilot_runs_workflow_id_workflow_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_workspace_id_workspace_id_fk": { + "name": "copilot_runs_workspace_id_workspace_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_workflow_read_hashes": { + "name": "copilot_workflow_read_hashes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_workflow_read_hashes_chat_id_idx": { + "name": "copilot_workflow_read_hashes_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_workflow_read_hashes_workflow_id_idx": { + "name": "copilot_workflow_read_hashes_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_workflow_read_hashes_chat_workflow_unique": { + "name": "copilot_workflow_read_hashes_chat_workflow_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_workflow_read_hashes_chat_id_copilot_chats_id_fk": { + "name": "copilot_workflow_read_hashes_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_workflow_read_hashes", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_workflow_read_hashes_workflow_id_workflow_id_fk": { + "name": "copilot_workflow_read_hashes_workflow_id_workflow_id_fk", + "tableFrom": "copilot_workflow_read_hashes", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credential": { + "name": "credential", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "slack_app_id": { + "name": "slack_app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "credential_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "unredacted": { + "name": "unredacted", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "env_key": { + "name": "env_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "env_owner_user_id": { + "name": "env_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_service_account_key": { + "name": "encrypted_service_account_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_personal_token": { + "name": "encrypted_personal_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_app_id": { + "name": "authorization_app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_group_enrollment_id": { + "name": "credential_group_enrollment_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_group_option_id": { + "name": "credential_group_option_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mcp_server_id": { + "name": "mcp_server_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mcp_oauth_config_version": { + "name": "mcp_oauth_config_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "managed_oauth_scope_version": { + "name": "managed_oauth_scope_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "provider_subject_id": { + "name": "provider_subject_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_tenant_id": { + "name": "provider_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_oauth_status": { + "name": "managed_oauth_status", + "type": "managed_oauth_credential_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "granted_scopes": { + "name": "granted_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "provider_metadata": { + "name": "provider_metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "encrypted_oauth_token_set": { + "name": "encrypted_oauth_token_set", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mcp_tools": { + "name": "mcp_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mcp_tools_refreshed_at": { + "name": "mcp_tools_refreshed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "granted_at": { + "name": "granted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_refreshed_at": { + "name": "last_refreshed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_organization_id_idx": { + "name": "credential_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_organization_account_unique": { + "name": "credential_organization_account_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credential\".\"account_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_org_personal_token_unique": { + "name": "credential_org_personal_token_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_subject_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credential\".\"type\" = 'personal_token'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_id_idx": { + "name": "credential_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_type_idx": { + "name": "credential_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_provider_id_idx": { + "name": "credential_provider_id_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_account_id_idx": { + "name": "credential_account_id_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_env_owner_user_id_idx": { + "name": "credential_env_owner_user_id_idx", + "columns": [ + { + "expression": "env_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_idx": { + "name": "credential_group_enrollment_idx", + "columns": [ + { + "expression": "credential_group_enrollment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_mcp_server_idx": { + "name": "credential_mcp_server_idx", + "columns": [ + { + "expression": "mcp_server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_option_unique": { + "name": "credential_group_option_unique", + "columns": [ + { + "expression": "credential_group_enrollment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "credential_group_option_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credential\".\"type\" = 'managed_oauth'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_managed_mcp_enrollment_server_unique": { + "name": "credential_managed_mcp_enrollment_server_unique", + "columns": [ + { + "expression": "credential_group_enrollment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "mcp_server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credential\".\"type\" = 'managed_mcp'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_account_unique": { + "name": "credential_workspace_account_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "account_id IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_env_unique": { + "name": "credential_workspace_env_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'env_workspace'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_personal_env_unique": { + "name": "credential_workspace_personal_env_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'env_personal'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_personal_token_identity_unique": { + "name": "credential_personal_token_identity_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_subject_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'personal_token'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_workspace_id_workspace_id_fk": { + "name": "credential_workspace_id_workspace_id_fk", + "tableFrom": "credential", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_organization_id_organization_id_fk": { + "name": "credential_organization_id_organization_id_fk", + "tableFrom": "credential", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_slack_app_id_slack_app_id_fk": { + "name": "credential_slack_app_id_slack_app_id_fk", + "tableFrom": "credential", + "tableTo": "slack_app", + "columnsFrom": ["slack_app_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "credential_account_id_account_id_fk": { + "name": "credential_account_id_account_id_fk", + "tableFrom": "credential", + "tableTo": "account", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_env_owner_user_id_user_id_fk": { + "name": "credential_env_owner_user_id_user_id_fk", + "tableFrom": "credential", + "tableTo": "user", + "columnsFrom": ["env_owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_credential_group_enrollment_id_credential_group_enrollment_id_fk": { + "name": "credential_credential_group_enrollment_id_credential_group_enrollment_id_fk", + "tableFrom": "credential", + "tableTo": "credential_group_enrollment", + "columnsFrom": ["credential_group_enrollment_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_mcp_server_id_mcp_servers_id_fk": { + "name": "credential_mcp_server_id_mcp_servers_id_fk", + "tableFrom": "credential", + "tableTo": "mcp_servers", + "columnsFrom": ["mcp_server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_created_by_user_id_fk": { + "name": "credential_created_by_user_id_fk", + "tableFrom": "credential", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "credential_owner_check": { + "name": "credential_owner_check", + "value": "num_nonnulls(\"credential\".\"workspace_id\", \"credential\".\"organization_id\") = 1" + }, + "credential_organization_type_check": { + "name": "credential_organization_type_check", + "value": "\"credential\".\"organization_id\" IS NULL OR \"credential\".\"type\" IN ('oauth', 'managed_oauth', 'managed_mcp', 'service_account', 'personal_token')" + }, + "credential_personal_token_source_check": { + "name": "credential_personal_token_source_check", + "value": "(type::text <> 'personal_token') OR (\n created_by IS NOT NULL\n AND provider_id IS NOT NULL\n AND provider_id = 'gitlab'\n AND provider_subject_id IS NOT NULL\n AND provider_tenant_id IS NOT NULL\n AND encrypted_personal_token IS NOT NULL\n AND granted_scopes IS NOT NULL\n AND cardinality(granted_scopes) > 0\n AND account_id IS NULL\n AND env_key IS NULL\n AND env_owner_user_id IS NULL\n AND authorization_app_id IS NULL\n AND encrypted_oauth_token_set IS NULL\n AND encrypted_service_account_key IS NULL\n AND unredacted = false\n )" + }, + "credential_oauth_source_check": { + "name": "credential_oauth_source_check", + "value": "(type <> 'oauth') OR (account_id IS NOT NULL AND provider_id IS NOT NULL)" + }, + "credential_managed_oauth_source_check": { + "name": "credential_managed_oauth_source_check", + "value": "(type::text <> 'managed_oauth') OR (\n account_id IS NULL\n AND provider_id IS NOT NULL\n AND authorization_app_id IS NOT NULL\n AND provider_subject_id IS NOT NULL\n AND managed_oauth_status IS NOT NULL\n AND granted_scopes IS NOT NULL\n AND encrypted_oauth_token_set IS NOT NULL\n AND granted_at IS NOT NULL\n )" + }, + "credential_managed_oauth_group_binding_check": { + "name": "credential_managed_oauth_group_binding_check", + "value": "(type::text <> 'managed_oauth') OR (\n credential_group_enrollment_id IS NOT NULL\n AND credential_group_option_id IS NOT NULL\n AND managed_oauth_scope_version IS NOT NULL\n AND managed_oauth_scope_version > 0\n )" + }, + "credential_managed_mcp_source_check": { + "name": "credential_managed_mcp_source_check", + "value": "(type::text <> 'managed_mcp') OR (\n id LIKE 'mcp-cg-%'\n AND account_id IS NULL\n AND provider_id IS NULL\n AND authorization_app_id IS NULL\n AND credential_group_enrollment_id IS NOT NULL\n AND credential_group_option_id IS NULL\n AND mcp_server_id IS NOT NULL\n AND managed_oauth_status IS NOT NULL\n AND (managed_oauth_status <> 'active' OR (\n encrypted_oauth_token_set IS NOT NULL\n AND mcp_tools IS NOT NULL\n ))\n AND granted_at IS NOT NULL\n AND managed_oauth_scope_version IS NULL\n AND provider_subject_id IS NULL\n AND provider_tenant_id IS NULL\n AND granted_scopes IS NULL\n AND provider_metadata IS NULL\n AND created_by IS NULL\n AND env_key IS NULL\n AND env_owner_user_id IS NULL\n AND encrypted_service_account_key IS NULL\n AND unredacted = false\n )" + }, + "credential_creator_source_check": { + "name": "credential_creator_source_check", + "value": "(type::text = 'managed_mcp') OR created_by IS NOT NULL" + }, + "credential_workspace_env_source_check": { + "name": "credential_workspace_env_source_check", + "value": "(type <> 'env_workspace') OR (env_key IS NOT NULL AND env_owner_user_id IS NULL)" + }, + "credential_personal_env_source_check": { + "name": "credential_personal_env_source_check", + "value": "(type <> 'env_personal') OR (env_key IS NOT NULL AND env_owner_user_id IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.credential_group": { + "name": "credential_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_id": { + "name": "public_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "encrypted_provider_configuration": { + "name": "encrypted_provider_configuration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "credential_group_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_group_organization_id_idx": { + "name": "credential_group_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_organization_unique": { + "name": "credential_group_organization_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_public_id_unique": { + "name": "credential_group_public_id_unique", + "columns": [ + { + "expression": "public_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_workspace_unique": { + "name": "credential_group_workspace_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_group_workspace_id_workspace_id_fk": { + "name": "credential_group_workspace_id_workspace_id_fk", + "tableFrom": "credential_group", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_group_organization_id_organization_id_fk": { + "name": "credential_group_organization_id_organization_id_fk", + "tableFrom": "credential_group", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_group_created_by_user_id_fk": { + "name": "credential_group_created_by_user_id_fk", + "tableFrom": "credential_group", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "credential_group_owner_check": { + "name": "credential_group_owner_check", + "value": "num_nonnulls(\"credential_group\".\"workspace_id\", \"credential_group\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.credential_group_enrollment": { + "name": "credential_group_enrollment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "credential_group_id": { + "name": "credential_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "credential_group_enrollment_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'invited'" + }, + "invitation_token_hash": { + "name": "invitation_token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invitation_expires_at": { + "name": "invitation_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "invited_at": { + "name": "invited_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_delivery_error": { + "name": "last_delivery_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_group_enrollment_group_user_unique": { + "name": "credential_group_enrollment_group_user_unique", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credential_group_enrollment\".\"user_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_user_id_idx": { + "name": "credential_group_enrollment_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_group_email_unique": { + "name": "credential_group_enrollment_group_email_unique", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_invitation_token_hash_unique": { + "name": "credential_group_enrollment_invitation_token_hash_unique", + "columns": [ + { + "expression": "invitation_token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_group_status_idx": { + "name": "credential_group_enrollment_group_status_idx", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_group_invited_at_id_idx": { + "name": "credential_group_enrollment_group_invited_at_id_idx", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invited_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_group_enrollment_credential_group_id_credential_group_id_fk": { + "name": "credential_group_enrollment_credential_group_id_credential_group_id_fk", + "tableFrom": "credential_group_enrollment", + "tableTo": "credential_group", + "columnsFrom": ["credential_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_group_enrollment_user_id_user_id_fk": { + "name": "credential_group_enrollment_user_id_user_id_fk", + "tableFrom": "credential_group_enrollment", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_group_enrollment_created_by_user_id_fk": { + "name": "credential_group_enrollment_created_by_user_id_fk", + "tableFrom": "credential_group_enrollment", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "credential_group_enrollment_normalized_email_check": { + "name": "credential_group_enrollment_normalized_email_check", + "value": "\"credential_group_enrollment\".\"email\" = lower(btrim(\"credential_group_enrollment\".\"email\")) AND length(\"credential_group_enrollment\".\"email\") BETWEEN 3 AND 320" + }, + "credential_group_enrollment_invitation_token_hash_length_check": { + "name": "credential_group_enrollment_invitation_token_hash_length_check", + "value": "length(\"credential_group_enrollment\".\"invitation_token_hash\") = 64" + } + }, + "isRLSEnabled": false + }, + "public.credential_member": { + "name": "credential_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "credential_member_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "status": { + "name": "status", + "type": "credential_member_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "joined_at": { + "name": "joined_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_member_user_id_idx": { + "name": "credential_member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_role_idx": { + "name": "credential_member_role_idx", + "columns": [ + { + "expression": "role", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_status_idx": { + "name": "credential_member_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_unique": { + "name": "credential_member_unique", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_member_credential_id_credential_id_fk": { + "name": "credential_member_credential_id_credential_id_fk", + "tableFrom": "credential_member", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_member_user_id_user_id_fk": { + "name": "credential_member_user_id_user_id_fk", + "tableFrom": "credential_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_member_invited_by_user_id_fk": { + "name": "credential_member_invited_by_user_id_fk", + "tableFrom": "credential_member", + "tableTo": "user", + "columnsFrom": ["invited_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custom_block": { + "name": "custom_block", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inputs": { + "name": "inputs", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "outputs": { + "name": "outputs", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "trace_child_runs": { + "name": "trace_child_runs", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "custom_block_organization_id_idx": { + "name": "custom_block_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_block_workflow_id_idx": { + "name": "custom_block_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_block_organization_type_unique": { + "name": "custom_block_organization_type_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "custom_block_organization_id_organization_id_fk": { + "name": "custom_block_organization_id_organization_id_fk", + "tableFrom": "custom_block", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_block_workflow_id_workflow_id_fk": { + "name": "custom_block_workflow_id_workflow_id_fk", + "tableFrom": "custom_block", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_block_created_by_user_id_fk": { + "name": "custom_block_created_by_user_id_fk", + "tableFrom": "custom_block", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custom_tools": { + "name": "custom_tools", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "custom_tools_workspace_id_idx": { + "name": "custom_tools_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_tools_workspace_title_unique": { + "name": "custom_tools_workspace_title_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "title", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "custom_tools_workspace_id_workspace_id_fk": { + "name": "custom_tools_workspace_id_workspace_id_fk", + "tableFrom": "custom_tools", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_tools_user_id_user_id_fk": { + "name": "custom_tools_user_id_user_id_fk", + "tableFrom": "custom_tools", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_drain_runs": { + "name": "data_drain_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "drain_id": { + "name": "drain_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "data_drain_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "trigger": { + "name": "trigger", + "type": "data_drain_run_trigger", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rows_exported": { + "name": "rows_exported", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bytes_written": { + "name": "bytes_written", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "cursor_before": { + "name": "cursor_before", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cursor_after": { + "name": "cursor_after", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locators": { + "name": "locators", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + } + }, + "indexes": { + "data_drain_runs_drain_started_idx": { + "name": "data_drain_runs_drain_started_idx", + "columns": [ + { + "expression": "drain_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_drain_runs_drain_id_data_drains_id_fk": { + "name": "data_drain_runs_drain_id_data_drains_id_fk", + "tableFrom": "data_drain_runs", + "tableTo": "data_drains", + "columnsFrom": ["drain_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_drains": { + "name": "data_drains", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "data_drain_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "destination_type": { + "name": "destination_type", + "type": "data_drain_destination", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "destination_config": { + "name": "destination_config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "destination_credentials": { + "name": "destination_credentials", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schedule_cadence": { + "name": "schedule_cadence", + "type": "data_drain_cadence", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_drains_org_idx": { + "name": "data_drains_org_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_drains_due_idx": { + "name": "data_drains_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_drains_org_name_unique": { + "name": "data_drains_org_name_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_drains_organization_id_organization_id_fk": { + "name": "data_drains_organization_id_organization_id_fk", + "tableFrom": "data_drains", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_drains_created_by_user_id_fk": { + "name": "data_drains_created_by_user_id_fk", + "tableFrom": "data_drains", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.docs_embeddings": { + "name": "docs_embeddings", + "schema": "", + "columns": { + "chunk_id": { + "name": "chunk_id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chunk_text": { + "name": "chunk_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_document": { + "name": "source_document", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_link": { + "name": "source_link", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "header_text": { + "name": "header_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "header_level": { + "name": "header_level", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": true + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "chunk_text_tsv": { + "name": "chunk_text_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "to_tsvector('english', \"docs_embeddings\".\"chunk_text\")", + "type": "stored" + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "docs_emb_source_document_idx": { + "name": "docs_emb_source_document_idx", + "columns": [ + { + "expression": "source_document", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_header_level_idx": { + "name": "docs_emb_header_level_idx", + "columns": [ + { + "expression": "header_level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_source_header_idx": { + "name": "docs_emb_source_header_idx", + "columns": [ + { + "expression": "source_document", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "header_level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_model_idx": { + "name": "docs_emb_model_idx", + "columns": [ + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_created_at_idx": { + "name": "docs_emb_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_embedding_vector_hnsw_idx": { + "name": "docs_embedding_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "docs_emb_metadata_gin_idx": { + "name": "docs_emb_metadata_gin_idx", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "docs_emb_chunk_text_fts_idx": { + "name": "docs_emb_chunk_text_fts_idx", + "columns": [ + { + "expression": "chunk_text_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "docs_embedding_not_null_check": { + "name": "docs_embedding_not_null_check", + "value": "\"embedding\" IS NOT NULL" + }, + "docs_header_level_check": { + "name": "docs_header_level_check", + "value": "\"header_level\" >= 1 AND \"header_level\" <= 6" + } + }, + "isRLSEnabled": false + }, + "public.document": { + "name": "document", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "file_url": { + "name": "file_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_count": { + "name": "chunk_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "character_count": { + "name": "character_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "processing_status": { + "name": "processing_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "processing_attempts": { + "name": "processing_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "processing_queued_at": { + "name": "processing_queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_queue_token": { + "name": "processing_queue_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_deferred_until": { + "name": "processing_deferred_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_completed_at": { + "name": "processing_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "processing_recovery_after": { + "name": "processing_recovery_after", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "user_excluded": { + "name": "user_excluded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "tag1": { + "name": "tag1", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag2": { + "name": "tag2", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag3": { + "name": "tag3", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag4": { + "name": "tag4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag5": { + "name": "tag5", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag6": { + "name": "tag6", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag7": { + "name": "tag7", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "number1": { + "name": "number1", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number2": { + "name": "number2", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number3": { + "name": "number3", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number4": { + "name": "number4", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number5": { + "name": "number5", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "date1": { + "name": "date1", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "date2": { + "name": "date2", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "boolean1": { + "name": "boolean1", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean2": { + "name": "boolean2", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean3": { + "name": "boolean3", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_provenance_version": { + "name": "secret_provenance_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "acl": { + "name": "acl", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{ws}'::text[]" + }, + "acl_requirements": { + "name": "acl_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "acl_verified_at": { + "name": "acl_verified_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "source_modified_at": { + "name": "source_modified_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "source_seen_at": { + "name": "source_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "doc_kb_id_idx": { + "name": "doc_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_source_modified_idx": { + "name": "doc_kb_source_modified_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_modified_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"deleted_at\" IS NULL", + "concurrently": true, + "method": "btree", + "with": {} + }, + "doc_acl_gin_idx": { + "name": "doc_acl_gin_idx", + "columns": [ + { + "expression": "acl", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "array_ops" + } + ], + "isUnique": false, + "where": "\"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "gin", + "with": {} + }, + "doc_filename_idx": { + "name": "doc_filename_idx", + "columns": [ + { + "expression": "filename", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_processing_status_idx": { + "name": "doc_processing_status_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "processing_status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_processing_recovery_idx": { + "name": "doc_processing_recovery_idx", + "columns": [ + { + "expression": "uploaded_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"processing_status\" IN ('pending', 'processing', 'failed') AND \"document\".\"connector_id\" IS NOT NULL AND \"document\".\"content_hash\" IS NOT NULL AND \"document\".\"storage_key\" IS NOT NULL AND \"document\".\"user_excluded\" = false AND \"document\".\"archived_at\" IS NULL AND \"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_processing_status_idx": { + "name": "doc_connector_processing_status_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "processing_status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"processing_status\" IN ('pending', 'processing', 'failed') AND \"document\".\"connector_id\" IS NOT NULL AND \"document\".\"user_excluded\" = false AND \"document\".\"archived_at\" IS NULL AND \"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_external_id_idx": { + "name": "doc_connector_external_id_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_source_lookup_idx": { + "name": "doc_connector_source_lookup_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_reconciliation_idx": { + "name": "doc_connector_reconciliation_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "COALESCE(\"source_seen_at\", '-infinity'::timestamp)", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"user_excluded\" = false AND \"document\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_active_kb_token_count_idx": { + "name": "doc_active_kb_token_count_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token_count", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"user_excluded\" = false AND \"document\".\"archived_at\" IS NULL AND \"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_storage_key_idx": { + "name": "doc_storage_key_idx", + "columns": [ + { + "expression": "storage_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"storage_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_archived_at_partial_idx": { + "name": "doc_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_deleted_at_partial_idx": { + "name": "doc_deleted_at_partial_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_tombstone_idx": { + "name": "doc_connector_tombstone_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"archived_at\" IS NULL AND (\"document\".\"deleted_at\" IS NOT NULL OR \"document\".\"content_hash\" IS NULL)", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_live_idx": { + "name": "doc_connector_live_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"user_excluded\" = false AND \"document\".\"archived_at\" IS NULL AND \"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag1_lower_idx": { + "name": "doc_kb_tag1_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag1\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag2_lower_idx": { + "name": "doc_kb_tag2_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag2\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag3_lower_idx": { + "name": "doc_kb_tag3_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag3\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag4_lower_idx": { + "name": "doc_kb_tag4_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag4\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag5_lower_idx": { + "name": "doc_kb_tag5_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag5\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag6_lower_idx": { + "name": "doc_kb_tag6_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag6\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag7_lower_idx": { + "name": "doc_kb_tag7_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag7\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number1_idx": { + "name": "doc_number1_idx", + "columns": [ + { + "expression": "number1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number2_idx": { + "name": "doc_number2_idx", + "columns": [ + { + "expression": "number2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number3_idx": { + "name": "doc_number3_idx", + "columns": [ + { + "expression": "number3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number4_idx": { + "name": "doc_number4_idx", + "columns": [ + { + "expression": "number4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number5_idx": { + "name": "doc_number5_idx", + "columns": [ + { + "expression": "number5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_date1_idx": { + "name": "doc_date1_idx", + "columns": [ + { + "expression": "date1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_date2_idx": { + "name": "doc_date2_idx", + "columns": [ + { + "expression": "date2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean1_idx": { + "name": "doc_boolean1_idx", + "columns": [ + { + "expression": "boolean1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean2_idx": { + "name": "doc_boolean2_idx", + "columns": [ + { + "expression": "boolean2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean3_idx": { + "name": "doc_boolean3_idx", + "columns": [ + { + "expression": "boolean3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_knowledge_base_id_knowledge_base_id_fk": { + "name": "document_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "document", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "document_connector_id_knowledge_connector_id_fk": { + "name": "document_connector_id_knowledge_connector_id_fk", + "tableFrom": "document", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "document_uploaded_by_user_id_fk": { + "name": "document_uploaded_by_user_id_fk", + "tableFrom": "document", + "tableTo": "user", + "columnsFrom": ["uploaded_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "doc_acl_token_shape_check": { + "name": "doc_acl_token_shape_check", + "value": "array_position(\"document\".\"acl\", NULL) IS NULL AND (cardinality(\"document\".\"acl\") = 0 OR (cardinality(\"document\".\"acl\") = array_length(string_to_array(array_to_string(\"document\".\"acl\", E'\\n'), E'\\n'), 1) AND array_to_string(\"document\".\"acl\", E'\\n') ~ '^((ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+)(\\n(ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+))*)$'))" + } + }, + "isRLSEnabled": false + }, + "public.document_secret_provenance": { + "name": "document_secret_provenance", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "source_hash": { + "name": "source_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "document_secret_provenance_document_id_document_id_fk": { + "name": "document_secret_provenance_document_id_document_id_fk", + "tableFrom": "document_secret_provenance", + "tableTo": "document", + "columnsFrom": ["document_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "document_secret_provenance_status_check": { + "name": "document_secret_provenance_status_check", + "value": "\"document_secret_provenance\".\"status\" IN ('exact', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.embedding": { + "name": "embedding", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_index": { + "name": "chunk_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "chunk_hash": { + "name": "chunk_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_provenance_version": { + "name": "secret_provenance_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "content_length": { + "name": "content_length", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "embedding_384": { + "name": "embedding_384", + "type": "vector(384)", + "primaryKey": false, + "notNull": false + }, + "embedding_768": { + "name": "embedding_768", + "type": "vector(768)", + "primaryKey": false, + "notNull": false + }, + "embedding_1024": { + "name": "embedding_1024", + "type": "vector(1024)", + "primaryKey": false, + "notNull": false + }, + "embedding_3072": { + "name": "embedding_3072", + "type": "vector(3072)", + "primaryKey": false, + "notNull": false + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "tag1": { + "name": "tag1", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag2": { + "name": "tag2", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag3": { + "name": "tag3", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag4": { + "name": "tag4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag5": { + "name": "tag5", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag6": { + "name": "tag6", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag7": { + "name": "tag7", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "number1": { + "name": "number1", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number2": { + "name": "number2", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number3": { + "name": "number3", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number4": { + "name": "number4", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number5": { + "name": "number5", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "date1": { + "name": "date1", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "date2": { + "name": "date2", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "boolean1": { + "name": "boolean1", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean2": { + "name": "boolean2", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean3": { + "name": "boolean3", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "content_tsv": { + "name": "content_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "to_tsvector('english', \"embedding\".\"content\")", + "type": "stored" + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "emb_kb_id_idx": { + "name": "emb_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_id_idx": { + "name": "emb_doc_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_chunk_idx": { + "name": "emb_doc_chunk_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chunk_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_model_idx": { + "name": "emb_kb_model_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_enabled_idx": { + "name": "emb_kb_enabled_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_enabled_idx": { + "name": "emb_doc_enabled_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag1_lower_idx": { + "name": "emb_kb_tag1_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag1\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag2_lower_idx": { + "name": "emb_kb_tag2_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag2\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag3_lower_idx": { + "name": "emb_kb_tag3_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag3\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag4_lower_idx": { + "name": "emb_kb_tag4_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag4\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag5_lower_idx": { + "name": "emb_kb_tag5_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag5\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag6_lower_idx": { + "name": "emb_kb_tag6_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag6\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag7_lower_idx": { + "name": "emb_kb_tag7_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag7\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number1_idx": { + "name": "emb_number1_idx", + "columns": [ + { + "expression": "number1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number2_idx": { + "name": "emb_number2_idx", + "columns": [ + { + "expression": "number2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number3_idx": { + "name": "emb_number3_idx", + "columns": [ + { + "expression": "number3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number4_idx": { + "name": "emb_number4_idx", + "columns": [ + { + "expression": "number4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number5_idx": { + "name": "emb_number5_idx", + "columns": [ + { + "expression": "number5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_date1_idx": { + "name": "emb_date1_idx", + "columns": [ + { + "expression": "date1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_date2_idx": { + "name": "emb_date2_idx", + "columns": [ + { + "expression": "date2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean1_idx": { + "name": "emb_boolean1_idx", + "columns": [ + { + "expression": "boolean1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean2_idx": { + "name": "emb_boolean2_idx", + "columns": [ + { + "expression": "boolean2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean3_idx": { + "name": "emb_boolean3_idx", + "columns": [ + { + "expression": "boolean3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_content_fts_idx": { + "name": "emb_content_fts_idx", + "columns": [ + { + "expression": "content_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "embedding_knowledge_base_id_knowledge_base_id_fk": { + "name": "embedding_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "embedding", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "embedding_document_id_document_id_fk": { + "name": "embedding_document_id_document_id_fk", + "tableFrom": "embedding", + "tableTo": "document", + "columnsFrom": ["document_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "embedding_width_check": { + "name": "embedding_width_check", + "value": "num_nonnulls(\"embedding\", \"embedding_384\", \"embedding_768\", \"embedding_1024\", \"embedding_3072\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.embedding_keyword_search": { + "name": "embedding_keyword_search", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "content_tsv": { + "name": "content_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "embedding_keyword_search_kb_idx": { + "name": "embedding_keyword_search_kb_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "embedding_keyword_search_document_idx": { + "name": "embedding_keyword_search_document_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "embedding_keyword_search_content_idx": { + "name": "embedding_keyword_search_content_idx", + "columns": [ + { + "expression": "content_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "embedding_keyword_search_id_embedding_id_fk": { + "name": "embedding_keyword_search_id_embedding_id_fk", + "tableFrom": "embedding_keyword_search", + "tableTo": "embedding", + "columnsFrom": ["id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.embedding_keyword_tin": { + "name": "embedding_keyword_tin", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "acl": { + "name": "acl", + "type": "text[]", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "embedding_keyword_tin_document_idx": { + "name": "embedding_keyword_tin_document_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "embedding_keyword_tin_id_embedding_id_fk": { + "name": "embedding_keyword_tin_id_embedding_id_fk", + "tableFrom": "embedding_keyword_tin", + "tableTo": "embedding", + "columnsFrom": ["id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.embedding_search": { + "name": "embedding_search", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "acl": { + "name": "acl", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "binary": { + "name": "binary", + "type": "bit(1536)", + "primaryKey": false, + "notNull": false + }, + "binary_384": { + "name": "binary_384", + "type": "bit(384)", + "primaryKey": false, + "notNull": false + }, + "binary_768": { + "name": "binary_768", + "type": "bit(768)", + "primaryKey": false, + "notNull": false + }, + "binary_1024": { + "name": "binary_1024", + "type": "bit(1024)", + "primaryKey": false, + "notNull": false + }, + "binary_3072": { + "name": "binary_3072", + "type": "bit(3072)", + "primaryKey": false, + "notNull": false + }, + "vector": { + "name": "vector", + "type": "halfvec(1536)", + "primaryKey": false, + "notNull": false + }, + "vector_384": { + "name": "vector_384", + "type": "halfvec(384)", + "primaryKey": false, + "notNull": false + }, + "vector_512": { + "name": "vector_512", + "type": "halfvec(512)", + "primaryKey": false, + "notNull": false + }, + "vector_768": { + "name": "vector_768", + "type": "halfvec(768)", + "primaryKey": false, + "notNull": false + }, + "vector_1024": { + "name": "vector_1024", + "type": "halfvec(1024)", + "primaryKey": false, + "notNull": false + }, + "vector_3072": { + "name": "vector_3072", + "type": "halfvec(3072)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "embedding_search_kb_idx": { + "name": "embedding_search_kb_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "embedding_search_document_lookup_idx": { + "name": "embedding_search_document_lookup_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"embedding_search\".\"enabled\"", + "concurrently": true, + "method": "btree", + "with": {} + }, + "embedding_search_cosine_hnsw_idx": { + "name": "embedding_search_cosine_hnsw_idx", + "columns": [ + { + "expression": "vector", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "halfvec_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "embedding_search_512_cosine_hnsw_idx": { + "name": "embedding_search_512_cosine_hnsw_idx", + "columns": [ + { + "expression": "vector_512", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "halfvec_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "embedding_search_384_cosine_hnsw_idx": { + "name": "embedding_search_384_cosine_hnsw_idx", + "columns": [ + { + "expression": "vector_384", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "halfvec_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "embedding_search_768_cosine_hnsw_idx": { + "name": "embedding_search_768_cosine_hnsw_idx", + "columns": [ + { + "expression": "vector_768", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "halfvec_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "embedding_search_1024_cosine_hnsw_idx": { + "name": "embedding_search_1024_cosine_hnsw_idx", + "columns": [ + { + "expression": "vector_1024", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "halfvec_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "embedding_search_3072_cosine_hnsw_idx": { + "name": "embedding_search_3072_cosine_hnsw_idx", + "columns": [ + { + "expression": "vector_3072", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "halfvec_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + } + }, + "foreignKeys": { + "embedding_search_id_embedding_id_fk": { + "name": "embedding_search_id_embedding_id_fk", + "tableFrom": "embedding_search", + "tableTo": "embedding", + "columnsFrom": ["id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "embedding_search_width_check": { + "name": "embedding_search_width_check", + "value": "num_nonnulls(\"binary\", \"binary_384\", \"binary_768\", \"binary_1024\", \"binary_3072\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.embedding_secret_provenance": { + "name": "embedding_secret_provenance", + "schema": "", + "columns": { + "embedding_id": { + "name": "embedding_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "embedding_secret_provenance_embedding_id_embedding_id_fk": { + "name": "embedding_secret_provenance_embedding_id_embedding_id_fk", + "tableFrom": "embedding_secret_provenance", + "tableTo": "embedding", + "columnsFrom": ["embedding_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "embedding_secret_provenance_status_check": { + "name": "embedding_secret_provenance_status_check", + "value": "\"embedding_secret_provenance\".\"status\" IN ('exact', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.environment": { + "name": "environment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "environment_user_id_user_id_fk": { + "name": "environment_user_id_user_id_fk", + "tableFrom": "environment", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "environment_user_id_unique": { + "name": "environment_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_value_dependencies": { + "name": "execution_large_value_dependencies", + "schema": "", + "columns": { + "parent_key": { + "name": "parent_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_key": { + "name": "child_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_large_value_dependencies_workspace_parent_key_idx": { + "name": "execution_large_value_dependencies_workspace_parent_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_value_dependencies_workspace_child_key_idx": { + "name": "execution_large_value_dependencies_workspace_child_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_value_dependencies_workspace_id_workspace_id_fk": { + "name": "execution_large_value_dependencies_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_value_dependencies", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_large_value_dependencies_parent_key_child_key_pk": { + "name": "execution_large_value_dependencies_parent_key_child_key_pk", + "columns": ["parent_key", "child_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_value_references": { + "name": "execution_large_value_references", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "execution_large_value_reference_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_large_value_references_workspace_execution_source_idx": { + "name": "execution_large_value_references_workspace_execution_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_value_references_workflow_id_idx": { + "name": "execution_large_value_references_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_value_references_workspace_id_workspace_id_fk": { + "name": "execution_large_value_references_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_value_references", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_large_value_references_workflow_id_workflow_id_fk": { + "name": "execution_large_value_references_workflow_id_workflow_id_fk", + "tableFrom": "execution_large_value_references", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_large_value_references_key_execution_id_source_pk": { + "name": "execution_large_value_references_key_execution_id_source_pk", + "columns": ["key", "execution_id", "source"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_values": { + "name": "execution_large_values", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_execution_id": { + "name": "owner_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "execution_large_values_owner_execution_id_idx": { + "name": "execution_large_values_owner_execution_id_idx", + "columns": [ + { + "expression": "owner_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_cleanup_idx": { + "name": "execution_large_values_cleanup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_large_values\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_tombstone_cleanup_idx": { + "name": "execution_large_values_tombstone_cleanup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_large_values\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_workflow_id_idx": { + "name": "execution_large_values_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_values_workspace_id_workspace_id_fk": { + "name": "execution_large_values_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_values", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_large_values_workflow_id_workflow_id_fk": { + "name": "execution_large_values_workflow_id_workflow_id_fk", + "tableFrom": "execution_large_values", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.folder": { + "name": "folder", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "folder_resource_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "folder_user_idx": { + "name": "folder_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_resource_parent_idx": { + "name": "folder_workspace_resource_parent_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_parent_sort_idx": { + "name": "folder_parent_sort_idx", + "columns": [ + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_deleted_at_idx": { + "name": "folder_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_deleted_partial_idx": { + "name": "folder_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"folder\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_resource_parent_name_active_unique": { + "name": "folder_workspace_resource_parent_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"parent_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"folder\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "folder_user_id_user_id_fk": { + "name": "folder_user_id_user_id_fk", + "tableFrom": "folder", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "folder_workspace_id_workspace_id_fk": { + "name": "folder_workspace_id_workspace_id_fk", + "tableFrom": "folder", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "folder_parent_id_folder_id_fk": { + "name": "folder_parent_id_folder_id_fk", + "tableFrom": "folder", + "tableTo": "folder", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.idempotency_key": { + "name": "idempotency_key", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idempotency_key_created_at_idx": { + "name": "idempotency_key_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation": { + "name": "invitation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "invitation_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'organization'" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "membership_intent": { + "name": "membership_intent", + "type": "invitation_membership_intent", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'internal'" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "invitation_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_status_idx": { + "name": "invitation_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_pending_email_org_unique": { + "name": "invitation_pending_email_org_unique", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"invitation\".\"status\" = 'pending' AND \"invitation\".\"organization_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "invitation_token_unique": { + "name": "invitation_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation_workspace_grant": { + "name": "invitation_workspace_grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "invitation_id": { + "name": "invitation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission": { + "name": "permission", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "invitation_workspace_grant_unique": { + "name": "invitation_workspace_grant_unique", + "columns": [ + { + "expression": "invitation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_workspace_grant_workspace_id_idx": { + "name": "invitation_workspace_grant_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_workspace_grant_invitation_id_invitation_id_fk": { + "name": "invitation_workspace_grant_invitation_id_invitation_id_fk", + "tableFrom": "invitation_workspace_grant", + "tableTo": "invitation", + "columnsFrom": ["invitation_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_workspace_grant_workspace_id_workspace_id_fk": { + "name": "invitation_workspace_grant_workspace_id_workspace_id_fk", + "tableFrom": "invitation_workspace_grant", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.job_execution_logs": { + "name": "job_execution_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "schedule_id": { + "name": "schedule_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_duration_ms": { + "name": "total_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "execution_data": { + "name": "execution_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "cost": { + "name": "cost", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "job_execution_logs_schedule_id_idx": { + "name": "job_execution_logs_schedule_id_idx", + "columns": [ + { + "expression": "schedule_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_workspace_started_at_idx": { + "name": "job_execution_logs_workspace_started_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_workspace_ended_at_id_idx": { + "name": "job_execution_logs_workspace_ended_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"ended_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_execution_id_unique": { + "name": "job_execution_logs_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_trigger_idx": { + "name": "job_execution_logs_trigger_idx", + "columns": [ + { + "expression": "trigger", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "job_execution_logs_schedule_id_workflow_schedule_id_fk": { + "name": "job_execution_logs_schedule_id_workflow_schedule_id_fk", + "tableFrom": "job_execution_logs", + "tableTo": "workflow_schedule", + "columnsFrom": ["schedule_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "job_execution_logs_workspace_id_workspace_id_fk": { + "name": "job_execution_logs_workspace_id_workspace_id_fk", + "tableFrom": "job_execution_logs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_base": { + "name": "knowledge_base", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_search_index": { + "name": "is_search_index", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "embedding_dimension": { + "name": "embedding_dimension", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1536 + }, + "chunking_config": { + "name": "chunking_config", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{\"maxSize\": 1024, \"minSize\": 1, \"overlap\": 200}'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kb_organization_id_idx": { + "name": "kb_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_organization_search_index_unique": { + "name": "kb_organization_search_index_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"knowledge_base\".\"is_search_index\" = true AND \"knowledge_base\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_organization_name_active_unique": { + "name": "kb_organization_name_active_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"knowledge_base\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_user_id_idx": { + "name": "kb_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_id_idx": { + "name": "kb_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_user_workspace_idx": { + "name": "kb_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_folder_id_idx": { + "name": "kb_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_deleted_at_idx": { + "name": "kb_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_deleted_partial_idx": { + "name": "kb_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_base\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_name_active_unique": { + "name": "kb_workspace_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"knowledge_base\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_search_index_unique": { + "name": "kb_workspace_search_index_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"knowledge_base\".\"is_search_index\" = true AND \"knowledge_base\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_base_user_id_user_id_fk": { + "name": "knowledge_base_user_id_user_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_workspace_id_workspace_id_fk": { + "name": "knowledge_base_workspace_id_workspace_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_organization_id_organization_id_fk": { + "name": "knowledge_base_organization_id_organization_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_folder_id_folder_id_fk": { + "name": "knowledge_base_folder_id_folder_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "kb_owner_check": { + "name": "kb_owner_check", + "value": "num_nonnulls(\"knowledge_base\".\"workspace_id\", \"knowledge_base\".\"organization_id\") = 1" + }, + "kb_organization_search_index_check": { + "name": "kb_organization_search_index_check", + "value": "\"knowledge_base\".\"organization_id\" IS NULL OR \"knowledge_base\".\"is_search_index\"" + }, + "kb_organization_folder_check": { + "name": "kb_organization_folder_check", + "value": "\"knowledge_base\".\"organization_id\" IS NULL OR \"knowledge_base\".\"folder_id\" IS NULL" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_base_tag_definitions": { + "name": "knowledge_base_tag_definitions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_slot": { + "name": "tag_slot", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "field_type": { + "name": "field_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kb_tag_definitions_kb_slot_idx": { + "name": "kb_tag_definitions_kb_slot_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tag_slot", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_tag_definitions_kb_display_name_idx": { + "name": "kb_tag_definitions_kb_display_name_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_tag_definitions_kb_id_idx": { + "name": "kb_tag_definitions_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_base_tag_definitions_knowledge_base_id_knowledge_base_id_fk": { + "name": "knowledge_base_tag_definitions_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "knowledge_base_tag_definitions", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_connector": { + "name": "knowledge_connector", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connector_type": { + "name": "connector_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_config": { + "name": "source_config", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "sync_mode": { + "name": "sync_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'full'" + }, + "sync_interval_minutes": { + "name": "sync_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1440 + }, + "access_mode": { + "name": "access_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'workspace'" + }, + "credential_group_id": { + "name": "credential_group_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_group_option_id": { + "name": "credential_group_option_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "member_sync_status": { + "name": "member_sync_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'idle'" + }, + "member_sync_lock_token": { + "name": "member_sync_lock_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "member_sync_lock_lease_at": { + "name": "member_sync_lock_lease_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "next_member_sync_at": { + "name": "next_member_sync_at", + "type": "timestamp (3)", + "primaryKey": false, + "notNull": false + }, + "last_member_sync_at": { + "name": "last_member_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_member_sync_error": { + "name": "last_member_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "member_sync_consecutive_failures": { + "name": "member_sync_consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "access_rewrite_pending": { + "name": "access_rewrite_pending", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "member_tombstone_cursor": { + "name": "member_tombstone_cursor", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_sync_doc_count": { + "name": "last_sync_doc_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "listing_checkpoint": { + "name": "listing_checkpoint", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "directory_checkpoint": { + "name": "directory_checkpoint", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "next_sync_at": { + "name": "next_sync_at", + "type": "timestamp (3)", + "primaryKey": false, + "notNull": false + }, + "next_directory_sync_at": { + "name": "next_directory_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "sync_lock_token": { + "name": "sync_lock_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_lock_lease_at": { + "name": "sync_lock_lease_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "detached_at": { + "name": "detached_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "detach_reserved_bytes": { + "name": "detach_reserved_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "kc_knowledge_base_id_idx": { + "name": "kc_knowledge_base_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_status_next_sync_idx": { + "name": "kc_status_next_sync_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_sync_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_archived_at_partial_idx": { + "name": "kc_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_deleted_at_partial_idx": { + "name": "kc_deleted_at_partial_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_member_sync_due_idx": { + "name": "kc_member_sync_due_idx", + "columns": [ + { + "expression": "member_sync_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_member_sync_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"access_mode\" = 'members' AND \"knowledge_connector\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_directory_sync_due_idx": { + "name": "kc_directory_sync_due_idx", + "columns": [ + { + "expression": "next_directory_sync_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"access_mode\" = 'admin' AND \"knowledge_connector\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_knowledge_base_id_knowledge_base_id_fk": { + "name": "knowledge_connector_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "knowledge_connector", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_connector_credential_group_id_credential_group_id_fk": { + "name": "knowledge_connector_credential_group_id_credential_group_id_fk", + "tableFrom": "knowledge_connector", + "tableTo": "credential_group", + "columnsFrom": ["credential_group_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "kc_access_mode_check": { + "name": "kc_access_mode_check", + "value": "\"knowledge_connector\".\"access_mode\" IN ('workspace', 'members', 'admin')" + }, + "kc_member_sync_status_check": { + "name": "kc_member_sync_status_check", + "value": "\"knowledge_connector\".\"member_sync_status\" IN ('idle', 'pending', 'running', 'error', 'disabled')" + }, + "kc_sync_lock_exclusive_check": { + "name": "kc_sync_lock_exclusive_check", + "value": "NOT (\"knowledge_connector\".\"sync_lock_token\" IS NOT NULL AND \"knowledge_connector\".\"member_sync_lock_token\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_connector_member": { + "name": "knowledge_connector_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "subject_token": { + "name": "subject_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_started_at": { + "name": "last_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_complete_listing_at": { + "name": "last_complete_listing_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_listed_count": { + "name": "last_listed_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "member_synced_through": { + "name": "member_synced_through", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope_renewed_at": { + "name": "scope_renewed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope_renewal_cursor": { + "name": "scope_renewal_cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope_renewal_started_at": { + "name": "scope_renewal_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "change_cursor": { + "name": "change_cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "listing_checkpoint": { + "name": "listing_checkpoint", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "suspended_at": { + "name": "suspended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kcm_organization_id_idx": { + "name": "kcm_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kcm_connector_credential_unique": { + "name": "kcm_connector_credential_unique", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kcm_connector_queue_idx": { + "name": "kcm_connector_queue_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "first" + }, + { + "expression": "last_started_at", + "isExpression": false, + "asc": true, + "nulls": "first" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kcm_credential_idx": { + "name": "kcm_credential_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_member_workspace_id_workspace_id_fk": { + "name": "knowledge_connector_member_workspace_id_workspace_id_fk", + "tableFrom": "knowledge_connector_member", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_connector_member_organization_id_organization_id_fk": { + "name": "knowledge_connector_member_organization_id_organization_id_fk", + "tableFrom": "knowledge_connector_member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_connector_member_connector_id_knowledge_connector_id_fk": { + "name": "knowledge_connector_member_connector_id_knowledge_connector_id_fk", + "tableFrom": "knowledge_connector_member", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_connector_member_credential_id_credential_id_fk": { + "name": "knowledge_connector_member_credential_id_credential_id_fk", + "tableFrom": "knowledge_connector_member", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "kcm_owner_check": { + "name": "kcm_owner_check", + "value": "num_nonnulls(\"knowledge_connector_member\".\"workspace_id\", \"knowledge_connector_member\".\"organization_id\") = 1" + }, + "kcm_status_check": { + "name": "kcm_status_check", + "value": "\"knowledge_connector_member\".\"status\" IN ('active', 'suspended', 'disabled')" + }, + "kcm_subject_token_shape_check": { + "name": "kcm_subject_token_shape_check", + "value": "\"knowledge_connector_member\".\"subject_token\" ~ '^s:[^:]+:[^:]+:.+$'" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_connector_member_sync_log": { + "name": "knowledge_connector_member_sync_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "members_claimed": { + "name": "members_claimed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "members_completed": { + "name": "members_completed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "members_incomplete": { + "name": "members_incomplete", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "members_failed": { + "name": "members_failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_failed": { + "name": "docs_failed", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "processing_dispatch_failed": { + "name": "processing_dispatch_failed", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "docs_listed": { + "name": "docs_listed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_added": { + "name": "docs_added", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_updated": { + "name": "docs_updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_unchanged": { + "name": "docs_unchanged", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_hydrated_once": { + "name": "docs_hydrated_once", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "observations_added": { + "name": "observations_added", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "observations_renewed": { + "name": "observations_renewed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "observations_removed": { + "name": "observations_removed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_tombstoned": { + "name": "docs_tombstoned", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_resurrected": { + "name": "docs_resurrected", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_purged": { + "name": "docs_purged", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "credentials_audited": { + "name": "credentials_audited", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kcmsl_connector_started_at_idx": { + "name": "kcmsl_connector_started_at_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"started_at\" DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kcmsl_started_at_partial_idx": { + "name": "kcmsl_started_at_partial_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector_member_sync_log\".\"status\" = 'started'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_member_sync_log_connector_id_knowledge_connector_id_fk": { + "name": "knowledge_connector_member_sync_log_connector_id_knowledge_connector_id_fk", + "tableFrom": "knowledge_connector_member_sync_log", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "kcmsl_status_check": { + "name": "kcmsl_status_check", + "value": "\"knowledge_connector_member_sync_log\".\"status\" IN ('started', 'partial', 'completed', 'failed')" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_connector_partition": { + "name": "knowledge_connector_partition", + "schema": "", + "columns": { + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "partition_key": { + "name": "partition_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "generation_id": { + "name": "generation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "context": { + "name": "context", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "retry_at": { + "name": "retry_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_served_at": { + "name": "last_served_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure": { + "name": "failure", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "permission_cursor": { + "name": "permission_cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "permission_attempts": { + "name": "permission_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "permission_retry_at": { + "name": "permission_retry_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "permission_last_served_at": { + "name": "permission_last_served_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "permission_started_at": { + "name": "permission_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "permission_failure": { + "name": "permission_failure", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kcp_content_due_idx": { + "name": "kcp_content_due_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retry_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_served_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kcp_permission_due_idx": { + "name": "kcp_permission_due_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "permission_retry_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "permission_last_served_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_partition_connector_id_knowledge_connector_id_fk": { + "name": "knowledge_connector_partition_connector_id_knowledge_connector_id_fk", + "tableFrom": "knowledge_connector_partition", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "kcp_pk": { + "name": "kcp_pk", + "columns": ["connector_id", "partition_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "kcp_partition_key_check": { + "name": "kcp_partition_key_check", + "value": "octet_length(\"knowledge_connector_partition\".\"partition_key\") BETWEEN 1 AND 1024" + }, + "kcp_context_check": { + "name": "kcp_context_check", + "value": "jsonb_typeof(\"knowledge_connector_partition\".\"context\") = 'object' AND octet_length(\"knowledge_connector_partition\".\"context\"::text) <= 16384" + }, + "kcp_status_check": { + "name": "kcp_status_check", + "value": "\"knowledge_connector_partition\".\"status\" IN ('pending', 'complete', 'blocked')" + }, + "kcp_cursor_check": { + "name": "kcp_cursor_check", + "value": "(\"knowledge_connector_partition\".\"cursor\" IS NULL OR octet_length(\"knowledge_connector_partition\".\"cursor\") <= 393216) AND (\"knowledge_connector_partition\".\"permission_cursor\" IS NULL OR octet_length(\"knowledge_connector_partition\".\"permission_cursor\") <= 393216)" + }, + "kcp_attempts_check": { + "name": "kcp_attempts_check", + "value": "\"knowledge_connector_partition\".\"attempts\" >= 0 AND \"knowledge_connector_partition\".\"permission_attempts\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_connector_permission_grant": { + "name": "knowledge_connector_permission_grant", + "schema": "", + "columns": { + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_key": { + "name": "group_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "subject_token": { + "name": "subject_token", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "kcpg_subject_idx": { + "name": "kcpg_subject_idx", + "columns": [ + { + "expression": "subject_token", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "group_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "kcpg_snapshot_fk": { + "name": "kcpg_snapshot_fk", + "tableFrom": "knowledge_connector_permission_grant", + "tableTo": "knowledge_connector_permission_snapshot", + "columnsFrom": ["connector_id"], + "columnsTo": ["connector_id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "kcpg_pk": { + "name": "kcpg_pk", + "columns": ["connector_id", "group_key", "subject_token"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "kcpg_group_check": { + "name": "kcpg_group_check", + "value": "length(\"knowledge_connector_permission_grant\".\"group_key\") BETWEEN 1 AND 255" + }, + "kcpg_subject_check": { + "name": "kcpg_subject_check", + "value": "\"knowledge_connector_permission_grant\".\"subject_token\" ~ '^u:[^[:space:]A-Z]+@[^[:space:]A-Z]+$'" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_connector_permission_snapshot": { + "name": "knowledge_connector_permission_snapshot", + "schema": "", + "columns": { + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "kcps_connector_fk": { + "name": "kcps_connector_fk", + "tableFrom": "knowledge_connector_permission_snapshot", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "kcps_revision_check": { + "name": "kcps_revision_check", + "value": "\"knowledge_connector_permission_snapshot\".\"revision\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_connector_sync_log": { + "name": "knowledge_connector_sync_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "docs_added": { + "name": "docs_added", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_updated": { + "name": "docs_updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_deleted": { + "name": "docs_deleted", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_unchanged": { + "name": "docs_unchanged", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_skipped": { + "name": "docs_skipped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_failed": { + "name": "docs_failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "listed_count": { + "name": "listed_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kcsl_connector_started_at_idx": { + "name": "kcsl_connector_started_at_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"started_at\" DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kcsl_started_at_partial_idx": { + "name": "kcsl_started_at_partial_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector_sync_log\".\"status\" = 'started'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_sync_log_connector_id_knowledge_connector_id_fk": { + "name": "knowledge_connector_sync_log_connector_id_knowledge_connector_id_fk", + "tableFrom": "knowledge_connector_sync_log", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_document_observation": { + "name": "knowledge_document_observation", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "member_id": { + "name": "member_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "kdo_member_idx": { + "name": "kdo_member_idx", + "columns": [ + { + "expression": "member_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_document_observation_document_id_document_id_fk": { + "name": "knowledge_document_observation_document_id_document_id_fk", + "tableFrom": "knowledge_document_observation", + "tableTo": "document", + "columnsFrom": ["document_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_document_observation_member_id_knowledge_connector_member_id_fk": { + "name": "knowledge_document_observation_member_id_knowledge_connector_member_id_fk", + "tableFrom": "knowledge_document_observation", + "tableTo": "knowledge_connector_member", + "columnsFrom": ["member_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "knowledge_document_observation_document_id_member_id_pk": { + "name": "knowledge_document_observation_document_id_member_id_pk", + "columns": ["document_id", "member_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_external_directory": { + "name": "knowledge_external_directory", + "schema": "", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sync_lock_token": { + "name": "sync_lock_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_lock_lease_at": { + "name": "sync_lock_lease_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_started_at": { + "name": "last_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_complete_sync_at": { + "name": "last_complete_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "ked_organization_id_idx": { + "name": "ked_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "ked_workspace_identity_unique": { + "name": "ked_workspace_identity_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "ked_organization_identity_unique": { + "name": "ked_organization_identity_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_external_directory_workspace_id_workspace_id_fk": { + "name": "knowledge_external_directory_workspace_id_workspace_id_fk", + "tableFrom": "knowledge_external_directory", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_external_directory_organization_id_organization_id_fk": { + "name": "knowledge_external_directory_organization_id_organization_id_fk", + "tableFrom": "knowledge_external_directory", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "ked_owner_check": { + "name": "ked_owner_check", + "value": "num_nonnulls(\"knowledge_external_directory\".\"workspace_id\", \"knowledge_external_directory\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_external_group": { + "name": "knowledge_external_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_group_id": { + "name": "external_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "keg_organization_id_idx": { + "name": "keg_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "keg_organization_identity_unique": { + "name": "keg_organization_identity_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "keg_organization_synced_idx": { + "name": "keg_organization_synced_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_synced_at", + "isExpression": false, + "asc": true, + "nulls": "first" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "keg_identity_unique": { + "name": "keg_identity_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "keg_workspace_synced_idx": { + "name": "keg_workspace_synced_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_synced_at", + "isExpression": false, + "asc": true, + "nulls": "first" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_external_group_organization_id_organization_id_fk": { + "name": "knowledge_external_group_organization_id_organization_id_fk", + "tableFrom": "knowledge_external_group", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "keg_workspace_fk": { + "name": "keg_workspace_fk", + "tableFrom": "knowledge_external_group", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "keg_owner_check": { + "name": "keg_owner_check", + "value": "num_nonnulls(\"knowledge_external_group\".\"workspace_id\", \"knowledge_external_group\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_external_group_member": { + "name": "knowledge_external_group_member", + "schema": "", + "columns": { + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "subject_token": { + "name": "subject_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kegm_subject_token_idx": { + "name": "kegm_subject_token_idx", + "columns": [ + { + "expression": "subject_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "kegm_group_fk": { + "name": "kegm_group_fk", + "tableFrom": "knowledge_external_group_member", + "tableTo": "knowledge_external_group", + "columnsFrom": ["group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "knowledge_external_group_member_group_id_subject_token_pk": { + "name": "knowledge_external_group_member_group_id_subject_token_pk", + "columns": ["group_id", "subject_token"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_projection_dirty": { + "name": "knowledge_projection_dirty", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "content": { + "name": "content", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "marked_at": { + "name": "marked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "knowledge_projection_dirty_marked_at_idx": { + "name": "knowledge_projection_dirty_marked_at_idx", + "columns": [ + { + "expression": "marked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_projection_dirty_document_id_document_id_fk": { + "name": "knowledge_projection_dirty_document_id_document_id_fk", + "tableFrom": "knowledge_projection_dirty", + "tableTo": "document", + "columnsFrom": ["document_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_server_oauth": { + "name": "mcp_server_oauth", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mcp_server_id": { + "name": "mcp_server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_information": { + "name": "client_information", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_verifier": { + "name": "code_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state_created_at": { + "name": "state_created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_refreshed_at": { + "name": "last_refreshed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_server_oauth_server_unique": { + "name": "mcp_server_oauth_server_unique", + "columns": [ + { + "expression": "mcp_server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_server_oauth_state_idx": { + "name": "mcp_server_oauth_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_server_oauth_mcp_server_id_mcp_servers_id_fk": { + "name": "mcp_server_oauth_mcp_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "mcp_servers", + "columnsFrom": ["mcp_server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_server_oauth_user_id_user_id_fk": { + "name": "mcp_server_oauth_user_id_user_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "mcp_server_oauth_workspace_id_workspace_id_fk": { + "name": "mcp_server_oauth_workspace_id_workspace_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_server_oauth_organization_id_organization_id_fk": { + "name": "mcp_server_oauth_organization_id_organization_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_server_oauth_owner_check": { + "name": "mcp_server_oauth_owner_check", + "value": "num_nonnulls(\"mcp_server_oauth\".\"workspace_id\", \"mcp_server_oauth\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_group_id": { + "name": "credential_group_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_connector_id": { + "name": "managed_connector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_config_version": { + "name": "oauth_config_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "transport": { + "name": "transport", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'headers'" + }, + "oauth_client_id": { + "name": "oauth_client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_secret": { + "name": "oauth_client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "headers": { + "name": "headers", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "timeout": { + "name": "timeout", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30000 + }, + "retries": { + "name": "retries", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 3 + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_connected": { + "name": "last_connected", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "connection_status": { + "name": "connection_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'disconnected'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status_config": { + "name": "status_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "tool_count": { + "name": "tool_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_tools_refresh": { + "name": "last_tools_refresh", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_used": { + "name": "last_used", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_servers_organization_id_idx": { + "name": "mcp_servers_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_servers_workspace_enabled_idx": { + "name": "mcp_servers_workspace_enabled_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_servers_credential_group_idx": { + "name": "mcp_servers_credential_group_idx", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_servers_credential_group_managed_connector_unique": { + "name": "mcp_servers_credential_group_managed_connector_unique", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "managed_connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"mcp_servers\".\"credential_group_id\" IS NOT NULL AND \"mcp_servers\".\"managed_connector_id\" IS NOT NULL AND \"mcp_servers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_servers_workspace_deleted_partial_idx": { + "name": "mcp_servers_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"mcp_servers\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_servers_workspace_id_workspace_id_fk": { + "name": "mcp_servers_workspace_id_workspace_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_servers_organization_id_organization_id_fk": { + "name": "mcp_servers_organization_id_organization_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_servers_credential_group_id_credential_group_id_fk": { + "name": "mcp_servers_credential_group_id_credential_group_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credential_group", + "columnsFrom": ["credential_group_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "mcp_servers_created_by_user_id_fk": { + "name": "mcp_servers_created_by_user_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_servers_owner_check": { + "name": "mcp_servers_owner_check", + "value": "num_nonnulls(\"mcp_servers\".\"workspace_id\", \"mcp_servers\".\"organization_id\") = 1" + }, + "mcp_servers_organization_managed_check": { + "name": "mcp_servers_organization_managed_check", + "value": "\"mcp_servers\".\"organization_id\" IS NULL OR \"mcp_servers\".\"credential_group_id\" IS NOT NULL" + }, + "mcp_servers_credential_group_managed_connector_check": { + "name": "mcp_servers_credential_group_managed_connector_check", + "value": "\"mcp_servers\".\"credential_group_id\" IS NULL OR \"mcp_servers\".\"managed_connector_id\" IS NOT NULL" + }, + "mcp_servers_managed_connector_oauth_check": { + "name": "mcp_servers_managed_connector_oauth_check", + "value": "\"mcp_servers\".\"managed_connector_id\" IS NULL OR \"mcp_servers\".\"auth_type\" = 'oauth'" + } + }, + "isRLSEnabled": false + }, + "public.member": { + "name": "member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "member_user_id_unique": { + "name": "member_user_id_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory": { + "name": "memory", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "storage_version": { + "name": "storage_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "encrypted_context_summary": { + "name": "encrypted_context_summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_provenance_version": { + "name": "secret_provenance_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "memory_key_idx": { + "name": "memory_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_idx": { + "name": "memory_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_key_idx": { + "name": "memory_workspace_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_deleted_partial_idx": { + "name": "memory_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"memory\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_workspace_id_workspace_id_fk": { + "name": "memory_workspace_id_workspace_id_fk", + "tableFrom": "memory", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_artifact": { + "name": "memory_artifact", + "schema": "", + "columns": { + "memory_id": { + "name": "memory_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "memory_artifact_key_idx": { + "name": "memory_artifact_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_artifact_memory_id_memory_id_fk": { + "name": "memory_artifact_memory_id_memory_id_fk", + "tableFrom": "memory_artifact", + "tableTo": "memory", + "columnsFrom": ["memory_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memory_artifact_key_execution_large_values_key_fk": { + "name": "memory_artifact_key_execution_large_values_key_fk", + "tableFrom": "memory_artifact", + "tableTo": "execution_large_values", + "columnsFrom": ["key"], + "columnsTo": ["key"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memory_artifact_memory_id_key_pk": { + "name": "memory_artifact_memory_id_key_pk", + "columns": ["memory_id", "key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_item": { + "name": "memory_item", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "memory_id": { + "name": "memory_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sequence": { + "name": "sequence", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "identity": { + "type": "always", + "name": "memory_item_sequence_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "append_key": { + "name": "append_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance_status": { + "name": "provenance_status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance_entries": { + "name": "provenance_entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memory_item_append_unique": { + "name": "memory_item_append_unique", + "columns": [ + { + "expression": "memory_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "append_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_item_sequence_idx": { + "name": "memory_item_sequence_idx", + "columns": [ + { + "expression": "memory_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sequence", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_item_memory_id_memory_id_fk": { + "name": "memory_item_memory_id_memory_id_fk", + "tableFrom": "memory_item", + "tableTo": "memory", + "columnsFrom": ["memory_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "memory_item_kind_check": { + "name": "memory_item_kind_check", + "value": "\"memory_item\".\"kind\" IN ('message', 'exchange')" + }, + "memory_item_provenance_status_check": { + "name": "memory_item_provenance_status_check", + "value": "\"memory_item\".\"provenance_status\" IN ('exact', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.memory_secret_provenance": { + "name": "memory_secret_provenance", + "schema": "", + "columns": { + "memory_id": { + "name": "memory_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "memory_secret_provenance_memory_id_memory_id_fk": { + "name": "memory_secret_provenance_memory_id_memory_id_fk", + "tableFrom": "memory_secret_provenance", + "tableTo": "memory", + "columnsFrom": ["memory_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "memory_secret_provenance_status_check": { + "name": "memory_secret_provenance_status_check", + "value": "\"memory_secret_provenance\".\"status\" IN ('exact', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.mothership_inbox_allowed_sender": { + "name": "mothership_inbox_allowed_sender", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "inbox_sender_ws_email_idx": { + "name": "inbox_sender_ws_email_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_inbox_allowed_sender_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_allowed_sender_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_allowed_sender", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mothership_inbox_allowed_sender_added_by_user_id_fk": { + "name": "mothership_inbox_allowed_sender_added_by_user_id_fk", + "tableFrom": "mothership_inbox_allowed_sender", + "tableTo": "user", + "columnsFrom": ["added_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_task": { + "name": "mothership_inbox_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_email": { + "name": "from_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_name": { + "name": "from_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body_preview": { + "name": "body_preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body_text": { + "name": "body_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body_html": { + "name": "body_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_message_id": { + "name": "email_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "in_reply_to": { + "name": "in_reply_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "response_message_id": { + "name": "response_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agentmail_message_id": { + "name": "agentmail_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "trigger_job_id": { + "name": "trigger_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "result_summary": { + "name": "result_summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rejection_reason": { + "name": "rejection_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "has_attachments": { + "name": "has_attachments", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cc_recipients": { + "name": "cc_recipients", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "inbox_task_ws_created_at_idx": { + "name": "inbox_task_ws_created_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_ws_status_idx": { + "name": "inbox_task_ws_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_response_msg_id_idx": { + "name": "inbox_task_response_msg_id_idx", + "columns": [ + { + "expression": "response_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_email_msg_id_idx": { + "name": "inbox_task_email_msg_id_idx", + "columns": [ + { + "expression": "email_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_inbox_task_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_task_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_task", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mothership_inbox_task_chat_id_copilot_chats_id_fk": { + "name": "mothership_inbox_task_chat_id_copilot_chats_id_fk", + "tableFrom": "mothership_inbox_task", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_webhook": { + "name": "mothership_inbox_webhook", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mothership_inbox_webhook_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_webhook_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_webhook", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mothership_inbox_webhook_workspace_id_unique": { + "name": "mothership_inbox_webhook_workspace_id_unique", + "nullsNotDistinct": false, + "columns": ["workspace_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_settings": { + "name": "mothership_settings", + "schema": "", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mcp_tool_refs": { + "name": "mcp_tool_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "custom_tool_refs": { + "name": "custom_tool_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "skill_refs": { + "name": "skill_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mothership_settings_workspace_id_workspace_id_fk": { + "name": "mothership_settings_workspace_id_workspace_id_fk", + "tableFrom": "mothership_settings", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauth_access_token_client_id_idx": { + "name": "oauth_access_token_client_id_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_access_token_session_id_idx": { + "name": "oauth_access_token_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_access_token_refresh_id_idx": { + "name": "oauth_access_token_refresh_id_idx", + "columns": [ + { + "expression": "refresh_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_access_token_user_client_idx": { + "name": "oauth_access_token_user_client_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_access_token_expires_at_idx": { + "name": "oauth_access_token_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_access_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": ["client_id"], + "columnsTo": ["client_id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_session_id_session_id_fk": { + "name": "oauth_access_token_session_id_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_user_id_user_id_fk": { + "name": "oauth_access_token_user_id_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refresh_id_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refresh_id_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": ["refresh_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": { + "oauth_access_token_search_resource_check": { + "name": "oauth_access_token_search_resource_check", + "value": "NOT ('search:read' = ANY(\"oauth_access_token\".\"scopes\")) OR \"oauth_access_token\".\"resource\" IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_types": { + "name": "grant_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "response_types": { + "name": "response_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauth_client_user_id_idx": { + "name": "oauth_client_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_user_id_user_id_fk": { + "name": "oauth_client_user_id_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_client_id_unique": { + "name": "oauth_client_client_id_unique", + "nullsNotDistinct": false, + "columns": ["client_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_consent_client_id_idx": { + "name": "oauth_consent_client_id_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_client_id_oauth_client_client_id_fk": { + "name": "oauth_consent_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": ["client_id"], + "columnsTo": ["client_id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_user_id_user_id_fk": { + "name": "oauth_consent_user_id_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_consent_user_client_reference_unique": { + "name": "oauth_consent_user_client_reference_unique", + "nullsNotDistinct": true, + "columns": ["user_id", "client_id", "reference_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auth_time": { + "name": "auth_time", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "family_id": { + "name": "family_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_refresh_token_client_id_idx": { + "name": "oauth_refresh_token_client_id_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_refresh_token_session_id_idx": { + "name": "oauth_refresh_token_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_refresh_token_user_client_idx": { + "name": "oauth_refresh_token_user_client_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_refresh_token_expires_at_idx": { + "name": "oauth_refresh_token_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_refresh_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": ["client_id"], + "columnsTo": ["client_id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_session_id_session_id_fk": { + "name": "oauth_refresh_token_session_id_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_user_id_user_id_fk": { + "name": "oauth_refresh_token_user_id_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_family_id_oauth_token_family_id_fk": { + "name": "oauth_refresh_token_family_id_oauth_token_family_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_token_family", + "columnsFrom": ["family_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + }, + "oauth_refresh_token_family_generation_unique": { + "name": "oauth_refresh_token_family_generation_unique", + "nullsNotDistinct": false, + "columns": ["family_id", "generation"] + } + }, + "policies": {}, + "checkConstraints": { + "oauth_refresh_token_generation_check": { + "name": "oauth_refresh_token_generation_check", + "value": "\"oauth_refresh_token\".\"generation\" BETWEEN 0 AND 1000" + }, + "oauth_refresh_token_search_resource_check": { + "name": "oauth_refresh_token_search_resource_check", + "value": "NOT ('search:read' = ANY(\"oauth_refresh_token\".\"scopes\")) OR \"oauth_refresh_token\".\"resource\" IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.oauth_token_family": { + "name": "oauth_token_family", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consent_id": { + "name": "consent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "current_generation": { + "name": "current_generation", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_token_family_client_id_idx": { + "name": "oauth_token_family_client_id_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_token_family_session_id_idx": { + "name": "oauth_token_family_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_token_family_user_client_idx": { + "name": "oauth_token_family_user_client_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_token_family_consent_id_idx": { + "name": "oauth_token_family_consent_id_idx", + "columns": [ + { + "expression": "consent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_token_family_expires_at_idx": { + "name": "oauth_token_family_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_token_family_client_id_oauth_client_client_id_fk": { + "name": "oauth_token_family_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_token_family", + "tableTo": "oauth_client", + "columnsFrom": ["client_id"], + "columnsTo": ["client_id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_token_family_session_id_session_id_fk": { + "name": "oauth_token_family_session_id_session_id_fk", + "tableFrom": "oauth_token_family", + "tableTo": "session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_token_family_user_id_user_id_fk": { + "name": "oauth_token_family_user_id_user_id_fk", + "tableFrom": "oauth_token_family", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_token_family_consent_id_oauth_consent_id_fk": { + "name": "oauth_token_family_consent_id_oauth_consent_id_fk", + "tableFrom": "oauth_token_family", + "tableTo": "oauth_consent", + "columnsFrom": ["consent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "oauth_token_family_generation_check": { + "name": "oauth_token_family_generation_check", + "value": "\"oauth_token_family\".\"current_generation\" BETWEEN 0 AND 1000" + } + }, + "isRLSEnabled": false + }, + "public.organization": { + "name": "organization", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "session_policy_settings": { + "name": "session_policy_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "security_policy_version": { + "name": "security_policy_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "require_sso": { + "name": "require_sso", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "whitelabel_settings": { + "name": "whitelabel_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "data_retention_settings": { + "name": "data_retention_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "org_usage_limit": { + "name": "org_usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "limit_notifications": { + "name": "limit_notifications", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "credit_balance": { + "name": "credit_balance", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_access_request_settings": { + "name": "organization_access_request_settings", + "schema": "", + "columns": { + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "allow_requests": { + "name": "allow_requests", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "organization_access_request_settings_organization_id_organization_id_fk": { + "name": "organization_access_request_settings_organization_id_organization_id_fk", + "tableFrom": "organization_access_request_settings", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_access_request_settings_updated_by_user_id_fk": { + "name": "organization_access_request_settings_updated_by_user_id_fk", + "tableFrom": "organization_access_request_settings", + "tableTo": "user", + "columnsFrom": ["updated_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_byok_keys": { + "name": "organization_byok_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organization_byok_organization_provider_idx": { + "name": "organization_byok_organization_provider_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "organization_byok_keys_organization_id_organization_id_fk": { + "name": "organization_byok_keys_organization_id_organization_id_fk", + "tableFrom": "organization_byok_keys", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_byok_keys_created_by_user_id_fk": { + "name": "organization_byok_keys_created_by_user_id_fk", + "tableFrom": "organization_byok_keys", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_member_usage_limit": { + "name": "organization_member_usage_limit", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "usage_limit": { + "name": "usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "set_by": { + "name": "set_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "org_member_usage_limit_org_user_unique": { + "name": "org_member_usage_limit_org_user_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_member_usage_limit_organization_id_idx": { + "name": "org_member_usage_limit_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "organization_member_usage_limit_organization_id_organization_id_fk": { + "name": "organization_member_usage_limit_organization_id_organization_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_member_usage_limit_user_id_user_id_fk": { + "name": "organization_member_usage_limit_user_id_user_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_member_usage_limit_set_by_user_id_fk": { + "name": "organization_member_usage_limit_set_by_user_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "user", + "columnsFrom": ["set_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_search_integration": { + "name": "organization_search_integration", + "schema": "", + "columns": { + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connector_type": { + "name": "connector_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approved": { + "name": "approved", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "organization_search_integration_organization_id_organization_id_fk": { + "name": "organization_search_integration_organization_id_organization_id_fk", + "tableFrom": "organization_search_integration", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "organization_search_integration_organization_id_connector_type_pk": { + "name": "organization_search_integration_organization_id_connector_type_pk", + "columns": ["organization_id", "connector_type"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_search_invocation": { + "name": "organization_search_invocation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "surface": { + "name": "surface", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_types": { + "name": "source_types", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "result_count": { + "name": "result_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organization_search_invocation_org_created_idx": { + "name": "organization_search_invocation_org_created_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "organization_search_invocation_user_idx": { + "name": "organization_search_invocation_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "organization_search_invocation_organization_id_organization_id_fk": { + "name": "organization_search_invocation_organization_id_organization_id_fk", + "tableFrom": "organization_search_invocation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_search_invocation_user_id_user_id_fk": { + "name": "organization_search_invocation_user_id_user_id_fk", + "tableFrom": "organization_search_invocation", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "organization_search_invocation_result_count_bounds": { + "name": "organization_search_invocation_result_count_bounds", + "value": "\"organization_search_invocation\".\"result_count\" BETWEEN 0 AND 100" + }, + "organization_search_invocation_source_types_bounds": { + "name": "organization_search_invocation_source_types_bounds", + "value": "cardinality(\"organization_search_invocation\".\"source_types\") <= 100" + } + }, + "isRLSEnabled": false + }, + "public.organization_search_mcp_invocation": { + "name": "organization_search_mcp_invocation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_kind": { + "name": "auth_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oauth_client_id": { + "name": "oauth_client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_name": { + "name": "client_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organization_search_mcp_invocation_org_created_idx": { + "name": "organization_search_mcp_invocation_org_created_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "organization_search_mcp_invocation_user_idx": { + "name": "organization_search_mcp_invocation_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "org_search_mcp_invocation_org_fk": { + "name": "org_search_mcp_invocation_org_fk", + "tableFrom": "organization_search_mcp_invocation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "org_search_mcp_invocation_user_fk": { + "name": "org_search_mcp_invocation_user_fk", + "tableFrom": "organization_search_mcp_invocation", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "organization_search_mcp_invocation_tool_check": { + "name": "organization_search_mcp_invocation_tool_check", + "value": "\"organization_search_mcp_invocation\".\"tool_name\" IN ('search', 'read_document', 'chat')" + }, + "organization_search_mcp_invocation_outcome_check": { + "name": "organization_search_mcp_invocation_outcome_check", + "value": "\"organization_search_mcp_invocation\".\"outcome\" IN ('success', 'error', 'cancelled', 'rate_limited')" + }, + "organization_search_mcp_invocation_duration_check": { + "name": "organization_search_mcp_invocation_duration_check", + "value": "\"organization_search_mcp_invocation\".\"duration_ms\" >= 0" + }, + "organization_search_mcp_invocation_client_name_check": { + "name": "organization_search_mcp_invocation_client_name_check", + "value": "length(\"organization_search_mcp_invocation\".\"client_name\") <= 256" + }, + "organization_search_mcp_invocation_auth_check": { + "name": "organization_search_mcp_invocation_auth_check", + "value": "(\"organization_search_mcp_invocation\".\"auth_kind\" = 'oauth_access_token' AND \"organization_search_mcp_invocation\".\"oauth_client_id\" IS NOT NULL)\n OR (\"organization_search_mcp_invocation\".\"auth_kind\" IN ('personal_api_key', 'workspace_api_key') AND \"organization_search_mcp_invocation\".\"oauth_client_id\" IS NULL AND \"organization_search_mcp_invocation\".\"client_name\" IS NULL)" + } + }, + "isRLSEnabled": false + }, + "public.outbox_event": { + "name": "outbox_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10 + }, + "available_at": { + "name": "available_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "locked_at": { + "name": "locked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "outbox_event_status_available_idx": { + "name": "outbox_event_status_available_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "available_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "outbox_event_pending_type_available_idx": { + "name": "outbox_event_pending_type_available_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "available_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"outbox_event\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "outbox_event_locked_at_idx": { + "name": "outbox_event_locked_at_idx", + "columns": [ + { + "expression": "locked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "outbox_event_type_created_idx": { + "name": "outbox_event_type_created_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.paused_executions": { + "name": "paused_executions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_snapshot": { + "name": "execution_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "pause_points": { + "name": "pause_points", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "total_pause_count": { + "name": "total_pause_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "resumed_count": { + "name": "resumed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "automatic_resume_retry_count": { + "name": "automatic_resume_retry_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paused'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "paused_at": { + "name": "paused_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "next_resume_at": { + "name": "next_resume_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "paused_executions_workflow_id_idx": { + "name": "paused_executions_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_status_idx": { + "name": "paused_executions_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_execution_id_unique": { + "name": "paused_executions_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_next_resume_at_idx": { + "name": "paused_executions_next_resume_at_idx", + "columns": [ + { + "expression": "next_resume_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'paused' AND next_resume_at IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "paused_executions_workflow_id_workflow_id_fk": { + "name": "paused_executions_workflow_id_workflow_id_fk", + "tableFrom": "paused_executions", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pending_credential_draft": { + "name": "pending_credential_draft", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_config": { + "name": "oauth_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "pending_draft_organization_id_idx": { + "name": "pending_draft_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pending_draft_user_provider_org": { + "name": "pending_draft_user_provider_org", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pending_draft_user_provider_ws": { + "name": "pending_draft_user_provider_ws", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pending_credential_draft_user_id_user_id_fk": { + "name": "pending_credential_draft_user_id_user_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_workspace_id_workspace_id_fk": { + "name": "pending_credential_draft_workspace_id_workspace_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_organization_id_organization_id_fk": { + "name": "pending_credential_draft_organization_id_organization_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_credential_id_credential_id_fk": { + "name": "pending_credential_draft_credential_id_credential_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "pending_draft_owner_check": { + "name": "pending_draft_owner_check", + "value": "num_nonnulls(\"pending_credential_draft\".\"workspace_id\", \"pending_credential_draft\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.permission_access_request": { + "name": "permission_access_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "requester_id": { + "name": "requester_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope_key": { + "name": "scope_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_key": { + "name": "target_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target": { + "name": "target", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "target_label": { + "name": "target_label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "membership_id": { + "name": "membership_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "group_name": { + "name": "group_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "decision_reason": { + "name": "decision_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "decided_by": { + "name": "decided_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "decision": { + "name": "decision", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "decided_at": { + "name": "decided_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "permission_access_request_pending_unique": { + "name": "permission_access_request_pending_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requester_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scope_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"permission_access_request\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_access_request_org_queue_idx": { + "name": "permission_access_request_org_queue_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_access_request_requester_idx": { + "name": "permission_access_request_requester_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requester_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scope_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_access_request_organization_id_organization_id_fk": { + "name": "permission_access_request_organization_id_organization_id_fk", + "tableFrom": "permission_access_request", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_access_request_requester_id_user_id_fk": { + "name": "permission_access_request_requester_id_user_id_fk", + "tableFrom": "permission_access_request", + "tableTo": "user", + "columnsFrom": ["requester_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_access_request_decided_by_user_id_fk": { + "name": "permission_access_request_decided_by_user_id_fk", + "tableFrom": "permission_access_request", + "tableTo": "user", + "columnsFrom": ["decided_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "permission_access_request_status_check": { + "name": "permission_access_request_status_check", + "value": "\"permission_access_request\".\"status\" in ('pending', 'fulfilled', 'declined', 'cancelled', 'closed')" + } + }, + "isRLSEnabled": false + }, + "public.permission_group": { + "name": "permission_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "membership_mode": { + "name": "membership_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'inherit'" + } + }, + "indexes": { + "permission_group_created_by_idx": { + "name": "permission_group_created_by_idx", + "columns": [ + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_organization_name_unique": { + "name": "permission_group_organization_name_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_organization_default_unique": { + "name": "permission_group_organization_default_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "is_default = true", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_organization_id_organization_id_fk": { + "name": "permission_group_organization_id_organization_id_fk", + "tableFrom": "permission_group", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_created_by_user_id_fk": { + "name": "permission_group_created_by_user_id_fk", + "tableFrom": "permission_group", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group_member": { + "name": "permission_group_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "assigned_by": { + "name": "assigned_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "assigned_at": { + "name": "assigned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permission_group_member_group_id_idx": { + "name": "permission_group_member_group_id_idx", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_member_group_user_unique": { + "name": "permission_group_member_group_user_unique", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_member_organization_user_idx": { + "name": "permission_group_member_organization_user_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_member_permission_group_id_permission_group_id_fk": { + "name": "permission_group_member_permission_group_id_permission_group_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_organization_id_organization_id_fk": { + "name": "permission_group_member_organization_id_organization_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_user_id_user_id_fk": { + "name": "permission_group_member_user_id_user_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_assigned_by_user_id_fk": { + "name": "permission_group_member_assigned_by_user_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "user", + "columnsFrom": ["assigned_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group_workspace": { + "name": "permission_group_workspace", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permission_group_workspace_workspace_id_idx": { + "name": "permission_group_workspace_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_workspace_group_workspace_unique": { + "name": "permission_group_workspace_group_workspace_unique", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_workspace_permission_group_id_permission_group_id_fk": { + "name": "permission_group_workspace_permission_group_id_permission_group_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_workspace_workspace_id_workspace_id_fk": { + "name": "permission_group_workspace_workspace_id_workspace_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_workspace_organization_id_organization_id_fk": { + "name": "permission_group_workspace_organization_id_organization_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permissions": { + "name": "permissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_type": { + "name": "entity_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission_type": { + "name": "permission_type", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permissions_user_id_idx": { + "name": "permissions_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_entity_idx": { + "name": "permissions_entity_idx", + "columns": [ + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_type_idx": { + "name": "permissions_user_entity_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_permission_idx": { + "name": "permissions_user_entity_permission_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "permission_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_unique_constraint": { + "name": "permissions_unique_constraint", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permissions_user_id_user_id_fk": { + "name": "permissions_user_id_user_id_fk", + "tableFrom": "permissions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pinned_item": { + "name": "pinned_item", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "pinned_item_user_workspace_idx": { + "name": "pinned_item_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pinned_item_resource_idx": { + "name": "pinned_item_resource_idx", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pinned_item_user_resource_unique": { + "name": "pinned_item_user_resource_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pinned_item_user_id_user_id_fk": { + "name": "pinned_item_user_id_user_id_fk", + "tableFrom": "pinned_item", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pinned_item_workspace_id_workspace_id_fk": { + "name": "pinned_item_workspace_id_workspace_id_fk", + "tableFrom": "pinned_item", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.public_share": { + "name": "public_share", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public'" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_emails": { + "name": "allowed_emails", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "public_share_token_unique": { + "name": "public_share_token_unique", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_resource_unique": { + "name": "public_share_resource_unique", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_resource_id_idx": { + "name": "public_share_resource_id_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_workspace_id_idx": { + "name": "public_share_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "public_share_workspace_id_workspace_id_fk": { + "name": "public_share_workspace_id_workspace_id_fk", + "tableFrom": "public_share", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "public_share_created_by_user_id_fk": { + "name": "public_share_created_by_user_id_fk", + "tableFrom": "public_share", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rate_limit_bucket": { + "name": "rate_limit_bucket", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tokens": { + "name": "tokens", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "blocked_until": { + "name": "blocked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "capacity_state": { + "name": "capacity_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.resource_policy": { + "name": "resource_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "document": { + "name": "document", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "resource_policy_organization_id_idx": { + "name": "resource_policy_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resource_policy_resource_unique": { + "name": "resource_policy_resource_unique", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resource_policy_workspace_id_idx": { + "name": "resource_policy_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "resource_policy_workspace_id_workspace_id_fk": { + "name": "resource_policy_workspace_id_workspace_id_fk", + "tableFrom": "resource_policy", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "resource_policy_organization_id_organization_id_fk": { + "name": "resource_policy_organization_id_organization_id_fk", + "tableFrom": "resource_policy", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "resource_policy_created_by_user_id_fk": { + "name": "resource_policy_created_by_user_id_fk", + "tableFrom": "resource_policy", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "resource_policy_updated_by_user_id_fk": { + "name": "resource_policy_updated_by_user_id_fk", + "tableFrom": "resource_policy", + "tableTo": "user", + "columnsFrom": ["updated_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "resource_policy_owner_check": { + "name": "resource_policy_owner_check", + "value": "num_nonnulls(\"resource_policy\".\"workspace_id\", \"resource_policy\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.resume_queue": { + "name": "resume_queue", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "paused_execution_id": { + "name": "paused_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_execution_id": { + "name": "parent_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "new_execution_id": { + "name": "new_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "context_id": { + "name": "context_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resume_input": { + "name": "resume_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "resume_queue_parent_status_idx": { + "name": "resume_queue_parent_status_idx", + "columns": [ + { + "expression": "parent_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resume_queue_new_execution_idx": { + "name": "resume_queue_new_execution_idx", + "columns": [ + { + "expression": "new_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "resume_queue_paused_execution_id_paused_executions_id_fk": { + "name": "resume_queue_paused_execution_id_paused_executions_id_fk", + "tableFrom": "resume_queue", + "tableTo": "paused_executions", + "columnsFrom": ["paused_execution_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandbox_image": { + "name": "sandbox_image", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "spec_hash": { + "name": "spec_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "spec": { + "name": "spec", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "sandbox_image_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "image_ref": { + "name": "image_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_image_id": { + "name": "provider_image_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "build_id": { + "name": "build_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "materialization_generation": { + "name": "materialization_generation", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_detail": { + "name": "error_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sandbox_image_provider_spec_unique": { + "name": "sandbox_image_provider_spec_unique", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "spec_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sandbox_image_status_idx": { + "name": "sandbox_image_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sandbox_image_last_used_idx": { + "name": "sandbox_image_last_used_idx", + "columns": [ + { + "expression": "last_used_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_connection": { + "name": "scim_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "settings": { + "name": "settings", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "last_request_at": { + "name": "last_request_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "reconcile_lock_token": { + "name": "reconcile_lock_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reconcile_lease_at": { + "name": "reconcile_lease_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_connection_organization_unique": { + "name": "scim_connection_organization_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_connection_reconcile_due_idx": { + "name": "scim_connection_reconcile_due_idx", + "columns": [ + { + "expression": "reconciled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_connection_organization_id_organization_id_fk": { + "name": "scim_connection_organization_id_organization_id_fk", + "tableFrom": "scim_connection", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_connection_created_by_user_id_fk": { + "name": "scim_connection_created_by_user_id_fk", + "tableFrom": "scim_connection", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_credential": { + "name": "scim_credential", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_prefix": { + "name": "token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_credential_token_hash_unique": { + "name": "scim_credential_token_hash_unique", + "columns": [ + { + "expression": "token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_credential_connection_idx": { + "name": "scim_credential_connection_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_credential_connection_id_scim_connection_id_fk": { + "name": "scim_credential_connection_id_scim_connection_id_fk", + "tableFrom": "scim_credential", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_credential_revoked_by_user_id_fk": { + "name": "scim_credential_revoked_by_user_id_fk", + "tableFrom": "scim_credential", + "tableTo": "user", + "columnsFrom": ["revoked_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "scim_credential_created_by_user_id_fk": { + "name": "scim_credential_created_by_user_id_fk", + "tableFrom": "scim_credential", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_group": { + "name": "scim_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name_key": { + "name": "display_name_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_group_connection_display_name_unique": { + "name": "scim_group_connection_display_name_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_connection_external_id_unique": { + "name": "scim_group_connection_external_id_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "external_id is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_connection_order_idx": { + "name": "scim_group_connection_order_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "order_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_group_connection_id_scim_connection_id_fk": { + "name": "scim_group_connection_id_scim_connection_id_fk", + "tableFrom": "scim_group", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_group_mapping": { + "name": "scim_group_mapping", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_kind": { + "name": "target_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "permission_type": { + "name": "permission_type", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_group_mapping_group_idx": { + "name": "scim_group_mapping_group_idx", + "columns": [ + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_mapping_permission_group_idx": { + "name": "scim_group_mapping_permission_group_idx", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_mapping_workspace_idx": { + "name": "scim_group_mapping_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_mapping_group_target_unique": { + "name": "scim_group_mapping_group_target_unique", + "columns": [ + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"permission_group_id\", \"workspace_id\", \"role\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_group_mapping_group_id_scim_group_id_fk": { + "name": "scim_group_mapping_group_id_scim_group_id_fk", + "tableFrom": "scim_group_mapping", + "tableTo": "scim_group", + "columnsFrom": ["group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_group_mapping_permission_group_id_permission_group_id_fk": { + "name": "scim_group_mapping_permission_group_id_permission_group_id_fk", + "tableFrom": "scim_group_mapping", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_group_mapping_workspace_id_workspace_id_fk": { + "name": "scim_group_mapping_workspace_id_workspace_id_fk", + "tableFrom": "scim_group_mapping", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_group_mapping_created_by_user_id_fk": { + "name": "scim_group_mapping_created_by_user_id_fk", + "tableFrom": "scim_group_mapping", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "scim_group_mapping_target_shape": { + "name": "scim_group_mapping_target_shape", + "value": "(\n (\"scim_group_mapping\".\"target_kind\" = 'permission_group' AND \"scim_group_mapping\".\"permission_group_id\" IS NOT NULL AND \"scim_group_mapping\".\"workspace_id\" IS NULL AND \"scim_group_mapping\".\"permission_type\" IS NULL AND \"scim_group_mapping\".\"role\" IS NULL)\n OR (\"scim_group_mapping\".\"target_kind\" = 'workspace' AND \"scim_group_mapping\".\"workspace_id\" IS NOT NULL AND \"scim_group_mapping\".\"permission_type\" IS NOT NULL AND \"scim_group_mapping\".\"permission_group_id\" IS NULL AND \"scim_group_mapping\".\"role\" IS NULL)\n OR (\"scim_group_mapping\".\"target_kind\" = 'org_role' AND \"scim_group_mapping\".\"role\" IS NOT NULL AND \"scim_group_mapping\".\"permission_group_id\" IS NULL AND \"scim_group_mapping\".\"workspace_id\" IS NULL AND \"scim_group_mapping\".\"permission_type\" IS NULL)\n )" + } + }, + "isRLSEnabled": false + }, + "public.scim_group_member": { + "name": "scim_group_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scim_user_id": { + "name": "scim_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_group_member_group_user_unique": { + "name": "scim_group_member_group_user_unique", + "columns": [ + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scim_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_member_scim_user_idx": { + "name": "scim_group_member_scim_user_idx", + "columns": [ + { + "expression": "scim_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_group_member_group_id_scim_group_id_fk": { + "name": "scim_group_member_group_id_scim_group_id_fk", + "tableFrom": "scim_group_member", + "tableTo": "scim_group", + "columnsFrom": ["group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_group_member_scim_user_id_scim_user_id_fk": { + "name": "scim_group_member_scim_user_id_scim_user_id_fk", + "tableFrom": "scim_group_member", + "tableTo": "scim_user", + "columnsFrom": ["scim_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_projection_grant": { + "name": "scim_projection_grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scim_user_id": { + "name": "scim_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_kind": { + "name": "target_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission_type": { + "name": "permission_type", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "baseline_permission": { + "name": "baseline_permission", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'directory'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_projection_grant_user_target_unique": { + "name": "scim_projection_grant_user_target_unique", + "columns": [ + { + "expression": "scim_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_projection_grant_connection_idx": { + "name": "scim_projection_grant_connection_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_projection_grant_connection_id_scim_connection_id_fk": { + "name": "scim_projection_grant_connection_id_scim_connection_id_fk", + "tableFrom": "scim_projection_grant", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_projection_grant_scim_user_id_scim_user_id_fk": { + "name": "scim_projection_grant_scim_user_id_scim_user_id_fk", + "tableFrom": "scim_projection_grant", + "tableTo": "scim_user", + "columnsFrom": ["scim_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_request_log": { + "name": "scim_request_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "scim_type": { + "name": "scim_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_request_log_connection_created_idx": { + "name": "scim_request_log_connection_created_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_request_log_connection_id_scim_connection_id_fk": { + "name": "scim_request_log_connection_id_scim_connection_id_fk", + "tableFrom": "scim_request_log", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_user": { + "name": "scim_user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_name": { + "name": "user_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "attributes": { + "name": "attributes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_user_connection_user_unique": { + "name": "scim_user_connection_user_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_user_connection_user_name_unique": { + "name": "scim_user_connection_user_name_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_user_connection_external_id_unique": { + "name": "scim_user_connection_external_id_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "external_id is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_user_connection_order_idx": { + "name": "scim_user_connection_order_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "order_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_user_user_idx": { + "name": "scim_user_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_user_connection_id_scim_connection_id_fk": { + "name": "scim_user_connection_id_scim_connection_id_fk", + "tableFrom": "scim_user", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_user_user_id_user_id_fk": { + "name": "scim_user_user_id_user_id_fk", + "tableFrom": "scim_user", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_user_tombstone": { + "name": "scim_user_tombstone", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_user_tombstone_connection_external_id_unique": { + "name": "scim_user_tombstone_connection_external_id_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_user_tombstone_user_idx": { + "name": "scim_user_tombstone_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_user_tombstone_connection_id_scim_connection_id_fk": { + "name": "scim_user_tombstone_connection_id_scim_connection_id_fk", + "tableFrom": "scim_user_tombstone", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_user_tombstone_user_id_user_id_fk": { + "name": "scim_user_tombstone_user_id_user_id_fk", + "tableFrom": "scim_user_tombstone", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.secret_usage": { + "name": "secret_usage", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_name": { + "name": "secret_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_scope": { + "name": "secret_scope", + "type": "secret_usage_scope", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "secret_owner_user_id": { + "name": "secret_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "source": { + "name": "source", + "type": "secret_usage_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "usage_date": { + "name": "usage_date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "use_count": { + "name": "use_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_execution_id": { + "name": "last_execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_trigger": { + "name": "last_trigger", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "secret_usage_bucket_unique": { + "name": "secret_usage_bucket_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "usage_date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "secret_usage_secret_recent_idx": { + "name": "secret_usage_secret_recent_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_used_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "secret_usage_workspace_id_workspace_id_fk": { + "name": "secret_usage_workspace_id_workspace_id_fk", + "tableFrom": "secret_usage", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_active_organization_id_organization_id_fk": { + "name": "session_active_organization_id_organization_id_fk", + "tableFrom": "session", + "tableTo": "organization", + "columnsFrom": ["active_organization_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.settings": { + "name": "settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "theme": { + "name": "theme", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'system'" + }, + "auto_connect": { + "name": "auto_connect", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "telemetry_enabled": { + "name": "telemetry_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "email_preferences": { + "name": "email_preferences", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "billing_usage_notifications_enabled": { + "name": "billing_usage_notifications_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "show_training_controls": { + "name": "show_training_controls", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "super_user_mode_enabled": { + "name": "super_user_mode_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "mothership_environment": { + "name": "mothership_environment", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'default'" + }, + "error_notifications_enabled": { + "name": "error_notifications_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "snap_to_grid_size": { + "name": "snap_to_grid_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "show_action_bar": { + "name": "show_action_bar", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "auto_focus_on_click": { + "name": "auto_focus_on_click", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "copilot_enabled_models": { + "name": "copilot_enabled_models", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "copilot_auto_allowed_tools": { + "name": "copilot_auto_allowed_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "last_active_workspace_id": { + "name": "last_active_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "settings_user_id_user_id_fk": { + "name": "settings_user_id_user_id_fk", + "tableFrom": "settings", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "settings_user_id_unique": { + "name": "settings_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sim_trigger_state": { + "name": "sim_trigger_state", + "schema": "", + "columns": { + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope_key": { + "name": "scope_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "last_fired_at": { + "name": "last_fired_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sim_trigger_state_workflow_id_workflow_id_fk": { + "name": "sim_trigger_state_workflow_id_workflow_id_fk", + "tableFrom": "sim_trigger_state", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "sim_trigger_state_workflow_id_block_id_scope_key_pk": { + "name": "sim_trigger_state_workflow_id_block_id_scope_key_pk", + "columns": ["workflow_id", "block_id", "scope_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill": { + "name": "skill", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_workspace_name_unique": { + "name": "skill_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_workspace_id_workspace_id_fk": { + "name": "skill_workspace_id_workspace_id_fk", + "tableFrom": "skill", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_user_id_user_id_fk": { + "name": "skill_user_id_user_id_fk", + "tableFrom": "skill", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_member": { + "name": "skill_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_member_user_id_idx": { + "name": "skill_member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skill_member_unique": { + "name": "skill_member_unique", + "columns": [ + { + "expression": "skill_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_member_skill_id_skill_id_fk": { + "name": "skill_member_skill_id_skill_id_fk", + "tableFrom": "skill_member", + "tableTo": "skill", + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_member_user_id_user_id_fk": { + "name": "skill_member_user_id_user_id_fk", + "tableFrom": "skill_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_member_invited_by_user_id_fk": { + "name": "skill_member_invited_by_user_id_fk", + "tableFrom": "skill_member", + "tableTo": "user", + "columnsFrom": ["invited_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.slack_app": { + "name": "slack_app", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_client_secret": { + "name": "encrypted_client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_signing_secret": { + "name": "encrypted_signing_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "slack_app_organization_id_organization_id_fk": { + "name": "slack_app_organization_id_organization_id_fk", + "tableFrom": "slack_app", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "slack_app_owner_check": { + "name": "slack_app_owner_check", + "value": "(\"slack_app\".\"kind\" = 'custom' AND \"slack_app\".\"organization_id\" IS NOT NULL) OR (\"slack_app\".\"kind\" = 'shared' AND \"slack_app\".\"organization_id\" IS NULL)" + }, + "slack_app_custom_credentials_check": { + "name": "slack_app_custom_credentials_check", + "value": "\"slack_app\".\"kind\" = 'shared' OR (\"slack_app\".\"client_id\" IS NOT NULL AND \"slack_app\".\"encrypted_client_secret\" IS NOT NULL AND \"slack_app\".\"encrypted_signing_secret\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.slack_search_installation": { + "name": "slack_search_installation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slack_app_id": { + "name": "slack_app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "team_id": { + "name": "team_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "team_name": { + "name": "team_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bot_user_id": { + "name": "bot_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enterprise_id": { + "name": "enterprise_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "credential_version": { + "name": "credential_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_outcome": { + "name": "last_outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_event_at": { + "name": "last_event_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "slack_search_installation_organization_idx": { + "name": "slack_search_installation_organization_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_search_installation_credential_unique": { + "name": "slack_search_installation_credential_unique", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_search_installation_app_team_unique": { + "name": "slack_search_installation_app_team_unique", + "columns": [ + { + "expression": "app_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "team_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_search_installation_active_team_unique": { + "name": "slack_search_installation_active_team_unique", + "columns": [ + { + "expression": "team_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"slack_search_installation\".\"enabled\" = true", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "slack_search_installation_organization_id_organization_id_fk": { + "name": "slack_search_installation_organization_id_organization_id_fk", + "tableFrom": "slack_search_installation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "slack_search_installation_credential_id_credential_id_fk": { + "name": "slack_search_installation_credential_id_credential_id_fk", + "tableFrom": "slack_search_installation", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "slack_search_installation_slack_app_id_slack_app_id_fk": { + "name": "slack_search_installation_slack_app_id_slack_app_id_fk", + "tableFrom": "slack_search_installation", + "tableTo": "slack_app", + "columnsFrom": ["slack_app_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.slack_search_turn": { + "name": "slack_search_turn", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true, + "identity": { + "type": "always", + "name": "slack_search_turn_ordinal_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "installation_id": { + "name": "installation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conversation_key": { + "name": "conversation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "lease_id": { + "name": "lease_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "slack_search_turn_event_unique": { + "name": "slack_search_turn_event_unique", + "columns": [ + { + "expression": "installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_search_turn_pending_idx": { + "name": "slack_search_turn_pending_idx", + "columns": [ + { + "expression": "installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_search_turn_thread_idx": { + "name": "slack_search_turn_thread_idx", + "columns": [ + { + "expression": "conversation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_search_turn_active_thread_unique": { + "name": "slack_search_turn_active_thread_unique", + "columns": [ + { + "expression": "conversation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"slack_search_turn\".\"status\" = 'running'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "slack_search_turn_installation_id_slack_search_installation_id_fk": { + "name": "slack_search_turn_installation_id_slack_search_installation_id_fk", + "tableFrom": "slack_search_turn", + "tableTo": "slack_search_installation", + "columnsFrom": ["installation_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_domain": { + "name": "sso_domain", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "verification_token": { + "name": "verification_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "primary_provider_id": { + "name": "primary_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sso_domain_organization_id_idx": { + "name": "sso_domain_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_domain_idx": { + "name": "sso_domain_domain_idx", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_org_domain_unique": { + "name": "sso_domain_org_domain_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_verified_unique": { + "name": "sso_domain_verified_unique", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "status = 'verified'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sso_domain_organization_id_organization_id_fk": { + "name": "sso_domain_organization_id_organization_id_fk", + "tableFrom": "sso_domain", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sso_domain_created_by_user_id_fk": { + "name": "sso_domain_created_by_user_id_fk", + "tableFrom": "sso_domain", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_provider": { + "name": "sso_provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain_verified": { + "name": "domain_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "jit_provisioning_enabled": { + "name": "jit_provisioning_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + } + }, + "indexes": { + "sso_provider_provider_id_unique": { + "name": "sso_provider_provider_id_unique", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_domain_idx": { + "name": "sso_provider_domain_idx", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_user_id_idx": { + "name": "sso_provider_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_organization_id_idx": { + "name": "sso_provider_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sso_provider_user_id_user_id_fk": { + "name": "sso_provider_user_id_user_id_fk", + "tableFrom": "sso_provider", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sso_provider_organization_id_organization_id_fk": { + "name": "sso_provider_organization_id_organization_id_fk", + "tableFrom": "sso_provider", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subscription": { + "name": "subscription", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "plan": { + "name": "plan", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_subscription_id": { + "name": "stripe_subscription_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "period_start": { + "name": "period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "period_end": { + "name": "period_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancel_at_period_end": { + "name": "cancel_at_period_end", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "cancel_at": { + "name": "cancel_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "canceled_at": { + "name": "canceled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "seats": { + "name": "seats", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "trial_start": { + "name": "trial_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "trial_end": { + "name": "trial_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "billing_interval": { + "name": "billing_interval", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_schedule_id": { + "name": "stripe_schedule_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "last_closed_period_start": { + "name": "last_closed_period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "subscription_reference_status_idx": { + "name": "subscription_reference_status_idx", + "columns": [ + { + "expression": "reference_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "subscription_cycle_close_lagging_idx": { + "name": "subscription_cycle_close_lagging_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"subscription\".\"status\" in ('active', 'past_due') and \"subscription\".\"period_start\" is not null and (\"subscription\".\"last_closed_period_start\" is null or \"subscription\".\"last_closed_period_start\" < \"subscription\".\"period_start\")", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "check_enterprise_metadata": { + "name": "check_enterprise_metadata", + "value": "plan != 'enterprise' OR metadata IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.table_jobs": { + "name": "table_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "rows_processed": { + "name": "rows_processed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "table_jobs_one_active_per_table": { + "name": "table_jobs_one_active_per_table", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"table_jobs\".\"status\" = 'running' AND \"table_jobs\".\"type\" <> 'export'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_jobs_watchdog_idx": { + "name": "table_jobs_watchdog_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_jobs_table_started_idx": { + "name": "table_jobs_table_started_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_jobs_table_id_user_table_definitions_id_fk": { + "name": "table_jobs_table_id_user_table_definitions_id_fk", + "tableFrom": "table_jobs", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_jobs_workspace_id_workspace_id_fk": { + "name": "table_jobs_workspace_id_workspace_id_fk", + "tableFrom": "table_jobs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_row_executions": { + "name": "table_row_executions", + "schema": "", + "columns": { + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "running_block_ids": { + "name": "running_block_ids", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'::text[]" + }, + "block_errors": { + "name": "block_errors", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "capability_governed_user_id": { + "name": "capability_governed_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enrichment_details": { + "name": "enrichment_details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "table_row_executions_table_status_idx": { + "name": "table_row_executions_table_status_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_row_executions\".\"status\" IN ('queued', 'running', 'pending')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_row_executions_execution_id_idx": { + "name": "table_row_executions_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_row_executions\".\"execution_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_row_executions_table_group_idx": { + "name": "table_row_executions_table_group_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_row_executions_table_id_user_table_definitions_id_fk": { + "name": "table_row_executions_table_id_user_table_definitions_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_row_executions_row_id_user_table_rows_id_fk": { + "name": "table_row_executions_row_id_user_table_rows_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user_table_rows", + "columnsFrom": ["row_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_row_executions_capability_governed_user_id_user_id_fk": { + "name": "table_row_executions_capability_governed_user_id_user_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user", + "columnsFrom": ["capability_governed_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "table_row_executions_row_id_group_id_pk": { + "name": "table_row_executions_row_id_group_id_pk", + "columns": ["row_id", "group_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_run_dispatches": { + "name": "table_run_dispatches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "cursor": { + "name": "cursor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "limit": { + "name": "limit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "processed_count": { + "name": "processed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_manual_run": { + "name": "is_manual_run", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "triggered_by_user_id": { + "name": "triggered_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "capability_governed_user_id": { + "name": "capability_governed_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "heartbeat_at": { + "name": "heartbeat_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "table_run_dispatches_active_idx": { + "name": "table_run_dispatches_active_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_run_dispatches_watchdog_idx": { + "name": "table_run_dispatches_watchdog_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_run_dispatches_governed_active_idx": { + "name": "table_run_dispatches_governed_active_idx", + "columns": [ + { + "expression": "capability_governed_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_run_dispatches\".\"status\" IN ('pending', 'dispatching')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_run_dispatches_table_id_user_table_definitions_id_fk": { + "name": "table_run_dispatches_table_id_user_table_definitions_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_run_dispatches_workspace_id_workspace_id_fk": { + "name": "table_run_dispatches_workspace_id_workspace_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_run_dispatches_triggered_by_user_id_user_id_fk": { + "name": "table_run_dispatches_triggered_by_user_id_user_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user", + "columnsFrom": ["triggered_by_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "table_run_dispatches_capability_governed_user_id_user_id_fk": { + "name": "table_run_dispatches_capability_governed_user_id_user_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user", + "columnsFrom": ["capability_governed_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_views": { + "name": "table_views", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "table_views_table_created_idx": { + "name": "table_views_table_created_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_views_workspace_created_idx": { + "name": "table_views_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_views_table_default_unique": { + "name": "table_views_table_default_unique", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "is_default = true", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_views_table_id_user_table_definitions_id_fk": { + "name": "table_views_table_id_user_table_definitions_id_fk", + "tableFrom": "table_views", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_views_workspace_id_workspace_id_fk": { + "name": "table_views_workspace_id_workspace_id_fk", + "tableFrom": "table_views", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_views_created_by_user_id_fk": { + "name": "table_views_created_by_user_id_fk", + "tableFrom": "table_views", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.upload_session": { + "name": "upload_session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "purpose": { + "name": "purpose", + "type": "upload_session_purpose", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "method": { + "name": "method", + "type": "upload_session_method", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "storage_context": { + "name": "storage_context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "final_key": { + "name": "final_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_provider": { + "name": "storage_provider", + "type": "upload_session_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider_upload_id": { + "name": "provider_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_object_version": { + "name": "provider_object_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "file_name": { + "name": "file_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "file_size": { + "name": "file_size", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "part_size": { + "name": "part_size", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "part_count": { + "name": "part_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "upload_session_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'uploading'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "processing_lease_id": { + "name": "processing_lease_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "processing_lease_expires_at": { + "name": "processing_lease_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_file_id": { + "name": "completed_file_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "upload_session_token_hash_unique": { + "name": "upload_session_token_hash_unique", + "columns": [ + { + "expression": "token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "upload_session_final_key_unique": { + "name": "upload_session_final_key_unique", + "columns": [ + { + "expression": "final_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "upload_session_status_expires_at_idx": { + "name": "upload_session_status_expires_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_log": { + "name": "usage_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "usage_log_category", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "usage_log_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "cost": { + "name": "cost", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_entity_type": { + "name": "billing_entity_type", + "type": "billing_entity_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "billing_entity_id": { + "name": "billing_entity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_period_start": { + "name": "billing_period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "billing_period_end": { + "name": "billing_period_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "usage_log_user_created_at_idx": { + "name": "usage_log_user_created_at_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_source_idx": { + "name": "usage_log_source_idx", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workspace_id_idx": { + "name": "usage_log_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workflow_id_idx": { + "name": "usage_log_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_event_key_unique": { + "name": "usage_log_event_key_unique", + "columns": [ + { + "expression": "event_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"usage_log\".\"event_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_entity_period_idx": { + "name": "usage_log_billing_entity_period_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_start", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_end", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_period_cost_idx": { + "name": "usage_log_billing_period_cost_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_start", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_end", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_entity_created_at_cost_idx": { + "name": "usage_log_billing_entity_created_at_cost_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workspace_created_at_idx": { + "name": "usage_log_workspace_created_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_execution_id_idx": { + "name": "usage_log_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_log_user_id_user_id_fk": { + "name": "usage_log_user_id_user_id_fk", + "tableFrom": "usage_log", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "usage_log_workspace_id_workspace_id_fk": { + "name": "usage_log_workspace_id_workspace_id_fk", + "tableFrom": "usage_log", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "usage_log_workflow_id_workflow_id_fk": { + "name": "usage_log_workflow_id_workflow_id_fk", + "tableFrom": "usage_log", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_log_billing_scope_all_or_none": { + "name": "usage_log_billing_scope_all_or_none", + "value": "(\n (\"usage_log\".\"billing_entity_type\" IS NULL AND \"usage_log\".\"billing_entity_id\" IS NULL AND \"usage_log\".\"billing_period_start\" IS NULL AND \"usage_log\".\"billing_period_end\" IS NULL)\n OR\n (\"usage_log\".\"billing_entity_type\" IS NOT NULL AND \"usage_log\".\"billing_entity_id\" IS NOT NULL AND \"usage_log\".\"billing_period_start\" IS NOT NULL AND \"usage_log\".\"billing_period_end\" IS NOT NULL AND \"usage_log\".\"billing_period_start\" < \"usage_log\".\"billing_period_end\")\n )" + } + }, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'user'" + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "suspended_at": { + "name": "suspended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "suspension_source": { + "name": "suspension_source", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "user_email_lower_idx": { + "name": "user_email_lower_idx", + "columns": [ + { + "expression": "lower(btrim(\"email\"))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": ["normalized_email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_stats": { + "name": "user_stats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "current_usage_limit": { + "name": "current_usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'5'" + }, + "usage_limit_updated_at": { + "name": "usage_limit_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "last_period_cost": { + "name": "last_period_cost", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "billed_overage_this_period": { + "name": "billed_overage_this_period", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "credit_balance": { + "name": "credit_balance", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_period_copilot_cost": { + "name": "last_period_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "billing_blocked": { + "name": "billing_blocked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "billing_blocked_reason": { + "name": "billing_blocked_reason", + "type": "billing_blocked_reason", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "limit_notifications": { + "name": "limit_notifications", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + } + }, + "indexes": {}, + "foreignKeys": { + "user_stats_user_id_user_id_fk": { + "name": "user_stats_user_id_user_id_fk", + "tableFrom": "user_stats", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_stats_user_id_unique": { + "name": "user_stats_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_table_definitions": { + "name": "user_table_definitions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "max_rows": { + "name": "max_rows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "row_count": { + "name": "row_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "rows_version": { + "name": "rows_version", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "schema_locked": { + "name": "schema_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "insert_locked": { + "name": "insert_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "update_locked": { + "name": "update_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "delete_locked": { + "name": "delete_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "user_table_def_workspace_id_idx": { + "name": "user_table_def_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_folder_id_idx": { + "name": "user_table_def_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_workspace_name_unique": { + "name": "user_table_def_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"user_table_definitions\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_archived_at_idx": { + "name": "user_table_def_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_workspace_archived_partial_idx": { + "name": "user_table_def_workspace_archived_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"user_table_definitions\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_table_definitions_workspace_id_workspace_id_fk": { + "name": "user_table_definitions_workspace_id_workspace_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_definitions_folder_id_folder_id_fk": { + "name": "user_table_definitions_folder_id_folder_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "user_table_definitions_created_by_user_id_fk": { + "name": "user_table_definitions_created_by_user_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_table_row_secret_provenance": { + "name": "user_table_row_secret_provenance", + "schema": "", + "columns": { + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "content_updated_at": { + "name": "content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_table_row_secret_provenance_row_id_user_table_rows_id_fk": { + "name": "user_table_row_secret_provenance_row_id_user_table_rows_id_fk", + "tableFrom": "user_table_row_secret_provenance", + "tableTo": "user_table_rows", + "columnsFrom": ["row_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "user_table_row_secret_provenance_status_check": { + "name": "user_table_row_secret_provenance_status_check", + "value": "\"user_table_row_secret_provenance\".\"status\" IN ('exact', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.user_table_rows": { + "name": "user_table_rows", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_provenance_version": { + "name": "secret_provenance_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "user_table_rows_tenant_data_gin_idx": { + "name": "user_table_rows_tenant_data_gin_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"data\" jsonb_path_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "user_table_rows_workspace_table_idx": { + "name": "user_table_rows_workspace_table_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_position_idx": { + "name": "user_table_rows_table_position_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "position", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_order_key_idx": { + "name": "user_table_rows_table_order_key_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "order_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_created_id_idx": { + "name": "user_table_rows_table_created_id_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_id_id_idx": { + "name": "user_table_rows_table_id_id_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_table_rows_table_id_user_table_definitions_id_fk": { + "name": "user_table_rows_table_id_user_table_definitions_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_rows_workspace_id_workspace_id_fk": { + "name": "user_table_rows_workspace_id_workspace_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_rows_created_by_user_id_fk": { + "name": "user_table_rows_created_by_user_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "verification_expires_at_idx": { + "name": "verification_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.waitlist": { + "name": "waitlist", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "waitlist_email_unique": { + "name": "waitlist_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook": { + "name": "webhook", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_status": { + "name": "registration_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_generation": { + "name": "registration_generation", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "config_fingerprint": { + "name": "config_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prepared_at": { + "name": "prepared_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "routing_key": { + "name": "routing_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_config": { + "name": "provider_config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "failed_count": { + "name": "failed_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_failed_at": { + "name": "last_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "path_deployment_unique": { + "name": "path_deployment_unique", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_workflow_deployment_idx": { + "name": "webhook_workflow_deployment_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_routing_key_active_idx": { + "name": "webhook_routing_key_active_idx", + "columns": [ + { + "expression": "routing_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"archived_at\" IS NULL AND \"webhook\".\"routing_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_archived_at_partial_idx": { + "name": "webhook_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_webhook_on_provider_is_active_workflow_id_deploym_bdeed5468": { + "name": "idx_webhook_on_provider_is_active_workflow_id_deploym_bdeed5468", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_webhook_on_workflow_id_block_id_updated_at_desc": { + "name": "idx_webhook_on_workflow_id_block_id_updated_at_desc", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_active_registration_unique": { + "name": "webhook_active_registration_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"registration_status\" = 'active' AND \"webhook\".\"block_id\" IS NOT NULL AND \"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_candidate_registration_unique": { + "name": "webhook_candidate_registration_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"registration_status\" = 'candidate' AND \"webhook\".\"block_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_registration_status_generation_idx": { + "name": "webhook_registration_status_generation_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_workflow_id_workflow_id_fk": { + "name": "webhook_workflow_id_workflow_id_fk", + "tableFrom": "webhook", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "webhook_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "webhook", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_registration_status_check": { + "name": "webhook_registration_status_check", + "value": "\"webhook\".\"registration_status\" IS NULL OR \"webhook\".\"registration_status\" IN ('active', 'candidate', 'retired', 'orphaned')" + }, + "webhook_registration_generation_check": { + "name": "webhook_registration_generation_check", + "value": "\"webhook\".\"registration_generation\" IS NULL OR \"webhook\".\"registration_generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.webhook_path_claim": { + "name": "webhook_path_claim", + "schema": "", + "columns": { + "path": { + "name": "path", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "webhook_path_claim_workflow_idx": { + "name": "webhook_path_claim_workflow_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_path_claim_workflow_id_workflow_id_fk": { + "name": "webhook_path_claim_workflow_id_workflow_id_fk", + "tableFrom": "webhook_path_claim", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_path_claim_generation_check": { + "name": "webhook_path_claim_generation_check", + "value": "\"webhook_path_claim\".\"generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow": { + "name": "workflow", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_synced": { + "name": "last_synced", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "is_deployed": { + "name": "is_deployed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deployed_at": { + "name": "deployed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "is_public_api": { + "name": "is_public_api", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "fork_sync_excluded": { + "name": "fork_sync_excluded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "run_count": { + "name": "run_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_user_id_idx": { + "name": "workflow_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_id_idx": { + "name": "workflow_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_user_workspace_idx": { + "name": "workflow_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_folder_name_active_unique": { + "name": "workflow_workspace_folder_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"folder_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_folder_sort_idx": { + "name": "workflow_folder_sort_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_active_workspace_sort_idx": { + "name": "workflow_active_workspace_sort_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_archived_at_idx": { + "name": "workflow_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_archived_partial_idx": { + "name": "workflow_workspace_archived_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_user_id_user_id_fk": { + "name": "workflow_user_id_user_id_fk", + "tableFrom": "workflow", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_workspace_id_workspace_id_fk": { + "name": "workflow_workspace_id_workspace_id_fk", + "tableFrom": "workflow", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_folder_id_folder_id_fk": { + "name": "workflow_folder_id_folder_id_fk", + "tableFrom": "workflow", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_blocks": { + "name": "workflow_blocks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position_x": { + "name": "position_x", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "position_y": { + "name": "position_y", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "horizontal_handles": { + "name": "horizontal_handles", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_wide": { + "name": "is_wide", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "advanced_mode": { + "name": "advanced_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "trigger_mode": { + "name": "trigger_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "error_enabled": { + "name": "error_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "retry": { + "name": "retry", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "height": { + "name": "height", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "sub_blocks": { + "name": "sub_blocks", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "outputs": { + "name": "outputs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_blocks_workflow_id_idx": { + "name": "workflow_blocks_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_blocks_type_idx": { + "name": "workflow_blocks_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_blocks_workflow_id_workflow_id_fk": { + "name": "workflow_blocks_workflow_id_workflow_id_fk", + "tableFrom": "workflow_blocks", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_checkpoints": { + "name": "workflow_checkpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_state": { + "name": "workflow_state", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_checkpoints_user_id_idx": { + "name": "workflow_checkpoints_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_workflow_id_idx": { + "name": "workflow_checkpoints_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_chat_id_idx": { + "name": "workflow_checkpoints_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_message_id_idx": { + "name": "workflow_checkpoints_message_id_idx", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_user_workflow_idx": { + "name": "workflow_checkpoints_user_workflow_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_workflow_chat_idx": { + "name": "workflow_checkpoints_workflow_chat_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_created_at_idx": { + "name": "workflow_checkpoints_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_chat_created_at_idx": { + "name": "workflow_checkpoints_chat_created_at_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_checkpoints_user_id_user_id_fk": { + "name": "workflow_checkpoints_user_id_user_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_checkpoints_workflow_id_workflow_id_fk": { + "name": "workflow_checkpoints_workflow_id_workflow_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_checkpoints_chat_id_copilot_chats_id_fk": { + "name": "workflow_checkpoints_chat_id_copilot_chats_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_deployment_operation": { + "name": "workflow_deployment_operation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "previous_active_version_id": { + "name": "previous_active_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "protocol_version": { + "name": "protocol_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'preparing'" + }, + "component_readiness": { + "name": "component_readiness", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_hash": { + "name": "request_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_deployment_operation_workflow_generation_unique": { + "name": "workflow_deployment_operation_workflow_generation_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_idempotency_unique": { + "name": "workflow_deployment_operation_workflow_idempotency_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_deployment_operation\".\"idempotency_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_in_flight_unique": { + "name": "workflow_deployment_operation_workflow_in_flight_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_deployment_operation\".\"status\" IN ('preparing', 'activating')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_status_idx": { + "name": "workflow_deployment_operation_workflow_status_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_deployment_version_idx": { + "name": "workflow_deployment_operation_deployment_version_idx", + "columns": [ + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_version_generation_idx": { + "name": "workflow_deployment_operation_workflow_version_generation_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_operation_workflow_id_workflow_id_fk": { + "name": "workflow_deployment_operation_workflow_id_workflow_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_operation_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_deployment_operation_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_operation_previous_active_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_deployment_operation_previous_active_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["previous_active_version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_deployment_operation_action_check": { + "name": "workflow_deployment_operation_action_check", + "value": "\"workflow_deployment_operation\".\"action\" IN ('deploy', 'activate')" + }, + "workflow_deployment_operation_status_check": { + "name": "workflow_deployment_operation_status_check", + "value": "\"workflow_deployment_operation\".\"status\" IN ('preparing', 'activating', 'active', 'failed', 'superseded')" + }, + "workflow_deployment_operation_generation_check": { + "name": "workflow_deployment_operation_generation_check", + "value": "\"workflow_deployment_operation\".\"generation\" > 0" + }, + "workflow_deployment_operation_protocol_version_check": { + "name": "workflow_deployment_operation_protocol_version_check", + "value": "\"workflow_deployment_operation\".\"protocol_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow_deployment_version": { + "name": "workflow_deployment_version", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_deployment_version_workflow_version_unique": { + "name": "workflow_deployment_version_workflow_version_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_version_workflow_active_idx": { + "name": "workflow_deployment_version_workflow_active_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_version_created_at_idx": { + "name": "workflow_deployment_version_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_version_workflow_id_workflow_id_fk": { + "name": "workflow_deployment_version_workflow_id_workflow_id_fk", + "tableFrom": "workflow_deployment_version", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_edges": { + "name": "workflow_edges", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_block_id": { + "name": "source_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_block_id": { + "name": "target_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_handle": { + "name": "source_handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_handle": { + "name": "target_handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_edges_workflow_id_idx": { + "name": "workflow_edges_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_edges_workflow_source_idx": { + "name": "workflow_edges_workflow_source_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_edges_workflow_target_idx": { + "name": "workflow_edges_workflow_target_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_edges_workflow_id_workflow_id_fk": { + "name": "workflow_edges_workflow_id_workflow_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_edges_source_block_id_workflow_blocks_id_fk": { + "name": "workflow_edges_source_block_id_workflow_blocks_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow_blocks", + "columnsFrom": ["source_block_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_edges_target_block_id_workflow_blocks_id_fk": { + "name": "workflow_edges_target_block_id_workflow_blocks_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow_blocks", + "columnsFrom": ["target_block_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_execution_logs": { + "name": "workflow_execution_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state_snapshot_id": { + "name": "state_snapshot_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "execution_deadline_at": { + "name": "execution_deadline_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_duration_ms": { + "name": "total_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "execution_data": { + "name": "execution_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "cost_total": { + "name": "cost_total", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "models_used": { + "name": "models_used", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "files": { + "name": "files", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_execution_logs_workflow_id_idx": { + "name": "workflow_execution_logs_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_state_snapshot_id_idx": { + "name": "workflow_execution_logs_state_snapshot_id_idx", + "columns": [ + { + "expression": "state_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_deployment_version_id_idx": { + "name": "workflow_execution_logs_deployment_version_id_idx", + "columns": [ + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_trigger_idx": { + "name": "workflow_execution_logs_trigger_idx", + "columns": [ + { + "expression": "trigger", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_level_idx": { + "name": "workflow_execution_logs_level_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_started_at_idx": { + "name": "workflow_execution_logs_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_execution_id_unique": { + "name": "workflow_execution_logs_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workflow_started_at_idx": { + "name": "workflow_execution_logs_workflow_started_at_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_started_at_idx": { + "name": "workflow_execution_logs_workspace_started_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_activity_idx": { + "name": "workflow_execution_logs_workspace_activity_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "total_duration_ms", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "trigger", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": true, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_started_at_id_desc_idx": { + "name": "workflow_execution_logs_workspace_started_at_id_desc_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"started_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "\"id\" DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_cost_total_idx": { + "name": "workflow_execution_logs_workspace_cost_total_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_total", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_models_used_idx": { + "name": "workflow_execution_logs_models_used_idx", + "columns": [ + { + "expression": "models_used", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "workflow_execution_logs_workspace_ended_at_id_idx": { + "name": "workflow_execution_logs_workspace_ended_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"ended_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_running_started_at_idx": { + "name": "workflow_execution_logs_running_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'running'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_running_deadline_idx": { + "name": "workflow_execution_logs_running_deadline_idx", + "columns": [ + { + "expression": "execution_deadline_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_execution_logs\".\"status\" = 'running' AND \"workflow_execution_logs\".\"execution_deadline_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_redacting_started_at_idx": { + "name": "workflow_execution_logs_redacting_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'redacting'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_redacting_deadline_idx": { + "name": "workflow_execution_logs_redacting_deadline_idx", + "columns": [ + { + "expression": "execution_deadline_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_execution_logs\".\"status\" = 'redacting' AND \"workflow_execution_logs\".\"execution_deadline_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_completed_ended_at_idx": { + "name": "workflow_execution_logs_completed_ended_at_idx", + "columns": [ + { + "expression": "ended_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_execution_logs\".\"status\" = 'completed' AND \"workflow_execution_logs\".\"level\" = 'info' AND \"workflow_execution_logs\".\"ended_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_execution_logs_workflow_id_workflow_id_fk": { + "name": "workflow_execution_logs_workflow_id_workflow_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_execution_logs_workspace_id_workspace_id_fk": { + "name": "workflow_execution_logs_workspace_id_workspace_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_execution_logs_state_snapshot_id_workflow_execution_snapshots_id_fk": { + "name": "workflow_execution_logs_state_snapshot_id_workflow_execution_snapshots_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow_execution_snapshots", + "columnsFrom": ["state_snapshot_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workflow_execution_logs_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_execution_logs_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_execution_snapshots": { + "name": "workflow_execution_snapshots", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state_hash": { + "name": "state_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state_data": { + "name": "state_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_snapshots_workflow_id_idx": { + "name": "workflow_snapshots_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_hash_idx": { + "name": "workflow_snapshots_hash_idx", + "columns": [ + { + "expression": "state_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_workflow_hash_idx": { + "name": "workflow_snapshots_workflow_hash_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_created_at_idx": { + "name": "workflow_snapshots_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_execution_snapshots_workflow_id_workflow_id_fk": { + "name": "workflow_execution_snapshots_workflow_id_workflow_id_fk", + "tableFrom": "workflow_execution_snapshots", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_mcp_server": { + "name": "workflow_mcp_server", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_public": { + "name": "is_public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_mcp_server_workspace_id_idx": { + "name": "workflow_mcp_server_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_created_by_idx": { + "name": "workflow_mcp_server_created_by_idx", + "columns": [ + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_deleted_at_idx": { + "name": "workflow_mcp_server_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_workspace_deleted_partial_idx": { + "name": "workflow_mcp_server_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_mcp_server\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_mcp_server_workspace_id_workspace_id_fk": { + "name": "workflow_mcp_server_workspace_id_workspace_id_fk", + "tableFrom": "workflow_mcp_server", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_mcp_server_created_by_user_id_fk": { + "name": "workflow_mcp_server_created_by_user_id_fk", + "tableFrom": "workflow_mcp_server", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_mcp_tool": { + "name": "workflow_mcp_tool", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_description": { + "name": "tool_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parameter_schema": { + "name": "parameter_schema", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "parameter_description_overrides": { + "name": "parameter_description_overrides", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'::json" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_mcp_tool_server_id_idx": { + "name": "workflow_mcp_tool_server_id_idx", + "columns": [ + { + "expression": "server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_workflow_id_idx": { + "name": "workflow_mcp_tool_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_server_workflow_unique": { + "name": "workflow_mcp_tool_server_workflow_unique", + "columns": [ + { + "expression": "server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_mcp_tool\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_archived_at_partial_idx": { + "name": "workflow_mcp_tool_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_mcp_tool\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_mcp_tool_server_id_workflow_mcp_server_id_fk": { + "name": "workflow_mcp_tool_server_id_workflow_mcp_server_id_fk", + "tableFrom": "workflow_mcp_tool", + "tableTo": "workflow_mcp_server", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_mcp_tool_workflow_id_workflow_id_fk": { + "name": "workflow_mcp_tool_workflow_id_workflow_id_fk", + "tableFrom": "workflow_mcp_tool", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_schedule": { + "name": "workflow_schedule", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deployment_operation_id": { + "name": "deployment_operation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cron_expression": { + "name": "cron_expression", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_ran_at": { + "name": "last_ran_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_queued_at": { + "name": "last_queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "trigger_type": { + "name": "trigger_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "failed_count": { + "name": "failed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "infra_retry_count": { + "name": "infra_retry_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "last_failed_at": { + "name": "last_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'workflow'" + }, + "job_title": { + "name": "job_title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lifecycle": { + "name": "lifecycle", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'persistent'" + }, + "success_condition": { + "name": "success_condition", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "max_runs": { + "name": "max_runs", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "run_count": { + "name": "run_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "source_chat_id": { + "name": "source_chat_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_task_name": { + "name": "source_task_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_user_id": { + "name": "source_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_workspace_id": { + "name": "source_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_scope": { + "name": "secret_scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'all'" + }, + "mounted_secrets": { + "name": "mounted_secrets", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "job_history": { + "name": "job_history", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "contexts": { + "name": "contexts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "excluded_dates": { + "name": "excluded_dates", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ends_at": { + "name": "ends_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_schedule_workflow_block_deployment_unique": { + "name": "workflow_schedule_workflow_block_deployment_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_workflow_deployment_idx": { + "name": "workflow_schedule_workflow_deployment_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_archived_at_partial_idx": { + "name": "workflow_schedule_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_workflow_schedule_on_source_workspace_id_source_t_c07f3bba6": { + "name": "idx_workflow_schedule_on_source_workspace_id_source_t_c07f3bba6", + "columns": [ + { + "expression": "source_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_due_workflow_idx": { + "name": "workflow_schedule_due_workflow_idx", + "columns": [ + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL AND \"workflow_schedule\".\"status\" NOT IN ('disabled', 'completed') AND (\"workflow_schedule\".\"source_type\" = 'workflow' OR \"workflow_schedule\".\"source_type\" IS NULL)", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_due_job_idx": { + "name": "workflow_schedule_due_job_idx", + "columns": [ + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL AND \"workflow_schedule\".\"status\" NOT IN ('disabled', 'completed') AND \"workflow_schedule\".\"source_type\" = 'job'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_schedule_workflow_id_workflow_id_fk": { + "name": "workflow_schedule_workflow_id_workflow_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_schedule_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_deployment_operation_id_workflow_deployment_operation_id_fk": { + "name": "workflow_schedule_deployment_operation_id_workflow_deployment_operation_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow_deployment_operation", + "columnsFrom": ["deployment_operation_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_schedule_source_user_id_user_id_fk": { + "name": "workflow_schedule_source_user_id_user_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "user", + "columnsFrom": ["source_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_source_workspace_id_workspace_id_fk": { + "name": "workflow_schedule_source_workspace_id_workspace_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workspace", + "columnsFrom": ["source_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_subflows": { + "name": "workflow_subflows", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_subflows_workflow_id_idx": { + "name": "workflow_subflows_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_subflows_workflow_type_idx": { + "name": "workflow_subflows_workflow_type_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_subflows_workflow_id_workflow_id_fk": { + "name": "workflow_subflows_workflow_id_workflow_id_fk", + "tableFrom": "workflow_subflows", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace": { + "name": "workspace", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'#33C482'" + }, + "logo_url": { + "name": "logo_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_mode": { + "name": "workspace_mode", + "type": "workspace_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'grandfathered_shared'" + }, + "billed_account_user_id": { + "name": "billed_account_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "allow_personal_api_keys": { + "name": "allow_personal_api_keys", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "inbox_enabled": { + "name": "inbox_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "inbox_address": { + "name": "inbox_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inbox_provider_id": { + "name": "inbox_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inbox_secret_scope": { + "name": "inbox_secret_scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'all'" + }, + "inbox_mounted_secrets": { + "name": "inbox_mounted_secrets", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "organization_assigned_at": { + "name": "organization_assigned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "forked_from_workspace_id": { + "name": "forked_from_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_owner_id_idx": { + "name": "workspace_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_organization_id_idx": { + "name": "workspace_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_mode_idx": { + "name": "workspace_mode_idx", + "columns": [ + { + "expression": "workspace_mode", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_forked_from_workspace_id_idx": { + "name": "workspace_forked_from_workspace_id_idx", + "columns": [ + { + "expression": "forked_from_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_inbox_provider_id_idx": { + "name": "workspace_inbox_provider_id_idx", + "columns": [ + { + "expression": "inbox_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace\".\"inbox_provider_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_owner_id_user_id_fk": { + "name": "workspace_owner_id_user_id_fk", + "tableFrom": "workspace", + "tableTo": "user", + "columnsFrom": ["owner_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_organization_id_organization_id_fk": { + "name": "workspace_organization_id_organization_id_fk", + "tableFrom": "workspace", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workspace_billed_account_user_id_user_id_fk": { + "name": "workspace_billed_account_user_id_user_id_fk", + "tableFrom": "workspace", + "tableTo": "user", + "columnsFrom": ["billed_account_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workspace_forked_from_workspace_id_workspace_id_fk": { + "name": "workspace_forked_from_workspace_id_workspace_id_fk", + "tableFrom": "workspace", + "tableTo": "workspace", + "columnsFrom": ["forked_from_workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workspace_storage_used_bytes_non_negative": { + "name": "workspace_storage_used_bytes_non_negative", + "value": "\"workspace\".\"storage_used_bytes\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workspace_byok_keys": { + "name": "workspace_byok_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_byok_workspace_provider_idx": { + "name": "workspace_byok_workspace_provider_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_byok_keys_workspace_id_workspace_id_fk": { + "name": "workspace_byok_keys_workspace_id_workspace_id_fk", + "tableFrom": "workspace_byok_keys", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_byok_keys_created_by_user_id_fk": { + "name": "workspace_byok_keys_created_by_user_id_fk", + "tableFrom": "workspace_byok_keys", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_environment": { + "name": "workspace_environment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_environment_workspace_unique": { + "name": "workspace_environment_workspace_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_environment_workspace_id_workspace_id_fk": { + "name": "workspace_environment_workspace_id_workspace_id_fk", + "tableFrom": "workspace_environment", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file": { + "name": "workspace_file", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_workspace_id_idx": { + "name": "workspace_file_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_deleted_at_idx": { + "name": "workspace_file_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_workspace_deleted_partial_idx": { + "name": "workspace_file_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_workspace_id_workspace_id_fk": { + "name": "workspace_file_workspace_id_workspace_id_fk", + "tableFrom": "workspace_file", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_uploaded_by_user_id_fk": { + "name": "workspace_file_uploaded_by_user_id_fk", + "tableFrom": "workspace_file", + "tableTo": "user", + "columnsFrom": ["uploaded_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_file_key_unique": { + "name": "workspace_file_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_collab_state": { + "name": "workspace_file_collab_state", + "schema": "", + "columns": { + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "doc_state": { + "name": "doc_state", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "source_hash": { + "name": "source_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_file_collab_state_file_id_workspace_files_id_fk": { + "name": "workspace_file_collab_state_file_id_workspace_files_id_fk", + "tableFrom": "workspace_file_collab_state", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_search_backfill": { + "name": "workspace_file_search_backfill", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "after_workspace_id": { + "name": "after_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "after_file_id": { + "name": "after_file_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_search_build": { + "name": "workspace_file_search_build", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_content_updated_at": { + "name": "source_content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workspace_file_search_build_file_idx": { + "name": "workspace_file_search_build_file_idx", + "columns": [ + { + "expression": "file_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_search_build_cleanup_idx": { + "name": "workspace_file_search_build_cleanup_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file_search_build\".\"expires_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_search_chunk": { + "name": "workspace_file_search_chunk", + "schema": "", + "columns": { + "build_id": { + "name": "build_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "line_start": { + "name": "line_start", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "fragment": { + "name": "fragment", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "overlap": { + "name": "overlap", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "workspace_file_search_chunk_line_idx": { + "name": "workspace_file_search_chunk_line_idx", + "columns": [ + { + "expression": "build_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "line_start", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "ordinal", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_search_chunk_content_idx": { + "name": "workspace_file_search_chunk_content_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "text_ops" + }, + { + "expression": "content", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "gin_trgm_ops" + } + ], + "isUnique": false, + "concurrently": true, + "method": "gin", + "with": { + "fastupdate": "off" + } + } + }, + "foreignKeys": { + "workspace_file_search_chunk_build_id_workspace_file_search_build_id_fk": { + "name": "workspace_file_search_chunk_build_id_workspace_file_search_build_id_fk", + "tableFrom": "workspace_file_search_chunk", + "tableTo": "workspace_file_search_build", + "columnsFrom": ["build_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "workspace_file_search_chunk_pk": { + "name": "workspace_file_search_chunk_pk", + "columns": ["build_id", "ordinal"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workspace_file_search_chunk_content_size": { + "name": "workspace_file_search_chunk_content_size", + "value": "octet_length(\"workspace_file_search_chunk\".\"content\") <= 8192" + }, + "workspace_file_search_chunk_position": { + "name": "workspace_file_search_chunk_position", + "value": "\"workspace_file_search_chunk\".\"ordinal\" >= 0 AND \"workspace_file_search_chunk\".\"line_start\" > 0 AND \"workspace_file_search_chunk\".\"overlap\" BETWEEN 0 AND 2" + } + }, + "isRLSEnabled": false + }, + "public.workspace_file_search_dispatch_queue": { + "name": "workspace_file_search_dispatch_queue", + "schema": "", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enqueued_at": { + "name": "enqueued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_dispatched_at": { + "name": "last_dispatched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_search_dispatch_queue_schedule_idx": { + "name": "workspace_file_search_dispatch_queue_schedule_idx", + "columns": [ + { + "expression": "last_dispatched_at", + "isExpression": false, + "asc": true, + "nulls": "first" + }, + { + "expression": "enqueued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_search_queue_workspace_fk": { + "name": "workspace_file_search_queue_workspace_fk", + "tableFrom": "workspace_file_search_dispatch_queue", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_search_index": { + "name": "workspace_file_search_index", + "schema": "", + "columns": { + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_content_updated_at": { + "name": "source_content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "workspace_file_search_index_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "partial": { + "name": "partial", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "line_count": { + "name": "line_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "indexed_bytes": { + "name": "indexed_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "dispatched_at": { + "name": "dispatched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_search_index_workspace_status_idx": { + "name": "workspace_file_search_index_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_content_updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_search_index_pending_dispatch_idx": { + "name": "workspace_file_search_index_pending_dispatch_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "file_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_content_updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file_search_index\".\"status\" = 'pending' AND \"workspace_file_search_index\".\"dispatched_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_search_index_active_dispatch_idx": { + "name": "workspace_file_search_index_active_dispatch_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dispatched_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file_search_index\".\"status\" = 'pending' AND \"workspace_file_search_index\".\"dispatched_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_search_index_file_fk": { + "name": "workspace_file_search_index_file_fk", + "tableFrom": "workspace_file_search_index", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_search_index_workspace_fk": { + "name": "workspace_file_search_index_workspace_fk", + "tableFrom": "workspace_file_search_index", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "workspace_file_search_index_pk": { + "name": "workspace_file_search_index_pk", + "columns": ["file_id", "source_content_updated_at"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_search_revision": { + "name": "workspace_file_search_revision", + "schema": "", + "columns": { + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_content_updated_at": { + "name": "source_content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "workspace_file_search_index_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "build_id": { + "name": "build_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "line_count": { + "name": "line_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "indexed_bytes": { + "name": "indexed_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "chunk_count": { + "name": "chunk_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "dispatched_at": { + "name": "dispatched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_search_revision_workspace_status_idx": { + "name": "workspace_file_search_revision_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_search_revision_build_idx": { + "name": "workspace_file_search_revision_build_idx", + "columns": [ + { + "expression": "build_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_search_revision_pending_idx": { + "name": "workspace_file_search_revision_pending_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "file_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_content_updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file_search_revision\".\"status\" = 'pending' AND \"workspace_file_search_revision\".\"dispatched_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_search_revision_active_idx": { + "name": "workspace_file_search_revision_active_idx", + "columns": [ + { + "expression": "dispatched_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file_search_revision\".\"status\" = 'pending' AND \"workspace_file_search_revision\".\"dispatched_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_search_revision_file_id_workspace_files_id_fk": { + "name": "workspace_file_search_revision_file_id_workspace_files_id_fk", + "tableFrom": "workspace_file_search_revision", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_search_revision_build_id_workspace_file_search_build_id_fk": { + "name": "workspace_file_search_revision_build_id_workspace_file_search_build_id_fk", + "tableFrom": "workspace_file_search_revision", + "tableTo": "workspace_file_search_build", + "columnsFrom": ["build_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_search_segment": { + "name": "workspace_file_search_segment", + "schema": "", + "columns": { + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_content_updated_at": { + "name": "source_content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "line_number": { + "name": "line_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "segment_number": { + "name": "segment_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "segment_start": { + "name": "segment_start", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "line_length": { + "name": "line_length", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "workspace_file_search_segment_workspace_revision_idx": { + "name": "workspace_file_search_segment_workspace_revision_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "file_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_content_updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_search_segment_workspace_content_trgm_idx": { + "name": "workspace_file_search_segment_workspace_content_trgm_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "text_ops" + }, + { + "expression": "content", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "gin_trgm_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_search_segment_file_fk": { + "name": "workspace_file_search_segment_file_fk", + "tableFrom": "workspace_file_search_segment", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_search_segment_workspace_fk": { + "name": "workspace_file_search_segment_workspace_fk", + "tableFrom": "workspace_file_search_segment", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "workspace_file_search_segment_pk": { + "name": "workspace_file_search_segment_pk", + "columns": ["file_id", "source_content_updated_at", "line_number", "segment_number"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_secret_provenance": { + "name": "workspace_file_secret_provenance", + "schema": "", + "columns": { + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "content_updated_at": { + "name": "content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_file_secret_provenance_file_id_workspace_files_id_fk": { + "name": "workspace_file_secret_provenance_file_id_workspace_files_id_fk", + "tableFrom": "workspace_file_secret_provenance", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workspace_file_secret_provenance_status_check": { + "name": "workspace_file_secret_provenance_status_check", + "value": "\"workspace_file_secret_provenance\".\"status\" IN ('exact', 'unknown', 'unrecorded')" + } + }, + "isRLSEnabled": false + }, + "public.workspace_file_version": { + "name": "workspace_file_version", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "superseded_at": { + "name": "superseded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "workspace_file_version_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "author_user_ids": { + "name": "author_user_ids", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'::text[]" + }, + "restored_from_version": { + "name": "restored_from_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "secret_provenance_status": { + "name": "secret_provenance_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_provenance_entries": { + "name": "secret_provenance_entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_version_file_version_unique": { + "name": "workspace_file_version_file_version_unique", + "columns": [ + { + "expression": "file_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_version_key_unique": { + "name": "workspace_file_version_key_unique", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_version_workspace_id_idx": { + "name": "workspace_file_version_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_version_workspace_superseded_idx": { + "name": "workspace_file_version_workspace_superseded_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "superseded_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file_version\".\"superseded_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_version_file_id_workspace_files_id_fk": { + "name": "workspace_file_version_file_id_workspace_files_id_fk", + "tableFrom": "workspace_file_version", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_version_workspace_id_workspace_id_fk": { + "name": "workspace_file_version_workspace_id_workspace_id_fk", + "tableFrom": "workspace_file_version", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workspace_file_version_provenance_status_check": { + "name": "workspace_file_version_provenance_status_check", + "value": "\"workspace_file_version\".\"secret_provenance_status\" IS NULL OR \"workspace_file_version\".\"secret_provenance_status\" IN ('exact', 'unknown', 'unrecorded')" + } + }, + "isRLSEnabled": false + }, + "public.workspace_files": { + "name": "workspace_files", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "context": { + "name": "context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "content_updated_at": { + "name": "content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "date_trunc('milliseconds', now())" + }, + "secret_provenance_version": { + "name": "secret_provenance_version", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workspace_files_key_active_unique": { + "name": "workspace_files_key_active_unique", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_folder_name_active_unique": { + "name": "workspace_files_workspace_folder_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"folder_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "original_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"deleted_at\" IS NULL AND \"workspace_files\".\"context\" = 'workspace' AND \"workspace_files\".\"workspace_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_active_keyset_idx": { + "name": "workspace_files_workspace_active_keyset_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_files\".\"deleted_at\" IS NULL AND \"workspace_files\".\"context\" = 'workspace' AND \"workspace_files\".\"workspace_id\" IS NOT NULL", + "concurrently": true, + "method": "btree", + "with": {} + }, + "workspace_files_chat_display_name_unique": { + "name": "workspace_files_chat_display_name_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"context\" = 'mothership' AND \"workspace_files\".\"chat_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_organization_id_idx": { + "name": "workspace_files_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_key_idx": { + "name": "workspace_files_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_user_id_idx": { + "name": "workspace_files_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_id_idx": { + "name": "workspace_files_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_folder_id_idx": { + "name": "workspace_files_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_context_idx": { + "name": "workspace_files_context_idx", + "columns": [ + { + "expression": "context", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_chat_id_idx": { + "name": "workspace_files_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_deleted_at_idx": { + "name": "workspace_files_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_deleted_partial_idx": { + "name": "workspace_files_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_files\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_files_user_id_user_id_fk": { + "name": "workspace_files_user_id_user_id_fk", + "tableFrom": "workspace_files", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_workspace_id_workspace_id_fk": { + "name": "workspace_files_workspace_id_workspace_id_fk", + "tableFrom": "workspace_files", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_organization_id_organization_id_fk": { + "name": "workspace_files_organization_id_organization_id_fk", + "tableFrom": "workspace_files", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_folder_id_folder_id_fk": { + "name": "workspace_files_folder_id_folder_id_fk", + "tableFrom": "workspace_files", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workspace_files_chat_id_copilot_chats_id_fk": { + "name": "workspace_files_chat_id_copilot_chats_id_fk", + "tableFrom": "workspace_files", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workspace_files_organization_binding_check": { + "name": "workspace_files_organization_binding_check", + "value": "\"workspace_files\".\"organization_id\" IS NULL OR (\"workspace_files\".\"workspace_id\" IS NULL AND \"workspace_files\".\"context\" = 'knowledge-base' AND \"workspace_files\".\"folder_id\" IS NULL AND \"workspace_files\".\"chat_id\" IS NULL)" + } + }, + "isRLSEnabled": false + }, + "public.workspace_fork_block_map": { + "name": "workspace_fork_block_map", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_workflow_id": { + "name": "parent_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_block_id": { + "name": "parent_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_workflow_id": { + "name": "child_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_block_id": { + "name": "child_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_block_map_child_ws_parent_unique": { + "name": "workspace_fork_block_map_child_ws_parent_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_child_unique": { + "name": "workspace_fork_block_map_child_ws_child_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_parent_wf_idx": { + "name": "workspace_fork_block_map_child_ws_parent_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_child_wf_idx": { + "name": "workspace_fork_block_map_child_ws_child_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_block_map_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_block_map_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_block_map", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_dependent_value": { + "name": "workspace_fork_dependent_value", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_workflow_id": { + "name": "target_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_block_id": { + "name": "target_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sub_block_key": { + "name": "sub_block_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_dependent_value_child_ws_wf_idx": { + "name": "workspace_fork_dependent_value_child_ws_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_dependent_value_field_unique": { + "name": "workspace_fork_dependent_value_field_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sub_block_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_dependent_value_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_dependent_value_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_dependent_value", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_promote_run": { + "name": "workspace_fork_promote_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_workspace_id": { + "name": "source_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_workspace_id": { + "name": "target_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "workspace_fork_promote_direction", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_promote_run_child_ws_target_unique": { + "name": "workspace_fork_promote_run_child_ws_target_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_promote_run_target_ws_idx": { + "name": "workspace_fork_promote_run_target_ws_idx", + "columns": [ + { + "expression": "target_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_promote_run_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_promote_run_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_promote_run", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_fork_promote_run_created_by_user_id_fk": { + "name": "workspace_fork_promote_run_created_by_user_id_fk", + "tableFrom": "workspace_fork_promote_run", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_resource_map": { + "name": "workspace_fork_resource_map", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "workspace_fork_resource_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "parent_resource_id": { + "name": "parent_resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_resource_id": { + "name": "child_resource_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_resource_map_child_ws_idx": { + "name": "workspace_fork_resource_map_child_ws_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_resource_map_child_ws_type_idx": { + "name": "workspace_fork_resource_map_child_ws_type_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_resource_map_child_type_parent_unique": { + "name": "workspace_fork_resource_map_child_type_parent_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_resource_map_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_resource_map_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_resource_map", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_fork_resource_map_created_by_user_id_fk": { + "name": "workspace_fork_resource_map_created_by_user_id_fk", + "tableFrom": "workspace_fork_resource_map", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_operation_receipt": { + "name": "workspace_operation_receipt", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_hash": { + "name": "request_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "report": { + "name": "report", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_operation_receipt_request_unique": { + "name": "workspace_operation_receipt_request_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_operation_receipt_workspace_created_idx": { + "name": "workspace_operation_receipt_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_operation_receipt_workspace_id_workspace_id_fk": { + "name": "workspace_operation_receipt_workspace_id_workspace_id_fk", + "tableFrom": "workspace_operation_receipt", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_sandbox": { + "name": "workspace_sandbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "language": { + "name": "language", + "type": "sandbox_language", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "dependencies": { + "name": "dependencies", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "cli_tools": { + "name": "cli_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "system_packages": { + "name": "system_packages", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "spec_hash": { + "name": "spec_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_sandbox_workspace_name_unique": { + "name": "workspace_sandbox_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_sandbox_workspace_idx": { + "name": "workspace_sandbox_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_sandbox_spec_hash_idx": { + "name": "workspace_sandbox_spec_hash_idx", + "columns": [ + { + "expression": "spec_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_sandbox_workspace_id_workspace_id_fk": { + "name": "workspace_sandbox_workspace_id_workspace_id_fk", + "tableFrom": "workspace_sandbox", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_sandbox_created_by_user_id_fk": { + "name": "workspace_sandbox_created_by_user_id_fk", + "tableFrom": "workspace_sandbox", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.academy_cert_status": { + "name": "academy_cert_status", + "schema": "public", + "values": ["active", "revoked", "expired"] + }, + "public.background_work_kind": { + "name": "background_work_kind", + "schema": "public", + "values": ["deployment_side_effects", "fork_content_copy", "fork_sync", "fork_rollback"] + }, + "public.background_work_status_value": { + "name": "background_work_status_value", + "schema": "public", + "values": ["pending", "processing", "completed", "completed_with_warnings", "failed"] + }, + "public.billing_blocked_reason": { + "name": "billing_blocked_reason", + "schema": "public", + "values": ["payment_failed", "dispute"] + }, + "public.billing_entity_type": { + "name": "billing_entity_type", + "schema": "public", + "values": ["user", "organization"] + }, + "public.chat_type": { + "name": "chat_type", + "schema": "public", + "values": ["mothership", "copilot"] + }, + "public.copilot_async_tool_status": { + "name": "copilot_async_tool_status", + "schema": "public", + "values": ["pending", "running", "completed", "failed", "cancelled", "delivered"] + }, + "public.copilot_run_status": { + "name": "copilot_run_status", + "schema": "public", + "values": ["active", "paused_waiting_for_tool", "resuming", "complete", "error", "cancelled"] + }, + "public.copilot_tool_permission_decision": { + "name": "copilot_tool_permission_decision", + "schema": "public", + "values": ["allow", "allow_chat", "always_allow", "skip"] + }, + "public.credential_group_enrollment_status": { + "name": "credential_group_enrollment_status", + "schema": "public", + "values": ["invited", "delivery_failed", "in_progress", "completed", "revoked"] + }, + "public.credential_group_status": { + "name": "credential_group_status", + "schema": "public", + "values": ["active", "disabled"] + }, + "public.credential_member_role": { + "name": "credential_member_role", + "schema": "public", + "values": ["admin", "member"] + }, + "public.credential_member_status": { + "name": "credential_member_status", + "schema": "public", + "values": ["active", "pending", "revoked"] + }, + "public.credential_type": { + "name": "credential_type", + "schema": "public", + "values": [ + "oauth", + "managed_oauth", + "managed_mcp", + "env_workspace", + "env_personal", + "service_account", + "personal_token" + ] + }, + "public.data_drain_cadence": { + "name": "data_drain_cadence", + "schema": "public", + "values": ["hourly", "daily"] + }, + "public.data_drain_destination": { + "name": "data_drain_destination", + "schema": "public", + "values": ["s3", "gcs", "azure_blob", "datadog", "bigquery", "snowflake", "webhook"] + }, + "public.data_drain_run_status": { + "name": "data_drain_run_status", + "schema": "public", + "values": ["running", "success", "failed"] + }, + "public.data_drain_run_trigger": { + "name": "data_drain_run_trigger", + "schema": "public", + "values": ["cron", "manual"] + }, + "public.data_drain_source": { + "name": "data_drain_source", + "schema": "public", + "values": ["workflow_logs", "job_logs", "audit_logs", "copilot_chats", "copilot_runs"] + }, + "public.execution_large_value_reference_source": { + "name": "execution_large_value_reference_source", + "schema": "public", + "values": ["execution_log", "paused_snapshot"] + }, + "public.folder_resource_type": { + "name": "folder_resource_type", + "schema": "public", + "values": ["workflow", "file", "knowledge_base", "table"] + }, + "public.invitation_kind": { + "name": "invitation_kind", + "schema": "public", + "values": ["organization", "workspace"] + }, + "public.invitation_membership_intent": { + "name": "invitation_membership_intent", + "schema": "public", + "values": ["internal", "external"] + }, + "public.invitation_status": { + "name": "invitation_status", + "schema": "public", + "values": ["pending", "accepted", "rejected", "cancelled", "expired"] + }, + "public.managed_oauth_credential_status": { + "name": "managed_oauth_credential_status", + "schema": "public", + "values": ["active", "needs_reauth", "revoked"] + }, + "public.permission_type": { + "name": "permission_type", + "schema": "public", + "values": ["admin", "write", "read"] + }, + "public.sandbox_image_status": { + "name": "sandbox_image_status", + "schema": "public", + "values": ["pending", "building", "ready", "failed"] + }, + "public.sandbox_language": { + "name": "sandbox_language", + "schema": "public", + "values": ["javascript", "python"] + }, + "public.secret_usage_scope": { + "name": "secret_usage_scope", + "schema": "public", + "values": ["workspace", "personal"] + }, + "public.secret_usage_source": { + "name": "secret_usage_source", + "schema": "public", + "values": ["workflow", "copilot", "mcp"] + }, + "public.upload_session_method": { + "name": "upload_session_method", + "schema": "public", + "values": ["put", "multipart"] + }, + "public.upload_session_provider": { + "name": "upload_session_provider", + "schema": "public", + "values": ["local", "s3", "blob", "gcs"] + }, + "public.upload_session_purpose": { + "name": "upload_session_purpose", + "schema": "public", + "values": [ + "workspace_file", + "table_import", + "knowledge_document", + "profile_picture", + "workspace_logo", + "organization_logo", + "mothership_attachment", + "execution_attachment" + ] + }, + "public.upload_session_status": { + "name": "upload_session_status", + "schema": "public", + "values": [ + "uploading", + "completing", + "finalizing", + "completed", + "aborting", + "aborted", + "failed", + "expired" + ] + }, + "public.usage_log_category": { + "name": "usage_log_category", + "schema": "public", + "values": ["model", "fixed", "tool", "model_unbilled"] + }, + "public.usage_log_source": { + "name": "usage_log_source", + "schema": "public", + "values": [ + "workflow", + "wand", + "copilot", + "workspace-chat", + "mcp_copilot", + "mothership_block", + "knowledge-base", + "voice-input", + "enrichment", + "voice-output", + "api-tool" + ] + }, + "public.workspace_file_search_index_status": { + "name": "workspace_file_search_index_status", + "schema": "public", + "values": ["pending", "ready", "skipped", "failed"] + }, + "public.workspace_file_version_source": { + "name": "workspace_file_version_source", + "schema": "public", + "values": ["upload", "user", "api", "copilot", "workflow", "collab", "revert", "unknown"] + }, + "public.workspace_fork_promote_direction": { + "name": "workspace_fork_promote_direction", + "schema": "public", + "values": ["push", "pull"] + }, + "public.workspace_fork_resource_type": { + "name": "workspace_fork_resource_type", + "schema": "public", + "values": [ + "workflow", + "oauth_credential", + "service_account_credential", + "env_var", + "table", + "knowledge_base", + "knowledge_document", + "file", + "file_folder", + "mcp_server", + "workflow_mcp_server", + "custom_block", + "custom_tool", + "skill", + "sandbox" + ] + }, + "public.workspace_mode": { + "name": "workspace_mode", + "schema": "public", + "values": ["personal", "organization", "grandfathered_shared"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index 9e1972c1099..01ff3c932ce 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -2640,6 +2640,13 @@ "when": 1790119588517, "tag": "0377_member_tombstone_cursor", "breakpoints": true + }, + { + "idx": 378, + "version": "7", + "when": 1790147511551, + "tag": "0378_knowledge_projection_dirty", + "breakpoints": true } ] } diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 6eca24b646c..388a9d68d4e 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -3596,8 +3596,8 @@ export const EMBEDDING_KEYWORD_TIN_INDEX = 'embedding_keyword_tin_content_idx' * BM25 keyword ranking for organization search indexes, served by the Tin text index where the * database provides the `tin` extension. `content` is the chunk's `english` lexemes in position * order, prefixed with a token naming its knowledge base, so ranking is scoped to one base inside - * the index and stems exactly as the GIN projection does. Access never enters the row, so ACL - * changes never rewrite it. Script migration `0019_tin_keyword_projection` installs the extension, + * the index and stems exactly as the GIN projection does. The row mirrors its document's source + * and ACL, like {@link embeddingSearch}. Script migration `0019_tin_keyword_projection` installs the extension, * the index, and the embedding and knowledge base triggers that own these rows, and only where * `tin` exists; elsewhere the table stays empty and keyword search keeps the GIN projection. */ @@ -3625,9 +3625,10 @@ export const embeddingKeywordTin = pgTable( ) /** - * Transactionally maintained candidate projection. Keeping identities and half-precision vectors apart - * from content prevents candidate scans from fetching full-precision TOAST values. - * The embedding write trigger owns this projection; application writers only change embedding. + * Candidate projection. Keeping identities and half-precision vectors apart from content prevents + * candidate scans from fetching full-precision TOAST values. Application writers only change + * `embedding`: its trigger writes this projection in the writer's transaction, or, for a writer that + * deferred it, the knowledge projector writes it after the commit (see {@link knowledgeProjectionDirty}). */ export const embeddingSearch = pgTable( 'embedding_search', @@ -3692,6 +3693,35 @@ export const embeddingSearch = pgTable( }) ) +/** + * Documents whose search projection rows may lag their source rows. The `document` and `embedding` + * triggers mark a document here whenever they change what its projection rows carry, in the + * writer's transaction; the projector rewrites the rows and then removes the mark, but only on the + * generation it read, so a change made while it ran leaves the mark in place. Search decides a + * marked document's rows on the document itself, so a mark never widens what a reader sees. + * + * A side table rather than a column on `document`: a mark is written by the writer that already + * holds the document row, but clearing it would otherwise take that row again, and readers probe + * this small table instead of joining `document` per ranked row. + */ +export const knowledgeProjectionDirty = pgTable( + 'knowledge_projection_dirty', + { + documentId: text('document_id') + .primaryKey() + .references(() => document.id, { onDelete: 'cascade' }), + /** Bumped by every mark; the projector removes the row only on the generation it read. */ + generation: bigint('generation', { mode: 'number' }).notNull().default(1), + /** Whether chunk content changed, not only the document's source or ACL. */ + content: boolean('content').notNull().default(false), + markedAt: timestamp('marked_at', { withTimezone: true }).notNull().defaultNow(), + }, + (table) => ({ + /** The projector claims the oldest marks first. */ + markedAtIdx: index('knowledge_projection_dirty_marked_at_idx').on(table.markedAt), + }) +) + /** Private provenance bound to one exact SHA-256 hash of the persisted chunk content. */ export const embeddingSecretProvenance = pgTable( 'embedding_secret_provenance', diff --git a/packages/db/script-migrations-paused-billing-attribution.test.ts b/packages/db/script-migrations-paused-billing-attribution.test.ts index 33e7588dfae..99230439736 100644 --- a/packages/db/script-migrations-paused-billing-attribution.test.ts +++ b/packages/db/script-migrations-paused-billing-attribution.test.ts @@ -455,6 +455,7 @@ describe('script migration registry', () => { '0019_tin_keyword_projection', '0022_projection_source_acl_backfill', '0023_projection_acl_skip_unfilled', + '0024_knowledge_projection_async', ]) }) }) diff --git a/packages/db/script-migrations/0016_backfill_search_vectors.postgres.test.ts b/packages/db/script-migrations/0016_backfill_search_vectors.postgres.test.ts index 4565d84e4c2..d9260a039bf 100644 --- a/packages/db/script-migrations/0016_backfill_search_vectors.postgres.test.ts +++ b/packages/db/script-migrations/0016_backfill_search_vectors.postgres.test.ts @@ -28,7 +28,13 @@ describe.runIf(Boolean(databaseUrl))('search projection upgrade in PostgreSQL', } admin = postgres(url.toString(), { max: 1, onnotice: () => undefined }) await admin.unsafe(`CREATE SCHEMA "${schemaName}"`) - for (const table of ['knowledge_base', 'embedding', 'embedding_search']) { + for (const table of [ + 'knowledge_base', + 'document', + 'embedding', + 'embedding_search', + 'knowledge_projection_dirty', + ]) { await admin.unsafe( `CREATE TABLE "${schemaName}"."${table}" (LIKE public."${table}" INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES INCLUDING GENERATED)` ) @@ -394,6 +400,7 @@ describe.runIf(Boolean(databaseUrl))('search projection upgrade in PostgreSQL', { name: '0021_embedding_search_connector' }, { name: '0022_projection_source_acl_backfill' }, { name: '0023_projection_acl_skip_unfilled' }, + { name: '0024_knowledge_projection_async' }, ]) const [{ complete }] = await sql`SELECT count(*)::int AS complete FROM embedding e JOIN embedding_search s ON s.id = e.id JOIN embedding_keyword_search k ON k.id = e.id diff --git a/packages/db/script-migrations/0016_backfill_search_vectors.ts b/packages/db/script-migrations/0016_backfill_search_vectors.ts index e40be56c391..36ee94c02ac 100644 --- a/packages/db/script-migrations/0016_backfill_search_vectors.ts +++ b/packages/db/script-migrations/0016_backfill_search_vectors.ts @@ -1,22 +1,35 @@ +import { + SEARCH_VECTOR_COLUMNS, + SYNCHRONOUS_PROJECTION_WHEN, + searchBinaryProjections, + searchVectorProjections, + searchVectorShortened, +} from '@sim/db/knowledge-projection' import type { ScriptMigration } from '@sim/db/script-migrations/types' import { createLogger } from '@sim/logger' -import postgres, { type Sql } from 'postgres' +import postgres, { type Sql, type TransactionSql } from 'postgres' const logger = createLogger('SearchVectorProjection') const BATCH_SIZE = 500 -const WIDTHS = [1536, 384, 512, 768, 1024, 3072] as const -const SOURCE_WIDTHS = [1536, 384, 768, 1024, 3072] as const -const field = (name: string, width: number) => (width === 1536 ? name : `${name}_${width}`) -const columns = WIDTHS.map((width) => field('vector', width)) +const columns = SEARCH_VECTOR_COLUMNS const missing = columns.map((column) => `s.${column} IS NULL`).join(' AND ') -/** Shortening is valid only for the two OpenAI models trained for prefix retrieval. */ -function projections(prefix: string, shortened: string): string { - return WIDTHS.map((width) => - width === 512 - ? `CASE WHEN ${shortened} THEN subvector(coalesce(${SOURCE_WIDTHS.map((size) => `${prefix}.${field('embedding', size)}`).join(', ')}), 1, 512)::halfvec(512) END` - : `CASE WHEN NOT (${shortened}) THEN ${prefix}.${field('embedding', width)}::halfvec(${width}) END` - ).join(', ') +/** + * The vector projection's trigger, skipped by a writer that declared the asynchronous projection + * mode; see `0024_knowledge_projection_async`. + */ +export async function installSearchVectorTrigger(tx: TransactionSql): Promise { + await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_search_sync + AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, + embedding, embedding_384, embedding_768, embedding_1024, embedding_3072 ON embedding + FOR EACH ROW WHEN (${SYNCHRONOUS_PROJECTION_WHEN}) EXECUTE FUNCTION sync_embedding_search()`) +} + +/** The keyword projection's trigger, skipped the same way as {@link installSearchVectorTrigger}. */ +export async function installSearchKeywordTrigger(tx: TransactionSql): Promise { + await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_keyword_search_sync + AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, content ON embedding + FOR EACH ROW WHEN (${SYNCHRONOUS_PROJECTION_WHEN}) EXECUTE FUNCTION sync_embedding_keyword_search()`) } /** Replaces the projection atomically, then fills only missing rows in independently committed pages. */ @@ -33,16 +46,13 @@ export async function backfillSearchVectors(sql: Sql): Promise { RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE shortened boolean; BEGIN - SELECT embedding_model IN ('text-embedding-3-small', 'text-embedding-3-large') - AND NEW.embedding_384 IS NULL INTO STRICT shortened + SELECT ${searchVectorShortened('embedding_model', 'NEW')} INTO STRICT shortened FROM knowledge_base WHERE id = NEW.knowledge_base_id; INSERT INTO embedding_search (id, knowledge_base_id, document_id, enabled, "binary", binary_384, binary_768, binary_1024, binary_3072, ${columns.join(', ')}) VALUES (NEW.id, NEW.knowledge_base_id, NEW.document_id, NEW.enabled, - binary_quantize(NEW.embedding)::bit(1536), binary_quantize(NEW.embedding_384)::bit(384), - binary_quantize(NEW.embedding_768)::bit(768), binary_quantize(NEW.embedding_1024)::bit(1024), - binary_quantize(NEW.embedding_3072)::bit(3072), ${projections('NEW', 'shortened')}) + ${searchBinaryProjections('NEW')}, ${searchVectorProjections('NEW', 'shortened')}) ON CONFLICT (id) DO UPDATE SET knowledge_base_id = EXCLUDED.knowledge_base_id, document_id = EXCLUDED.document_id, enabled = EXCLUDED.enabled, "binary" = EXCLUDED."binary", binary_384 = EXCLUDED.binary_384, @@ -52,10 +62,7 @@ export async function backfillSearchVectors(sql: Sql): Promise { RETURN NEW; END; $$`) - await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_search_sync - AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, - embedding, embedding_384, embedding_768, embedding_1024, embedding_3072 ON embedding - FOR EACH ROW EXECUTE FUNCTION sync_embedding_search()`) + await installSearchVectorTrigger(tx) }) let afterId = '' @@ -77,8 +84,7 @@ export async function backfillSearchVectors(sql: Sql): Promise { ), batch AS MATERIALIZED ( SELECT e.id, e.knowledge_base_id, e.document_id, e.enabled, e.embedding, e.embedding_384, e.embedding_768, e.embedding_1024, e.embedding_3072, - k.embedding_model IN ('text-embedding-3-small', 'text-embedding-3-large') - AND e.embedding_384 IS NULL AS shortened + ${searchVectorShortened('k.embedding_model', 'e')} AS shortened FROM missing m INNER JOIN embedding e ON e.id = m.id INNER JOIN knowledge_base k ON k.id = e.knowledge_base_id ORDER BY e.id FOR KEY SHARE OF e @@ -87,9 +93,7 @@ export async function backfillSearchVectors(sql: Sql): Promise { (id, knowledge_base_id, document_id, enabled, "binary", binary_384, binary_768, binary_1024, binary_3072, ${columns.join(', ')}) SELECT id, knowledge_base_id, document_id, enabled, - binary_quantize(embedding)::bit(1536), binary_quantize(embedding_384)::bit(384), - binary_quantize(embedding_768)::bit(768), binary_quantize(embedding_1024)::bit(1024), - binary_quantize(embedding_3072)::bit(3072), ${projections('batch', 'shortened')} + ${searchBinaryProjections('batch')}, ${searchVectorProjections('batch', 'shortened')} FROM batch ON CONFLICT (id) DO UPDATE SET ${columns.map((column) => `${column} = EXCLUDED.${column}`).join(', ')} WHERE ${missing} RETURNING id @@ -135,9 +139,7 @@ export async function backfillSearchKeywords(sql: Sql): Promise { RETURN NEW; END; $$`) - await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_keyword_search_sync - AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, content ON embedding - FOR EACH ROW EXECUTE FUNCTION sync_embedding_keyword_search()`) + await installSearchKeywordTrigger(tx) }) let afterId = '' let count = 0 @@ -192,10 +194,10 @@ export async function buildSearchIndexes(sql: Sql): Promise { table: 'embedding_search', definition: 'ON embedding_search (document_id, knowledge_base_id, id) WHERE enabled', }, - ...WIDTHS.map((width) => ({ - name: `embedding_search${width === 1536 ? '' : `_${width}`}_cosine_hnsw_idx`, + ...columns.map((column) => ({ + name: `embedding_search${column.replace(/^vector/, '')}_cosine_hnsw_idx`, table: 'embedding_search', - definition: `ON embedding_search USING hnsw (${field('vector', width)} halfvec_cosine_ops) WITH (m=16, ef_construction=64)`, + definition: `ON embedding_search USING hnsw (${column} halfvec_cosine_ops) WITH (m=16, ef_construction=64)`, })), { name: 'embedding_keyword_search_kb_idx', diff --git a/packages/db/script-migrations/0019_tin_keyword_projection.ts b/packages/db/script-migrations/0019_tin_keyword_projection.ts index b73efe42d78..8dc5a2b4b3c 100644 --- a/packages/db/script-migrations/0019_tin_keyword_projection.ts +++ b/packages/db/script-migrations/0019_tin_keyword_projection.ts @@ -1,7 +1,8 @@ +import { SYNCHRONOUS_PROJECTION_WHEN } from '@sim/db/knowledge-projection' import { EMBEDDING_KEYWORD_TIN_INDEX } from '@sim/db/schema' import { type ScriptMigration, ScriptMigrationDeferred } from '@sim/db/script-migrations/types' import { createLogger } from '@sim/logger' -import postgres, { type Sql } from 'postgres' +import postgres, { type Sql, type TransactionSql } from 'postgres' const logger = createLogger('TinKeywordProjection') const BATCH_SIZE = 500 @@ -38,6 +39,16 @@ async function tinAvailable(sql: Sql): Promise { return rows.length > 0 } +/** + * The Tin projection's embedding trigger, skipped by a writer that declared the asynchronous + * projection mode; see `0024_knowledge_projection_async`. + */ +export async function installTinKeywordTrigger(tx: TransactionSql): Promise { + await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_keyword_tin_sync + AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, content ON embedding + FOR EACH ROW WHEN (${SYNCHRONOUS_PROJECTION_WHEN}) EXECUTE FUNCTION sync_embedding_keyword_tin()`) +} + /** * Installs the stream and base-token functions shared by the triggers and the query path, and the * embedding and knowledge base triggers, atomically with respect to embedding writers. @@ -97,9 +108,7 @@ export async function installProjection(sql: Sql): Promise { RETURN NEW; END; $$`) - await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_keyword_tin_sync - AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, content ON embedding - FOR EACH ROW EXECUTE FUNCTION sync_embedding_keyword_tin()`) + await installTinKeywordTrigger(tx) await tx.unsafe(`CREATE OR REPLACE FUNCTION sync_knowledge_base_keyword_tin() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN diff --git a/packages/db/script-migrations/0021_embedding_search_connector.postgres.test.ts b/packages/db/script-migrations/0021_embedding_search_connector.postgres.test.ts index 626c8ad4d18..24fd78d089f 100644 --- a/packages/db/script-migrations/0021_embedding_search_connector.postgres.test.ts +++ b/packages/db/script-migrations/0021_embedding_search_connector.postgres.test.ts @@ -1,23 +1,19 @@ -import { - backfillProjectionSourceAcl, - PROJECTION_SOURCE_ACL_TABLES, - replaceProjectionSourceAclSync, -} from '@sim/db/script-migrations/0021_embedding_search_connector' +import { PROJECTION_SOURCE_ACL_TABLES } from '@sim/db/script-migrations/0021_embedding_search_connector' import { projectionSourceAclBackfillMigration as embeddingSearchConnectorMigration } from '@sim/db/script-migrations/0022_projection_source_acl_backfill' import { projectionAclSkipUnfilledMigration } from '@sim/db/script-migrations/0023_projection_acl_skip_unfilled' -import { sleep } from '@sim/utils/helpers' +import { installKnowledgeProjectionMarking } from '@sim/db/script-migrations/0024_knowledge_projection_async' import { generateId } from '@sim/utils/id' -import postgres, { type Sql, type TransactionSql } from 'postgres' +import postgres, { type Sql } from 'postgres' import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' const databaseUrl = process.env.KNOWLEDGE_ACL_TEST_DATABASE_URL /** - * The projections here carry only the columns the source and ACL triggers and backfill touch; the - * vector and lexeme columns, and their indexes, are what make the backfill slow, not what decides - * which rows it writes. + * The projections here carry only the columns the source and ACL triggers touch; the vector and + * lexeme columns, and their indexes, are what make a projection write slow, not what decides + * which rows the triggers write or which documents they mark. */ -describe.runIf(Boolean(databaseUrl))('projection source and ACL backfill in PostgreSQL', () => { +describe.runIf(Boolean(databaseUrl))('projection source and ACL triggers in PostgreSQL', () => { let admin: Sql let sql: Sql const schemaName = `projection_acl_${generateId().replaceAll('-', '')}` @@ -26,6 +22,17 @@ describe.runIf(Boolean(databaseUrl))('projection source and ACL backfill in Post sql<{ id: string; connector_id: string | null; acl: string[] | null }[]>` SELECT id, connector_id, acl FROM ${sql(projection)} ORDER BY id` + const marks = () => + sql<{ document_id: string; generation: string; content: boolean }[]>` + SELECT document_id, generation, content FROM knowledge_projection_dirty ORDER BY document_id` + + /** Runs `write` in a transaction that declared the asynchronous projection mode. */ + const asynchronously = (write: (tx: postgres.TransactionSql) => Promise) => + sql.begin(async (tx) => { + await tx`SELECT set_config('sim.projection_mode', 'async', true)` + await write(tx) + }) + beforeAll(async () => { const url = new URL(databaseUrl!) if ( @@ -44,6 +51,11 @@ describe.runIf(Boolean(databaseUrl))('projection source and ACL backfill in Post await sql`CREATE TABLE document ( id text PRIMARY KEY, connector_id text, acl text[] NOT NULL DEFAULT '{ws}' )` + await sql`CREATE TABLE knowledge_projection_dirty ( + document_id text PRIMARY KEY REFERENCES document (id) ON DELETE CASCADE, + generation bigint NOT NULL DEFAULT 1, content boolean NOT NULL DEFAULT false, + marked_at timestamptz NOT NULL DEFAULT now() + )` for (const projection of ['embedding_search', 'embedding_keyword_tin']) { await sql`CREATE TABLE ${sql(projection)} ( id text PRIMARY KEY, document_id text NOT NULL, enabled boolean NOT NULL DEFAULT true, @@ -60,11 +72,35 @@ describe.runIf(Boolean(databaseUrl))('projection source and ACL backfill in Post }) beforeEach(async () => { - await sql`TRUNCATE embedding_search, embedding_keyword_tin, document` + await sql`TRUNCATE embedding_search, embedding_keyword_tin, knowledge_projection_dirty, document` await sql`ALTER TABLE embedding_search DISABLE TRIGGER embedding_search_source_acl_set` await sql`ALTER TABLE embedding_keyword_tin DISABLE TRIGGER embedding_keyword_tin_source_acl_set` }) + it('runs every migration up to 0023, and every write between them, before 0024 exists', async () => { + /** A database that has run `0023` and not yet `0024`: no mark function exists. */ + await sql`DROP FUNCTION IF EXISTS mark_knowledge_projection(text[], boolean)` + await embeddingSearchConnectorMigration.up(sql) + await projectionAclSkipUnfilledMigration.up(sql) + const [before] = await sql<{ installed: boolean }[]>` + SELECT to_regprocedure('mark_knowledge_projection(text[], boolean)') IS NOT NULL AS installed` + expect(before?.installed).toBe(false) + await sql`INSERT INTO document (id, connector_id, acl) VALUES ('doc', 'src', ARRAY['u:alice'])` + await sql`INSERT INTO embedding_search (id, document_id, connector_id, acl) + VALUES ('filled', 'doc', 'src', ARRAY['u:alice'])` + + await sql`UPDATE document SET acl = ARRAY['u:bob'] WHERE id = 'doc'` + expect(await projected('embedding_search')).toEqual([ + { id: 'filled', connector_id: 'src', acl: ['u:bob'] }, + ]) + expect(await marks()).toEqual([]) + + await installKnowledgeProjectionMarking(sql) + await sql`UPDATE document SET acl = ARRAY['u:carol'] WHERE id = 'doc'` + expect((await projected('embedding_search'))[0]?.acl).toEqual(['u:carol']) + expect(await marks()).toEqual([{ document_id: 'doc', generation: '1', content: false }]) + }) + it('installs its triggers and indexes again without failing, so a cut-short deploy completes', async () => { await expect(embeddingSearchConnectorMigration.up(sql)).resolves.toBeUndefined() const indexes = await sql<{ indexname: string }[]>` @@ -80,104 +116,10 @@ describe.runIf(Boolean(databaseUrl))('projection source and ACL backfill in Post ) }) - it('fills only the rows still unset, in pages, and leaves a chunk that changed documents to its trigger', async () => { - await sql`INSERT INTO document (id, connector_id, acl) VALUES - ('doc-a', 'src-a', ARRAY['u:alice']), ('doc-b', NULL, ARRAY['ws']), ('doc-c', 'src-c', ARRAY['u:carol'])` - await sql`INSERT INTO embedding_search (id, document_id, connector_id, acl) VALUES - ('c1', 'doc-a', NULL, NULL), ('c2', 'doc-b', NULL, NULL), - ('c3', 'doc-c', 'src-old', ARRAY['u:stale']), ('c4', 'doc-a', NULL, NULL), ('c5', 'missing', NULL, NULL)` - const progress = await backfillProjectionSourceAcl(sql, 'embedding_search', { - pageSize: 2, - pauseMs: 0, - }) - expect(progress).toEqual({ - projection: 'embedding_search', - scanned: 3, - written: 3, - afterId: 'c4', - done: true, - }) - expect(await projected('embedding_search')).toEqual([ - { id: 'c1', connector_id: 'src-a', acl: ['u:alice'] }, - { id: 'c2', connector_id: null, acl: ['ws'] }, - { id: 'c3', connector_id: 'src-old', acl: ['u:stale'] }, - { id: 'c4', connector_id: 'src-a', acl: ['u:alice'] }, - { id: 'c5', connector_id: null, acl: null }, - ]) - expect(await projected('embedding_keyword_tin')).toEqual([]) - }) - - it('stops at its budget with the cursor to resume from, and resumes after it', async () => { - await sql`INSERT INTO document (id, connector_id, acl) VALUES ('doc', 'src', ARRAY['u:alice'])` - await sql`INSERT INTO embedding_keyword_tin (id, document_id) VALUES - ('k1', 'doc'), ('k2', 'doc'), ('k3', 'doc')` - const paused = await backfillProjectionSourceAcl(sql, 'embedding_keyword_tin', { - pageSize: 1, - pauseMs: 0, - budgetMs: 0, - }) - expect(paused).toMatchObject({ scanned: 1, written: 1, afterId: 'k1', done: false }) - const resumed = await backfillProjectionSourceAcl(sql, 'embedding_keyword_tin', { - afterId: paused.afterId, - pageSize: 1, - pauseMs: 0, - }) - expect(resumed).toMatchObject({ scanned: 2, written: 2, afterId: 'k3', done: true }) - expect((await projected('embedding_keyword_tin')).map((row) => row.acl)).toEqual([ - ['u:alice'], - ['u:alice'], - ['u:alice'], - ]) - const again = await backfillProjectionSourceAcl(sql, 'embedding_keyword_tin', { pauseMs: 0 }) - expect(again).toMatchObject({ scanned: 0, written: 0, afterId: '', done: true }) - }) - describe('a document change on chunks the backfill has not filled', () => { - /** A promise the test resolves by hand, to hold a transaction open at a chosen point. */ - const gate = () => { - let resolve = () => {} - const promise = new Promise((done) => { - resolve = done - }) - return { promise, resolve } - } - - /** Waits until `pid` is blocked on a lock, so the interleaving under test really happened. */ - const blockedOnLock = async (pid: number) => { - for (let attempt = 0; attempt < 100; attempt++) { - const [row] = await admin<{ waiting: boolean }[]>` - SELECT wait_event_type = 'Lock' AS waiting FROM pg_stat_activity WHERE pid = ${pid}` - if (row?.waiting) return true - await sleep(20) - } - return false - } - - /** The backfill's page statement, run in a transaction the test holds open. */ - const backfillPage = (tx: TransactionSql) => - tx.unsafe(`WITH page AS ( - SELECT s.id, s.document_id, d.connector_id, d.acl - FROM embedding_search s JOIN document d ON d.id = s.document_id - WHERE s.acl IS NULL ORDER BY s.id LIMIT 100 - FOR SHARE OF d - ) - UPDATE embedding_search s SET connector_id = page.connector_id, acl = page.acl - FROM page WHERE s.id = page.id AND s.document_id = page.document_id AND s.acl IS NULL`) - - let other: Sql - beforeAll(() => { - other = postgres(databaseUrl!, { - max: 1, - onnotice: () => undefined, - connection: { search_path: schemaName }, - }) - }) - afterAll(async () => { - await other?.end() - }) - + describe('a document change', () => { beforeEach(async () => { - /** A test below installs an older body; each starts from the current one. */ - await replaceProjectionSourceAclSync(sql) + /** Tests here install older bodies; each starts from the one `0024` installs. */ + await installKnowledgeProjectionMarking(sql) await sql`INSERT INTO document (id, connector_id, acl) VALUES ('doc', 'src', ARRAY['u:alice'])` for (const projection of PROJECTION_SOURCE_ACL_TABLES) { await sql`INSERT INTO ${sql(projection)} (id, document_id, connector_id, acl) VALUES @@ -187,7 +129,7 @@ describe.runIf(Boolean(databaseUrl))('projection source and ACL backfill in Post }) it('replaces the body a database already has when its own migration runs', async () => { - /** The body `0022` installed before this change. */ + /** The body `0022` installed before `0023`. */ await sql.unsafe(`CREATE OR REPLACE FUNCTION sync_projection_source_acl() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN @@ -217,7 +159,7 @@ describe.runIf(Boolean(databaseUrl))('projection source and ACL backfill in Post ]) }) - it('writes a changed ACL onto filled chunks only, leaving unfilled ones to their document', async () => { + it('writes a changed ACL onto filled chunks only and marks the document', async () => { await sql`UPDATE document SET acl = ARRAY['u:bob'] WHERE id = 'doc'` for (const projection of PROJECTION_SOURCE_ACL_TABLES) { expect(await projected(projection)).toEqual([ @@ -226,6 +168,7 @@ describe.runIf(Boolean(databaseUrl))('projection source and ACL backfill in Post { id: 'unfilled-sourced', connector_id: 'src', acl: null }, ]) } + expect(await marks()).toEqual([{ document_id: 'doc', generation: '1', content: false }]) }) it('still carries a changed source onto unfilled chunks, whose source filters read the row', async () => { @@ -237,59 +180,33 @@ describe.runIf(Boolean(databaseUrl))('projection source and ACL backfill in Post { id: 'unfilled-sourced', connector_id: 'moved', acl: null }, ]) } + expect(await marks()).toEqual([{ document_id: 'doc', generation: '1', content: false }]) }) - it('fills the current ACL when the change commits before the backfill reads the document', async () => { - await sql`UPDATE document SET acl = ARRAY['u:bob'] WHERE id = 'doc'` - await backfillProjectionSourceAcl(sql, 'embedding_search', { pauseMs: 0 }) - expect((await projected('embedding_search')).map((row) => row.acl)).toEqual([ - ['u:bob'], - ['u:bob'], - ['u:bob'], - ]) + it('leaves every row to the projector in the asynchronous mode, and only marks the document', async () => { + await asynchronously((tx) => tx`UPDATE document SET acl = ARRAY['u:bob'] WHERE id = 'doc'`) + await asynchronously((tx) => tx`UPDATE document SET connector_id = 'moved' WHERE id = 'doc'`) + for (const projection of PROJECTION_SOURCE_ACL_TABLES) { + expect(await projected(projection)).toEqual([ + { id: 'filled', connector_id: 'src', acl: ['u:alice'] }, + { id: 'unfilled', connector_id: null, acl: null }, + { id: 'unfilled-sourced', connector_id: 'src', acl: null }, + ]) + } + expect(await marks()).toEqual([{ document_id: 'doc', generation: '2', content: false }]) }) - it('fans the change out after a backfill page that read the old ACL commits', async () => { - const [pageRead, release] = [gate(), gate()] - const page = sql.begin(async (tx) => { - await backfillPage(tx) - pageRead.resolve() - await release.promise - }) - await pageRead.promise - const [{ pid }] = await other<{ pid: number }[]>`SELECT pg_backend_pid() AS pid` - /** Blocks on the page's share lock on the document until the page commits. */ - const change = other`UPDATE document SET acl = ARRAY['u:bob'] WHERE id = 'doc'`.execute() - expect(await blockedOnLock(pid)).toBe(true) - release.resolve() - await page - await change - expect((await projected('embedding_search')).map((row) => row.acl)).toEqual([ - ['u:bob'], - ['u:bob'], - ['u:bob'], - ]) + it('marks nothing when an assignment leaves the source and ACL as they were', async () => { + await sql`UPDATE document SET acl = ARRAY['u:alice'], connector_id = 'src' WHERE id = 'doc'` + expect(await marks()).toEqual([]) }) - it('fills the new ACL when the backfill waits on a change that has not committed yet', async () => { - const [changed, commit] = [gate(), gate()] - const change = other.begin(async (tx) => { + it('keeps the synchronous fan-out for a transaction that set another mode', async () => { + await sql.begin(async (tx) => { + await tx`SELECT set_config('sim.projection_mode', 'sync', true)` await tx`UPDATE document SET acl = ARRAY['u:bob'] WHERE id = 'doc'` - changed.resolve() - await commit.promise }) - await changed.promise - const [{ pid }] = await sql<{ pid: number }[]>`SELECT pg_backend_pid() AS pid` - const fill = backfillProjectionSourceAcl(sql, 'embedding_search', { pauseMs: 0 }) - expect(await blockedOnLock(pid)).toBe(true) - commit.resolve() - await change - await fill - expect((await projected('embedding_search')).map((row) => row.acl)).toEqual([ - ['u:bob'], - ['u:bob'], - ['u:bob'], - ]) + expect((await projected('embedding_search'))[0]?.acl).toEqual(['u:bob']) }) }) }) diff --git a/packages/db/script-migrations/0021_embedding_search_connector.test.ts b/packages/db/script-migrations/0021_embedding_search_connector.test.ts deleted file mode 100644 index de39d4282b4..00000000000 --- a/packages/db/script-migrations/0021_embedding_search_connector.test.ts +++ /dev/null @@ -1,187 +0,0 @@ -/** - * @vitest-environment node - */ -import { - backfillProjectionSourceAcl, - PROJECTION_SOURCE_ACL_PAGE_RETRIES, -} from '@sim/db/script-migrations/0021_embedding_search_connector' -import type { Sql } from 'postgres' -import { afterEach, describe, expect, it, vi } from 'vitest' - -/** A session that must never be reached: every case below is refused before the first page. */ -const untouched = { begin: vi.fn() } as unknown as Sql - -type PageRow = { scanned: number; filled: number; last_id: string | null } - -/** The message the database pairs with each SQLSTATE below. */ -const SERVER_MESSAGES: Record = { - '55P03': 'canceling statement due to lock timeout', - '57014': 'canceling statement due to statement timeout', - '40P01': 'deadlock detected', -} - -/** - * The error `postgres` throws for `code`: a SQLSTATE carries the server's message as is, and a - * lost connection is the driver's own connection error (`write `). - */ -function postgresError(code: string, message = SERVER_MESSAGES[code] ?? 'failed'): Error { - if (code.startsWith('CONNECTION_')) { - return Object.assign(new Error(`write ${code} localhost:5432`), { - code, - errno: code, - address: 'localhost', - port: 5432, - }) - } - return Object.assign(new Error(message), { code }) -} - -/** - * A session whose page statement answers from `outcomes` in order — a row, or an error to throw — - * and records the cursor each page was bound to. `beforePage` runs with the page's index before - * it answers. - */ -function sessionOf(outcomes: Array, beforePage?: (index: number) => void) { - const cursors: string[] = [] - const tx = { - unsafe: vi.fn(async (query: string, params?: unknown[]) => { - if (!query.includes('WITH page')) return [] - beforePage?.(cursors.length) - cursors.push(String(params?.[0])) - const outcome = outcomes.shift() - if (outcome === undefined) throw new Error('No outcome left for this page') - if (outcome instanceof Error) throw outcome - return [outcome] - }), - } - const session = { - begin: vi.fn(async (work: (tx: unknown) => Promise) => work(tx)), - unsafe: vi.fn(async () => []), - } as unknown as Sql - return { session, cursors } -} - -/** Runs a backfill under fake timers, so its retry pauses pass without waiting. */ -async function backfillNow(...args: Parameters) { - vi.useFakeTimers() - const result = backfillProjectionSourceAcl(...args) - /** A rejection must not surface as unhandled while the timers are still being drained. */ - const settled = result.catch(() => undefined) - await vi.runAllTimersAsync() - await settled - return result -} - -describe('backfillProjectionSourceAcl', () => { - afterEach(() => { - vi.useRealTimers() - }) - - it.each(['55P03', '57014', '40P01', 'CONNECTION_CLOSED'])( - 'retries the page after a %s failure and moves the cursor only once it commits', - async (code) => { - const { session, cursors } = sessionOf([ - { scanned: 2, filled: 2, last_id: 'id-2' }, - postgresError(code), - postgresError(code), - { scanned: 1, filled: 1, last_id: 'id-3' }, - { scanned: 0, filled: 0, last_id: null }, - ]) - await expect( - backfillNow(session, 'embedding_keyword_tin', { pauseMs: 0 }) - ).resolves.toMatchObject({ scanned: 3, written: 3, afterId: 'id-3', done: true }) - expect(cursors).toEqual(['', 'id-2', 'id-2', 'id-2', 'id-3']) - } - ) - - it('gives up on a page that times out more than the retry limit in a row', async () => { - const { session, cursors } = sessionOf( - Array.from({ length: PROJECTION_SOURCE_ACL_PAGE_RETRIES + 1 }, () => postgresError('55P03')) - ) - await expect( - backfillNow(session, 'embedding_keyword_tin', { pauseMs: 0 }) - ).rejects.toMatchObject({ code: '55P03' }) - expect(cursors).toHaveLength(PROJECTION_SOURCE_ACL_PAGE_RETRIES + 1) - expect(new Set(cursors)).toEqual(new Set([''])) - }) - - it('propagates an error that is not a timeout without retrying', async () => { - const { session, cursors } = sessionOf([postgresError('42P01')]) - await expect(backfillNow(session, 'embedding_search', { pauseMs: 0 })).rejects.toMatchObject({ - code: '42P01', - }) - expect(cursors).toEqual(['']) - }) - - it('propagates an explicit cancellation, which shares the statement timeout SQLSTATE', async () => { - const { session, cursors } = sessionOf([ - postgresError('57014', 'canceling statement due to user request'), - ]) - await expect(backfillNow(session, 'embedding_search', { pauseMs: 0 })).rejects.toThrow( - 'user request' - ) - expect(cursors).toEqual(['']) - }) - - it('does not start another page when the budget ran out during the retry pause', async () => { - const { session, cursors } = sessionOf([ - { scanned: 1, filled: 1, last_id: 'id-1' }, - postgresError('57014'), - ]) - await expect( - backfillNow(session, 'embedding_search', { pauseMs: 0, budgetMs: 1000 }) - ).resolves.toMatchObject({ afterId: 'id-1', done: false }) - expect(cursors).toEqual(['', 'id-1']) - }) - - it('leaves a page still failing at the budget to the continuation, from the last committed page', async () => { - const { session, cursors } = sessionOf( - [{ scanned: 1, filled: 1, last_id: 'id-1' }, postgresError('57014')], - /** The second page spends the budget before the database cancels it. */ - (index) => { - if (index === 1) vi.advanceTimersByTime(1000) - } - ) - await expect( - backfillNow(session, 'embedding_search', { pauseMs: 0, budgetMs: 1000 }) - ).resolves.toMatchObject({ afterId: 'id-1', done: false }) - expect(cursors).toEqual(['', 'id-1']) - }) - - it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( - 'refuses a page size of %s instead of reporting the projection filled', - async (pageSize) => { - await expect( - backfillProjectionSourceAcl(untouched, 'embedding_search', { pageSize }) - ).rejects.toThrow('page size must be a positive integer') - expect(untouched.begin).not.toHaveBeenCalled() - } - ) - - it.each([-1, Number.NaN])('refuses a pause of %s', async (pauseMs) => { - await expect( - backfillProjectionSourceAcl(untouched, 'embedding_search', { pauseMs }) - ).rejects.toThrow('pause must be a non-negative number') - expect(untouched.begin).not.toHaveBeenCalled() - }) - - it('binds the range it was given to every page, so shards never meet', async () => { - const statements: Array<{ query: string; params?: unknown[] }> = [] - const tx = { - unsafe: vi.fn(async (query: string, params?: unknown[]) => { - statements.push({ query, params }) - return query.includes('WITH page') ? [{ scanned: 0, filled: 0, last_id: null }] : [] - }), - } - const session = { - begin: vi.fn(async (work: (tx: unknown) => Promise) => work(tx)), - unsafe: vi.fn(async () => []), - } as unknown as Sql - await expect( - backfillProjectionSourceAcl(session, 'embedding_search', { afterId: '4', beforeId: '8' }) - ).resolves.toMatchObject({ done: true }) - const page = statements.find((statement) => statement.query.includes('WITH page'))! - expect(page.query).toContain('($2::text IS NULL OR s.id < $2)') - expect(page.params).toEqual(['4', '8']) - }) -}) diff --git a/packages/db/script-migrations/0021_embedding_search_connector.ts b/packages/db/script-migrations/0021_embedding_search_connector.ts index 3fdfec7027d..3b5326c7902 100644 --- a/packages/db/script-migrations/0021_embedding_search_connector.ts +++ b/packages/db/script-migrations/0021_embedding_search_connector.ts @@ -1,58 +1,23 @@ -import { createLogger } from '@sim/logger' -import { getPostgresErrorCode, getTransientDatabaseFailure } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' -import { backoffWithJitter } from '@sim/utils/retry' +import { SYNCHRONOUS_PROJECTION_WHEN } from '@sim/db/knowledge-projection' import postgres, { type Sql, type TransactionSql } from 'postgres' -const logger = createLogger('ProjectionSourceAcl') - -/** - * Chunks filled per page. Every write to `embedding_search` re-inserts the row into each of its - * HNSW indexes, so a page's cost is index maintenance rather than the plan, and a small page keeps - * each transaction short and its locks brief. - */ -export const PROJECTION_SOURCE_ACL_PAGE_SIZE = 100 - -/** Pause between pages, so the backfill shares the database with the search it serves. */ -export const PROJECTION_SOURCE_ACL_PAGE_PAUSE_MS = 250 - -/** - * Longest a page may run before the database cancels it; the page is then retried in place. A - * caller that bounds a run leaves at least this much headroom after its budget, plus the longest - * retry pause, since the budget is checked between pages and the page in flight runs to this limit. - */ -export const PROJECTION_SOURCE_ACL_PAGE_TIMEOUT_MS = 60_000 - -/** - * How many times in a row one page may time out before the run fails. Index maintenance was - * observed holding the page for several minutes; with the pauses below a page waits roughly - * twelve minutes, about fourteen at the jitter's worst, so the budget covers one such pass. - */ -export const PROJECTION_SOURCE_ACL_PAGE_RETRIES = 12 - -/** Pause before a page is retried: 10 s, doubling to a 60 s base with up to 20% jitter (about 72 s). */ -const PAGE_RETRY_PAUSE = { baseMs: 10_000, maxMs: 60_000 } as const - -/** Pages between progress log lines. */ -const PROGRESS_EVERY_PAGES = 100 - /** The projections that carry their document's source and ACL. */ export const PROJECTION_SOURCE_ACL_TABLES = ['embedding_search', 'embedding_keyword_tin'] as const export type ProjectionSourceAclTable = (typeof PROJECTION_SOURCE_ACL_TABLES)[number] /** - * The document trigger's body: fans a document's source and ACL out to its enabled chunks. + * The document trigger's fan-out: copies a document's source and ACL onto its enabled chunks. * - * A chunk the backfill has not filled yet (`acl IS NULL`) keeps a NULL ACL. Search decides such a - * row on its document, so writing the ACL there changes no answer, while every write to + * A chunk the source and ACL fill has not reached (`acl IS NULL`) keeps a NULL ACL. Search decides + * such a row on its document, so writing the ACL there changes no answer, while every write to * `embedding_search` re-inserts the row into its vector index: a document whose ACL changed would - * otherwise rewrite each of its unfilled chunks inside the writer's statement. The backfill fills - * the row later from the document under a share lock, so it copies whichever ACL is current. A - * document that moves to another source still carries the source onto its unfilled chunks, because - * source filters read it from the row; an ACL change alone leaves them untouched. + * otherwise rewrite each of its unfilled chunks inside the writer's statement. A document that + * moves to another source still carries the source onto its unfilled chunks, because source + * filters read it from the row; an ACL change alone leaves them untouched. Expects `moved` in scope. */ -export async function replaceProjectionSourceAclSync(sql: Sql | TransactionSql): Promise { - const fanOut = (projection: ProjectionSourceAclTable) => ` +export function projectionSourceAclFanOut(): string { + return PROJECTION_SOURCE_ACL_TABLES.map( + (projection) => ` UPDATE ${projection} SET connector_id = NEW.connector_id, acl = CASE WHEN acl IS NULL THEN NULL ELSE NEW.acl END WHERE document_id = NEW.id AND enabled @@ -60,16 +25,38 @@ export async function replaceProjectionSourceAclSync(sql: Sql | TransactionSql): THEN moved AND connector_id IS DISTINCT FROM NEW.connector_id ELSE connector_id IS DISTINCT FROM NEW.connector_id OR acl IS DISTINCT FROM NEW.acl END;` + ).join('') +} + +/** + * The document trigger's body as `0022` and `0023` install it: the fan-out alone. It depends on no + * object a later migration creates, so every migration up to `0023` runs, and every write between + * them succeeds, on its own. `0024_knowledge_projection_async` replaces it with the body that also + * marks the document for the knowledge projector. + */ +export async function replaceProjectionSourceAclSync(sql: Sql | TransactionSql): Promise { await sql.unsafe(`CREATE OR REPLACE FUNCTION sync_projection_source_acl() RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE moved boolean := TG_OP = 'UPDATE' AND OLD.connector_id IS DISTINCT FROM NEW.connector_id; - BEGIN${PROJECTION_SOURCE_ACL_TABLES.map(fanOut).join('')} + BEGIN${projectionSourceAclFanOut()} RETURN NEW; END; $$`) } +/** + * The projection triggers that copy a chunk's source and ACL from its document on every write, + * skipped in the asynchronous mode, where the projector writes both itself. + */ +export async function installProjectionSourceAclSetTriggers(tx: TransactionSql): Promise { + for (const projection of PROJECTION_SOURCE_ACL_TABLES) { + await tx.unsafe(`CREATE OR REPLACE TRIGGER ${projection}_source_acl_set + BEFORE INSERT OR UPDATE OF document_id, enabled ON ${projection} + FOR EACH ROW WHEN (${SYNCHRONOUS_PROJECTION_WHEN}) EXECUTE FUNCTION set_projection_source_acl()`) + } +} + /** * Carries a chunk's source and ACL onto the ranking projections and keeps them there. * @@ -98,11 +85,7 @@ export async function installProjectionSourceAcl(sql: Sql): Promise { RETURN NEW; END; $$`) - for (const projection of PROJECTION_SOURCE_ACL_TABLES) { - await tx.unsafe(`CREATE OR REPLACE TRIGGER ${projection}_source_acl_set - BEFORE INSERT OR UPDATE OF document_id, enabled ON ${projection} - FOR EACH ROW EXECUTE FUNCTION set_projection_source_acl()`) - } + await installProjectionSourceAclSetTriggers(tx) /** The earlier shape of this migration, where the source alone was carried, and only on vectors. */ await tx.unsafe('DROP TRIGGER IF EXISTS embedding_search_connector_sync ON document') await tx.unsafe('DROP TRIGGER IF EXISTS embedding_search_connector_set ON embedding_search') @@ -111,187 +94,17 @@ export async function installProjectionSourceAcl(sql: Sql): Promise { }) } -export interface ProjectionSourceAclBackfillOptions { - /** Resume after this chunk id; the projection's first page otherwise. */ - afterId?: string - /** Stop before this chunk id; the projection's end otherwise. Lets workers fill disjoint ranges. */ - beforeId?: string - pageSize?: number - pauseMs?: number - /** Stop once this much time has passed and report where to resume; unbounded otherwise. */ - budgetMs?: number -} - -export interface ProjectionSourceAclBackfillProgress { - projection: ProjectionSourceAclTable - /** Unfilled chunks this run read, including any a concurrent write filled first. */ - scanned: number - written: number - /** The last chunk id this run reached; the next run resumes after it while `done` is false. */ - afterId: string - done: boolean -} - -/** - * Fills a projection's source and ACL for chunks written before the trigger existed, in - * independently committed keyset pages of unfilled rows. Each page is one statement that bounds - * its own locking and runtime and writes only rows still unset, so an interrupted run resumes by - * rerunning and a row the trigger has since written is left alone. The documents are share-locked - * before their values are copied, so a change in flight waits for the page and then fans its own - * values out; and a chunk that moved to another document meanwhile is left to that document's - * trigger, since the write requires the document the values were read from. - * - * On `embedding_search` every filled row is re-inserted into each HNSW index, which is the whole - * cost of a page and far more than a deploy can wait for; the run paces itself with a pause between - * pages and stops at its budget so a background task can chain runs until the projection is filled. - * Search does not wait: an unfilled row is decided on its document by the on-row candidate - * predicate, the join per candidate every row paid before the columns existed. A run fills the - * range it was given and reports that range done; whether the projection as a whole is done, and - * the analysis the planner then needs, is the caller's, since several runs may share a projection. - * - * A page the database cancels — on a lock timeout, because the keyword index's background - * maintenance holds the index page the row's write needs, or on a statement timeout — is retried - * in place after a pause, up to {@link PROJECTION_SOURCE_ACL_PAGE_RETRIES} times in a row, and - * the cursor stays on the last page that committed. A retry from the run's original cursor would - * instead walk every row the run had filled, past the index entries those writes left behind, - * into a statement timeout of its own; and the maintenance that cancelled the page outlasts the - * few attempts a run gets, so the fill would end where it stalled. A page that is still failing - * when the budget runs out is left to the continuation rather than retried past it. - */ -export async function backfillProjectionSourceAcl( - sql: Sql, - projection: ProjectionSourceAclTable, - options: ProjectionSourceAclBackfillOptions = {} -): Promise { - const pageSize = options.pageSize ?? PROJECTION_SOURCE_ACL_PAGE_SIZE - const pauseMs = options.pauseMs ?? PROJECTION_SOURCE_ACL_PAGE_PAUSE_MS - /** - * The page size is interpolated into the statement and a page of nothing would report the - * projection filled; a payload that asks for either is refused rather than quietly reshaped. - */ - if (!Number.isSafeInteger(pageSize) || pageSize < 1) { - throw new Error(`Projection backfill page size must be a positive integer, got ${pageSize}`) - } - if (!Number.isFinite(pauseMs) || pauseMs < 0) { - throw new Error(`Projection backfill pause must be a non-negative number, got ${pauseMs}`) - } - const startedAt = Date.now() - const deadline = - options.budgetMs === undefined ? Number.POSITIVE_INFINITY : startedAt + options.budgetMs - let afterId = options.afterId ?? '' - const beforeId = options.beforeId ?? null - let scanned = 0 - let written = 0 - let pages = 0 - let done = false - /** Timeouts in a row on the page after `afterId`; reset once it commits. */ - let timeouts = 0 - for (;;) { - let page: { scanned: number; filled: number; last_id: string | null } - try { - page = await sql.begin(async (tx) => { - await tx.unsafe("SET LOCAL lock_timeout = '5s'") - await tx.unsafe(`SET LOCAL statement_timeout = ${PROJECTION_SOURCE_ACL_PAGE_TIMEOUT_MS}`) - const [row] = await tx.unsafe< - Array<{ scanned: number; filled: number; last_id: string | null }> - >( - `WITH page AS ( - SELECT s.id, s.document_id, d.connector_id, d.acl - FROM ${projection} s JOIN document d ON d.id = s.document_id - WHERE s.id > $1 AND ($2::text IS NULL OR s.id < $2) AND s.acl IS NULL - ORDER BY s.id LIMIT ${pageSize} - FOR SHARE OF d - ), updated AS ( - UPDATE ${projection} s SET connector_id = page.connector_id, acl = page.acl - FROM page - WHERE s.id = page.id AND s.document_id = page.document_id AND s.acl IS NULL - RETURNING s.id - ) - SELECT (SELECT count(*)::int FROM page) AS scanned, - (SELECT count(*)::int FROM updated) AS filled, - (SELECT max(id) FROM page) AS last_id`, - [afterId, beforeId] - ) - return row - }) - } catch (error) { - /** - * The page is cancelled on `lock_timeout` (55P03) while its index write waits on a lock the - * index's background maintenance holds, and on `statement_timeout` (57014) when it runs past - * {@link PROJECTION_SOURCE_ACL_PAGE_TIMEOUT_MS}; both pass once the maintenance moves on. A - * deadlock, a serialization failure, or a dropped connection rolls the page back the same - * way, and a page only fills rows still unset, so all are retried in place. 57014 is also - * what an explicit cancellation raises, and that is not retried. - */ - const failure = getTransientDatabaseFailure(error) - if (!failure) throw error - const code = getPostgresErrorCode(error) - timeouts += 1 - if (timeouts > PROJECTION_SOURCE_ACL_PAGE_RETRIES) throw error - if (Date.now() >= deadline) break - const pauseMs = backoffWithJitter(timeouts, null, PAGE_RETRY_PAUSE) - logger.warn('Projection source and ACL backfill page failed transiently; retrying', { - projection, - afterId, - failure, - code, - attempt: timeouts, - retryInMs: Math.round(pauseMs), - }) - await sleep(pauseMs) - /** Checked again after the pause, so a timeout at the budget cannot start another page. */ - if (Date.now() >= deadline) break - continue - } - timeouts = 0 - if (page.last_id === null) { - done = true - break - } - afterId = page.last_id - scanned += page.scanned - written += page.filled - pages += 1 - if (pages % PROGRESS_EVERY_PAGES === 0) { - logger.info('Projection source and ACL backfill progress', { - projection, - scanned, - written, - afterId, - elapsedMs: Date.now() - startedAt, - }) - } - if (pauseMs > 0) await sleep(pauseMs) - /** Checked after the pause, so the pause cannot carry a run past its budget into another page. */ - if (Date.now() >= deadline) break - } - logger.info( - done - ? 'Projection source and ACL range backfilled' - : 'Projection source and ACL backfill paused', - { - projection, - beforeId, - scanned, - written, - afterId, - elapsedMs: Date.now() - startedAt, - } - ) - return { projection, scanned, written, afterId, done } -} - /** * The indexes exact ranking of a readable set needs: the ACL index on each projection, and on the * vector projection the source index that lets the planner lead with a few sources when the * caller's tokens alone would match most of the index. Built concurrently, so the triggers and the - * backfill keep writing. `CONCURRENTLY` cannot run in a transaction, and the pool's lock timeout + * projector keep writing. `CONCURRENTLY` cannot run in a transaction, and the pool's lock timeout * would cancel a build that merely waits for a long transaction to finish. The timeout is a * session setting, so one connection is reserved for it, the builds, and the reset — a pool * would otherwise hand the builds to connections that never saw the setting. * - * The unfilled index on each projection lists the rows the backfill has not reached: each page - * reads its rows from it instead of walking past every filled one, and the on-row predicate's + * The unfilled index on each projection lists the rows the fill has not reached: the projector's + * fill reads them from it instead of walking past every filled row, and the on-row predicate's * unfilled branch, an `OR` beside the ACL overlap, stays an index probe for the planner — once the * projection is filled, a probe of an empty index. */ @@ -330,20 +143,17 @@ export async function indexProjectionAcl(pool: Sql): Promise { } /** - * Run directly — `db:push`, or an operator filling a database by hand — the projections are filled - * here, paced the same way, rather than left to the app. The registered migration is - * `0022_projection_source_acl_backfill`, which supersedes this file's earlier, synchronous shape. + * Run directly — `db:push`, or an operator repairing a database by hand — the triggers and indexes + * are installed here; rows they have not filled are marked and filled by the knowledge projector. + * The registered migration is `0022_projection_source_acl_backfill`. */ if (import.meta.main) { const url = process.env.MIGRATION_DATABASE_URL ?? process.env.DATABASE_URL - if (!url) throw new Error('DATABASE_URL is required to backfill the projection source and ACL') + if (!url) throw new Error('DATABASE_URL is required to install the projection source and ACL') const sql = postgres(url, { max: 1, max_lifetime: null, onnotice: () => undefined }) try { await installProjectionSourceAcl(sql) await indexProjectionAcl(sql) - for (const projection of PROJECTION_SOURCE_ACL_TABLES) { - await backfillProjectionSourceAcl(sql, projection) - } } finally { await sql.end() } diff --git a/packages/db/script-migrations/0022_projection_source_acl_backfill.ts b/packages/db/script-migrations/0022_projection_source_acl_backfill.ts index 9d3718ec84f..2d0aa70e3e8 100644 --- a/packages/db/script-migrations/0022_projection_source_acl_backfill.ts +++ b/packages/db/script-migrations/0022_projection_source_acl_backfill.ts @@ -8,13 +8,10 @@ import type { ScriptMigration } from '@sim/db/script-migrations/types' * Installs the projection source and ACL triggers and builds their indexes; both are idempotent, * so a run that was cut short completes on the next deploy. The columns are not filled here: on * `embedding_search` every filled row is re-inserted into each HNSW index, which puts the full - * projection far beyond what a deploy job can wait for. The backfill is started after the deploy - * with `bun apps/sim/scripts/backfill-projection-source-acl.ts`, which enqueues the - * `projection-source-acl-backfill` Trigger.dev task the way the table backfill is enqueued (or runs - * the same backfill detached in the app where no worker is configured); the task chains bounded - * runs until both projections are filled and is safe to start again at any time. Until it - * completes, an unfilled row is decided on its document, the join per candidate every row paid - * before the columns existed. + * projection far beyond what a deploy job can wait for. The knowledge projector fills them after + * the deploy, marking the documents of unfilled rows in bounded batches and converging them as it + * converges any other document. Until then, an unfilled row is decided on its document, the join + * per candidate every row paid before the columns existed. * * Supersedes `0021_embedding_search_connector`, whose synchronous backfill this replaces: a * database that recorded it still gains the unfilled indexes, and one where it was cut short diff --git a/packages/db/script-migrations/0023_projection_acl_skip_unfilled.ts b/packages/db/script-migrations/0023_projection_acl_skip_unfilled.ts index 00a20a35eda..aed566f1d67 100644 --- a/packages/db/script-migrations/0023_projection_acl_skip_unfilled.ts +++ b/packages/db/script-migrations/0023_projection_acl_skip_unfilled.ts @@ -3,7 +3,7 @@ import type { ScriptMigration } from '@sim/db/script-migrations/types' /** * Replaces the document trigger's body so a document's ACL change no longer writes the ACL onto - * chunks the projection backfill has not filled yet; see {@link replaceProjectionSourceAclSync}. + * chunks the source and ACL fill has not reached yet; see {@link replaceProjectionSourceAclSync}. * The function is replaced in place, so the trigger that calls it and every other object from * `0022_projection_source_acl_backfill` stay as they are. A database that runs `0022` now installs * the same body, so this is a no-op there. diff --git a/packages/db/script-migrations/0024_knowledge_projection_async.ts b/packages/db/script-migrations/0024_knowledge_projection_async.ts new file mode 100644 index 00000000000..d6bccb4225e --- /dev/null +++ b/packages/db/script-migrations/0024_knowledge_projection_async.ts @@ -0,0 +1,190 @@ +import { KNOWLEDGE_PROJECTION_DEFERRED } from '@sim/db/knowledge-projection' +import { + installSearchKeywordTrigger, + installSearchVectorTrigger, +} from '@sim/db/script-migrations/0016_backfill_search_vectors' +import { installTinKeywordTrigger } from '@sim/db/script-migrations/0019_tin_keyword_projection' +import { + installProjectionSourceAclSetTriggers, + projectionSourceAclFanOut, +} from '@sim/db/script-migrations/0021_embedding_search_connector' +import type { ScriptMigration } from '@sim/db/script-migrations/types' +import { createLogger } from '@sim/logger' +import postgres, { type Sql, type TransactionSql } from 'postgres' +import { retryOnLockTimeout } from '../scripts/lock-timeout-retry' + +const logger = createLogger('KnowledgeProjectionAsync') + +/** Each attempt's wait for the table locks the triggers need; the retries find a quiet moment. */ +const TRIGGER_LOCK_TIMEOUT = '5s' +const TRIGGER_LOCK_RETRY_BUDGET_MS = 20 * 60_000 +const TRIGGER_LOCK_RETRY_BACKOFF = { baseMs: 2_000, maxMs: 30_000 } as const + +/** + * Marks documents for the knowledge projector: a new mark starts at generation 1, and a mark + * already waiting gains a generation and keeps whether any change touched chunk content. Documents + * are marked in id order so writers marking several at once take the rows in one order. + */ +async function installMarkFunctions(tx: Sql | TransactionSql): Promise { + await tx.unsafe(`CREATE OR REPLACE FUNCTION mark_knowledge_projection(p_document_ids text[], p_content boolean) + RETURNS void LANGUAGE sql AS $$ + INSERT INTO knowledge_projection_dirty (document_id, content) + SELECT marked.id, p_content + FROM (SELECT DISTINCT unnest(p_document_ids) AS id) AS marked + WHERE marked.id IS NOT NULL + ORDER BY marked.id + ON CONFLICT (document_id) DO UPDATE SET + generation = knowledge_projection_dirty.generation + 1, + content = knowledge_projection_dirty.content OR EXCLUDED.content + $$`) + await tx.unsafe(`CREATE OR REPLACE FUNCTION mark_inserted_embedding_projection() + RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + PERFORM mark_knowledge_projection(ARRAY(SELECT document_id FROM inserted_embeddings), ${KNOWLEDGE_PROJECTION_DEFERRED}); + RETURN NULL; + END; + $$`) + await tx.unsafe(`CREATE OR REPLACE FUNCTION mark_updated_embedding_projection() + RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + PERFORM mark_knowledge_projection(ARRAY[NEW.document_id, OLD.document_id], ${KNOWLEDGE_PROJECTION_DEFERRED}); + RETURN NEW; + END; + $$`) +} + +/** + * The document trigger's body once documents are marked: a document whose source or ACL changed + * is marked, whatever the writer's mode, and a writer that did not defer its projection still fans + * the new values out to its enabled chunks in its own statement, as every writer before the + * projector did. A chunk write marks its document with content to project only when it deferred: + * a synchronous writer has already written the rows, and its mark exists only so a projector pass + * that read the document earlier cannot settle over the write. + */ +async function installDocumentMarking(tx: Sql | TransactionSql): Promise { + await tx.unsafe(`CREATE OR REPLACE FUNCTION sync_projection_source_acl() + RETURNS trigger LANGUAGE plpgsql AS $$ + DECLARE + moved boolean := TG_OP = 'UPDATE' AND OLD.connector_id IS DISTINCT FROM NEW.connector_id; + BEGIN + IF moved OR (TG_OP = 'UPDATE' AND OLD.acl IS DISTINCT FROM NEW.acl) THEN + PERFORM mark_knowledge_projection(ARRAY[NEW.id], false); + END IF; + IF ${KNOWLEDGE_PROJECTION_DEFERRED} THEN + RETURN NEW; + END IF;${projectionSourceAclFanOut()} + RETURN NEW; + END; + $$`) +} + +/** + * The marking installed on its own, for a test schema that holds only `document` and the + * projections: the mark function, then the document trigger's marking body. + */ +export async function installKnowledgeProjectionMarking(tx: Sql | TransactionSql): Promise { + await installMarkFunctions(tx) + await installDocumentMarking(tx) +} + +/** The chunk columns any projection carries: the columns the synchronous triggers fire on. */ +const PROJECTED_COLUMNS = [ + 'knowledge_base_id', + 'document_id', + 'enabled', + 'content', + 'embedding', + 'embedding_384', + 'embedding_768', + 'embedding_1024', + 'embedding_3072', +] as const + +/** + * Chunk writes mark their documents. An insert marks once per statement, from the statement's + * inserted rows, so a processing commit that inserts a document's chunks in batches marks it once + * per batch rather than once per chunk; the statement's inserted rows hold their vectors as stored + * references, and measured on a 1,000-chunk commit the trigger added no time and no temporary + * files next to none at all. An update marks when a column the projections carry actually changed. + * A delete marks nothing: the projections' foreign keys remove their rows in the deleting + * statement. + */ +async function installMarkTriggers(tx: TransactionSql): Promise { + await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_projection_mark_insert + AFTER INSERT ON embedding REFERENCING NEW TABLE AS inserted_embeddings + FOR EACH STATEMENT EXECUTE FUNCTION mark_inserted_embedding_projection()`) + await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_projection_mark_update + AFTER UPDATE OF ${PROJECTED_COLUMNS.join(', ')} ON embedding + FOR EACH ROW WHEN (${PROJECTED_COLUMNS.map((column) => `OLD.${column} IS DISTINCT FROM NEW.${column}`).join(' OR ')}) + EXECUTE FUNCTION mark_updated_embedding_projection()`) +} + +/** Whether the Tin projection's embedding trigger exists here: `0019` installs it only where `tin` does. */ +async function tinTriggerInstalled(tx: TransactionSql): Promise { + const [row] = await tx>` + SELECT EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgname = 'embedding_keyword_tin_sync' AND tgrelid = 'embedding'::regclass + ) AS installed` + return Boolean(row?.installed) +} + +/** + * Lets a writer defer its search projection rows to the knowledge projector, and marks every + * document whose projection rows a write changes, in either mode. + * + * The triggers that write projection rows in the writer's transaction — the embedding triggers of + * each projection, the projection triggers that copy a chunk's source and ACL, and the document + * trigger's fan-out — are skipped when the transaction set `sim.projection_mode` to `async`, and + * run as before otherwise. Every writer deployed before this release sets nothing, so it keeps + * writing the rows itself; the app sets the mode only while its feature flag is on. The marks are + * written in both modes, so a projector pass that read a document before a synchronous writer + * changed it cannot settle over that change. + * + * All of it is installed in one transaction, so no write sees the guards without the marks. The + * trigger DDL takes a short lock on `embedding` and on each projection, in the order writers take + * them; each attempt waits at most {@link TRIGGER_LOCK_TIMEOUT} and is retried within the budget, + * so it never queues writers behind it for longer than one attempt. Replacing the functions takes + * no table lock. Idempotent. + */ +export async function installKnowledgeProjectionAsync(sql: Sql): Promise { + await retryOnLockTimeout( + () => + sql.begin(async (tx) => { + await tx.unsafe(`SET LOCAL lock_timeout = '${TRIGGER_LOCK_TIMEOUT}'`) + await installMarkFunctions(tx) + await installDocumentMarking(tx) + await installSearchVectorTrigger(tx) + await installSearchKeywordTrigger(tx) + if (await tinTriggerInstalled(tx)) await installTinKeywordTrigger(tx) + await installMarkTriggers(tx) + await installProjectionSourceAclSetTriggers(tx) + }), + { + budgetMs: TRIGGER_LOCK_RETRY_BUDGET_MS, + backoff: TRIGGER_LOCK_RETRY_BACKOFF, + onRetry: ({ attempt, delayMs }) => + logger.warn('Knowledge projection triggers waited out their lock timeout; retrying', { + attempt, + retryInMs: Math.round(delayMs), + }), + } + ) +} + +export const knowledgeProjectionAsyncMigration: ScriptMigration = { + name: '0024_knowledge_projection_async', + up: installKnowledgeProjectionAsync, +} + +/** Run directly by `db:push`, after the projection migrations it builds on. */ +if (import.meta.main) { + const url = process.env.MIGRATION_DATABASE_URL ?? process.env.DATABASE_URL + if (!url) throw new Error('DATABASE_URL is required to install the knowledge projection triggers') + const sql = postgres(url, { max: 1, max_lifetime: null, onnotice: () => undefined }) + try { + await installKnowledgeProjectionAsync(sql) + } finally { + await sql.end() + } +} diff --git a/packages/db/script-migrations/index.ts b/packages/db/script-migrations/index.ts index 4304a9336d4..c38b01a7464 100644 --- a/packages/db/script-migrations/index.ts +++ b/packages/db/script-migrations/index.ts @@ -7,6 +7,7 @@ import { repairWorkspaceFileContentRevisionMigration } from '@sim/db/script-migr import { tinKeywordProjectionMigration } from '@sim/db/script-migrations/0019_tin_keyword_projection' import { projectionSourceAclBackfillMigration } from '@sim/db/script-migrations/0022_projection_source_acl_backfill' import { projectionAclSkipUnfilledMigration } from '@sim/db/script-migrations/0023_projection_acl_skip_unfilled' +import { knowledgeProjectionAsyncMigration } from '@sim/db/script-migrations/0024_knowledge_projection_async' import type { Sql } from 'postgres' import { backfillTableOrderKeys } from './0001_backfill_table_order_keys' import { backfillPausedBillingAttribution } from './0002_backfill_paused_billing_attribution' @@ -49,6 +50,8 @@ export const scriptMigrations: readonly ScriptMigration[] = [ projectionSourceAclBackfillMigration, /** 0023 stops document ACL changes from writing chunks the 0022 backfill has not filled. */ projectionAclSkipUnfilledMigration, + /** 0024 marks changed documents for the knowledge projector and lets writers defer to it. */ + knowledgeProjectionAsyncMigration, ] /** diff --git a/packages/db/scripts/push.test.ts b/packages/db/scripts/push.test.ts index aed4e5c0002..3451d0b466f 100644 --- a/packages/db/scripts/push.test.ts +++ b/packages/db/scripts/push.test.ts @@ -18,9 +18,10 @@ describe('pushCommands', () => { it('rebuilds the projections a push drops, since their indexes and triggers are not in the schema', () => { const scripts = pushCommands([]).map((command) => command.at(-1)) - expect(scripts.slice(-2)).toEqual([ + expect(scripts.slice(-3)).toEqual([ './script-migrations/0019_tin_keyword_projection.ts', './script-migrations/0021_embedding_search_connector.ts', + './script-migrations/0024_knowledge_projection_async.ts', ]) }) }) diff --git a/packages/db/scripts/push.ts b/packages/db/scripts/push.ts index c76f3b5104f..9f7ff13295e 100644 --- a/packages/db/scripts/push.ts +++ b/packages/db/scripts/push.ts @@ -11,6 +11,7 @@ export function pushCommands(flags: readonly string[]): string[][] { ['bun', '--env-file=.env', 'run', './script-migrations/0016_backfill_search_vectors.ts'], ['bun', '--env-file=.env', 'run', './script-migrations/0019_tin_keyword_projection.ts'], ['bun', '--env-file=.env', 'run', './script-migrations/0021_embedding_search_connector.ts'], + ['bun', '--env-file=.env', 'run', './script-migrations/0024_knowledge_projection_async.ts'], ] } diff --git a/packages/testing/src/mocks/schema.mock.ts b/packages/testing/src/mocks/schema.mock.ts index 1cda246782e..4bcdfd71848 100644 --- a/packages/testing/src/mocks/schema.mock.ts +++ b/packages/testing/src/mocks/schema.mock.ts @@ -1876,6 +1876,12 @@ export const schemaMock = { lastSeenAt: 'knowledgeDocumentObservation.lastSeenAt', runId: 'knowledgeDocumentObservation.runId', }, + knowledgeProjectionDirty: { + documentId: 'knowledgeProjectionDirty.documentId', + generation: 'knowledgeProjectionDirty.generation', + content: 'knowledgeProjectionDirty.content', + markedAt: 'knowledgeProjectionDirty.markedAt', + }, knowledgeConnectorMemberSyncLog: { id: 'knowledgeConnectorMemberSyncLog.id', connectorId: 'knowledgeConnectorMemberSyncLog.connectorId', diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 1d75e87ed76..06c6d4ea475 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -118,6 +118,7 @@ const INDIRECT_ZOD_ROUTES = new Set([ 'apps/sim/app/api/cron/run-data-drains/route.ts', // Returns immediately after Trigger.dev accepts the asynchronous dispatcher task. 'apps/sim/app/api/cron/workspace-file-search-dispatch/route.ts', + 'apps/sim/app/api/cron/knowledge-projection/route.ts', 'apps/sim/app/api/logs/cleanup/route.ts', 'apps/sim/app/api/knowledge/connectors/sync/route.ts', 'apps/sim/app/api/knowledge/connectors/member-sync/route.ts', diff --git a/scripts/check-tool-registry-boundary.baseline.json b/scripts/check-tool-registry-boundary.baseline.json index af6dad97f18..419b0c4e172 100644 --- a/scripts/check-tool-registry-boundary.baseline.json +++ b/scripts/check-tool-registry-boundary.baseline.json @@ -308,7 +308,7 @@ } }, "app/workspace/[workspaceId]/knowledge/page.tsx": { - "modules": 2389, + "modules": 2439, "gateways": { "apps/sim/triggers/registry.ts": 485, "apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts": 408, From 6309546ae843ee60a7828788899f45ae340458a3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 03:05:59 -0700 Subject: [PATCH 2/3] fix(knowledge): keep projection guards out of historical migrations Restores 0016, 0019 and 0021 to their staging bodies; 0024 alone re-creates the projection triggers with the mode guard, in the same transaction that installs the marks. The fill starts only inside the pass budget and reports what it marked as remaining. Passes dispatch to Trigger.dev by the rule document processing uses, and the inline coalescer clears its running flag in the same step it reads the owed flag. Bumps the Helm chart for the projection cron job. --- .../knowledge-projection.integration.ts | 61 ++++ .../projection/enqueue-inline.test.ts | 24 ++ .../lib/knowledge/projection/enqueue.test.ts | 26 ++ apps/sim/lib/knowledge/projection/enqueue.ts | 43 ++- apps/sim/lib/knowledge/projection/run.test.ts | 36 ++- apps/sim/lib/knowledge/projection/run.ts | 4 +- helm/sim/Chart.yaml | 2 +- packages/db/knowledge-projection.ts | 14 +- .../0016_backfill_search_vectors.ts | 66 +++-- .../0019_tin_keyword_projection.ts | 17 +- ...mbedding_search_connector.postgres.test.ts | 55 +++- .../0021_embedding_search_connector.test.ts | 187 ++++++++++++ .../0021_embedding_search_connector.ts | 272 +++++++++++++++--- .../0024_knowledge_projection_async.ts | 64 ++++- 14 files changed, 739 insertions(+), 132 deletions(-) create mode 100644 packages/db/script-migrations/0021_embedding_search_connector.test.ts diff --git a/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts b/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts index f5654cedaf1..81b65c24f5e 100644 --- a/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts @@ -548,6 +548,67 @@ describe('the projector', () => { expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('alice')) }) + it('keeps the mark when a synchronous ACL change commits under a page that then writes stale values', async () => { + /** A pending revocation of Bob: the rows still name both until a pass. */ + await write('async', (tx) => + tx + .update(document) + .set({ acl: aclOf('alice') }) + .where(eq(document.id, documentId)) + ) + const before = await markOf() + const [{ pid }] = await projector>`SELECT pg_backend_pid() AS pid` + /** + * A synchronous writer revokes Alice too and holds its transaction open: its fan-out has + * locked the projection rows and its mark bump is not yet visible. + */ + const writer = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) + let release: () => void = () => {} + const held = new Promise((resolve) => { + release = resolve + }) + let locked: () => void = () => {} + const fannedOut = new Promise((resolve) => { + locked = resolve + }) + try { + const writing = writer.begin(async (tx) => { + await tx`UPDATE document SET acl = ${aclOf('bob')} WHERE id = ${documentId}` + locked() + await held + }) + await fannedOut + /** + * The pass reads the committed document, Alice alone, and its page blocks on the rows the + * writer holds. Once the writer commits, the page rewrites those rows from what it read. + */ + const pass = project() + await vi.waitFor(async () => { + const [activity] = await db.execute<{ waiting: boolean }>( + sql`SELECT wait_event_type = 'Lock' AS waiting FROM pg_stat_activity WHERE pid = ${pid}` + ) + expect(activity?.waiting).toBe(true) + }) + release() + await writing + await pass + } finally { + release() + await writer.end() + } + /** The page wrote the value it read over the writer's newer one. */ + expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('alice')) + /** The writer's bump is what the pass could not settle over: the rows stay decided on the document. */ + expect((await markOf())?.generation).toBe((before?.generation ?? 0) + 1) + expect(await admitted()).toEqual({ vector: [], keyword: [] }) + expect(await vectorIds()).toEqual([]) + await project() + expect(await markOf()).toBeUndefined() + expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('bob')) + expect((await rowAcl(embeddingKeywordTin))?.acl).toEqual(aclOf('bob')) + expect(await admitted()).toEqual({ vector: [], keyword: [] }) + }) + it('passes over a document deleted while it is being projected', async () => { const doomed = generateId() await db.insert(document).values({ diff --git a/apps/sim/lib/knowledge/projection/enqueue-inline.test.ts b/apps/sim/lib/knowledge/projection/enqueue-inline.test.ts index 607b1c950e1..d67233abaf0 100644 --- a/apps/sim/lib/knowledge/projection/enqueue-inline.test.ts +++ b/apps/sim/lib/knowledge/projection/enqueue-inline.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { sleep } from '@sim/utils/helpers' import { describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ runPass: vi.fn(), trigger: vi.fn() })) @@ -10,6 +11,7 @@ vi.mock('@/lib/core/config/feature-flags', () => ({ isFeatureEnabled: async () = vi.mock('@trigger.dev/sdk', () => ({ tasks: { trigger: mocks.trigger } })) vi.mock('@/lib/core/config/env-flags', () => ({ isTriggerDevEnabled: false })) +vi.mock('@/lib/core/config/trigger-runtime', () => ({ isInsideTriggerRun: () => false })) vi.mock('@/lib/knowledge/projection/run', () => ({ runKnowledgeProjectionPass: mocks.runPass })) import { @@ -48,6 +50,28 @@ describe('knowledge projection without a Trigger.dev worker', () => { expect(mocks.trigger).not.toHaveBeenCalled() }) + it('runs a pass for a request that arrives at any point while the last pass is finishing', async () => { + for (let hops = 0; hops < 8; hops++) { + mocks.runPass.mockReset() + /** The pass asks for another once it has settled, `hops` microtasks later. */ + mocks.runPass + .mockImplementationOnce(() => { + void (async () => { + for (let hop = 0; hop < hops; hop++) await Promise.resolve() + await requestKnowledgeProjection() + })() + return Promise.resolve() + }) + .mockResolvedValue(undefined) + await requestKnowledgeProjection() + await vi.waitFor(() => expect(mocks.runPass.mock.calls.length).toBeGreaterThanOrEqual(2), { + timeout: 200, + }) + /** Lets the loop finish before the next interleaving starts. */ + await sleep(1) + } + }) + it('logs a failed pass and still runs the pass owed after it', async () => { mocks.runPass.mockReset() const failFirst = heldPass() diff --git a/apps/sim/lib/knowledge/projection/enqueue.test.ts b/apps/sim/lib/knowledge/projection/enqueue.test.ts index 60c8886ade1..2953aeb2aa9 100644 --- a/apps/sim/lib/knowledge/projection/enqueue.test.ts +++ b/apps/sim/lib/knowledge/projection/enqueue.test.ts @@ -9,6 +9,8 @@ const mocks = vi.hoisted(() => ({ trigger: vi.fn(), execute: vi.fn(), isFeatureEnabled: vi.fn(), + env: { TRIGGER_SECRET_KEY: 'fixture-key' as string | undefined }, + insideRun: vi.fn(), })) vi.mock('@sim/db', () => ({ db: { execute: mocks.execute } })) @@ -17,6 +19,8 @@ vi.mock('@/lib/core/config/feature-flags', () => ({ isFeatureEnabled: mocks.isFe vi.mock('@trigger.dev/sdk', () => ({ tasks: { trigger: mocks.trigger } })) vi.mock('@/lib/core/async-jobs/region', () => ({ resolveTriggerRegion: mocks.resolveRegion })) vi.mock('@/lib/core/config/env-flags', () => ({ isTriggerDevEnabled: true })) +vi.mock('@/lib/core/config/env', () => ({ env: mocks.env })) +vi.mock('@/lib/core/config/trigger-runtime', () => ({ isInsideTriggerRun: mocks.insideRun })) vi.mock('@/lib/knowledge/projection/run', () => ({ runKnowledgeProjectionPass: mocks.runPass })) import { @@ -33,6 +37,8 @@ describe('knowledge projection enqueue', () => { mocks.trigger.mockResolvedValue({ id: 'run-1' }) mocks.execute.mockResolvedValue([{ pending: true }]) mocks.isFeatureEnabled.mockResolvedValue(false) + mocks.env.TRIGGER_SECRET_KEY = 'fixture-key' + mocks.insideRun.mockReturnValue(false) }) afterEach(() => { @@ -90,6 +96,26 @@ describe('knowledge projection enqueue', () => { expect(mocks.trigger).toHaveBeenCalledTimes(2) }) + it('runs the pass in this process where Trigger.dev is enabled without its secret key', async () => { + mocks.env.TRIGGER_SECRET_KEY = undefined + mocks.runPass.mockResolvedValue(undefined) + await expect(enqueueKnowledgeProjectionSweep()).resolves.toEqual({ + triggered: true, + backend: 'inline', + jobId: null, + }) + expect(mocks.trigger).not.toHaveBeenCalled() + await vi.waitFor(() => expect(mocks.runPass).toHaveBeenCalledTimes(1)) + }) + + it('enqueues from inside a Trigger.dev run whatever the environment holds', async () => { + mocks.env.TRIGGER_SECRET_KEY = undefined + mocks.insideRun.mockReturnValue(true) + await expect(enqueueKnowledgeProjectionSweep()).resolves.toMatchObject({ + backend: 'trigger-dev', + }) + }) + it('never fails the write that asked when the request is refused', async () => { vi.advanceTimersByTime(60_000) mocks.trigger.mockRejectedValueOnce(new Error('trigger unavailable')) diff --git a/apps/sim/lib/knowledge/projection/enqueue.ts b/apps/sim/lib/knowledge/projection/enqueue.ts index e34d420781f..36892f3a654 100644 --- a/apps/sim/lib/knowledge/projection/enqueue.ts +++ b/apps/sim/lib/knowledge/projection/enqueue.ts @@ -3,8 +3,10 @@ import { SOURCE_ACL_PROJECTIONS } from '@sim/db/knowledge-projection' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { sql } from 'drizzle-orm' +import { env } from '@/lib/core/config/env' import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' +import { isInsideTriggerRun } from '@/lib/core/config/trigger-runtime' const logger = createLogger('KnowledgeProjectionEnqueue') @@ -31,24 +33,26 @@ const PROMPT_REQUEST_INTERVAL_MS = 5_000 let lastPromptAt = 0 -/** The inline pass in flight when no Trigger.dev worker is configured, and whether another is owed. */ -let inlinePass: Promise | undefined +/** Whether an inline pass is running when no Trigger.dev worker is configured, and whether another is owed. */ +let inlineRunning = false let inlinePassOwed = false /** * Starts a pass in this process without waiting for it: at most one runs at a time, a request while * one runs is folded into a single pass after it, and a failed pass is logged without dropping one - * owed after it. The - * pass module loads on first use. For deployments without a Trigger.dev worker, whose sweep and - * writes run passes here, as their document processing does. + * owed after it. The loop reads the owed flag and clears the running flag in the same synchronous + * step, so a request can never land between the two and be dropped. The pass module loads on first + * use. For deployments without a Trigger.dev worker, whose sweep and writes run passes here, as + * their document processing does. */ function runInline(): void { - if (inlinePass) { + if (inlineRunning) { inlinePassOwed = true return } - inlinePass = (async () => { - do { + inlineRunning = true + void (async () => { + for (;;) { inlinePassOwed = false try { const { runKnowledgeProjectionPass } = await import('@/lib/knowledge/projection/run') @@ -56,10 +60,21 @@ function runInline(): void { } catch (error) { logger.error('Inline knowledge projection pass failed', { error: getErrorMessage(error) }) } - } while (inlinePassOwed) - })().finally(() => { - inlinePass = undefined - }) + if (!inlinePassOwed) { + inlineRunning = false + return + } + } + })() +} + +/** + * Whether passes run on Trigger.dev, by the rule document processing dispatches with: inside a + * Trigger.dev run always, and otherwise only where Trigger.dev is enabled and the secret key the + * SDK authenticates with is set. + */ +function projectsOnTrigger(): boolean { + return isInsideTriggerRun() || Boolean(isTriggerDevEnabled && env.TRIGGER_SECRET_KEY) } /** @@ -70,7 +85,7 @@ function runInline(): void { * leaves the marks to the sweep, which keeps enqueueing a pass every minute until one runs. */ export async function requestKnowledgeProjection(): Promise { - if (!isTriggerDevEnabled) { + if (!projectsOnTrigger()) { runInline() return } @@ -125,7 +140,7 @@ export async function enqueueKnowledgeProjectionSweep(): Promise ({ runProjection: vi.fn(), @@ -106,6 +106,40 @@ describe('runKnowledgeProjectionPass', () => { }) }) + describe('at the budget', () => { + beforeEach(() => { + vi.useFakeTimers() + mocks.isFeatureEnabled.mockResolvedValue(true) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('does not start the fill once the budget is spent', async () => { + mocks.runProjection.mockImplementation(async () => { + vi.advanceTimersByTime(60_001) + return drained + }) + await expect(runKnowledgeProjectionPass({ budgetMs: 60_000 })).resolves.toMatchObject({ + remaining: false, + filled: 0, + }) + expect(mocks.markUnfilled).not.toHaveBeenCalled() + }) + + it('reports documents the fill marked but no round settled as remaining', async () => { + mocks.markUnfilled.mockImplementation(async () => { + vi.advanceTimersByTime(60_001) + return { marked: 2, cursor: { projection: 0, afterId: 'row-2' } } + }) + await expect(runKnowledgeProjectionPass({ budgetMs: 60_000 })).resolves.toMatchObject({ + remaining: true, + filled: 2, + }) + }) + }) + it('does not fill while marks remain, so writers are converged first', async () => { mocks.isFeatureEnabled.mockResolvedValue(true) mocks.runProjection.mockResolvedValue({ ...drained, settled: 0, remaining: true }) diff --git a/apps/sim/lib/knowledge/projection/run.ts b/apps/sim/lib/knowledge/projection/run.ts index 9f7fc12fdfc..0729502088f 100644 --- a/apps/sim/lib/knowledge/projection/run.ts +++ b/apps/sim/lib/knowledge/projection/run.ts @@ -106,10 +106,12 @@ export async function runKnowledgeProjectionPass(options: { result.deferred = round.reduce((most, progress) => Math.max(most, progress.deferred), 0) break } - if (fillCursor === null) break + /** The fill starts only inside the budget, and what it marks is left for a round to settle. */ + if (fillCursor === null || Date.now() >= deadline) break const fill = await markUnfilledProjectionDocuments(sessions[0], fillCursor) result.filled += fill.marked fillCursor = fill.cursor + if (fill.marked > 0) result.remaining = true if (fill.marked === 0 && fillCursor === null) break } logger.info('Knowledge projection pass finished', result) diff --git a/helm/sim/Chart.yaml b/helm/sim/Chart.yaml index e163c7aa150..cf743708295 100644 --- a/helm/sim/Chart.yaml +++ b/helm/sim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sim description: A Helm chart for Sim - the open-source AI workspace where teams build, deploy, and manage AI agents type: application -version: 1.11.3 +version: 1.11.4 appVersion: "v0.8.26" kubeVersion: ">=1.25.0-0" home: https://sim.ai diff --git a/packages/db/knowledge-projection.ts b/packages/db/knowledge-projection.ts index 07c2d6014a1..9862c118ae3 100644 --- a/packages/db/knowledge-projection.ts +++ b/packages/db/knowledge-projection.ts @@ -60,18 +60,18 @@ const SOURCE_VECTOR_WIDTHS = [1536, 384, 768, 1024, 3072] as const const widthColumn = (name: string, width: number) => (width === 1536 ? name : `${name}_${width}`) /** The halfvec columns of `embedding_search`, in width order. */ -export const SEARCH_VECTOR_COLUMNS = SEARCH_VECTOR_WIDTHS.map((width) => - widthColumn('vector', width) -) +const SEARCH_VECTOR_COLUMNS = SEARCH_VECTOR_WIDTHS.map((width) => widthColumn('vector', width)) /** The bit columns of `embedding_search`, whose width check requires exactly one. */ const SEARCH_BINARY_COLUMNS = ['"binary"', 'binary_384', 'binary_768', 'binary_1024', 'binary_3072'] /** * The halfvec projections of an embedding row, in {@link SEARCH_VECTOR_COLUMNS} order. Shortening - * is valid only for the two OpenAI models trained for prefix retrieval. + * is valid only for the two OpenAI models trained for prefix retrieval. These and the bit + * projections match what `sync_embedding_search()` from `0016_backfill_search_vectors` writes, so + * a pass over rows a synchronous writer wrote finds nothing to rewrite. */ -export function searchVectorProjections(prefix: string, shortened: string): string { +function searchVectorProjections(prefix: string, shortened: string): string { return SEARCH_VECTOR_WIDTHS.map((width) => width === 512 ? `CASE WHEN ${shortened} THEN subvector(coalesce(${SOURCE_VECTOR_WIDTHS.map((size) => `${prefix}.${widthColumn('embedding', size)}`).join(', ')}), 1, 512)::halfvec(512) END` @@ -80,14 +80,14 @@ export function searchVectorProjections(prefix: string, shortened: string): stri } /** The bit projections of an embedding row, in {@link SEARCH_BINARY_COLUMNS} order. */ -export function searchBinaryProjections(prefix: string): string { +function searchBinaryProjections(prefix: string): string { return `binary_quantize(${prefix}.embedding)::bit(1536), binary_quantize(${prefix}.embedding_384)::bit(384), binary_quantize(${prefix}.embedding_768)::bit(768), binary_quantize(${prefix}.embedding_1024)::bit(1024), binary_quantize(${prefix}.embedding_3072)::bit(3072)` } /** Whether a knowledge base's model is shortened for the 512 projection, for an embedding row. */ -export function searchVectorShortened(model: string, prefix: string): string { +function searchVectorShortened(model: string, prefix: string): string { return `${model} IN ${SHORTENED_EMBEDDING_MODELS} AND ${prefix}.embedding_384 IS NULL` } diff --git a/packages/db/script-migrations/0016_backfill_search_vectors.ts b/packages/db/script-migrations/0016_backfill_search_vectors.ts index 36ee94c02ac..e40be56c391 100644 --- a/packages/db/script-migrations/0016_backfill_search_vectors.ts +++ b/packages/db/script-migrations/0016_backfill_search_vectors.ts @@ -1,35 +1,22 @@ -import { - SEARCH_VECTOR_COLUMNS, - SYNCHRONOUS_PROJECTION_WHEN, - searchBinaryProjections, - searchVectorProjections, - searchVectorShortened, -} from '@sim/db/knowledge-projection' import type { ScriptMigration } from '@sim/db/script-migrations/types' import { createLogger } from '@sim/logger' -import postgres, { type Sql, type TransactionSql } from 'postgres' +import postgres, { type Sql } from 'postgres' const logger = createLogger('SearchVectorProjection') const BATCH_SIZE = 500 -const columns = SEARCH_VECTOR_COLUMNS +const WIDTHS = [1536, 384, 512, 768, 1024, 3072] as const +const SOURCE_WIDTHS = [1536, 384, 768, 1024, 3072] as const +const field = (name: string, width: number) => (width === 1536 ? name : `${name}_${width}`) +const columns = WIDTHS.map((width) => field('vector', width)) const missing = columns.map((column) => `s.${column} IS NULL`).join(' AND ') -/** - * The vector projection's trigger, skipped by a writer that declared the asynchronous projection - * mode; see `0024_knowledge_projection_async`. - */ -export async function installSearchVectorTrigger(tx: TransactionSql): Promise { - await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_search_sync - AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, - embedding, embedding_384, embedding_768, embedding_1024, embedding_3072 ON embedding - FOR EACH ROW WHEN (${SYNCHRONOUS_PROJECTION_WHEN}) EXECUTE FUNCTION sync_embedding_search()`) -} - -/** The keyword projection's trigger, skipped the same way as {@link installSearchVectorTrigger}. */ -export async function installSearchKeywordTrigger(tx: TransactionSql): Promise { - await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_keyword_search_sync - AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, content ON embedding - FOR EACH ROW WHEN (${SYNCHRONOUS_PROJECTION_WHEN}) EXECUTE FUNCTION sync_embedding_keyword_search()`) +/** Shortening is valid only for the two OpenAI models trained for prefix retrieval. */ +function projections(prefix: string, shortened: string): string { + return WIDTHS.map((width) => + width === 512 + ? `CASE WHEN ${shortened} THEN subvector(coalesce(${SOURCE_WIDTHS.map((size) => `${prefix}.${field('embedding', size)}`).join(', ')}), 1, 512)::halfvec(512) END` + : `CASE WHEN NOT (${shortened}) THEN ${prefix}.${field('embedding', width)}::halfvec(${width}) END` + ).join(', ') } /** Replaces the projection atomically, then fills only missing rows in independently committed pages. */ @@ -46,13 +33,16 @@ export async function backfillSearchVectors(sql: Sql): Promise { RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE shortened boolean; BEGIN - SELECT ${searchVectorShortened('embedding_model', 'NEW')} INTO STRICT shortened + SELECT embedding_model IN ('text-embedding-3-small', 'text-embedding-3-large') + AND NEW.embedding_384 IS NULL INTO STRICT shortened FROM knowledge_base WHERE id = NEW.knowledge_base_id; INSERT INTO embedding_search (id, knowledge_base_id, document_id, enabled, "binary", binary_384, binary_768, binary_1024, binary_3072, ${columns.join(', ')}) VALUES (NEW.id, NEW.knowledge_base_id, NEW.document_id, NEW.enabled, - ${searchBinaryProjections('NEW')}, ${searchVectorProjections('NEW', 'shortened')}) + binary_quantize(NEW.embedding)::bit(1536), binary_quantize(NEW.embedding_384)::bit(384), + binary_quantize(NEW.embedding_768)::bit(768), binary_quantize(NEW.embedding_1024)::bit(1024), + binary_quantize(NEW.embedding_3072)::bit(3072), ${projections('NEW', 'shortened')}) ON CONFLICT (id) DO UPDATE SET knowledge_base_id = EXCLUDED.knowledge_base_id, document_id = EXCLUDED.document_id, enabled = EXCLUDED.enabled, "binary" = EXCLUDED."binary", binary_384 = EXCLUDED.binary_384, @@ -62,7 +52,10 @@ export async function backfillSearchVectors(sql: Sql): Promise { RETURN NEW; END; $$`) - await installSearchVectorTrigger(tx) + await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_search_sync + AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, + embedding, embedding_384, embedding_768, embedding_1024, embedding_3072 ON embedding + FOR EACH ROW EXECUTE FUNCTION sync_embedding_search()`) }) let afterId = '' @@ -84,7 +77,8 @@ export async function backfillSearchVectors(sql: Sql): Promise { ), batch AS MATERIALIZED ( SELECT e.id, e.knowledge_base_id, e.document_id, e.enabled, e.embedding, e.embedding_384, e.embedding_768, e.embedding_1024, e.embedding_3072, - ${searchVectorShortened('k.embedding_model', 'e')} AS shortened + k.embedding_model IN ('text-embedding-3-small', 'text-embedding-3-large') + AND e.embedding_384 IS NULL AS shortened FROM missing m INNER JOIN embedding e ON e.id = m.id INNER JOIN knowledge_base k ON k.id = e.knowledge_base_id ORDER BY e.id FOR KEY SHARE OF e @@ -93,7 +87,9 @@ export async function backfillSearchVectors(sql: Sql): Promise { (id, knowledge_base_id, document_id, enabled, "binary", binary_384, binary_768, binary_1024, binary_3072, ${columns.join(', ')}) SELECT id, knowledge_base_id, document_id, enabled, - ${searchBinaryProjections('batch')}, ${searchVectorProjections('batch', 'shortened')} + binary_quantize(embedding)::bit(1536), binary_quantize(embedding_384)::bit(384), + binary_quantize(embedding_768)::bit(768), binary_quantize(embedding_1024)::bit(1024), + binary_quantize(embedding_3072)::bit(3072), ${projections('batch', 'shortened')} FROM batch ON CONFLICT (id) DO UPDATE SET ${columns.map((column) => `${column} = EXCLUDED.${column}`).join(', ')} WHERE ${missing} RETURNING id @@ -139,7 +135,9 @@ export async function backfillSearchKeywords(sql: Sql): Promise { RETURN NEW; END; $$`) - await installSearchKeywordTrigger(tx) + await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_keyword_search_sync + AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, content ON embedding + FOR EACH ROW EXECUTE FUNCTION sync_embedding_keyword_search()`) }) let afterId = '' let count = 0 @@ -194,10 +192,10 @@ export async function buildSearchIndexes(sql: Sql): Promise { table: 'embedding_search', definition: 'ON embedding_search (document_id, knowledge_base_id, id) WHERE enabled', }, - ...columns.map((column) => ({ - name: `embedding_search${column.replace(/^vector/, '')}_cosine_hnsw_idx`, + ...WIDTHS.map((width) => ({ + name: `embedding_search${width === 1536 ? '' : `_${width}`}_cosine_hnsw_idx`, table: 'embedding_search', - definition: `ON embedding_search USING hnsw (${column} halfvec_cosine_ops) WITH (m=16, ef_construction=64)`, + definition: `ON embedding_search USING hnsw (${field('vector', width)} halfvec_cosine_ops) WITH (m=16, ef_construction=64)`, })), { name: 'embedding_keyword_search_kb_idx', diff --git a/packages/db/script-migrations/0019_tin_keyword_projection.ts b/packages/db/script-migrations/0019_tin_keyword_projection.ts index 8dc5a2b4b3c..b73efe42d78 100644 --- a/packages/db/script-migrations/0019_tin_keyword_projection.ts +++ b/packages/db/script-migrations/0019_tin_keyword_projection.ts @@ -1,8 +1,7 @@ -import { SYNCHRONOUS_PROJECTION_WHEN } from '@sim/db/knowledge-projection' import { EMBEDDING_KEYWORD_TIN_INDEX } from '@sim/db/schema' import { type ScriptMigration, ScriptMigrationDeferred } from '@sim/db/script-migrations/types' import { createLogger } from '@sim/logger' -import postgres, { type Sql, type TransactionSql } from 'postgres' +import postgres, { type Sql } from 'postgres' const logger = createLogger('TinKeywordProjection') const BATCH_SIZE = 500 @@ -39,16 +38,6 @@ async function tinAvailable(sql: Sql): Promise { return rows.length > 0 } -/** - * The Tin projection's embedding trigger, skipped by a writer that declared the asynchronous - * projection mode; see `0024_knowledge_projection_async`. - */ -export async function installTinKeywordTrigger(tx: TransactionSql): Promise { - await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_keyword_tin_sync - AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, content ON embedding - FOR EACH ROW WHEN (${SYNCHRONOUS_PROJECTION_WHEN}) EXECUTE FUNCTION sync_embedding_keyword_tin()`) -} - /** * Installs the stream and base-token functions shared by the triggers and the query path, and the * embedding and knowledge base triggers, atomically with respect to embedding writers. @@ -108,7 +97,9 @@ export async function installProjection(sql: Sql): Promise { RETURN NEW; END; $$`) - await installTinKeywordTrigger(tx) + await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_keyword_tin_sync + AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, content ON embedding + FOR EACH ROW EXECUTE FUNCTION sync_embedding_keyword_tin()`) await tx.unsafe(`CREATE OR REPLACE FUNCTION sync_knowledge_base_keyword_tin() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN diff --git a/packages/db/script-migrations/0021_embedding_search_connector.postgres.test.ts b/packages/db/script-migrations/0021_embedding_search_connector.postgres.test.ts index 24fd78d089f..f77fb91c62a 100644 --- a/packages/db/script-migrations/0021_embedding_search_connector.postgres.test.ts +++ b/packages/db/script-migrations/0021_embedding_search_connector.postgres.test.ts @@ -1,7 +1,10 @@ import { PROJECTION_SOURCE_ACL_TABLES } from '@sim/db/script-migrations/0021_embedding_search_connector' import { projectionSourceAclBackfillMigration as embeddingSearchConnectorMigration } from '@sim/db/script-migrations/0022_projection_source_acl_backfill' import { projectionAclSkipUnfilledMigration } from '@sim/db/script-migrations/0023_projection_acl_skip_unfilled' -import { installKnowledgeProjectionMarking } from '@sim/db/script-migrations/0024_knowledge_projection_async' +import { + installKnowledgeProjectionMarking, + knowledgeProjectionAsyncMigration, +} from '@sim/db/script-migrations/0024_knowledge_projection_async' import { generateId } from '@sim/utils/id' import postgres, { type Sql } from 'postgres' import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' @@ -11,7 +14,8 @@ const databaseUrl = process.env.KNOWLEDGE_ACL_TEST_DATABASE_URL /** * The projections here carry only the columns the source and ACL triggers touch; the vector and * lexeme columns, and their indexes, are what make a projection write slow, not what decides - * which rows the triggers write or which documents they mark. + * which rows the triggers write or which documents they mark. `embedding` and its projection + * functions are stand-ins, so `0024` can re-create the triggers it guards. */ describe.runIf(Boolean(databaseUrl))('projection source and ACL triggers in PostgreSQL', () => { let admin: Sql @@ -62,9 +66,30 @@ describe.runIf(Boolean(databaseUrl))('projection source and ACL triggers in Post connector_id text, acl text[] )` } + await sql`CREATE TABLE embedding ( + id text PRIMARY KEY, knowledge_base_id text, document_id text, enabled boolean, content text, + embedding text, embedding_384 text, embedding_768 text, embedding_1024 text, embedding_3072 text + )` + for (const name of ['sync_embedding_search', 'sync_embedding_keyword_search']) { + await sql.unsafe(`CREATE FUNCTION ${name}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RETURN NULL; END; $$`) + } await embeddingSearchConnectorMigration.up(sql) }, 60_000) + /** Each trigger `0024` guards, with whether its definition carries the mode guard. */ + const guarded = async () => + Object.fromEntries( + ( + await sql<{ name: string; guarded: boolean }[]>` + SELECT tgname AS name, pg_get_triggerdef(oid) LIKE '%sim.projection_mode%' AS guarded + FROM pg_trigger + WHERE tgname IN ('embedding_search_sync', 'embedding_keyword_search_sync', + 'embedding_search_source_acl_set', 'embedding_keyword_tin_source_acl_set') + AND tgrelid::regclass::text IN ('embedding', 'embedding_search', 'embedding_keyword_tin')` + ).map((row) => [row.name, row.guarded]) + ) + afterAll(async () => { await sql?.end() await admin?.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`) @@ -77,25 +102,43 @@ describe.runIf(Boolean(databaseUrl))('projection source and ACL triggers in Post await sql`ALTER TABLE embedding_keyword_tin DISABLE TRIGGER embedding_keyword_tin_source_acl_set` }) - it('runs every migration up to 0023, and every write between them, before 0024 exists', async () => { - /** A database that has run `0023` and not yet `0024`: no mark function exists. */ + it('runs everything synchronously through 0023, and 0024 adds the guards with the marks', async () => { + /** A database that has run `0023` and not yet `0024`. */ await sql`DROP FUNCTION IF EXISTS mark_knowledge_projection(text[], boolean)` await embeddingSearchConnectorMigration.up(sql) await projectionAclSkipUnfilledMigration.up(sql) + /** The embedding triggers as `0016` installs them. */ + for (const projection of ['embedding_search', 'embedding_keyword_search']) { + await sql.unsafe(`CREATE OR REPLACE TRIGGER ${projection}_sync AFTER INSERT ON embedding + FOR EACH ROW EXECUTE FUNCTION sync_${projection}()`) + } const [before] = await sql<{ installed: boolean }[]>` SELECT to_regprocedure('mark_knowledge_projection(text[], boolean)') IS NOT NULL AS installed` expect(before?.installed).toBe(false) + expect(await guarded()).toEqual({ + embedding_search_sync: false, + embedding_keyword_search_sync: false, + embedding_search_source_acl_set: false, + embedding_keyword_tin_source_acl_set: false, + }) await sql`INSERT INTO document (id, connector_id, acl) VALUES ('doc', 'src', ARRAY['u:alice'])` await sql`INSERT INTO embedding_search (id, document_id, connector_id, acl) VALUES ('filled', 'doc', 'src', ARRAY['u:alice'])` - await sql`UPDATE document SET acl = ARRAY['u:bob'] WHERE id = 'doc'` + /** Even a writer that asks for the asynchronous mode is synchronous until `0024`. */ + await asynchronously((tx) => tx`UPDATE document SET acl = ARRAY['u:bob'] WHERE id = 'doc'`) expect(await projected('embedding_search')).toEqual([ { id: 'filled', connector_id: 'src', acl: ['u:bob'] }, ]) expect(await marks()).toEqual([]) - await installKnowledgeProjectionMarking(sql) + await knowledgeProjectionAsyncMigration.up(sql) + expect(await guarded()).toEqual({ + embedding_search_sync: true, + embedding_keyword_search_sync: true, + embedding_search_source_acl_set: true, + embedding_keyword_tin_source_acl_set: true, + }) await sql`UPDATE document SET acl = ARRAY['u:carol'] WHERE id = 'doc'` expect((await projected('embedding_search'))[0]?.acl).toEqual(['u:carol']) expect(await marks()).toEqual([{ document_id: 'doc', generation: '1', content: false }]) diff --git a/packages/db/script-migrations/0021_embedding_search_connector.test.ts b/packages/db/script-migrations/0021_embedding_search_connector.test.ts new file mode 100644 index 00000000000..de39d4282b4 --- /dev/null +++ b/packages/db/script-migrations/0021_embedding_search_connector.test.ts @@ -0,0 +1,187 @@ +/** + * @vitest-environment node + */ +import { + backfillProjectionSourceAcl, + PROJECTION_SOURCE_ACL_PAGE_RETRIES, +} from '@sim/db/script-migrations/0021_embedding_search_connector' +import type { Sql } from 'postgres' +import { afterEach, describe, expect, it, vi } from 'vitest' + +/** A session that must never be reached: every case below is refused before the first page. */ +const untouched = { begin: vi.fn() } as unknown as Sql + +type PageRow = { scanned: number; filled: number; last_id: string | null } + +/** The message the database pairs with each SQLSTATE below. */ +const SERVER_MESSAGES: Record = { + '55P03': 'canceling statement due to lock timeout', + '57014': 'canceling statement due to statement timeout', + '40P01': 'deadlock detected', +} + +/** + * The error `postgres` throws for `code`: a SQLSTATE carries the server's message as is, and a + * lost connection is the driver's own connection error (`write `). + */ +function postgresError(code: string, message = SERVER_MESSAGES[code] ?? 'failed'): Error { + if (code.startsWith('CONNECTION_')) { + return Object.assign(new Error(`write ${code} localhost:5432`), { + code, + errno: code, + address: 'localhost', + port: 5432, + }) + } + return Object.assign(new Error(message), { code }) +} + +/** + * A session whose page statement answers from `outcomes` in order — a row, or an error to throw — + * and records the cursor each page was bound to. `beforePage` runs with the page's index before + * it answers. + */ +function sessionOf(outcomes: Array, beforePage?: (index: number) => void) { + const cursors: string[] = [] + const tx = { + unsafe: vi.fn(async (query: string, params?: unknown[]) => { + if (!query.includes('WITH page')) return [] + beforePage?.(cursors.length) + cursors.push(String(params?.[0])) + const outcome = outcomes.shift() + if (outcome === undefined) throw new Error('No outcome left for this page') + if (outcome instanceof Error) throw outcome + return [outcome] + }), + } + const session = { + begin: vi.fn(async (work: (tx: unknown) => Promise) => work(tx)), + unsafe: vi.fn(async () => []), + } as unknown as Sql + return { session, cursors } +} + +/** Runs a backfill under fake timers, so its retry pauses pass without waiting. */ +async function backfillNow(...args: Parameters) { + vi.useFakeTimers() + const result = backfillProjectionSourceAcl(...args) + /** A rejection must not surface as unhandled while the timers are still being drained. */ + const settled = result.catch(() => undefined) + await vi.runAllTimersAsync() + await settled + return result +} + +describe('backfillProjectionSourceAcl', () => { + afterEach(() => { + vi.useRealTimers() + }) + + it.each(['55P03', '57014', '40P01', 'CONNECTION_CLOSED'])( + 'retries the page after a %s failure and moves the cursor only once it commits', + async (code) => { + const { session, cursors } = sessionOf([ + { scanned: 2, filled: 2, last_id: 'id-2' }, + postgresError(code), + postgresError(code), + { scanned: 1, filled: 1, last_id: 'id-3' }, + { scanned: 0, filled: 0, last_id: null }, + ]) + await expect( + backfillNow(session, 'embedding_keyword_tin', { pauseMs: 0 }) + ).resolves.toMatchObject({ scanned: 3, written: 3, afterId: 'id-3', done: true }) + expect(cursors).toEqual(['', 'id-2', 'id-2', 'id-2', 'id-3']) + } + ) + + it('gives up on a page that times out more than the retry limit in a row', async () => { + const { session, cursors } = sessionOf( + Array.from({ length: PROJECTION_SOURCE_ACL_PAGE_RETRIES + 1 }, () => postgresError('55P03')) + ) + await expect( + backfillNow(session, 'embedding_keyword_tin', { pauseMs: 0 }) + ).rejects.toMatchObject({ code: '55P03' }) + expect(cursors).toHaveLength(PROJECTION_SOURCE_ACL_PAGE_RETRIES + 1) + expect(new Set(cursors)).toEqual(new Set([''])) + }) + + it('propagates an error that is not a timeout without retrying', async () => { + const { session, cursors } = sessionOf([postgresError('42P01')]) + await expect(backfillNow(session, 'embedding_search', { pauseMs: 0 })).rejects.toMatchObject({ + code: '42P01', + }) + expect(cursors).toEqual(['']) + }) + + it('propagates an explicit cancellation, which shares the statement timeout SQLSTATE', async () => { + const { session, cursors } = sessionOf([ + postgresError('57014', 'canceling statement due to user request'), + ]) + await expect(backfillNow(session, 'embedding_search', { pauseMs: 0 })).rejects.toThrow( + 'user request' + ) + expect(cursors).toEqual(['']) + }) + + it('does not start another page when the budget ran out during the retry pause', async () => { + const { session, cursors } = sessionOf([ + { scanned: 1, filled: 1, last_id: 'id-1' }, + postgresError('57014'), + ]) + await expect( + backfillNow(session, 'embedding_search', { pauseMs: 0, budgetMs: 1000 }) + ).resolves.toMatchObject({ afterId: 'id-1', done: false }) + expect(cursors).toEqual(['', 'id-1']) + }) + + it('leaves a page still failing at the budget to the continuation, from the last committed page', async () => { + const { session, cursors } = sessionOf( + [{ scanned: 1, filled: 1, last_id: 'id-1' }, postgresError('57014')], + /** The second page spends the budget before the database cancels it. */ + (index) => { + if (index === 1) vi.advanceTimersByTime(1000) + } + ) + await expect( + backfillNow(session, 'embedding_search', { pauseMs: 0, budgetMs: 1000 }) + ).resolves.toMatchObject({ afterId: 'id-1', done: false }) + expect(cursors).toEqual(['', 'id-1']) + }) + + it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( + 'refuses a page size of %s instead of reporting the projection filled', + async (pageSize) => { + await expect( + backfillProjectionSourceAcl(untouched, 'embedding_search', { pageSize }) + ).rejects.toThrow('page size must be a positive integer') + expect(untouched.begin).not.toHaveBeenCalled() + } + ) + + it.each([-1, Number.NaN])('refuses a pause of %s', async (pauseMs) => { + await expect( + backfillProjectionSourceAcl(untouched, 'embedding_search', { pauseMs }) + ).rejects.toThrow('pause must be a non-negative number') + expect(untouched.begin).not.toHaveBeenCalled() + }) + + it('binds the range it was given to every page, so shards never meet', async () => { + const statements: Array<{ query: string; params?: unknown[] }> = [] + const tx = { + unsafe: vi.fn(async (query: string, params?: unknown[]) => { + statements.push({ query, params }) + return query.includes('WITH page') ? [{ scanned: 0, filled: 0, last_id: null }] : [] + }), + } + const session = { + begin: vi.fn(async (work: (tx: unknown) => Promise) => work(tx)), + unsafe: vi.fn(async () => []), + } as unknown as Sql + await expect( + backfillProjectionSourceAcl(session, 'embedding_search', { afterId: '4', beforeId: '8' }) + ).resolves.toMatchObject({ done: true }) + const page = statements.find((statement) => statement.query.includes('WITH page'))! + expect(page.query).toContain('($2::text IS NULL OR s.id < $2)') + expect(page.params).toEqual(['4', '8']) + }) +}) diff --git a/packages/db/script-migrations/0021_embedding_search_connector.ts b/packages/db/script-migrations/0021_embedding_search_connector.ts index 3b5326c7902..3fdfec7027d 100644 --- a/packages/db/script-migrations/0021_embedding_search_connector.ts +++ b/packages/db/script-migrations/0021_embedding_search_connector.ts @@ -1,23 +1,58 @@ -import { SYNCHRONOUS_PROJECTION_WHEN } from '@sim/db/knowledge-projection' +import { createLogger } from '@sim/logger' +import { getPostgresErrorCode, getTransientDatabaseFailure } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { backoffWithJitter } from '@sim/utils/retry' import postgres, { type Sql, type TransactionSql } from 'postgres' +const logger = createLogger('ProjectionSourceAcl') + +/** + * Chunks filled per page. Every write to `embedding_search` re-inserts the row into each of its + * HNSW indexes, so a page's cost is index maintenance rather than the plan, and a small page keeps + * each transaction short and its locks brief. + */ +export const PROJECTION_SOURCE_ACL_PAGE_SIZE = 100 + +/** Pause between pages, so the backfill shares the database with the search it serves. */ +export const PROJECTION_SOURCE_ACL_PAGE_PAUSE_MS = 250 + +/** + * Longest a page may run before the database cancels it; the page is then retried in place. A + * caller that bounds a run leaves at least this much headroom after its budget, plus the longest + * retry pause, since the budget is checked between pages and the page in flight runs to this limit. + */ +export const PROJECTION_SOURCE_ACL_PAGE_TIMEOUT_MS = 60_000 + +/** + * How many times in a row one page may time out before the run fails. Index maintenance was + * observed holding the page for several minutes; with the pauses below a page waits roughly + * twelve minutes, about fourteen at the jitter's worst, so the budget covers one such pass. + */ +export const PROJECTION_SOURCE_ACL_PAGE_RETRIES = 12 + +/** Pause before a page is retried: 10 s, doubling to a 60 s base with up to 20% jitter (about 72 s). */ +const PAGE_RETRY_PAUSE = { baseMs: 10_000, maxMs: 60_000 } as const + +/** Pages between progress log lines. */ +const PROGRESS_EVERY_PAGES = 100 + /** The projections that carry their document's source and ACL. */ export const PROJECTION_SOURCE_ACL_TABLES = ['embedding_search', 'embedding_keyword_tin'] as const export type ProjectionSourceAclTable = (typeof PROJECTION_SOURCE_ACL_TABLES)[number] /** - * The document trigger's fan-out: copies a document's source and ACL onto its enabled chunks. + * The document trigger's body: fans a document's source and ACL out to its enabled chunks. * - * A chunk the source and ACL fill has not reached (`acl IS NULL`) keeps a NULL ACL. Search decides - * such a row on its document, so writing the ACL there changes no answer, while every write to + * A chunk the backfill has not filled yet (`acl IS NULL`) keeps a NULL ACL. Search decides such a + * row on its document, so writing the ACL there changes no answer, while every write to * `embedding_search` re-inserts the row into its vector index: a document whose ACL changed would - * otherwise rewrite each of its unfilled chunks inside the writer's statement. A document that - * moves to another source still carries the source onto its unfilled chunks, because source - * filters read it from the row; an ACL change alone leaves them untouched. Expects `moved` in scope. + * otherwise rewrite each of its unfilled chunks inside the writer's statement. The backfill fills + * the row later from the document under a share lock, so it copies whichever ACL is current. A + * document that moves to another source still carries the source onto its unfilled chunks, because + * source filters read it from the row; an ACL change alone leaves them untouched. */ -export function projectionSourceAclFanOut(): string { - return PROJECTION_SOURCE_ACL_TABLES.map( - (projection) => ` +export async function replaceProjectionSourceAclSync(sql: Sql | TransactionSql): Promise { + const fanOut = (projection: ProjectionSourceAclTable) => ` UPDATE ${projection} SET connector_id = NEW.connector_id, acl = CASE WHEN acl IS NULL THEN NULL ELSE NEW.acl END WHERE document_id = NEW.id AND enabled @@ -25,38 +60,16 @@ export function projectionSourceAclFanOut(): string { THEN moved AND connector_id IS DISTINCT FROM NEW.connector_id ELSE connector_id IS DISTINCT FROM NEW.connector_id OR acl IS DISTINCT FROM NEW.acl END;` - ).join('') -} - -/** - * The document trigger's body as `0022` and `0023` install it: the fan-out alone. It depends on no - * object a later migration creates, so every migration up to `0023` runs, and every write between - * them succeeds, on its own. `0024_knowledge_projection_async` replaces it with the body that also - * marks the document for the knowledge projector. - */ -export async function replaceProjectionSourceAclSync(sql: Sql | TransactionSql): Promise { await sql.unsafe(`CREATE OR REPLACE FUNCTION sync_projection_source_acl() RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE moved boolean := TG_OP = 'UPDATE' AND OLD.connector_id IS DISTINCT FROM NEW.connector_id; - BEGIN${projectionSourceAclFanOut()} + BEGIN${PROJECTION_SOURCE_ACL_TABLES.map(fanOut).join('')} RETURN NEW; END; $$`) } -/** - * The projection triggers that copy a chunk's source and ACL from its document on every write, - * skipped in the asynchronous mode, where the projector writes both itself. - */ -export async function installProjectionSourceAclSetTriggers(tx: TransactionSql): Promise { - for (const projection of PROJECTION_SOURCE_ACL_TABLES) { - await tx.unsafe(`CREATE OR REPLACE TRIGGER ${projection}_source_acl_set - BEFORE INSERT OR UPDATE OF document_id, enabled ON ${projection} - FOR EACH ROW WHEN (${SYNCHRONOUS_PROJECTION_WHEN}) EXECUTE FUNCTION set_projection_source_acl()`) - } -} - /** * Carries a chunk's source and ACL onto the ranking projections and keeps them there. * @@ -85,7 +98,11 @@ export async function installProjectionSourceAcl(sql: Sql): Promise { RETURN NEW; END; $$`) - await installProjectionSourceAclSetTriggers(tx) + for (const projection of PROJECTION_SOURCE_ACL_TABLES) { + await tx.unsafe(`CREATE OR REPLACE TRIGGER ${projection}_source_acl_set + BEFORE INSERT OR UPDATE OF document_id, enabled ON ${projection} + FOR EACH ROW EXECUTE FUNCTION set_projection_source_acl()`) + } /** The earlier shape of this migration, where the source alone was carried, and only on vectors. */ await tx.unsafe('DROP TRIGGER IF EXISTS embedding_search_connector_sync ON document') await tx.unsafe('DROP TRIGGER IF EXISTS embedding_search_connector_set ON embedding_search') @@ -94,17 +111,187 @@ export async function installProjectionSourceAcl(sql: Sql): Promise { }) } +export interface ProjectionSourceAclBackfillOptions { + /** Resume after this chunk id; the projection's first page otherwise. */ + afterId?: string + /** Stop before this chunk id; the projection's end otherwise. Lets workers fill disjoint ranges. */ + beforeId?: string + pageSize?: number + pauseMs?: number + /** Stop once this much time has passed and report where to resume; unbounded otherwise. */ + budgetMs?: number +} + +export interface ProjectionSourceAclBackfillProgress { + projection: ProjectionSourceAclTable + /** Unfilled chunks this run read, including any a concurrent write filled first. */ + scanned: number + written: number + /** The last chunk id this run reached; the next run resumes after it while `done` is false. */ + afterId: string + done: boolean +} + +/** + * Fills a projection's source and ACL for chunks written before the trigger existed, in + * independently committed keyset pages of unfilled rows. Each page is one statement that bounds + * its own locking and runtime and writes only rows still unset, so an interrupted run resumes by + * rerunning and a row the trigger has since written is left alone. The documents are share-locked + * before their values are copied, so a change in flight waits for the page and then fans its own + * values out; and a chunk that moved to another document meanwhile is left to that document's + * trigger, since the write requires the document the values were read from. + * + * On `embedding_search` every filled row is re-inserted into each HNSW index, which is the whole + * cost of a page and far more than a deploy can wait for; the run paces itself with a pause between + * pages and stops at its budget so a background task can chain runs until the projection is filled. + * Search does not wait: an unfilled row is decided on its document by the on-row candidate + * predicate, the join per candidate every row paid before the columns existed. A run fills the + * range it was given and reports that range done; whether the projection as a whole is done, and + * the analysis the planner then needs, is the caller's, since several runs may share a projection. + * + * A page the database cancels — on a lock timeout, because the keyword index's background + * maintenance holds the index page the row's write needs, or on a statement timeout — is retried + * in place after a pause, up to {@link PROJECTION_SOURCE_ACL_PAGE_RETRIES} times in a row, and + * the cursor stays on the last page that committed. A retry from the run's original cursor would + * instead walk every row the run had filled, past the index entries those writes left behind, + * into a statement timeout of its own; and the maintenance that cancelled the page outlasts the + * few attempts a run gets, so the fill would end where it stalled. A page that is still failing + * when the budget runs out is left to the continuation rather than retried past it. + */ +export async function backfillProjectionSourceAcl( + sql: Sql, + projection: ProjectionSourceAclTable, + options: ProjectionSourceAclBackfillOptions = {} +): Promise { + const pageSize = options.pageSize ?? PROJECTION_SOURCE_ACL_PAGE_SIZE + const pauseMs = options.pauseMs ?? PROJECTION_SOURCE_ACL_PAGE_PAUSE_MS + /** + * The page size is interpolated into the statement and a page of nothing would report the + * projection filled; a payload that asks for either is refused rather than quietly reshaped. + */ + if (!Number.isSafeInteger(pageSize) || pageSize < 1) { + throw new Error(`Projection backfill page size must be a positive integer, got ${pageSize}`) + } + if (!Number.isFinite(pauseMs) || pauseMs < 0) { + throw new Error(`Projection backfill pause must be a non-negative number, got ${pauseMs}`) + } + const startedAt = Date.now() + const deadline = + options.budgetMs === undefined ? Number.POSITIVE_INFINITY : startedAt + options.budgetMs + let afterId = options.afterId ?? '' + const beforeId = options.beforeId ?? null + let scanned = 0 + let written = 0 + let pages = 0 + let done = false + /** Timeouts in a row on the page after `afterId`; reset once it commits. */ + let timeouts = 0 + for (;;) { + let page: { scanned: number; filled: number; last_id: string | null } + try { + page = await sql.begin(async (tx) => { + await tx.unsafe("SET LOCAL lock_timeout = '5s'") + await tx.unsafe(`SET LOCAL statement_timeout = ${PROJECTION_SOURCE_ACL_PAGE_TIMEOUT_MS}`) + const [row] = await tx.unsafe< + Array<{ scanned: number; filled: number; last_id: string | null }> + >( + `WITH page AS ( + SELECT s.id, s.document_id, d.connector_id, d.acl + FROM ${projection} s JOIN document d ON d.id = s.document_id + WHERE s.id > $1 AND ($2::text IS NULL OR s.id < $2) AND s.acl IS NULL + ORDER BY s.id LIMIT ${pageSize} + FOR SHARE OF d + ), updated AS ( + UPDATE ${projection} s SET connector_id = page.connector_id, acl = page.acl + FROM page + WHERE s.id = page.id AND s.document_id = page.document_id AND s.acl IS NULL + RETURNING s.id + ) + SELECT (SELECT count(*)::int FROM page) AS scanned, + (SELECT count(*)::int FROM updated) AS filled, + (SELECT max(id) FROM page) AS last_id`, + [afterId, beforeId] + ) + return row + }) + } catch (error) { + /** + * The page is cancelled on `lock_timeout` (55P03) while its index write waits on a lock the + * index's background maintenance holds, and on `statement_timeout` (57014) when it runs past + * {@link PROJECTION_SOURCE_ACL_PAGE_TIMEOUT_MS}; both pass once the maintenance moves on. A + * deadlock, a serialization failure, or a dropped connection rolls the page back the same + * way, and a page only fills rows still unset, so all are retried in place. 57014 is also + * what an explicit cancellation raises, and that is not retried. + */ + const failure = getTransientDatabaseFailure(error) + if (!failure) throw error + const code = getPostgresErrorCode(error) + timeouts += 1 + if (timeouts > PROJECTION_SOURCE_ACL_PAGE_RETRIES) throw error + if (Date.now() >= deadline) break + const pauseMs = backoffWithJitter(timeouts, null, PAGE_RETRY_PAUSE) + logger.warn('Projection source and ACL backfill page failed transiently; retrying', { + projection, + afterId, + failure, + code, + attempt: timeouts, + retryInMs: Math.round(pauseMs), + }) + await sleep(pauseMs) + /** Checked again after the pause, so a timeout at the budget cannot start another page. */ + if (Date.now() >= deadline) break + continue + } + timeouts = 0 + if (page.last_id === null) { + done = true + break + } + afterId = page.last_id + scanned += page.scanned + written += page.filled + pages += 1 + if (pages % PROGRESS_EVERY_PAGES === 0) { + logger.info('Projection source and ACL backfill progress', { + projection, + scanned, + written, + afterId, + elapsedMs: Date.now() - startedAt, + }) + } + if (pauseMs > 0) await sleep(pauseMs) + /** Checked after the pause, so the pause cannot carry a run past its budget into another page. */ + if (Date.now() >= deadline) break + } + logger.info( + done + ? 'Projection source and ACL range backfilled' + : 'Projection source and ACL backfill paused', + { + projection, + beforeId, + scanned, + written, + afterId, + elapsedMs: Date.now() - startedAt, + } + ) + return { projection, scanned, written, afterId, done } +} + /** * The indexes exact ranking of a readable set needs: the ACL index on each projection, and on the * vector projection the source index that lets the planner lead with a few sources when the * caller's tokens alone would match most of the index. Built concurrently, so the triggers and the - * projector keep writing. `CONCURRENTLY` cannot run in a transaction, and the pool's lock timeout + * backfill keep writing. `CONCURRENTLY` cannot run in a transaction, and the pool's lock timeout * would cancel a build that merely waits for a long transaction to finish. The timeout is a * session setting, so one connection is reserved for it, the builds, and the reset — a pool * would otherwise hand the builds to connections that never saw the setting. * - * The unfilled index on each projection lists the rows the fill has not reached: the projector's - * fill reads them from it instead of walking past every filled row, and the on-row predicate's + * The unfilled index on each projection lists the rows the backfill has not reached: each page + * reads its rows from it instead of walking past every filled one, and the on-row predicate's * unfilled branch, an `OR` beside the ACL overlap, stays an index probe for the planner — once the * projection is filled, a probe of an empty index. */ @@ -143,17 +330,20 @@ export async function indexProjectionAcl(pool: Sql): Promise { } /** - * Run directly — `db:push`, or an operator repairing a database by hand — the triggers and indexes - * are installed here; rows they have not filled are marked and filled by the knowledge projector. - * The registered migration is `0022_projection_source_acl_backfill`. + * Run directly — `db:push`, or an operator filling a database by hand — the projections are filled + * here, paced the same way, rather than left to the app. The registered migration is + * `0022_projection_source_acl_backfill`, which supersedes this file's earlier, synchronous shape. */ if (import.meta.main) { const url = process.env.MIGRATION_DATABASE_URL ?? process.env.DATABASE_URL - if (!url) throw new Error('DATABASE_URL is required to install the projection source and ACL') + if (!url) throw new Error('DATABASE_URL is required to backfill the projection source and ACL') const sql = postgres(url, { max: 1, max_lifetime: null, onnotice: () => undefined }) try { await installProjectionSourceAcl(sql) await indexProjectionAcl(sql) + for (const projection of PROJECTION_SOURCE_ACL_TABLES) { + await backfillProjectionSourceAcl(sql, projection) + } } finally { await sql.end() } diff --git a/packages/db/script-migrations/0024_knowledge_projection_async.ts b/packages/db/script-migrations/0024_knowledge_projection_async.ts index d6bccb4225e..4a85c34ae6b 100644 --- a/packages/db/script-migrations/0024_knowledge_projection_async.ts +++ b/packages/db/script-migrations/0024_knowledge_projection_async.ts @@ -1,13 +1,8 @@ -import { KNOWLEDGE_PROJECTION_DEFERRED } from '@sim/db/knowledge-projection' import { - installSearchKeywordTrigger, - installSearchVectorTrigger, -} from '@sim/db/script-migrations/0016_backfill_search_vectors' -import { installTinKeywordTrigger } from '@sim/db/script-migrations/0019_tin_keyword_projection' -import { - installProjectionSourceAclSetTriggers, - projectionSourceAclFanOut, -} from '@sim/db/script-migrations/0021_embedding_search_connector' + KNOWLEDGE_PROJECTION_DEFERRED, + SOURCE_ACL_PROJECTIONS, + SYNCHRONOUS_PROJECTION_WHEN, +} from '@sim/db/knowledge-projection' import type { ScriptMigration } from '@sim/db/script-migrations/types' import { createLogger } from '@sim/logger' import postgres, { type Sql, type TransactionSql } from 'postgres' @@ -53,6 +48,22 @@ async function installMarkFunctions(tx: Sql | TransactionSql): Promise { $$`) } +/** + * The document trigger's fan-out, as `0021_embedding_search_connector` wrote it: copies a + * document's source and ACL onto its enabled chunks, leaving a chunk the source and ACL fill has not + * reached with a NULL ACL unless the document moved. Expects `moved` in scope. + */ +const SOURCE_ACL_FAN_OUT = SOURCE_ACL_PROJECTIONS.map( + (projection) => ` + UPDATE ${projection} + SET connector_id = NEW.connector_id, acl = CASE WHEN acl IS NULL THEN NULL ELSE NEW.acl END + WHERE document_id = NEW.id AND enabled + AND CASE WHEN acl IS NULL + THEN moved AND connector_id IS DISTINCT FROM NEW.connector_id + ELSE connector_id IS DISTINCT FROM NEW.connector_id OR acl IS DISTINCT FROM NEW.acl + END;` +).join('') + /** * The document trigger's body once documents are marked: a document whose source or ACL changed * is marked, whatever the writer's mode, and a writer that did not defer its projection still fans @@ -72,7 +83,7 @@ async function installDocumentMarking(tx: Sql | TransactionSql): Promise { END IF; IF ${KNOWLEDGE_PROJECTION_DEFERRED} THEN RETURN NEW; - END IF;${projectionSourceAclFanOut()} + END IF;${SOURCE_ACL_FAN_OUT} RETURN NEW; END; $$`) @@ -119,6 +130,34 @@ async function installMarkTriggers(tx: TransactionSql): Promise { EXECUTE FUNCTION mark_updated_embedding_projection()`) } +/** + * The triggers that write projection rows in the writer's transaction, re-created with the guard + * that skips them for a writer that deferred its projection. Their functions are the ones + * `0016_backfill_search_vectors`, `0019_tin_keyword_projection` and + * `0021_embedding_search_connector` installed, and their events are the same; only the `WHEN` is + * new. The Tin trigger is re-created only where `0019` installed it. + */ +async function installGuardedProjectionTriggers(tx: TransactionSql): Promise { + const guard = `FOR EACH ROW WHEN (${SYNCHRONOUS_PROJECTION_WHEN})` + await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_search_sync + AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, + embedding, embedding_384, embedding_768, embedding_1024, embedding_3072 ON embedding + ${guard} EXECUTE FUNCTION sync_embedding_search()`) + await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_keyword_search_sync + AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, content ON embedding + ${guard} EXECUTE FUNCTION sync_embedding_keyword_search()`) + if (await tinTriggerInstalled(tx)) { + await tx.unsafe(`CREATE OR REPLACE TRIGGER embedding_keyword_tin_sync + AFTER INSERT OR UPDATE OF knowledge_base_id, document_id, enabled, content ON embedding + ${guard} EXECUTE FUNCTION sync_embedding_keyword_tin()`) + } + for (const projection of SOURCE_ACL_PROJECTIONS) { + await tx.unsafe(`CREATE OR REPLACE TRIGGER ${projection}_source_acl_set + BEFORE INSERT OR UPDATE OF document_id, enabled ON ${projection} + ${guard} EXECUTE FUNCTION set_projection_source_acl()`) + } +} + /** Whether the Tin projection's embedding trigger exists here: `0019` installs it only where `tin` does. */ async function tinTriggerInstalled(tx: TransactionSql): Promise { const [row] = await tx>` @@ -154,11 +193,8 @@ export async function installKnowledgeProjectionAsync(sql: Sql): Promise { await tx.unsafe(`SET LOCAL lock_timeout = '${TRIGGER_LOCK_TIMEOUT}'`) await installMarkFunctions(tx) await installDocumentMarking(tx) - await installSearchVectorTrigger(tx) - await installSearchKeywordTrigger(tx) - if (await tinTriggerInstalled(tx)) await installTinKeywordTrigger(tx) + await installGuardedProjectionTriggers(tx) await installMarkTriggers(tx) - await installProjectionSourceAclSetTriggers(tx) }), { budgetMs: TRIGGER_LOCK_RETRY_BUDGET_MS, From 5b54733b77b1697faff586a1fa27a43e3da389cd Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 03:15:33 -0700 Subject: [PATCH 3/3] test(knowledge): count projection requests per connector ACL case --- .../__integration__/knowledge-projection.integration.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts b/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts index 81b65c24f5e..a69bdc86311 100644 --- a/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts @@ -418,6 +418,7 @@ afterAll(async () => { * so no synchronous trigger would write the Tin row. */ beforeEach(async () => { + requestKnowledgeProjection.mockClear() await db.delete(embedding).where(eq(embedding.documentId, documentId)) await db .update(document) @@ -666,7 +667,7 @@ describe('the projector', () => { } finally { enabledFlags.delete('knowledge-async-projection') } - expect(requestKnowledgeProjection).toHaveBeenCalled() + expect(requestKnowledgeProjection).toHaveBeenCalledOnce() expect(await markOf()).toMatchObject({ content: false }) expect((await rowAcl(embeddingSearch))?.acl).toEqual( flagOn ? aclOf('alice', 'bob') : aclOf('bob')