From cc951d4cfaf778664b13ffd169a6e85fabf2c7e3 Mon Sep 17 00:00:00 2001 From: bercianor Date: Mon, 28 Sep 2026 15:38:35 +0200 Subject: [PATCH] feat(audit): prepare stores for journal history backfill --- .../store/couchbase/CouchbaseAuditStore.java | 69 +++++-- .../internal/CouchbaseAuditPersistence.java | 73 +++---- .../couchbase/internal/CouchbaseAuditor.java | 2 +- .../internal/CouchbaseJournalWriter.java | 36 ++++ .../CouchbaseAuditStoreFacadeTest.java | 155 +++++++++++++++ .../CouchbaseAuditPersistenceJournalTest.java | 29 ++- .../store/dynamodb/DynamoDBAuditStore.java | 61 ++++-- .../internal/DynamoDBAuditPersistence.java | 56 +++--- .../internal/DynamoDBAuditRepository.java | 2 +- .../internal/DynamoDBJournalWriter.java | 35 ++++ .../DynamoDBAuditStoreJournalTest.java | 82 +++++++- .../DynamoDBAuditPersistenceJournalTest.java | 14 +- .../reactive/MongoDBReactiveAuditStore.java | 66 +++++-- .../MongoDBReactiveAuditPersistence.java | 43 ++--- .../MongoDBReactiveAuditRepository.java | 2 +- .../MongoDBReactiveJournalWriter.java | 43 +++++ .../MongoDBReactiveAuditStoreJournalTest.java | 73 +++++-- ...DBReactiveAuditPersistenceJournalTest.java | 33 ++-- .../MongoDBReactiveAuditPersistenceTest.java | 6 +- .../mongodb/sync/MongoDBSyncAuditStore.java | 62 ++++-- .../internal/MongoDBSyncAuditPersistence.java | 54 ++---- .../internal/MongoDBSyncAuditRepository.java | 10 +- .../MongoDBSyncJournalEventStore.java | 2 +- .../internal/MongoDBSyncJournalWriter.java | 43 +++++ .../sync/MongoDBSyncHistoryProviderTest.java | 182 ++++++++++++++++++ ...ongoDBSyncAuditPersistenceJournalTest.java | 39 ++-- .../flamingock/store/sql/SqlAuditStore.java | 51 ++++- .../sql/internal/SqlAuditPersistence.java | 49 ++--- .../internal/SqlJournalHistoryAppender.java | 66 +++++++ .../store/sql/SqlAuditStoreTest.java | 105 +++++++++- .../SqlAuditPersistenceJournalTest.java | 55 +++++- .../internal/SqlJournalDialectHelperTest.java | 8 +- .../core/audit/AuditHistoryAppender.java | 25 +++ .../core/audit/JournalHistoryAppender.java | 25 +++ .../builder/AbstractChangeRunnerBuilder.java | 20 ++ .../store/HistoryAppenderProvider.java | 29 +++ .../AbstractCommunityAuditPersistence.java | 8 +- .../core/journal/JournalEventSequencer.java | 38 +++- .../journal/JournalEventSequencerFactory.java | 14 +- .../builder/HistoryAppenderProviderTest.java | 62 ++++++ .../journal/JournalEventSequencerTest.java | 86 +++++++++ 41 files changed, 1570 insertions(+), 343 deletions(-) create mode 100644 community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseJournalWriter.java create mode 100644 community/flamingock-couchbase-auditstore/src/test/java/io/flamingock/store/couchbase/CouchbaseAuditStoreFacadeTest.java create mode 100644 community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBJournalWriter.java create mode 100644 community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveJournalWriter.java create mode 100644 community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncJournalWriter.java create mode 100644 community/flamingock-mongodb-sync-auditstore/src/test/java/io/flamingock/store/mongodb/sync/MongoDBSyncHistoryProviderTest.java create mode 100644 community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/internal/SqlJournalHistoryAppender.java create mode 100644 core/flamingock-core-commons/src/main/java/io/flamingock/internal/common/core/audit/AuditHistoryAppender.java create mode 100644 core/flamingock-core-commons/src/main/java/io/flamingock/internal/common/core/audit/JournalHistoryAppender.java create mode 100644 core/flamingock-core/src/main/java/io/flamingock/internal/core/external/store/HistoryAppenderProvider.java create mode 100644 core/flamingock-core/src/test/java/io/flamingock/internal/core/builder/HistoryAppenderProviderTest.java diff --git a/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/CouchbaseAuditStore.java b/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/CouchbaseAuditStore.java index 789703207..9e554beaf 100644 --- a/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/CouchbaseAuditStore.java +++ b/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/CouchbaseAuditStore.java @@ -20,12 +20,17 @@ import com.couchbase.client.java.Cluster; import com.couchbase.client.java.transactions.TransactionAttemptContext; import io.flamingock.internal.common.core.audit.AuditPersistenceFactory; +import io.flamingock.internal.common.core.audit.AuditHistoryAppender; +import io.flamingock.internal.common.core.audit.JournalHistoryAppender; import io.flamingock.internal.common.core.audit.AuditReader; +import io.flamingock.internal.common.core.audit.AuditEntry; +import io.flamingock.internal.common.core.journal.JournalEvent; import io.flamingock.internal.common.core.context.ContextResolver; import io.flamingock.internal.common.core.error.FlamingockException; import io.flamingock.internal.common.core.feature.Features; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; import io.flamingock.internal.core.external.store.CommunityAuditStore; +import io.flamingock.internal.core.external.store.HistoryAppenderProvider; +import io.flamingock.internal.core.context.BasicRuntimeContext; import io.flamingock.internal.core.external.store.audit.community.CommunityAuditPersistence; import io.flamingock.internal.core.external.store.lock.community.CommunityLockService; import io.flamingock.internal.core.journal.JournalEventSequencer; @@ -39,19 +44,19 @@ import io.flamingock.store.couchbase.internal.CouchbaseAuditPersistence; import io.flamingock.store.couchbase.internal.CouchbaseAuditor; import io.flamingock.store.couchbase.internal.CouchbaseJournalEventStore; +import io.flamingock.store.couchbase.internal.CouchbaseJournalWriter; import io.flamingock.store.couchbase.internal.CouchbaseLockService; import io.flamingock.externalsystem.couchbase.api.CouchbaseExternalSystem; import java.util.Collections; import java.util.Set; -public class CouchbaseAuditStore implements CommunityAuditStore { +public class CouchbaseAuditStore implements CommunityAuditStore, HistoryAppenderProvider { private final CouchbaseExternalSystem targetSystem; private final Cluster cluster; private final String bucketName; private RunnerId runnerId; - private CommunityConfigurable communityConfiguration; private CouchbaseLockService lockService; private Bucket bucket; private String scopeName = CollectionIdentifier.DEFAULT_SCOPE; @@ -122,42 +127,65 @@ public CouchbaseAuditStore withAutoCreate(boolean autoCreate) { public void initialize(ContextResolver baseContext) { this.validate(); runnerId = baseContext.getRequiredDependencyValue(RunnerId.class); - communityConfiguration = baseContext.getRequiredDependencyValue(CommunityConfigurable.class); auditor = new CouchbaseAuditor(cluster, bucket); journalEventStore = new CouchbaseJournalEventStore(cluster, bucket); journalEventSequencerFactory = new JournalEventSequencerFactory(journalEventStore); lockService = new CouchbaseLockService(cluster, bucket, TimeService.getDefault()); + auditor.initialize(autoCreate, scopeName, auditRepositoryName); lockService.initialize(autoCreate, scopeName, lockRepositoryName); + FeatureFlag.ifEnabled(Features.JOURNAL_EVENTS, + () -> journalEventStore.initialize(autoCreate, scopeName, journalRepositoryName)); } @Override public AuditPersistenceFactory getPersistenceFactory() { return stageId -> { - // Must run before forStream(stageId): forStream seeds the sequence from the last persisted event, - // which Couchbase reports as empty until the journal store is initialized. Idempotent and - // synchronized, so repeating it in CouchbaseAuditPersistence#doInitialize is safe. - FeatureFlag.ifEnabled(Features.JOURNAL_EVENTS, () -> journalEventStore.initialize(autoCreate, scopeName, journalRepositoryName)); - JournalEventSequencer journalEventSequencer = journalEventSequencerFactory.forStream(stageId); + JournalEventSequencer journalEventSequencer = FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false) + ? journalEventSequencerFactory.forStream(stageId) : null; CouchbaseAuditPersistence persistence = new CouchbaseAuditPersistence( - communityConfiguration, auditor, - journalEventStore, journalEventSequencer, targetSystem.getTxWrapper(), - scopeName, - auditRepositoryName, - journalRepositoryName, - autoCreate); + new CouchbaseJournalWriter(journalEventStore)); persistence.initialize(runnerId); return persistence; }; } + @Override + public AuditHistoryAppender getAuditHistoryAppender() { + return auditor::append; + } + + @Override + public JournalHistoryAppender getJournalHistoryAppender() { + return (streamId, entry) -> { + if (!FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + throw new IllegalStateException("Journal events must be enabled to write journal history"); + } + JournalEventSequencer sequencer = journalEventSequencerFactory.forStream(streamId); + CouchbaseJournalWriter writer = new CouchbaseJournalWriter(journalEventStore); + synchronized (sequencer) { + try { + JournalEvent event = sequencer.newEvent(entry); + io.flamingock.internal.util.Result result = targetSystem.getTxWrapper().wrapExecution( + new BasicRuntimeContext("write-journal-" + entry.getChangeId()), runtimeContext -> + writer.write(runtimeContext.getContext().getRequiredDependencyValue( + TransactionAttemptContext.class), event)); + sequencer.confirm(); + return result; + } catch (RuntimeException | Error failure) { + sequencer.markWriteOutcomeUncertain(); + throw failure; + } + } + }; + } + @Override public AuditReader getAuditReader() { - auditor.initialize(autoCreate, scopeName, auditRepositoryName); return () -> auditor.getAuditHistory(); } @@ -198,7 +226,8 @@ private void validate() { throw new FlamingockException("The 'lockRepositoryName' property is required."); } - if (journalRepositoryName == null || journalRepositoryName.trim().isEmpty()) { + if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false) + && (journalRepositoryName == null || journalRepositoryName.trim().isEmpty())) { throw new FlamingockException("The 'journalRepositoryName' property is required."); } @@ -206,11 +235,13 @@ private void validate() { throw new FlamingockException("The 'auditRepositoryName' and 'lockRepositoryName' properties must not be the same."); } - if (journalRepositoryName.trim().equalsIgnoreCase(auditRepositoryName.trim())) { + if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false) + && journalRepositoryName.trim().equalsIgnoreCase(auditRepositoryName.trim())) { throw new FlamingockException("The 'journalRepositoryName' and 'auditRepositoryName' properties must not be the same."); } - if (journalRepositoryName.trim().equalsIgnoreCase(lockRepositoryName.trim())) { + if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false) + && journalRepositoryName.trim().equalsIgnoreCase(lockRepositoryName.trim())) { throw new FlamingockException("The 'journalRepositoryName' and 'lockRepositoryName' properties must not be the same."); } } diff --git a/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseAuditPersistence.java b/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseAuditPersistence.java index ce9f09202..08020fad1 100644 --- a/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseAuditPersistence.java +++ b/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseAuditPersistence.java @@ -21,61 +21,32 @@ import io.flamingock.internal.common.core.feature.Features; import io.flamingock.internal.common.core.journal.JournalEvent; import io.flamingock.internal.common.core.external.ExecutionWrapper; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; import io.flamingock.internal.core.context.BasicRuntimeContext; import io.flamingock.internal.core.external.store.audit.community.AbstractCommunityAuditPersistence; import io.flamingock.internal.core.journal.JournalEventSequencer; import io.flamingock.internal.util.FeatureFlag; import io.flamingock.internal.util.Result; -import io.flamingock.internal.util.id.RunnerId; import java.util.List; public class CouchbaseAuditPersistence extends AbstractCommunityAuditPersistence { private final CouchbaseAuditor auditor; - private final CouchbaseJournalEventStore journalEventStore; + private final CouchbaseJournalWriter journalWriter; private final JournalEventSequencer journalEventSequencer; private final ExecutionWrapper txWrapper; - private final String scopeName; - private final String auditRepositoryName; - private final String journalRepositoryName; - private final boolean autoCreate; - public CouchbaseAuditPersistence(CommunityConfigurable localConfiguration, - CouchbaseAuditor auditor, - CouchbaseJournalEventStore journalEventStore, + public CouchbaseAuditPersistence(CouchbaseAuditor auditor, JournalEventSequencer journalEventSequencer, ExecutionWrapper txWrapper, - String scopeName, - String auditRepositoryName, - String journalRepositoryName, - boolean autoCreate) { - super(localConfiguration); + CouchbaseJournalWriter journalWriter) { this.auditor = auditor; - this.journalEventStore = journalEventStore; + this.journalWriter = journalWriter; this.journalEventSequencer = journalEventSequencer; this.txWrapper = txWrapper; - this.scopeName = scopeName; - this.auditRepositoryName = auditRepositoryName; - this.journalRepositoryName = journalRepositoryName; - this.autoCreate = autoCreate; } - @Override - protected void doInitialize(RunnerId runnerId) { - auditor.initialize(autoCreate, scopeName, auditRepositoryName); - // Creating the collection/indexes is what brings the journal collection into existence, so skipping - // this keeps it from ever appearing while the flag is off. It must stay in step with the append in - // writeEntry: skipping setup while still appending would let ctx.insert create the collection - // implicitly and without indexes, voiding the stream-position and eventId-lookup guarantees. - // Also repeated (idempotently) in CouchbaseAuditStore#getPersistenceFactory, which must run this - // before seeding the JournalEventSequencer via forStream(stageId). - FeatureFlag.ifEnabled(Features.JOURNAL_EVENTS, () -> journalEventStore.initialize(autoCreate, scopeName, journalRepositoryName)); - } - - @Override public List getAuditHistory() { return auditor.getAuditHistory(); @@ -88,24 +59,24 @@ public Result writeEntry(AuditEntry auditEntry) { // without them, the audit record set is itself the history. if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS)) { RuntimeContext baseContext = new BasicRuntimeContext("write-changeState-" + auditEntry.getChangeId()); - Result result = txWrapper.wrapExecution(baseContext, runtimeContext -> { - TransactionAttemptContext ctx = runtimeContext.getContext().getRequiredDependencyValue(TransactionAttemptContext.class); - JournalEvent journalEvent = journalEventSequencer.newEvent(auditEntry); - journalEventStore.contributeToTransaction(ctx, journalEvent); - return auditor.contributeToTransaction(ctx, auditEntry); - }); - // Spends the stream position, and only a committed transaction attempt may reach this line. A - // normal return from wrapExecution does NOT in general mean commit — CouchbaseTxWrapper - // returns normally after a deliberate rollback too, when the operation's result is a FailedStep. - // It is sound here because this operation returns a Result, which can never be a FailedStep, so - // the only way to return normally is a committed attempt; a failing attempt is caught and - // rethrown as TransactionFailedException (see CouchbaseTxWrapper — it doesn't yet wrap that as - // DatabaseTransactionException, a known deviation from the ExecutionWrapper contract, tracked - // separately from this ticket). Keep that true: an operation that could return a failed step - // would silently burn a position and gap the stream, and a contiguous sequence is what lets a - // consumer tell "in flight" from "lost". - journalEventSequencer.confirm(); - return result; + synchronized (journalEventSequencer) { + try { + JournalEvent event = journalEventSequencer.newEvent(auditEntry); + Result result = txWrapper.wrapExecution(baseContext, runtimeContext -> { + TransactionAttemptContext ctx = runtimeContext.getContext() + .getRequiredDependencyValue(TransactionAttemptContext.class); + journalWriter.write(ctx, event); + return auditor.contributeToTransaction(ctx, auditEntry); + }); + // Result cannot be a FailedStep: a normal wrapper return means commit. Keep the + // transaction and confirmation under the same stream lock as journal-only writes. + journalEventSequencer.confirm(); + return result; + } catch (RuntimeException | Error failure) { + journalEventSequencer.markWriteOutcomeUncertain(); + throw failure; + } + } } else { return auditor.append(auditEntry); } diff --git a/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseAuditor.java b/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseAuditor.java index e67f8928b..941c1d281 100644 --- a/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseAuditor.java +++ b/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseAuditor.java @@ -83,7 +83,7 @@ public synchronized void initialize(boolean autoCreate, String scopeName, String * {@link #contributeToTransaction} would collapse them onto each other, discarding the very history being * imported. */ - Result append(AuditEntry auditEntry) { + public Result append(AuditEntry auditEntry) { String key = toKey(auditEntry); logger.debug("Saving audit entry with key {}", key); diff --git a/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseJournalWriter.java b/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseJournalWriter.java new file mode 100644 index 000000000..0fd953f79 --- /dev/null +++ b/community/flamingock-couchbase-auditstore/src/main/java/io/flamingock/store/couchbase/internal/CouchbaseJournalWriter.java @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Flamingock (https://www.flamingock.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.flamingock.store.couchbase.internal; + +import com.couchbase.client.java.transactions.TransactionAttemptContext; +import io.flamingock.internal.common.core.audit.AuditEntry; +import io.flamingock.internal.common.core.journal.JournalEvent; +import io.flamingock.internal.util.Result; + +/** Stages an event in the caller's transaction; confirmation belongs to the transaction owner. */ +public class CouchbaseJournalWriter { + + private final CouchbaseJournalEventStore store; + + public CouchbaseJournalWriter(CouchbaseJournalEventStore store) { + this.store = store; + } + + public Result write(TransactionAttemptContext context, JournalEvent event) { + store.contributeToTransaction(context, event); + return Result.OK(); + } +} diff --git a/community/flamingock-couchbase-auditstore/src/test/java/io/flamingock/store/couchbase/CouchbaseAuditStoreFacadeTest.java b/community/flamingock-couchbase-auditstore/src/test/java/io/flamingock/store/couchbase/CouchbaseAuditStoreFacadeTest.java new file mode 100644 index 000000000..36494a0f6 --- /dev/null +++ b/community/flamingock-couchbase-auditstore/src/test/java/io/flamingock/store/couchbase/CouchbaseAuditStoreFacadeTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2026 Flamingock (https://www.flamingock.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.flamingock.store.couchbase; + +import com.couchbase.client.core.io.CollectionIdentifier; +import com.couchbase.client.java.Cluster; +import com.couchbase.client.java.ClusterOptions; +import io.flamingock.core.kit.audit.AuditEntryTestFactory; +import io.flamingock.internal.common.core.audit.AuditEntry; +import io.flamingock.internal.common.core.audit.AuditTxType; +import io.flamingock.internal.common.core.context.ContextResolver; +import io.flamingock.internal.common.core.feature.Features; +import io.flamingock.internal.common.core.journal.JournalEvent; +import io.flamingock.internal.common.couchbase.CouchbaseCollectionHelper; +import io.flamingock.internal.common.couchbase.CouchbaseJournalEventMapper; +import io.flamingock.internal.core.configuration.community.CommunityConfigurable; +import io.flamingock.internal.core.configuration.community.CommunityConfiguration; +import io.flamingock.internal.util.FeatureFlag; +import io.flamingock.internal.util.id.RunnerId; +import io.flamingock.targetsystem.couchbase.CouchbaseTargetSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.couchbase.BucketDefinition; +import org.testcontainers.couchbase.CouchbaseContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Testcontainers +class CouchbaseAuditStoreFacadeTest { + private static final String BUCKET = "test"; + private static final String SCOPE = CollectionIdentifier.DEFAULT_SCOPE; + private static final String AUDIT = "facadeAudit"; + private static final String LOCK = "facadeLock"; + private static final String JOURNAL = "facadeJournal"; + private static final String STREAM = "facade-stage"; + + @Container + static final CouchbaseContainer container = new CouchbaseContainer("couchbase/server:7.2.4") + .withBucket(new BucketDefinition(BUCKET)); + private static Cluster cluster; + private CouchbaseAuditStore store; + + @BeforeAll + static void connect() { + container.start(); + cluster = Cluster.connect(container.getConnectionString(), + ClusterOptions.clusterOptions(container.getUsername(), container.getPassword()) + .environment(env -> env.timeoutConfig(timeout -> timeout.kvTimeout(Duration.ofSeconds(10))))); + cluster.bucket(BUCKET).waitUntilReady(Duration.ofSeconds(10)); + } + + private void initialize(boolean journalEnabled) { + if (journalEnabled) { + FeatureFlag.enable(Features.JOURNAL_EVENTS); + } + ContextResolver context = mock(ContextResolver.class); + when(context.getDependencyValue(io.flamingock.internal.core.builder.FlamingockEdition.class)) + .thenReturn(Optional.empty()); + when(context.getRequiredDependencyValue(RunnerId.class)).thenReturn(RunnerId.generate()); + when(context.getRequiredDependencyValue(CommunityConfigurable.class)).thenReturn(new CommunityConfiguration()); + CouchbaseTargetSystem target = new CouchbaseTargetSystem("couchbase", cluster, BUCKET); + store = CouchbaseAuditStore.from(target).withAuditRepositoryName(AUDIT) + .withLockRepositoryName(LOCK).withJournalRepositoryName(JOURNAL); + target.initialize(context); + store.initialize(context); + } + + @AfterEach + void cleanup() { + FeatureFlag.remove(Features.JOURNAL_EVENTS); + CouchbaseCollectionHelper.dropCollectionIfExists(cluster, BUCKET, SCOPE, AUDIT); + CouchbaseCollectionHelper.dropCollectionIfExists(cluster, BUCKET, SCOPE, LOCK); + CouchbaseCollectionHelper.dropCollectionIfExists(cluster, BUCKET, SCOPE, JOURNAL); + } + + @Test + void disabledAppenderPreservesHistoryAndRejectsJournalWrites() { + initialize(false); + AuditEntry started = entry("change", AuditEntry.Status.STARTED); + AuditEntry applied = entry("change", AuditEntry.Status.APPLIED); + store.getAuditHistoryAppender().append(started); + store.getAuditHistoryAppender().append(applied); + + assertEquals(2, store.getAuditReader().getAuditHistory().size()); + assertThrows(IllegalStateException.class, () -> store.getJournalHistoryAppender().append(STREAM, applied)); + assertFalse(CouchbaseCollectionHelper.collectionExists(cluster, BUCKET, SCOPE, JOURNAL)); + } + + @Test + void enabledJournalOnlyAndNormalWriteShareEventConstruction() { + initialize(true); + // Stage persistence is built before the journal-only append, so both paths must share its sequencer. + io.flamingock.internal.core.external.store.audit.community.CommunityAuditPersistence persistence = + store.getPersistenceFactory().get(STREAM); + AuditEntry first = entry("first", AuditEntry.Status.STARTED); + persistence.writeEntry(first); + AuditEntry journalOnly = entry("journal-only", AuditEntry.Status.STARTED); + store.getJournalHistoryAppender().append(STREAM, journalOnly); + assertEquals(1, store.getAuditReader().getAuditHistory().size()); + + AuditEntry normal = entry("normal", AuditEntry.Status.APPLIED); + persistence.writeEntry(normal); + assertEquals(2, store.getAuditReader().getAuditHistory().size()); + + List> events = new ArrayList<>(); + CouchbaseJournalEventMapper mapper = new CouchbaseJournalEventMapper(); + CouchbaseCollectionHelper.selectAllDocuments(cluster, BUCKET, SCOPE, JOURNAL) + .forEach(document -> events.add(mapper.fromDocument(document))); + assertEquals(3, events.size()); + events.sort((left, right) -> Long.compare(left.getStreamSequence(), right.getStreamSequence())); + assertEquals(1L, events.get(0).getStreamSequence()); + assertEquals(2L, events.get(1).getStreamSequence()); + assertEquals(3L, events.get(2).getStreamSequence()); + assertEquals(events.get(0).getEventType(), events.get(1).getEventType()); + assertEquals(events.get(0).getEventVersion(), events.get(1).getEventVersion()); + assertEquals(STREAM, events.get(0).getStreamId()); + assertEquals(STREAM, events.get(1).getStreamId()); + assertEquals(STREAM, events.get(2).getStreamId()); + assertEquals("first", events.get(0).getData().getChangeId()); + assertEquals("journal-only", events.get(1).getData().getChangeId()); + assertEquals("normal", events.get(2).getData().getChangeId()); + assertFalse(events.get(0).isAcknowledged()); + assertFalse(events.get(1).isAcknowledged()); + assertFalse(events.get(2).isAcknowledged()); + } + + private static AuditEntry entry(String changeId, AuditEntry.Status status) { + return AuditEntryTestFactory.createTestAuditEntry(changeId, status, AuditTxType.NON_TX, (Class) null); + } +} diff --git a/community/flamingock-couchbase-auditstore/src/test/java/io/flamingock/store/couchbase/internal/CouchbaseAuditPersistenceJournalTest.java b/community/flamingock-couchbase-auditstore/src/test/java/io/flamingock/store/couchbase/internal/CouchbaseAuditPersistenceJournalTest.java index 50503c4fe..e68cba12d 100644 --- a/community/flamingock-couchbase-auditstore/src/test/java/io/flamingock/store/couchbase/internal/CouchbaseAuditPersistenceJournalTest.java +++ b/community/flamingock-couchbase-auditstore/src/test/java/io/flamingock/store/couchbase/internal/CouchbaseAuditPersistenceJournalTest.java @@ -29,7 +29,6 @@ import io.flamingock.internal.common.core.journal.JournalEventType; import io.flamingock.internal.common.couchbase.CouchbaseCollectionHelper; import io.flamingock.internal.common.couchbase.CouchbaseJournalEventMapper; -import io.flamingock.internal.core.configuration.community.CommunityConfiguration; import io.flamingock.internal.core.journal.JournalEventSequencer; import io.flamingock.internal.core.journal.JournalEventSequencerFactory; import io.flamingock.internal.core.transaction.TransactionManager; @@ -236,16 +235,40 @@ void failedWriteLeavesNoGapInTheStream() { "the stream must stay contiguous, so consumers can tell in-flight from lost"); } + @Test + @DisplayName("shared journal writer stages an event without changing audit current state") + void journalWriterStagesEventWithoutAuditState() { + FeatureFlag.enable(Features.JOURNAL_EVENTS); + journalEventStore.initialize(true, SCOPE_NAME, JOURNAL_COLLECTION); + JournalEventSequencer sequencer = new JournalEventSequencerFactory(journalEventStore).forStream(STREAM_ID); + CouchbaseJournalWriter writer = new CouchbaseJournalWriter(journalEventStore); + JournalEvent event = sequencer.newEvent(auditEntry("journal-only-change")); + + txWrapper.wrapExecution(new io.flamingock.internal.core.context.BasicRuntimeContext("journal-only"), context -> + writer.write(context.getContext().getRequiredDependencyValue(TransactionAttemptContext.class), event)); + sequencer.confirm(); + + assertEquals(1, storedEvents().size()); + assertEquals(1L, storedEvents().get(0).getStreamSequence()); + assertFalse(CouchbaseCollectionHelper.collectionExists(cluster, BUCKET_NAME, SCOPE_NAME, AUDIT_COLLECTION)); + } + // ----------------------------- helpers ----------------------------- private CouchbaseAuditPersistence persistenceFor(CouchbaseAuditor auditor) { + if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + journalEventStore.initialize(true, SCOPE_NAME, JOURNAL_COLLECTION); + } return persistenceFor(auditor, new JournalEventSequencerFactory(journalEventStore).forStream(STREAM_ID)); } private CouchbaseAuditPersistence persistenceFor(CouchbaseAuditor auditor, JournalEventSequencer sequencer) { + if (auditor == this.auditor) { + auditor.initialize(true, SCOPE_NAME, AUDIT_COLLECTION); + } CouchbaseAuditPersistence persistence = new CouchbaseAuditPersistence( - new CommunityConfiguration(), auditor, journalEventStore, sequencer, txWrapper, - SCOPE_NAME, AUDIT_COLLECTION, JOURNAL_COLLECTION, true); + auditor, sequencer, txWrapper, + new CouchbaseJournalWriter(journalEventStore)); persistence.initialize(RunnerId.generate()); return persistence; } diff --git a/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/DynamoDBAuditStore.java b/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/DynamoDBAuditStore.java index 4474d68ab..a600889c0 100644 --- a/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/DynamoDBAuditStore.java +++ b/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/DynamoDBAuditStore.java @@ -17,11 +17,13 @@ import io.flamingock.internal.common.core.audit.AuditPersistenceFactory; import io.flamingock.internal.common.core.audit.AuditReader; +import io.flamingock.internal.common.core.audit.AuditHistoryAppender; +import io.flamingock.internal.common.core.audit.JournalHistoryAppender; import io.flamingock.internal.common.core.context.ContextResolver; import io.flamingock.internal.common.core.error.FlamingockException; import io.flamingock.internal.common.core.feature.Features; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; import io.flamingock.internal.core.external.store.CommunityAuditStore; +import io.flamingock.internal.core.external.store.HistoryAppenderProvider; import io.flamingock.internal.core.external.store.audit.community.CommunityAuditPersistence; import io.flamingock.internal.core.external.store.lock.community.CommunityLockService; import io.flamingock.internal.core.journal.JournalEventSequencer; @@ -35,17 +37,18 @@ import io.flamingock.store.dynamodb.internal.DynamoDBAuditPersistence; import io.flamingock.store.dynamodb.internal.DynamoDBAuditRepository; import io.flamingock.store.dynamodb.internal.DynamoDBJournalEventStore; +import io.flamingock.store.dynamodb.internal.DynamoDBJournalWriter; +import io.flamingock.internal.util.dynamodb.DynamoDBUtil; +import software.amazon.awssdk.enhanced.dynamodb.model.TransactWriteItemsEnhancedRequest; import io.flamingock.store.dynamodb.internal.DynamoDBLockService; import io.flamingock.externalsystem.dynamodb.api.DynamoDBExternalSystem; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -public class DynamoDBAuditStore implements CommunityAuditStore { +public class DynamoDBAuditStore implements CommunityAuditStore, HistoryAppenderProvider { private final DynamoDBExternalSystem targetSystem; private RunnerId runnerId; - private CommunityConfigurable communityConfiguration; - private DynamoDBAuditPersistence persistence; private DynamoDBLockService lockService; private final DynamoDbClient client; private String auditRepositoryName = CommunityPersistenceConstants.DEFAULT_AUDIT_STORE_NAME; @@ -57,6 +60,7 @@ public class DynamoDBAuditStore implements CommunityAuditStore { private DynamoDBAuditRepository auditRepository; private DynamoDBJournalEventStore journalEventStore; private JournalEventSequencerFactory journalEventSequencerFactory; + private DynamoDBJournalWriter journalWriter; private DynamoDBAuditStore(DynamoDBExternalSystem targetSystem) { this.targetSystem = targetSystem; @@ -114,7 +118,6 @@ public DynamoDBAuditStore withAutoCreate(boolean autoCreate) { @Override public void initialize(ContextResolver baseContext) { runnerId = baseContext.getRequiredDependencyValue(RunnerId.class); - communityConfiguration = baseContext.getRequiredDependencyValue(CommunityConfigurable.class); auditRepository = new DynamoDBAuditRepository(client, auditRepositoryName, readCapacityUnits, writeCapacityUnits); journalEventStore = new DynamoDBJournalEventStore( client, @@ -123,6 +126,7 @@ public void initialize(ContextResolver baseContext) { writeCapacityUnits ); journalEventSequencerFactory = new JournalEventSequencerFactory(journalEventStore); + journalWriter = new DynamoDBJournalWriter(journalEventStore); lockService = new DynamoDBLockService( client, @@ -131,34 +135,59 @@ public void initialize(ContextResolver baseContext) { writeCapacityUnits, TimeService.getDefault() ); - lockService.initialize(autoCreate); this.validate(); + auditRepository.initialize(autoCreate); + lockService.initialize(autoCreate); + if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + journalEventStore.initialize(autoCreate); + } } @Override public AuditPersistenceFactory getPersistenceFactory() { return stageId -> { - auditRepository.initialize(autoCreate); - if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { - journalEventStore.initialize(autoCreate); - } - JournalEventSequencer journalEventSequencer = journalEventSequencerFactory.forStream(stageId); - persistence = new DynamoDBAuditPersistence( - communityConfiguration, + JournalEventSequencer journalEventSequencer = FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false) + ? journalEventSequencerFactory.forStream(stageId) : null; + DynamoDBAuditPersistence persistence = new DynamoDBAuditPersistence( auditRepository, - journalEventStore, journalEventSequencer, targetSystem.getTxWrapper(), - autoCreate + journalWriter ); persistence.initialize(runnerId); return persistence; }; } + @Override + public AuditHistoryAppender getAuditHistoryAppender() { + return auditRepository::append; + } + + @Override + public JournalHistoryAppender getJournalHistoryAppender() { + return (streamId, entry) -> { + if (!FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + throw new IllegalStateException("Journal events must be enabled to write journal history"); + } + JournalEventSequencer sequencer = journalEventSequencerFactory.forStream(streamId); + synchronized (sequencer) { + try { + TransactWriteItemsEnhancedRequest.Builder builder = TransactWriteItemsEnhancedRequest.builder(); + io.flamingock.internal.util.Result result = journalWriter.write(builder, sequencer, entry); + new DynamoDBUtil(client).getEnhancedClient().transactWriteItems(builder.build()); + sequencer.confirm(); + return result; + } catch (RuntimeException | Error exception) { + sequencer.markWriteOutcomeUncertain(); + throw exception; + } + } + }; + } + @Override public AuditReader getAuditReader() { - auditRepository.initialize(autoCreate); return () -> auditRepository.getAuditHistory(); } diff --git a/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditPersistence.java b/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditPersistence.java index 68a65e0dd..fe876b213 100644 --- a/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditPersistence.java +++ b/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditPersistence.java @@ -18,15 +18,12 @@ import io.flamingock.internal.common.core.audit.AuditEntry; import io.flamingock.internal.common.core.context.RuntimeContext; import io.flamingock.internal.common.core.feature.Features; -import io.flamingock.internal.common.core.journal.JournalEvent; import io.flamingock.internal.common.core.external.ExecutionWrapper; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; import io.flamingock.internal.core.context.BasicRuntimeContext; import io.flamingock.internal.core.external.store.audit.community.AbstractCommunityAuditPersistence; import io.flamingock.internal.core.journal.JournalEventSequencer; import io.flamingock.internal.util.FeatureFlag; import io.flamingock.internal.util.Result; -import io.flamingock.internal.util.id.RunnerId; import software.amazon.awssdk.enhanced.dynamodb.model.TransactWriteItemsEnhancedRequest; import java.util.List; @@ -34,41 +31,26 @@ public class DynamoDBAuditPersistence extends AbstractCommunityAuditPersistence { private final DynamoDBAuditRepository auditRepository; - private final DynamoDBJournalEventStore journalEventStore; + private final DynamoDBJournalWriter journalWriter; private JournalEventSequencer journalEventSequencer; private final ExecutionWrapper txWrapper; - private final boolean autoCreate; /** * Creates a persistence over explicitly supplied audit, journal and transaction collaborators. * - * @param localConfiguration community configuration * @param auditRepository repository for audits - * @param journalEventStore journal store receiving staged events * @param journalEventSequencer sequencer for the stage journal stream * @param txWrapper transaction wrapper shared with the target system - * @param autoCreate whether missing tables may be created + * @param journalWriter explicitly configured journal writer */ - public DynamoDBAuditPersistence(CommunityConfigurable localConfiguration, - DynamoDBAuditRepository auditRepository, - DynamoDBJournalEventStore journalEventStore, + public DynamoDBAuditPersistence(DynamoDBAuditRepository auditRepository, JournalEventSequencer journalEventSequencer, ExecutionWrapper txWrapper, - boolean autoCreate) { - super(localConfiguration); + DynamoDBJournalWriter journalWriter) { this.auditRepository = auditRepository; - this.journalEventStore = journalEventStore; + this.journalWriter = journalWriter; this.journalEventSequencer = journalEventSequencer; this.txWrapper = txWrapper; - this.autoCreate = autoCreate; - } - - @Override - protected void doInitialize(RunnerId runnerId) { - auditRepository.initialize(autoCreate); - if (isJournalEventsEnabled()) { - journalEventStore.initialize(autoCreate); - } } @Override @@ -79,18 +61,24 @@ public List getAuditHistory() { @Override public Result writeEntry(AuditEntry auditEntry) { if (isJournalEventsEnabled()) { - RuntimeContext baseContext = new BasicRuntimeContext("write-changeState-" + auditEntry.getChangeId()); - Result result = txWrapper.wrapExecution(baseContext, runtimeContext -> { - TransactWriteItemsEnhancedRequest.Builder builder = runtimeContext.getContext() - .getRequiredDependencyValue(TransactWriteItemsEnhancedRequest.Builder.class); - JournalEvent journalEvent = journalEventSequencer.newEvent(auditEntry); - journalEventStore.contributeToTransaction(builder, journalEvent); - return auditRepository.contributeToTransaction(builder, auditEntry); - }); - journalEventSequencer.confirm(); - return result; + synchronized (journalEventSequencer) { + try { + RuntimeContext baseContext = new BasicRuntimeContext("write-changeState-" + auditEntry.getChangeId()); + Result result = txWrapper.wrapExecution(baseContext, runtimeContext -> { + TransactWriteItemsEnhancedRequest.Builder builder = runtimeContext.getContext() + .getRequiredDependencyValue(TransactWriteItemsEnhancedRequest.Builder.class); + journalWriter.write(builder, journalEventSequencer, auditEntry); + return auditRepository.contributeToTransaction(builder, auditEntry); + }); + journalEventSequencer.confirm(); + return result; + } catch (RuntimeException | Error exception) { + journalEventSequencer.markWriteOutcomeUncertain(); + throw exception; + } + } } - return auditRepository.writeEntry(auditEntry); + return auditRepository.append(auditEntry); } private static boolean isJournalEventsEnabled() { diff --git a/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditRepository.java b/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditRepository.java index d6a52fb32..f51c8be84 100644 --- a/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditRepository.java +++ b/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditRepository.java @@ -116,7 +116,7 @@ private boolean hasAttribute(AttributeDefinition attribute, String name, ScalarA * @param auditEntry entry to append * @return successful write result */ - Result writeEntry(AuditEntry auditEntry) { + public Result append(AuditEntry auditEntry) { AuditEntryEntity entity = new AuditEntryEntity(auditEntry); logger.debug("Saving audit entry with key {}", entity.getPartitionKey()); table.putItem(PutItemEnhancedRequest.builder(AuditEntryEntity.class) diff --git a/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBJournalWriter.java b/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBJournalWriter.java new file mode 100644 index 000000000..3ca4b977c --- /dev/null +++ b/community/flamingock-dynamodb-auditstore/src/main/java/io/flamingock/store/dynamodb/internal/DynamoDBJournalWriter.java @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Flamingock (https://www.flamingock.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.flamingock.store.dynamodb.internal; + +import io.flamingock.internal.common.core.audit.AuditEntry; +import io.flamingock.internal.core.journal.JournalEventSequencer; +import io.flamingock.internal.util.Result; +import software.amazon.awssdk.enhanced.dynamodb.model.TransactWriteItemsEnhancedRequest; + +/** Stages journal events in a caller-owned transaction; the caller confirms only after commit. */ +public class DynamoDBJournalWriter { + private final DynamoDBJournalEventStore store; + + public DynamoDBJournalWriter(DynamoDBJournalEventStore store) { + this.store = store; + } + + public Result write(TransactWriteItemsEnhancedRequest.Builder builder, + JournalEventSequencer sequencer, AuditEntry entry) { + return store.contributeToTransaction(builder, sequencer.newEvent(entry)); + } +} diff --git a/community/flamingock-dynamodb-auditstore/src/test/java/io/flamingock/store/dynamodb/DynamoDBAuditStoreJournalTest.java b/community/flamingock-dynamodb-auditstore/src/test/java/io/flamingock/store/dynamodb/DynamoDBAuditStoreJournalTest.java index 435a251f9..d8ccba055 100644 --- a/community/flamingock-dynamodb-auditstore/src/test/java/io/flamingock/store/dynamodb/DynamoDBAuditStoreJournalTest.java +++ b/community/flamingock-dynamodb-auditstore/src/test/java/io/flamingock/store/dynamodb/DynamoDBAuditStoreJournalTest.java @@ -26,6 +26,7 @@ import io.flamingock.internal.core.external.store.audit.community.CommunityAuditPersistence; import io.flamingock.internal.util.FeatureFlag; import io.flamingock.internal.util.dynamodb.DynamoDBUtil; +import io.flamingock.internal.util.dynamodb.entities.AuditEntryEntity; import io.flamingock.internal.util.dynamodb.entities.journal.DynamoDBJournalEventMapper; import io.flamingock.internal.util.dynamodb.entities.journal.JournalEventEntity; import io.flamingock.internal.util.id.RunnerId; @@ -86,6 +87,66 @@ void tearDown() { } } + @Test + @DisplayName("historical append preserves the legacy key without creating a journal table") + void historicalAppenderWorksWithJournalDisabled() { + SimpleContext context = newContext(); + DynamoDBAuditStore store = initializeStore(context); + AuditEntry entry = auditEntry("history-only"); + store.getAuditHistoryAppender().append(entry); + + List rows = new DynamoDBUtil(client).getEnhancedClient() + .table(auditTableName, TableSchema.fromBean(AuditEntryEntity.class)) + .scan().items().stream().collect(Collectors.toList()); + assertEquals(1, rows.size()); + assertEquals(new AuditEntryEntity(entry).getPartitionKey(), rows.get(0).getPartitionKey()); + assertFalse(client.listTables().tableNames().contains(journalTableName)); + assertThrows(IllegalStateException.class, + () -> store.getJournalHistoryAppender().append(STAGE_ONE, auditEntry("rejected"))); + assertFalse(client.listTables().tableNames().contains(journalTableName)); + } + + @Test + @DisplayName("journal-only history uses independent per-stream transactions without audit state") + void journalHistoryWritesOnlyEventsAndReseedsStreams() { + FeatureFlag.enable(Features.JOURNAL_EVENTS); + DynamoDBAuditStore store = initializeStore(newContext()); + store.getJournalHistoryAppender().append(STAGE_ONE, auditEntry("first")); + store.getJournalHistoryAppender().append(STAGE_TWO, auditEntry("other")); + store.getJournalHistoryAppender().append(STAGE_ONE, auditEntry("second")); + + assertEquals(3, storedEvents().size()); + assertTrue(storedEvents().stream().anyMatch(event -> STAGE_ONE.equals(event.getStreamId()) + && event.getStreamSequence() == 2L && "second".equals(event.getData().getChangeId()))); + assertTrue(storedEvents().stream().anyMatch(event -> STAGE_TWO.equals(event.getStreamId()) + && event.getStreamSequence() == 1L)); + assertTrue(store.getAuditReader().getAuditHistory().isEmpty()); + } + + @Test + @DisplayName("persistence created before journal-only writes shares the stream sequence") + void persistenceCreatedFirstSharesJournalHistorySequence() { + FeatureFlag.enable(Features.JOURNAL_EVENTS); + DynamoDBAuditStore store = initializeStore(newContext()); + CommunityAuditPersistence persistence = store.getPersistenceFactory().get(STAGE_ONE); + + store.getJournalHistoryAppender().append(STAGE_ONE, auditEntry("history-first")); + store.getJournalHistoryAppender().append(STAGE_ONE, auditEntry("history-second")); + persistence.writeEntry(auditEntry("normal-third")); + + List> events = storedEvents().stream() + .filter(event -> STAGE_ONE.equals(event.getStreamId())) + .collect(Collectors.toList()); + assertEquals(3, events.size()); + assertTrue(events.stream().anyMatch(event -> event.getStreamSequence() == 1L + && "history-first".equals(event.getData().getChangeId()))); + assertTrue(events.stream().anyMatch(event -> event.getStreamSequence() == 2L + && "history-second".equals(event.getData().getChangeId()))); + assertTrue(events.stream().anyMatch(event -> event.getStreamSequence() == 3L + && "normal-third".equals(event.getData().getChangeId()))); + assertEquals(1, store.getAuditReader().getAuditHistory().size()); + } + @Test @DisplayName("journal-enabled stores create independent persistence streams for each stage") void journalEnabledStoreCreatesPerStagePersistenceStreams() { @@ -141,6 +202,7 @@ void journalEnabledAutoCreatesJournalForAuditOnlyInstallation() { assertTrue(auditStore.getAuditReader().getAuditHistory().isEmpty()); assertTrue(client.listTables().tableNames().contains(auditTableName)); + assertTrue(client.listTables().tableNames().contains(journalTableName)); auditStore.getPersistenceFactory().get(STAGE_ONE).writeEntry(auditEntry("audit-only-change")); auditStore.getPersistenceFactory().get(STAGE_TWO); @@ -166,8 +228,8 @@ void nonPositiveCapacitiesFailDynamoDbSetup() { .withJournalRepositoryName(journalTableName) .withWriteCapacityUnits(-1L); - assertThrows(DynamoDbException.class, () -> invalidReadCapacity.initialize(context)); - assertThrows(DynamoDbException.class, () -> invalidWriteCapacity.initialize(context)); + assertThrows(FlamingockException.class, () -> invalidReadCapacity.initialize(context)); + assertThrows(FlamingockException.class, () -> invalidWriteCapacity.initialize(context)); assertTrue(client.listTables().tableNames().isEmpty()); } @@ -181,11 +243,9 @@ void manualSetupValidatesAuditTableBeforeJournalSetup() { .withLockRepositoryName(lockTableName) .withJournalRepositoryName(journalTableName) .withAutoCreate(false); - auditStore.initialize(context); - IllegalStateException exception = assertThrows( IllegalStateException.class, - () -> auditStore.getPersistenceFactory().get(STAGE_ONE)); + () -> auditStore.initialize(context)); assertTrue(exception.getMessage().contains("audit table")); assertFalse(client.listTables().tableNames().contains(journalTableName)); @@ -221,6 +281,18 @@ void journalRepositoryNameMustBeUnique() { .withJournalRepositoryName(auditTableName); assertThrows(FlamingockException.class, () -> auditStore.initialize(context)); + + DynamoDBAuditStore lockCollision = DynamoDBAuditStore.from(targetSystem) + .withAuditRepositoryName(auditTableName) + .withLockRepositoryName(lockTableName) + .withJournalRepositoryName(" " + lockTableName.toUpperCase() + " "); + assertThrows(FlamingockException.class, () -> lockCollision.initialize(context)); + + DynamoDBAuditStore missingJournalName = DynamoDBAuditStore.from(targetSystem) + .withAuditRepositoryName(auditTableName) + .withLockRepositoryName(lockTableName) + .withJournalRepositoryName(null); + assertThrows(FlamingockException.class, () -> missingJournalName.initialize(context)); } private List> storedEvents() { diff --git a/community/flamingock-dynamodb-auditstore/src/test/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditPersistenceJournalTest.java b/community/flamingock-dynamodb-auditstore/src/test/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditPersistenceJournalTest.java index 08d0bd189..f1fd3cc87 100644 --- a/community/flamingock-dynamodb-auditstore/src/test/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditPersistenceJournalTest.java +++ b/community/flamingock-dynamodb-auditstore/src/test/java/io/flamingock/store/dynamodb/internal/DynamoDBAuditPersistenceJournalTest.java @@ -22,7 +22,6 @@ import io.flamingock.internal.common.core.feature.Features; import io.flamingock.internal.common.core.journal.JournalEvent; import io.flamingock.internal.common.core.journal.JournalEventType; -import io.flamingock.internal.core.configuration.community.CommunityConfiguration; import io.flamingock.internal.core.journal.JournalEventSequencer; import io.flamingock.internal.core.journal.JournalEventSequencerFactory; import io.flamingock.internal.core.transaction.TransactionManager; @@ -245,18 +244,19 @@ void canceledTransactionLeavesNoGapForRetry() { } private DynamoDBAuditPersistence persistenceFor(JournalEventSequencer sequencer) { + DynamoDBAuditRepository repository = new DynamoDBAuditRepository(client, auditTableName, 5L, 5L); + repository.initialize(true); DynamoDBAuditPersistence persistence = new DynamoDBAuditPersistence( - new CommunityConfiguration(), - new DynamoDBAuditRepository(client, auditTableName, 5L, 5L), - journalEventStore, - sequencer, - txWrapper, - true); + repository, sequencer, txWrapper, + new DynamoDBJournalWriter(journalEventStore)); persistence.initialize(io.flamingock.internal.util.id.RunnerId.generate()); return persistence; } private JournalEventSequencer newSequencer() { + if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + journalEventStore.initialize(true); + } return new JournalEventSequencerFactory(journalEventStore).forStream(STREAM_ID); } diff --git a/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/MongoDBReactiveAuditStore.java b/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/MongoDBReactiveAuditStore.java index 91c3c33bc..39e4b0947 100644 --- a/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/MongoDBReactiveAuditStore.java +++ b/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/MongoDBReactiveAuditStore.java @@ -23,11 +23,15 @@ import io.flamingock.externalsystem.mongodb.reactive.api.MongoDBReactiveExternalSystem; import io.flamingock.internal.common.core.audit.AuditPersistenceFactory; import io.flamingock.internal.common.core.audit.AuditReader; +import io.flamingock.internal.common.core.audit.AuditHistoryAppender; +import io.flamingock.internal.common.core.audit.JournalHistoryAppender; +import io.flamingock.internal.common.core.context.RuntimeContext; +import io.flamingock.internal.core.context.BasicRuntimeContext; import io.flamingock.internal.common.core.context.ContextResolver; import io.flamingock.internal.common.core.error.FlamingockException; import io.flamingock.internal.common.core.feature.Features; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; import io.flamingock.internal.core.external.store.CommunityAuditStore; +import io.flamingock.internal.core.external.store.HistoryAppenderProvider; import io.flamingock.internal.core.external.store.audit.community.CommunityAuditPersistence; import io.flamingock.internal.core.external.store.lock.community.CommunityLockService; import io.flamingock.internal.core.journal.JournalEventSequencer; @@ -39,6 +43,7 @@ import io.flamingock.store.mongodb.reactive.internal.MongoDBReactiveAuditPersistence; import io.flamingock.store.mongodb.reactive.internal.MongoDBReactiveAuditRepository; import io.flamingock.store.mongodb.reactive.internal.MongoDBReactiveJournalEventStore; +import io.flamingock.store.mongodb.reactive.internal.MongoDBReactiveJournalWriter; import io.flamingock.store.mongodb.reactive.internal.MongoDBReactiveLockService; import io.flamingock.internal.common.core.external.ExecutionWrapper; @@ -51,13 +56,11 @@ import static io.flamingock.internal.util.constants.CommunityPersistenceConstants.DEFAULT_AUDIT_STORE_NAME; import static io.flamingock.internal.util.constants.CommunityPersistenceConstants.DEFAULT_LOCK_STORE_NAME; -public class MongoDBReactiveAuditStore implements CommunityAuditStore { +public class MongoDBReactiveAuditStore implements CommunityAuditStore, HistoryAppenderProvider { private final MongoDBReactiveExternalSystem mongoDBTargetSystem; protected RunnerId runnerId; - private CommunityConfigurable communityConfiguration; - private CommunityAuditPersistence persistence; private MongoDBReactiveLockService lockService; private MongoDatabase database; private String auditRepositoryName = DEFAULT_AUDIT_STORE_NAME; @@ -70,6 +73,7 @@ public class MongoDBReactiveAuditStore implements CommunityAuditStore { private MongoDBReactiveAuditRepository auditRepository; private MongoDBReactiveJournalEventStore journalEventStore; private JournalEventSequencerFactory journalEventSequencerFactory; + private MongoDBReactiveJournalWriter journalWriter; private MongoDBReactiveAuditStore(MongoDBReactiveExternalSystem mongoDBTargetSystem) { this.mongoDBTargetSystem = mongoDBTargetSystem; @@ -132,7 +136,6 @@ public MongoDBReactiveAuditStore withAutoCreate(boolean autoCreate) { @Override public void initialize(ContextResolver baseContext) { runnerId = baseContext.getRequiredDependencyValue(RunnerId.class); - communityConfiguration = baseContext.getRequiredDependencyValue(CommunityConfigurable.class); database = mongoDBTargetSystem.getMongoDatabase(); this.validate(); @@ -141,6 +144,7 @@ public void initialize(ContextResolver baseContext) { journalEventStore = new MongoDBReactiveJournalEventStore( database, journalRepositoryName, readConcern, readPreference, writeConcern); journalEventSequencerFactory = new JournalEventSequencerFactory(journalEventStore); + journalWriter = new MongoDBReactiveJournalWriter(journalEventStore); lockService = new MongoDBReactiveLockService( database, @@ -151,38 +155,66 @@ public void initialize(ContextResolver baseContext) { TimeService.getDefault() ); lockService.initialize(autoCreate); + auditRepository.initialize(autoCreate); + if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + journalEventStore.initialize(autoCreate); + } } @Override public AuditPersistenceFactory getPersistenceFactory() { return stageId -> { - auditRepository.initialize(autoCreate); - if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { - journalEventStore.initialize(autoCreate); - } - JournalEventSequencer journalEventSequencer = journalEventSequencerFactory.forStream(stageId); + JournalEventSequencer journalEventSequencer = FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false) + ? journalEventSequencerFactory.forStream(stageId) : null; boolean supportsTransactions = mongoDBTargetSystem.supportsTransactions(); ExecutionWrapper txWrapper = supportsTransactions ? mongoDBTargetSystem.getTxWrapper() : null; MongoDBReactiveAuditPersistence stagePersistence = new MongoDBReactiveAuditPersistence( - communityConfiguration, auditRepository, journalEventStore, journalEventSequencer, supportsTransactions, - txWrapper, - autoCreate + txWrapper ); stagePersistence.initialize(runnerId); - if (persistence == null) { - persistence = stagePersistence; - } return stagePersistence; }; } + @Override + public AuditHistoryAppender getAuditHistoryAppender() { + return auditRepository::append; + } + + @Override + public JournalHistoryAppender getJournalHistoryAppender() { + return (streamId, entry) -> { + if (!FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + throw new IllegalStateException("Journal events must be enabled to write journal history"); + } + JournalEventSequencer sequencer = journalEventSequencerFactory.forStream(streamId); + synchronized (sequencer) { + try { + if (!mongoDBTargetSystem.supportsTransactions()) { + return journalWriter.write(sequencer, entry); + } + ExecutionWrapper wrapper = mongoDBTargetSystem.getTxWrapper(); + RuntimeContext context = new BasicRuntimeContext("write-journalHistory-" + entry.getChangeId()); + io.flamingock.internal.util.Result result = wrapper.wrapExecution(context, runtime -> { + ClientSession session = runtime.getContext().getRequiredDependencyValue(ClientSession.class); + return journalWriter.write(session, sequencer, entry); + }); + sequencer.confirm(); + return result; + } catch (RuntimeException | Error failure) { + sequencer.markWriteOutcomeUncertain(); + throw failure; + } + } + }; + } + @Override public AuditReader getAuditReader() { - auditRepository.initialize(autoCreate); return () -> auditRepository.getAuditHistory(); } diff --git a/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistence.java b/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistence.java index eed5d1072..1f3720f2d 100644 --- a/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistence.java +++ b/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistence.java @@ -19,15 +19,12 @@ import io.flamingock.internal.common.core.audit.AuditEntry; import io.flamingock.internal.common.core.context.RuntimeContext; import io.flamingock.internal.common.core.feature.Features; -import io.flamingock.internal.common.core.journal.JournalEvent; import io.flamingock.internal.common.core.external.ExecutionWrapper; import io.flamingock.internal.core.context.BasicRuntimeContext; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; import io.flamingock.internal.core.external.store.audit.community.AbstractCommunityAuditPersistence; import io.flamingock.internal.core.journal.JournalEventSequencer; import io.flamingock.internal.util.FeatureFlag; import io.flamingock.internal.util.Result; -import io.flamingock.internal.util.id.RunnerId; import java.util.List; import java.util.Objects; @@ -36,34 +33,29 @@ public class MongoDBReactiveAuditPersistence extends AbstractCommunityAuditPersi private final MongoDBReactiveAuditRepository auditRepository; private final MongoDBReactiveJournalEventStore journalEventStore; + private final MongoDBReactiveJournalWriter journalWriter; private final JournalEventSequencer journalEventSequencer; private final boolean supportsTransactions; private final ExecutionWrapper txWrapper; - private final boolean autoCreate; /** - * @param localConfiguration local Community configuration * @param auditRepository audit state repository * @param journalEventStore journal event store * @param journalEventSequencer sequencer for this persistence stream * @param supportsTransactions whether journal and audit writes must share a MongoDB transaction * @param txWrapper transaction wrapper; must be non-null if and only if transactions are supported - * @param autoCreate whether required MongoDB collections and indexes may be created */ - public MongoDBReactiveAuditPersistence(CommunityConfigurable localConfiguration, - MongoDBReactiveAuditRepository auditRepository, + public MongoDBReactiveAuditPersistence(MongoDBReactiveAuditRepository auditRepository, MongoDBReactiveJournalEventStore journalEventStore, JournalEventSequencer journalEventSequencer, boolean supportsTransactions, - ExecutionWrapper txWrapper, - boolean autoCreate) { - super(localConfiguration); + ExecutionWrapper txWrapper) { this.auditRepository = auditRepository; this.journalEventStore = journalEventStore; + this.journalWriter = new MongoDBReactiveJournalWriter(journalEventStore); this.journalEventSequencer = journalEventSequencer; this.supportsTransactions = supportsTransactions; this.txWrapper = validateTransactionWrapper(supportsTransactions, txWrapper); - this.autoCreate = autoCreate; } private static ExecutionWrapper validateTransactionWrapper(boolean supportsTransactions, @@ -80,14 +72,6 @@ private static ExecutionWrapper validateTransactionWrapper(boolean supportsTrans return null; } - @Override - protected void doInitialize(RunnerId runnerId) { - auditRepository.initialize(autoCreate); - if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { - journalEventStore.initialize(autoCreate); - } - } - @Override public List getAuditHistory() { return auditRepository.getAuditHistory(); @@ -103,19 +87,22 @@ public Result writeEntry(AuditEntry auditEntry) { throw new IllegalStateException("MongoDB reactive journal writes require a sequencer"); } - if (!supportsTransactions) { - return writeJournalAndAuditWithoutTransaction(auditEntry); + synchronized (journalEventSequencer) { + try { + return supportsTransactions ? writeJournalAndAuditInTransaction(auditEntry) + : writeJournalAndAuditWithoutTransaction(auditEntry); + } catch (RuntimeException | Error failure) { + journalEventSequencer.markWriteOutcomeUncertain(); + throw failure; + } } - - return writeJournalAndAuditInTransaction(auditEntry); } private Result writeJournalAndAuditInTransaction(AuditEntry auditEntry) { RuntimeContext baseContext = new BasicRuntimeContext("write-changeState-" + auditEntry.getChangeId()); Result result = txWrapper.wrapExecution(baseContext, runtimeContext -> { ClientSession clientSession = runtimeContext.getContext().getRequiredDependencyValue(ClientSession.class); - JournalEvent journalEvent = journalEventSequencer.newEvent(auditEntry); - journalEventStore.append(clientSession, journalEvent); + journalWriter.write(clientSession, journalEventSequencer, auditEntry); return auditRepository.save(clientSession, auditEntry); }); @@ -131,9 +118,7 @@ private Result writeJournalAndAuditInTransaction(AuditEntry auditEntry) { * audit state without its corresponding historical event. */ private Result writeJournalAndAuditWithoutTransaction(AuditEntry auditEntry) { - JournalEvent journalEvent = journalEventSequencer.newEvent(auditEntry); - journalEventStore.append(journalEvent); - journalEventSequencer.confirm(); + journalWriter.write(journalEventSequencer, auditEntry); return auditRepository.save(auditEntry); } } diff --git a/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditRepository.java b/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditRepository.java index 1c6579fca..0739c1391 100644 --- a/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditRepository.java +++ b/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditRepository.java @@ -137,7 +137,7 @@ Result save(AuditEntry auditEntry) { * @param auditEntry entry to append or replace * @return successful write result; driver failures are propagated */ - Result append(AuditEntry auditEntry) { + public Result append(AuditEntry auditEntry) { Bson filter = Filters.and( Filters.eq(KEY_EXECUTION_ID, auditEntry.getExecutionId()), Filters.eq(KEY_CHANGE_ID, auditEntry.getChangeId()), diff --git a/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveJournalWriter.java b/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveJournalWriter.java new file mode 100644 index 000000000..4e877783b --- /dev/null +++ b/community/flamingock-mongodb-reactive-auditstore/src/main/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveJournalWriter.java @@ -0,0 +1,43 @@ +/* + * Copyright 2026 Flamingock (https://www.flamingock.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.flamingock.store.mongodb.reactive.internal; + +import com.mongodb.reactivestreams.client.ClientSession; +import io.flamingock.internal.common.core.audit.AuditEntry; +import io.flamingock.internal.common.core.journal.JournalEvent; +import io.flamingock.internal.core.journal.JournalEventSequencer; +import io.flamingock.internal.util.Result; + +/** Shares event construction and persistence between normal and history-only writes. */ +public class MongoDBReactiveJournalWriter { + private final MongoDBReactiveJournalEventStore store; + + public MongoDBReactiveJournalWriter(MongoDBReactiveJournalEventStore store) { + this.store = store; + } + + public Result write(ClientSession session, JournalEventSequencer sequencer, AuditEntry entry) { + JournalEvent event = sequencer.newEvent(entry); + return store.append(session, event); + } + + public Result write(JournalEventSequencer sequencer, AuditEntry entry) { + JournalEvent event = sequencer.newEvent(entry); + Result result = store.append(event); + sequencer.confirm(); + return result; + } +} diff --git a/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/MongoDBReactiveAuditStoreJournalTest.java b/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/MongoDBReactiveAuditStoreJournalTest.java index 2ed577fb6..d4279c09e 100644 --- a/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/MongoDBReactiveAuditStoreJournalTest.java +++ b/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/MongoDBReactiveAuditStoreJournalTest.java @@ -94,6 +94,8 @@ void tearDown() { @DisplayName("the flag-off store keeps historical audit rows and does not create the journal") void flagOffKeepsHistoricalAuditRows() { MongoDBReactiveAuditStore auditStore = initializeStore(); + assertTrue(collectionExists(AUDIT_COLLECTION)); + assertFalse(collectionExists(JOURNAL_COLLECTION)); CommunityAuditPersistence persistence = auditStore.getPersistenceFactory().get("stage-one"); persistence.writeEntry(auditEntry("change-1", AuditEntry.Status.STARTED)); @@ -103,11 +105,62 @@ void flagOffKeepsHistoricalAuditRows() { assertFalse(collectionExists(JOURNAL_COLLECTION)); } + @Test + @DisplayName("history appender preserves legacy rows without creating a journal") + void historyAppenderPreservesLegacyRows() { + MongoDBReactiveAuditStore store = initializeStore(); + store.getAuditHistoryAppender().append(auditEntry("history-change", AuditEntry.Status.STARTED)); + store.getAuditHistoryAppender().append(auditEntry("history-change", AuditEntry.Status.APPLIED)); + + assertTrue(collectionExists(AUDIT_COLLECTION)); + assertTrue(indexesByName(AUDIT_COLLECTION).values().stream() + .anyMatch(index -> Boolean.TRUE.equals(index.getBoolean("unique")))); + assertEquals(2, store.getAuditReader().getAuditHistory().size()); + assertFalse(collectionExists(JOURNAL_COLLECTION)); + } + + @Test + @DisplayName("journal history rejects writes when the flag is disabled") + void journalHistoryRejectsDisabledFlag() { + MongoDBReactiveAuditStore store = initializeStore(); + assertThrows(IllegalStateException.class, () -> store.getJournalHistoryAppender() + .append("history-stage", auditEntry("history-change", AuditEntry.Status.APPLIED))); + assertFalse(collectionExists(JOURNAL_COLLECTION)); + } + + @Test + @DisplayName("journal history writes its own stream without changing current audit state") + void journalHistoryWritesWithoutAuditState() { + FeatureFlag.enable(Features.JOURNAL_EVENTS); + MongoDBReactiveAuditStore store = initializeStore(); + store.getJournalHistoryAppender().append("history-stage", auditEntry("first", AuditEntry.Status.APPLIED)); + store.getJournalHistoryAppender().append("history-stage", auditEntry("second", AuditEntry.Status.APPLIED)); + + assertTrue(store.getAuditReader().getAuditHistory().isEmpty()); + assertEquals(Arrays.asList(1L, 2L), sequences(storedEvents())); + assertTrue(storedEvents().stream().allMatch(event -> "history-stage".equals(event.getStreamId()))); + } + + @Test + void stagePersistenceAndHistoryWriterShareStreamSequence() { + FeatureFlag.enable(Features.JOURNAL_EVENTS); + MongoDBReactiveAuditStore store = initializeStore(); + CommunityAuditPersistence persistence = store.getPersistenceFactory().get("history-stage"); + persistence.writeEntry(auditEntry("first", AuditEntry.Status.APPLIED)); + store.getJournalHistoryAppender().append("history-stage", auditEntry("second", AuditEntry.Status.APPLIED)); + assertEquals(1, store.getAuditReader().getAuditHistory().size()); + persistence.writeEntry(auditEntry("third", AuditEntry.Status.APPLIED)); + assertEquals(Arrays.asList(1L, 2L, 3L), sequences(storedEvents())); + assertEquals(2, store.getAuditReader().getAuditHistory().size()); + } + @Test @DisplayName("journal-enabled store creates independent streams for each stage") void flagOnCreatesStageAwareJournalStreams() { FeatureFlag.enable(Features.JOURNAL_EVENTS); MongoDBReactiveAuditStore auditStore = initializeStore(); + assertTrue(collectionExists(AUDIT_COLLECTION)); + assertTrue(collectionExists(JOURNAL_COLLECTION)); AuditPersistenceFactory factory = auditStore.getPersistenceFactory(); factory.get("stage-one").writeEntry(auditEntry("change-one", AuditEntry.Status.APPLIED)); @@ -141,8 +194,8 @@ void recreatedStagePersistenceReseedsOnlyItsOwnStream() { } @Test - @DisplayName("an audit-only installation remains readable until journal persistence is first accessed") - void auditOnlyUpgradeCreatesJournalSchemaLazily() { + @DisplayName("an audit-only installation gains journal schema during store initialization") + void auditOnlyUpgradeCreatesJournalSchemaDuringInitialization() { MongoDBReactiveAuditStore legacyStore = initializeStore(); CommunityAuditPersistence legacyPersistence = legacyStore.getPersistenceFactory().get("legacy-stage"); legacyPersistence.writeEntry(auditEntry("legacy-started", AuditEntry.Status.STARTED)); @@ -154,7 +207,7 @@ void auditOnlyUpgradeCreatesJournalSchemaLazily() { FeatureFlag.enable(Features.JOURNAL_EVENTS); MongoDBReactiveAuditStore upgradedStore = initializeStore(); assertEquals(Arrays.asList("legacy-applied", "legacy-started"), auditChangeIds(upgradedStore.getAuditReader().getAuditHistory())); - assertFalse(collectionExists(JOURNAL_COLLECTION)); + assertTrue(collectionExists(JOURNAL_COLLECTION)); upgradedStore.getPersistenceFactory().get("upgrade-stage"); @@ -169,8 +222,8 @@ void auditOnlyUpgradeCreatesJournalSchemaLazily() { } @Test - @DisplayName("manual schemas validate at their lazy access boundaries") - void manualSchemaValidationIsLazyAndAcceptsValidSchema() { + @DisplayName("manual schemas validate during store initialization") + void manualSchemaValidationOccursDuringInitialization() { createManualSchema(false); MongoDBReactiveAuditStore validStore = initializeManualStore(); validStore.getAuditReader().getAuditHistory(); @@ -181,16 +234,12 @@ void manualSchemaValidationIsLazyAndAcceptsValidSchema() { PublisherSync.complete(database.drop()); createLockSchema(); - MongoDBReactiveAuditStore missingAuditStore = initializeManualStore(); - assertInvalidSchema(AUDIT_COLLECTION, () -> missingAuditStore.getAuditReader()); - assertInvalidSchema(AUDIT_COLLECTION, () -> missingAuditStore.getPersistenceFactory().get("manual-stage")); + assertInvalidSchema(AUDIT_COLLECTION, () -> initializeManualStore()); PublisherSync.complete(database.drop()); createLockSchema(); createAuditSchema(); - MongoDBReactiveAuditStore missingJournalStore = initializeManualStore(); - missingJournalStore.getAuditReader().getAuditHistory(); - assertInvalidSchema(JOURNAL_COLLECTION, () -> missingJournalStore.getPersistenceFactory().get("manual-stage")); + assertInvalidSchema(JOURNAL_COLLECTION, () -> initializeManualStore()); } @Test @@ -289,7 +338,7 @@ private void assertInvalidJournalIndex(Executable mutation) { } catch (Throwable throwable) { throw new RuntimeException(throwable); } - assertInvalidSchema(JOURNAL_COLLECTION, () -> initializeManualStore().getPersistenceFactory().get("invalid-stage")); + assertInvalidSchema(JOURNAL_COLLECTION, () -> initializeManualStore()); } private void assertInvalidSchema(String collectionName, Executable access) { diff --git a/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistenceJournalTest.java b/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistenceJournalTest.java index 94aa63ccf..293628523 100644 --- a/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistenceJournalTest.java +++ b/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistenceJournalTest.java @@ -29,7 +29,6 @@ import io.flamingock.internal.common.core.feature.Features; import io.flamingock.internal.common.core.journal.JournalEvent; import io.flamingock.internal.common.core.journal.JournalEventType; -import io.flamingock.internal.core.configuration.community.CommunityConfiguration; import io.flamingock.internal.core.journal.JournalEventSequencer; import io.flamingock.internal.core.journal.JournalEventSequencerFactory; import io.flamingock.internal.core.transaction.TransactionManager; @@ -99,6 +98,7 @@ void setUp() { ReadConcern.MAJORITY, ReadPreference.primary(), WriteConcern.MAJORITY.withJournal(true)); txWrapper = new MongoDBReactiveTxWrapper( new TransactionManager<>(() -> PublisherSync.first(mongoClient.startSession()))); + auditRepository.initialize(true); } @AfterEach @@ -112,32 +112,32 @@ void tearDown() { @DisplayName("constructor accepts a transaction wrapper when transactions are supported") void constructorAcceptsWrapperWhenTransactionsAreSupported() { assertDoesNotThrow(() -> new MongoDBReactiveAuditPersistence( - new CommunityConfiguration(), auditRepository, journalEventStore, - mock(JournalEventSequencer.class), true, txWrapper, true)); + auditRepository, journalEventStore, + mock(JournalEventSequencer.class), true, txWrapper)); } @Test @DisplayName("constructor rejects a missing transaction wrapper when transactions are supported") void constructorRejectsMissingWrapperWhenTransactionsAreSupported() { assertThrows(NullPointerException.class, () -> new MongoDBReactiveAuditPersistence( - new CommunityConfiguration(), auditRepository, journalEventStore, - mock(JournalEventSequencer.class), true, null, true)); + auditRepository, journalEventStore, + mock(JournalEventSequencer.class), true, null)); } @Test @DisplayName("constructor accepts no transaction wrapper when transactions are not supported") void constructorAcceptsMissingWrapperWhenTransactionsAreNotSupported() { assertDoesNotThrow(() -> new MongoDBReactiveAuditPersistence( - new CommunityConfiguration(), auditRepository, journalEventStore, - mock(JournalEventSequencer.class), false, null, true)); + auditRepository, journalEventStore, + mock(JournalEventSequencer.class), false, null)); } @Test @DisplayName("constructor rejects a transaction wrapper when transactions are not supported") void constructorRejectsWrapperWhenTransactionsAreNotSupported() { assertThrows(IllegalArgumentException.class, () -> new MongoDBReactiveAuditPersistence( - new CommunityConfiguration(), auditRepository, journalEventStore, - mock(JournalEventSequencer.class), false, txWrapper, true)); + auditRepository, journalEventStore, + mock(JournalEventSequencer.class), false, txWrapper)); } @Test @@ -219,6 +219,7 @@ void auditFailureRollsBackJournalEvent() { @DisplayName("a failed write does not consume its stream position") void failedWriteLeavesNoGap() { FeatureFlag.enable(Features.JOURNAL_EVENTS); + prepareJournal(); JournalEventSequencer sequencer = new JournalEventSequencerFactory(journalEventStore).forStream(STREAM_ID); MongoDBReactiveAuditRepository failingRepository = mock(MongoDBReactiveAuditRepository.class); doThrow(new IllegalStateException("audit write failed")) @@ -241,6 +242,7 @@ void failedWriteLeavesNoGap() { @DisplayName("journal enabled without transactions writes journal and audit in best-effort order") void nonTransactionalJournalWritesEventThenAuditState() { FeatureFlag.enable(Features.JOURNAL_EVENTS); + prepareJournal(); MongoDBReactiveAuditPersistence persistence = persistenceWithoutTransactions( auditRepository, journalEventStore, new JournalEventSequencerFactory(journalEventStore).forStream(STREAM_ID)); @@ -255,6 +257,7 @@ void nonTransactionalJournalWritesEventThenAuditState() { @DisplayName("journal enabled without transactions prevents audit write when journal append fails") void nonTransactionalJournalFailurePreventsAuditWrite() { FeatureFlag.enable(Features.JOURNAL_EVENTS); + prepareJournal(); JournalEventSequencer sequencer = new JournalEventSequencerFactory(journalEventStore).forStream(STREAM_ID); MongoDBReactiveJournalEventStore failingJournal = mock(MongoDBReactiveJournalEventStore.class); doThrow(new IllegalStateException("journal write failed")) @@ -272,6 +275,7 @@ void nonTransactionalJournalFailurePreventsAuditWrite() { @DisplayName("journal enabled without transactions preserves and confirms event when audit write fails") void nonTransactionalAuditFailurePreservesAndConfirmsJournalEvent() { FeatureFlag.enable(Features.JOURNAL_EVENTS); + prepareJournal(); JournalEventSequencer sequencer = new JournalEventSequencerFactory(journalEventStore).forStream(STREAM_ID); MongoDBReactiveAuditRepository failingRepository = mock(MongoDBReactiveAuditRepository.class); doThrow(new IllegalStateException("audit write failed")) @@ -293,23 +297,30 @@ void nonTransactionalAuditFailurePreservesAndConfirmsJournalEvent() { } private MongoDBReactiveAuditPersistence persistenceFor(MongoDBReactiveAuditRepository repository) { + prepareJournal(); return persistenceFor(repository, new JournalEventSequencerFactory(journalEventStore).forStream(STREAM_ID)); } private MongoDBReactiveAuditPersistence persistenceFor(MongoDBReactiveAuditRepository repository, JournalEventSequencer sequencer) { MongoDBReactiveAuditPersistence persistence = new MongoDBReactiveAuditPersistence( - new CommunityConfiguration(), repository, journalEventStore, sequencer, true, txWrapper, true); + repository, journalEventStore, sequencer, true, txWrapper); persistence.initialize(RunnerId.generate()); return persistence; } + private void prepareJournal() { + if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + journalEventStore.initialize(true); + } + } + private MongoDBReactiveAuditPersistence persistenceWithoutTransactions( MongoDBReactiveAuditRepository repository, MongoDBReactiveJournalEventStore journalStore, JournalEventSequencer sequencer) { MongoDBReactiveAuditPersistence persistence = new MongoDBReactiveAuditPersistence( - new CommunityConfiguration(), repository, journalStore, sequencer, false, null, true); + repository, journalStore, sequencer, false, null); persistence.initialize(RunnerId.generate()); return persistence; } diff --git a/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistenceTest.java b/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistenceTest.java index 039917a1e..e5a256d6c 100644 --- a/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistenceTest.java +++ b/community/flamingock-mongodb-reactive-auditstore/src/test/java/io/flamingock/store/mongodb/reactive/internal/MongoDBReactiveAuditPersistenceTest.java @@ -25,7 +25,6 @@ import io.flamingock.internal.common.core.audit.AuditEntry; import io.flamingock.internal.common.core.audit.AuditTxType; import io.flamingock.internal.common.core.feature.Features; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; import io.flamingock.internal.core.journal.JournalEventSequencer; import io.flamingock.internal.common.core.external.ExecutionWrapper; import io.flamingock.internal.util.FeatureFlag; @@ -73,14 +72,13 @@ void beforeEach() { MongoDBReactiveJournalEventStore journalEventStore = new MongoDBReactiveJournalEventStore( database, JOURNAL_COLLECTION, ReadConcern.MAJORITY, ReadPreference.primary(), WriteConcern.MAJORITY.withJournal(true)); + auditRepository.initialize(true); persistence = new MongoDBReactiveAuditPersistence( - mock(CommunityConfigurable.class), auditRepository, journalEventStore, mock(JournalEventSequencer.class), true, - mock(ExecutionWrapper.class), - true); + mock(ExecutionWrapper.class)); persistence.initialize(RunnerId.fromString("runner-1")); } diff --git a/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/MongoDBSyncAuditStore.java b/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/MongoDBSyncAuditStore.java index 4a9336890..b9bf8aa7e 100644 --- a/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/MongoDBSyncAuditStore.java +++ b/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/MongoDBSyncAuditStore.java @@ -21,12 +21,16 @@ import com.mongodb.client.ClientSession; import com.mongodb.client.MongoDatabase; import io.flamingock.internal.common.core.audit.AuditPersistenceFactory; +import io.flamingock.internal.common.core.audit.AuditHistoryAppender; +import io.flamingock.internal.common.core.audit.JournalHistoryAppender; +import io.flamingock.internal.common.core.context.RuntimeContext; +import io.flamingock.internal.core.context.BasicRuntimeContext; +import io.flamingock.internal.core.external.store.HistoryAppenderProvider; import io.flamingock.internal.common.core.audit.AuditReader; import io.flamingock.internal.common.core.context.ContextResolver; import io.flamingock.internal.common.core.error.FlamingockException; import io.flamingock.internal.common.core.feature.Features; import io.flamingock.internal.common.core.external.ExecutionWrapper; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; import io.flamingock.internal.core.external.store.CommunityAuditStore; import io.flamingock.internal.core.external.store.audit.community.CommunityAuditPersistence; import io.flamingock.internal.core.external.store.lock.community.CommunityLockService; @@ -40,6 +44,7 @@ import io.flamingock.store.mongodb.sync.internal.MongoDBSyncAuditPersistence; import io.flamingock.store.mongodb.sync.internal.MongoDBSyncAuditRepository; import io.flamingock.store.mongodb.sync.internal.MongoDBSyncJournalEventStore; +import io.flamingock.store.mongodb.sync.internal.MongoDBSyncJournalWriter; import io.flamingock.store.mongodb.sync.internal.MongoDBSyncLockService; import io.flamingock.externalsystem.mongodb.api.MongoDBExternalSystem; @@ -53,15 +58,13 @@ import static io.flamingock.internal.util.constants.CommunityPersistenceConstants.DEFAULT_AUDIT_STORE_NAME; import static io.flamingock.internal.util.constants.CommunityPersistenceConstants.DEFAULT_LOCK_STORE_NAME; -public class MongoDBSyncAuditStore implements CommunityAuditStore { +public class MongoDBSyncAuditStore implements CommunityAuditStore, HistoryAppenderProvider { private static final Logger logger = FlamingockLoggerFactory.getLogger("MongoDBSyncAuditStore"); private final MongoDBExternalSystem mongoDBTargetSystem; protected RunnerId runnerId; - private CommunityConfigurable communityConfiguration; - private MongoDBSyncAuditPersistence persistence; private MongoDBSyncLockService lockService; private MongoDatabase database; private String auditRepositoryName = DEFAULT_AUDIT_STORE_NAME; @@ -74,6 +77,7 @@ public class MongoDBSyncAuditStore implements CommunityAuditStore { private MongoDBSyncAuditRepository auditRepository; private MongoDBSyncJournalEventStore journalEventStore; private JournalEventSequencerFactory journalEventSequencerFactory; + private MongoDBSyncJournalWriter journalWriter; private MongoDBSyncAuditStore(MongoDBExternalSystem mongoDBTargetSystem) { @@ -137,11 +141,11 @@ public MongoDBSyncAuditStore withAutoCreate(boolean autoCreate) { @Override public void initialize(ContextResolver baseContext) { runnerId = baseContext.getRequiredDependencyValue(RunnerId.class); - communityConfiguration = baseContext.getRequiredDependencyValue(CommunityConfigurable.class); database = mongoDBTargetSystem.getMongoDatabase(); auditRepository = new MongoDBSyncAuditRepository(database, auditRepositoryName, readConcern, readPreference, writeConcern); journalEventStore = new MongoDBSyncJournalEventStore(database, journalRepositoryName, readConcern, readPreference, writeConcern); journalEventSequencerFactory = new JournalEventSequencerFactory(journalEventStore); + journalWriter = new MongoDBSyncJournalWriter(journalEventStore); lockService = new MongoDBSyncLockService( database, @@ -151,31 +155,67 @@ public void initialize(ContextResolver baseContext) { writeConcern, TimeService.getDefault() ); - lockService.initialize(autoCreate); this.validate(); + lockService.initialize(autoCreate); + auditRepository.initialize(autoCreate); + if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + journalEventStore.initialize(autoCreate); + } warnIfJournalWritesAreNonAtomic(); } @Override public AuditPersistenceFactory getPersistenceFactory() { return stageId -> { - JournalEventSequencer journalEventSequencer = journalEventSequencerFactory.forStream(stageId); + JournalEventSequencer journalEventSequencer = FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false) + ? journalEventSequencerFactory.forStream(stageId) : null; boolean supportsTransactions = mongoDBTargetSystem.supportsTransactions(); ExecutionWrapper txWrapper = supportsTransactions ? mongoDBTargetSystem.getTxWrapper() : null; - persistence = new MongoDBSyncAuditPersistence( - communityConfiguration, + MongoDBSyncAuditPersistence persistence = new MongoDBSyncAuditPersistence( auditRepository, journalEventStore, journalEventSequencer, supportsTransactions, - txWrapper, - autoCreate + txWrapper ); persistence.initialize(runnerId); return persistence; }; } + @Override + public AuditHistoryAppender getAuditHistoryAppender() { + return auditRepository::append; + } + + @Override + public JournalHistoryAppender getJournalHistoryAppender() { + return (streamId, entry) -> { + if (!FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + throw new IllegalStateException("Journal events must be enabled to write journal history"); + } + JournalEventSequencer sequencer = journalEventSequencerFactory.forStream(streamId); + synchronized (sequencer) { + try { + if (!mongoDBTargetSystem.supportsTransactions()) { + return journalWriter.write(sequencer, entry); + } + ExecutionWrapper wrapper = mongoDBTargetSystem.getTxWrapper(); + RuntimeContext context = new BasicRuntimeContext("write-journalHistory-" + entry.getChangeId()); + io.flamingock.internal.util.Result result = wrapper.wrapExecution(context, runtime -> { + ClientSession session = runtime.getContext().getRequiredDependencyValue(ClientSession.class); + return journalWriter.write(session, sequencer, entry); + }); + sequencer.confirm(); + return result; + } catch (RuntimeException | Error failure) { + sequencer.markWriteOutcomeUncertain(); + throw failure; + } + } + }; + } + @Override public AuditReader getAuditReader() { return () -> auditRepository.getAuditHistory(); diff --git a/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditPersistence.java b/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditPersistence.java index fc3d976fd..06d40f086 100644 --- a/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditPersistence.java +++ b/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditPersistence.java @@ -21,13 +21,11 @@ import io.flamingock.internal.common.core.feature.Features; import io.flamingock.internal.common.core.journal.JournalEvent; import io.flamingock.internal.common.core.external.ExecutionWrapper; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; import io.flamingock.internal.core.context.BasicRuntimeContext; import io.flamingock.internal.core.external.store.audit.community.AbstractCommunityAuditPersistence; import io.flamingock.internal.core.journal.JournalEventSequencer; import io.flamingock.internal.util.FeatureFlag; import io.flamingock.internal.util.Result; -import io.flamingock.internal.util.id.RunnerId; import java.util.List; import java.util.Objects; @@ -35,35 +33,28 @@ public class MongoDBSyncAuditPersistence extends AbstractCommunityAuditPersistence { private final MongoDBSyncAuditRepository auditRepository; - private final MongoDBSyncJournalEventStore journalEventStore; + private final MongoDBSyncJournalWriter journalWriter; private final JournalEventSequencer journalEventSequencer; private final boolean supportsTransactions; private final ExecutionWrapper txWrapper; - private final boolean autoCreate; /** - * @param localConfiguration local Community configuration * @param auditRepository audit state repository * @param journalEventStore journal event store * @param journalEventSequencer sequencer for this persistence stream * @param supportsTransactions whether journal and audit writes must share a MongoDB transaction * @param txWrapper transaction wrapper; must be non-null if and only if transactions are supported - * @param autoCreate whether required MongoDB collections and indexes may be created */ - public MongoDBSyncAuditPersistence(CommunityConfigurable localConfiguration, - MongoDBSyncAuditRepository auditRepository, + public MongoDBSyncAuditPersistence(MongoDBSyncAuditRepository auditRepository, MongoDBSyncJournalEventStore journalEventStore, JournalEventSequencer journalEventSequencer, boolean supportsTransactions, - ExecutionWrapper txWrapper, - boolean autoCreate) { - super(localConfiguration); + ExecutionWrapper txWrapper) { this.auditRepository = auditRepository; - this.journalEventStore = journalEventStore; + this.journalWriter = new MongoDBSyncJournalWriter(journalEventStore); this.journalEventSequencer = journalEventSequencer; this.supportsTransactions = supportsTransactions; this.txWrapper = validateTransactionWrapper(supportsTransactions, txWrapper); - this.autoCreate = autoCreate; } private static ExecutionWrapper validateTransactionWrapper(boolean supportsTransactions, @@ -80,17 +71,6 @@ private static ExecutionWrapper validateTransactionWrapper(boolean supportsTrans return null; } - @Override - protected void doInitialize(RunnerId runnerId) { - auditRepository.initialize(autoCreate); - // Creating the indexes is what brings the journal collection into existence — there is no explicit - // createCollection call — so skipping this keeps it from ever appearing. It must stay in step with the - // append in writeEntry: skipping setup while still appending would let insertOne create the collection - // implicitly and without indexes, voiding the unique (streamId, streamSequence) and eventId guarantees. - FeatureFlag.ifEnabled(Features.JOURNAL_EVENTS, () -> journalEventStore.initialize(autoCreate)); - } - - @Override public List getAuditHistory() { return auditRepository.getAuditHistory(); @@ -101,22 +81,28 @@ public Result writeEntry(AuditEntry auditEntry) { // Read once rather than per branch: the journal append and the audit write shape are two halves of one // model. With events, the audit record is the change's current state and the journal is the history; // without them, the audit record set is itself the history. - if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS)) { - if (!supportsTransactions) { - return writeJournalAndAuditWithoutTransaction(auditEntry); - } - return writeJournalAndAuditInTransaction(auditEntry); - } else { + if (FeatureFlag.isDisabled(Features.JOURNAL_EVENTS, false)) { return auditRepository.append(auditEntry); } + if (journalEventSequencer == null) { + throw new IllegalStateException("MongoDB sync journal writes require a sequencer"); + } + synchronized (journalEventSequencer) { + try { + return supportsTransactions ? writeJournalAndAuditInTransaction(auditEntry) + : writeJournalAndAuditWithoutTransaction(auditEntry); + } catch (RuntimeException | Error failure) { + journalEventSequencer.markWriteOutcomeUncertain(); + throw failure; + } + } } private Result writeJournalAndAuditInTransaction(AuditEntry auditEntry) { RuntimeContext baseContext = new BasicRuntimeContext("write-changeState-" + auditEntry.getChangeId()); Result result = txWrapper.wrapExecution(baseContext, runtimeContext -> { ClientSession clientSession = runtimeContext.getContext().getRequiredDependencyValue(ClientSession.class); - JournalEvent journalEvent = journalEventSequencer.newEvent(auditEntry); - journalEventStore.write(clientSession, journalEvent); + journalWriter.write(clientSession, journalEventSequencer, auditEntry); return auditRepository.save(clientSession, auditEntry); }); // Spends the stream position, and only a committed transaction may reach this line. In general a @@ -138,9 +124,7 @@ private Result writeJournalAndAuditInTransaction(AuditEntry auditEntry) { * accepted best-effort behavior for non-transactional deployments. */ private Result writeJournalAndAuditWithoutTransaction(AuditEntry auditEntry) { - JournalEvent journalEvent = journalEventSequencer.newEvent(auditEntry); - journalEventStore.write(journalEvent); - journalEventSequencer.confirm(); + journalWriter.write(journalEventSequencer, auditEntry); return auditRepository.save(auditEntry); } diff --git a/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditRepository.java b/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditRepository.java index 318277ffb..6e80f66b8 100644 --- a/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditRepository.java +++ b/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditRepository.java @@ -49,6 +49,7 @@ public class MongoDBSyncAuditRepository { private final MongoCollection collection; private final MongoDBAuditMapper mapper = new MongoDBAuditMapper<>(() -> new MongoDBDocumentHelper(new Document())); + private boolean initialized; public MongoDBSyncAuditRepository(MongoDatabase database, String collectionName, @@ -61,7 +62,10 @@ public MongoDBSyncAuditRepository(MongoDatabase database, .withWriteConcern(writeConcern); } - protected void initialize(boolean autoCreate) { + public synchronized void initialize(boolean autoCreate) { + if (initialized) { + return; + } CollectionInitializator initializer = new CollectionInitializator<>( new MongoDBSyncCollectionHelper(collection), () -> new MongoDBDocumentHelper(new Document()), @@ -72,7 +76,7 @@ protected void initialize(boolean autoCreate) { } else { initializer.justValidateCollection(); } - + initialized = true; } /** @@ -133,7 +137,7 @@ Result save(AuditEntry auditEntry) { * regardless: a legacy changelog can hold several entries for the same change across executions, and * {@link #save} would collapse them onto each other, discarding the very history being imported. */ - Result append(AuditEntry auditEntry) { + public Result append(AuditEntry auditEntry) { Bson filter = Filters.and( Filters.eq(KEY_EXECUTION_ID, auditEntry.getExecutionId()), Filters.eq(KEY_CHANGE_ID, auditEntry.getChangeId()), diff --git a/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncJournalEventStore.java b/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncJournalEventStore.java index 267260ffd..86aac2b96 100644 --- a/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncJournalEventStore.java +++ b/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncJournalEventStore.java @@ -84,7 +84,7 @@ public MongoDBSyncJournalEventStore(MongoDatabase database, .withWriteConcern(writeConcern); } - protected void initialize(boolean autoCreate) { + public void initialize(boolean autoCreate) { CollectionInitializator initializer = new CollectionInitializator<>( new MongoDBSyncCollectionHelper(collection), () -> new MongoDBDocumentHelper(new Document()), diff --git a/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncJournalWriter.java b/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncJournalWriter.java new file mode 100644 index 000000000..3c99bcc9c --- /dev/null +++ b/community/flamingock-mongodb-sync-auditstore/src/main/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncJournalWriter.java @@ -0,0 +1,43 @@ +/* + * Copyright 2026 Flamingock (https://www.flamingock.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.flamingock.store.mongodb.sync.internal; + +import com.mongodb.client.ClientSession; +import io.flamingock.internal.common.core.audit.AuditEntry; +import io.flamingock.internal.common.core.journal.JournalEvent; +import io.flamingock.internal.core.journal.JournalEventSequencer; +import io.flamingock.internal.util.Result; + +/** Shares event construction and persistence across normal and history-only writes. */ +public class MongoDBSyncJournalWriter { + private final MongoDBSyncJournalEventStore store; + + public MongoDBSyncJournalWriter(MongoDBSyncJournalEventStore store) { + this.store = store; + } + + public Result write(ClientSession session, JournalEventSequencer sequencer, AuditEntry entry) { + JournalEvent event = sequencer.newEvent(entry); + return store.write(session, event); + } + + public Result write(JournalEventSequencer sequencer, AuditEntry entry) { + JournalEvent event = sequencer.newEvent(entry); + Result result = store.write(event); + sequencer.confirm(); + return result; + } +} diff --git a/community/flamingock-mongodb-sync-auditstore/src/test/java/io/flamingock/store/mongodb/sync/MongoDBSyncHistoryProviderTest.java b/community/flamingock-mongodb-sync-auditstore/src/test/java/io/flamingock/store/mongodb/sync/MongoDBSyncHistoryProviderTest.java new file mode 100644 index 000000000..22913b2ee --- /dev/null +++ b/community/flamingock-mongodb-sync-auditstore/src/test/java/io/flamingock/store/mongodb/sync/MongoDBSyncHistoryProviderTest.java @@ -0,0 +1,182 @@ +/* + * Copyright 2026 Flamingock (https://www.flamingock.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.flamingock.store.mongodb.sync; + +import com.mongodb.client.ClientSession; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoDatabase; +import io.flamingock.core.kit.audit.AuditEntryTestFactory; +import io.flamingock.internal.common.core.audit.AuditEntry; +import io.flamingock.internal.common.core.audit.AuditTxType; +import io.flamingock.internal.common.core.context.ContextResolver; +import io.flamingock.internal.common.core.feature.Features; +import io.flamingock.internal.common.core.error.DatabaseTransactionException; +import io.flamingock.internal.common.core.journal.JournalEvent; +import io.flamingock.store.mongodb.sync.internal.MongoDBSyncJournalWriter; +import java.lang.reflect.Field; +import io.flamingock.internal.common.mongodb.MongoDBJournalEventMapper; +import io.flamingock.internal.core.configuration.community.CommunityConfigurable; +import io.flamingock.internal.core.configuration.community.CommunityConfiguration; +import io.flamingock.internal.util.FeatureFlag; +import io.flamingock.internal.util.id.RunnerId; +import io.flamingock.targetsystem.mongodb.sync.MongoDBSyncTargetSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MongoDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Testcontainers +class MongoDBSyncHistoryProviderTest { + @Container + static final MongoDBContainer MONGO = new MongoDBContainer(DockerImageName.parse("mongo:6")); + + private MongoClient client; + private MongoDatabase database; + private MongoDBSyncAuditStore store; + + @BeforeEach + void setUp() { + client = MongoClients.create(MONGO.getConnectionString()); + database = client.getDatabase("history-provider-test"); + MongoDBSyncTargetSystem target = new MongoDBSyncTargetSystem("mongodb", client, "history-provider-test"); + ContextResolver context = mock(ContextResolver.class); + when(context.getRequiredDependencyValue(RunnerId.class)).thenReturn(RunnerId.generate()); + when(context.getRequiredDependencyValue(CommunityConfigurable.class)).thenReturn(new CommunityConfiguration()); + target.initialize(context); + store = MongoDBSyncAuditStore.from(target); + store.initialize(context); + assertFalse(database.listCollectionNames().into(new ArrayList<>()).contains("flamingockJournalEvents")); + } + + @AfterEach + void tearDown() { + FeatureFlag.remove(Features.JOURNAL_EVENTS); + database.drop(); + client.close(); + } + + @Test + void historicalAppenderWorksWithoutJournalFlag() { + store.getAuditHistoryAppender().append(entry("one")); + store.getAuditHistoryAppender().append(entry("one")); + assertEquals(2, database.getCollection("flamingockAuditLog").countDocuments()); + assertEquals(2, store.getAuditReader().getAuditHistory().size()); + assertFalse(database.listCollectionNames().into(new ArrayList<>()).contains("flamingockJournalEvents")); + assertThrows(IllegalStateException.class, () -> store.getJournalHistoryAppender().append("stage", entry("two"))); + } + + @Test + void flagOffStagePersistenceWritesHistoricalAuditRows() { + io.flamingock.internal.core.external.store.audit.community.CommunityAuditPersistence persistence = + store.getPersistenceFactory().get("stage"); + persistence.writeEntry(entry("first")); + persistence.writeEntry(entry("second")); + + assertEquals(2, persistence.getAuditHistory().size()); + assertFalse(database.listCollectionNames().into(new ArrayList<>()).contains("flamingockJournalEvents")); + } + + @Test + void earlyAppenderValidatesMissingAuditSchemaWhenAutoCreateIsDisabled() { + database.drop(); + MongoDBSyncTargetSystem target = new MongoDBSyncTargetSystem("mongodb", client, "history-provider-test"); + ContextResolver context = mock(ContextResolver.class); + when(context.getRequiredDependencyValue(RunnerId.class)).thenReturn(RunnerId.generate()); + when(context.getRequiredDependencyValue(CommunityConfigurable.class)).thenReturn(new CommunityConfiguration()); + target.initialize(context); + MongoDBSyncAuditStore manualStore = MongoDBSyncAuditStore.from(target).withAutoCreate(false); + assertThrows(RuntimeException.class, () -> manualStore.initialize(context)); + assertFalse(database.listCollectionNames().into(new ArrayList<>()).contains("flamingockAuditLog")); + } + + @Test + void journalOnlyDoesNotUpdateStateAndNormalWriteUsesNextPosition() { + FeatureFlag.enable(Features.JOURNAL_EVENTS); + initializeJournalEnabledStore(); + io.flamingock.internal.core.external.store.audit.community.CommunityAuditPersistence persistence = + store.getPersistenceFactory().get("stage"); + persistence.writeEntry(entry("one")); + store.getJournalHistoryAppender().append("stage", entry("two")); + assertEquals(1, store.getAuditReader().getAuditHistory().size()); + persistence.writeEntry(entry("three")); + assertEquals(2, store.getAuditReader().getAuditHistory().size()); + MongoDBJournalEventMapper mapper = new MongoDBJournalEventMapper(); + java.util.List sequences = new ArrayList<>(); + database.getCollection("flamingockJournalEvents").find().forEach(doc -> { + assertEquals("CHANGE_STATE", mapper.fromDocument(doc).getEventType().name()); + sequences.add(mapper.fromDocument(doc).getStreamSequence()); + }); + assertEquals(java.util.Arrays.asList(1L, 2L, 3L), sequences); + } + + @Test + void failedJournalTransactionDoesNotConsumePosition() throws Exception { + FeatureFlag.enable(Features.JOURNAL_EVENTS); + initializeJournalEnabledStore(); + store.getJournalHistoryAppender().append("stage", entry("first")); + Field writerField = MongoDBSyncAuditStore.class.getDeclaredField("journalWriter"); + writerField.setAccessible(true); + MongoDBSyncJournalWriter original = (MongoDBSyncJournalWriter) writerField.get(store); + MongoDBSyncJournalWriter failingWriter = org.mockito.Mockito.mock(MongoDBSyncJournalWriter.class); + org.mockito.Mockito.doThrow(new IllegalStateException("failed insert")) + .when(failingWriter).write(org.mockito.ArgumentMatchers.any(ClientSession.class), + org.mockito.ArgumentMatchers.any(io.flamingock.internal.core.journal.JournalEventSequencer.class), + org.mockito.ArgumentMatchers.any(AuditEntry.class)); + writerField.set(store, failingWriter); + try { + assertThrows(DatabaseTransactionException.class, + () -> store.getJournalHistoryAppender().append("stage", entry("failed"))); + } finally { + writerField.set(store, original); + } + store.getJournalHistoryAppender().append("stage", entry("second")); + assertEquals(0, store.getAuditReader().getAuditHistory().size()); + java.util.List positions = new ArrayList<>(); + MongoDBJournalEventMapper mapper = new MongoDBJournalEventMapper(); + database.getCollection("flamingockJournalEvents").find().forEach(doc -> + positions.add(mapper.fromDocument(doc).getStreamSequence())); + assertEquals(java.util.Arrays.asList(1L, 2L), positions); + } + + private void initializeJournalEnabledStore() { + MongoDBSyncTargetSystem target = new MongoDBSyncTargetSystem("mongodb", client, "history-provider-test"); + ContextResolver context = mock(ContextResolver.class); + when(context.getRequiredDependencyValue(RunnerId.class)).thenReturn(RunnerId.generate()); + when(context.getRequiredDependencyValue(CommunityConfigurable.class)).thenReturn(new CommunityConfiguration()); + target.initialize(context); + store = MongoDBSyncAuditStore.from(target); + store.initialize(context); + org.junit.jupiter.api.Assertions.assertTrue( + database.listCollectionNames().into(new ArrayList<>()).contains("flamingockJournalEvents")); + } + + private static AuditEntry entry(String id) { + return AuditEntryTestFactory.createTestAuditEntry(id, AuditEntry.Status.APPLIED, AuditTxType.NON_TX, + (Class) null); + } +} diff --git a/community/flamingock-mongodb-sync-auditstore/src/test/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditPersistenceJournalTest.java b/community/flamingock-mongodb-sync-auditstore/src/test/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditPersistenceJournalTest.java index 4740d39d5..87e759ff4 100644 --- a/community/flamingock-mongodb-sync-auditstore/src/test/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditPersistenceJournalTest.java +++ b/community/flamingock-mongodb-sync-auditstore/src/test/java/io/flamingock/store/mongodb/sync/internal/MongoDBSyncAuditPersistenceJournalTest.java @@ -30,7 +30,6 @@ import io.flamingock.internal.common.core.journal.JournalEvent; import io.flamingock.internal.common.core.journal.JournalEventType; import io.flamingock.internal.common.mongodb.MongoDBJournalEventMapper; -import io.flamingock.internal.core.configuration.community.CommunityConfiguration; import io.flamingock.internal.core.journal.JournalEventSequencer; import io.flamingock.internal.core.journal.JournalEventSequencerFactory; import io.flamingock.internal.core.transaction.TransactionManager; @@ -97,6 +96,11 @@ void setUp() { database, JOURNAL_COLLECTION, ReadConcern.MAJORITY, ReadPreference.primary(), WriteConcern.MAJORITY.withJournal(true)); txWrapper = new MongoDBSyncTxWrapper(new TransactionManager<>(mongoClient::startSession)); + auditRepository.initialize(true); + // Prepare the real collaborators before any fixture obtains a stream sequencer. + if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + journalEventStore.initialize(true); + } } @AfterEach @@ -112,32 +116,32 @@ void tearDown() { @DisplayName("constructor accepts a transaction wrapper when transactions are supported") void constructorAcceptsWrapperWhenTransactionsAreSupported() { assertDoesNotThrow(() -> new MongoDBSyncAuditPersistence( - new CommunityConfiguration(), auditRepository, journalEventStore, - mock(JournalEventSequencer.class), true, txWrapper, true)); + auditRepository, journalEventStore, + mock(JournalEventSequencer.class), true, txWrapper)); } @Test @DisplayName("constructor rejects a missing transaction wrapper when transactions are supported") void constructorRejectsMissingWrapperWhenTransactionsAreSupported() { assertThrows(NullPointerException.class, () -> new MongoDBSyncAuditPersistence( - new CommunityConfiguration(), auditRepository, journalEventStore, - mock(JournalEventSequencer.class), true, null, true)); + auditRepository, journalEventStore, + mock(JournalEventSequencer.class), true, null)); } @Test @DisplayName("constructor accepts no transaction wrapper when transactions are not supported") void constructorAcceptsMissingWrapperWhenTransactionsAreNotSupported() { assertDoesNotThrow(() -> new MongoDBSyncAuditPersistence( - new CommunityConfiguration(), auditRepository, journalEventStore, - mock(JournalEventSequencer.class), false, null, true)); + auditRepository, journalEventStore, + mock(JournalEventSequencer.class), false, null)); } @Test @DisplayName("constructor rejects a transaction wrapper when transactions are not supported") void constructorRejectsWrapperWhenTransactionsAreNotSupported() { assertThrows(IllegalArgumentException.class, () -> new MongoDBSyncAuditPersistence( - new CommunityConfiguration(), auditRepository, journalEventStore, - mock(JournalEventSequencer.class), false, txWrapper, true)); + auditRepository, journalEventStore, + mock(JournalEventSequencer.class), false, txWrapper)); } @Test @@ -312,28 +316,35 @@ void nonTransactionalAuditFailurePreservesAndConfirmsJournalEvent() { * Builds a persistence over {@code STREAM_ID}. The sequencer has to come from the factory because * {@link JournalEventSequencer}'s constructor is package-private in another package. *

- * Call only after deciding the flag state: {@code initialize} is where the journal collection setup is - * gated. + * Prepare the journal collaborator after deciding the flag state and before obtaining a sequencer. */ private MongoDBSyncAuditPersistence persistenceFor(MongoDBSyncAuditRepository repository) { + prepareJournal(); return persistenceFor(repository, new JournalEventSequencerFactory(journalEventStore).forStream(STREAM_ID)); } private MongoDBSyncAuditPersistence persistenceFor(MongoDBSyncAuditRepository repository, JournalEventSequencer sequencer) { + prepareJournal(); MongoDBSyncAuditPersistence persistence = new MongoDBSyncAuditPersistence( - new CommunityConfiguration(), repository, journalEventStore, sequencer, - true, txWrapper, true); + repository, journalEventStore, sequencer, + true, txWrapper); persistence.initialize(RunnerId.generate()); return persistence; } + private void prepareJournal() { + if (FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) { + journalEventStore.initialize(true); + } + } + private MongoDBSyncAuditPersistence persistenceWithoutTransactions( MongoDBSyncAuditRepository repository, MongoDBSyncJournalEventStore journalStore, JournalEventSequencer sequencer) { MongoDBSyncAuditPersistence persistence = new MongoDBSyncAuditPersistence( - new CommunityConfiguration(), repository, journalStore, sequencer, false, null, true); + repository, journalStore, sequencer, false, null); persistence.initialize(RunnerId.generate()); return persistence; } diff --git a/community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/SqlAuditStore.java b/community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/SqlAuditStore.java index a1754fd1e..47fc9a386 100644 --- a/community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/SqlAuditStore.java +++ b/community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/SqlAuditStore.java @@ -15,13 +15,15 @@ */ package io.flamingock.store.sql; +import io.flamingock.internal.common.core.audit.AuditHistoryAppender; +import io.flamingock.internal.common.core.audit.JournalHistoryAppender; import io.flamingock.internal.common.core.audit.AuditPersistenceFactory; import io.flamingock.internal.common.core.audit.AuditReader; import io.flamingock.internal.common.core.context.ContextResolver; import io.flamingock.internal.common.core.error.FlamingockException; import io.flamingock.internal.common.core.feature.Features; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; import io.flamingock.internal.core.external.store.CommunityAuditStore; +import io.flamingock.internal.core.external.store.HistoryAppenderProvider; import io.flamingock.internal.core.external.store.audit.community.CommunityAuditPersistence; import io.flamingock.internal.core.external.store.lock.community.CommunityLockService; import io.flamingock.internal.core.journal.JournalEventSequencer; @@ -34,21 +36,26 @@ import io.flamingock.store.sql.internal.SqlAuditRepository; import io.flamingock.store.sql.internal.SqlLockService; import io.flamingock.store.sql.internal.SqlJournalEventStore; +import io.flamingock.store.sql.internal.SqlJournalHistoryAppender; +import io.flamingock.internal.core.transaction.TransactionManager; +import io.flamingock.targetsystem.sql.SqlTxWrapper; +import java.sql.Connection; +import java.sql.SQLException; import io.flamingock.externalsystem.sql.api.SqlExternalSystem; import javax.sql.DataSource; -public class SqlAuditStore implements CommunityAuditStore { +public class SqlAuditStore implements CommunityAuditStore, HistoryAppenderProvider { private static final String SQL_IDENTIFIER_PATTERN = "[A-Za-z][A-Za-z0-9_]*"; private static final String DEFAULT_JOURNAL_REPOSITORY_NAME = "flamingockJournalEvents"; private final SqlExternalSystem targetSystem; private final DataSource dataSource; - private CommunityConfigurable communityConfiguration; private RunnerId runnerId; private SqlLockService lockService; private SqlJournalEventStore journalEventStore; + private SqlJournalHistoryAppender journalHistoryWriter; private JournalEventSequencerFactory journalEventSequencerFactory; private SqlAuditRepository auditRepository; private String auditRepositoryName = CommunityPersistenceConstants.DEFAULT_AUDIT_STORE_NAME; @@ -103,18 +110,22 @@ public SqlAuditStore withAutoCreate(boolean autoCreate) { @Override public void initialize(ContextResolver baseContext) { runnerId = baseContext.getRequiredDependencyValue(RunnerId.class); - communityConfiguration = baseContext.getRequiredDependencyValue(CommunityConfigurable.class); validate(); auditRepository = new SqlAuditRepository(dataSource, auditRepositoryName); journalEventStore = new SqlJournalEventStore( dataSource, journalRepositoryName, - targetSystem.getTxWrapper()); + new SqlTxWrapper(new TransactionManager<>(this::openAuditConnection))); journalEventSequencerFactory = new JournalEventSequencerFactory(journalEventStore); + journalHistoryWriter = new SqlJournalHistoryAppender(journalEventStore, journalEventSequencerFactory, + new SqlTxWrapper(new TransactionManager<>(this::openAuditConnection))); auditRepository.initialize(autoCreate); lockService = new SqlLockService(dataSource, lockRepositoryName); lockService.initialize(autoCreate); + if (isJournalEventsEnabled()) { + journalEventStore.initialize(autoCreate); + } } @Override @@ -123,22 +134,44 @@ public AuditPersistenceFactory getPersistenceFactory( boolean journalEventsEnabled = isJournalEventsEnabled(); JournalEventSequencer journalEventSequencer = null; if (journalEventsEnabled) { - journalEventStore.initialize(autoCreate); journalEventSequencer = journalEventSequencerFactory.forStream(stageId); } SqlAuditPersistence persistence = new SqlAuditPersistence( - communityConfiguration, auditRepository, journalEventStore, journalEventSequencer, - targetSystem.getTxWrapper(), - journalEventsEnabled); + new SqlTxWrapper(new TransactionManager<>(this::openAuditConnection)), + journalEventsEnabled, + journalHistoryWriter); persistence.initialize(runnerId); return persistence; }; } + @Override + public AuditHistoryAppender getAuditHistoryAppender() { + return auditRepository::append; + } + + @Override + public JournalHistoryAppender getJournalHistoryAppender() { + return (streamId, entry) -> { + if (!isJournalEventsEnabled()) { + throw new IllegalStateException("Journal Events are disabled"); + } + return journalHistoryWriter.append(streamId, entry); + }; + } + + private Connection openAuditConnection() { + try { + return dataSource.getConnection(); + } catch (SQLException exception) { + throw new IllegalStateException("Could not open audit SQL connection", exception); + } + } + @Override public synchronized AuditReader getAuditReader() { return () -> auditRepository.getAuditHistory(); diff --git a/community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/internal/SqlAuditPersistence.java b/community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/internal/SqlAuditPersistence.java index 91fe9730a..0cc40ddfd 100644 --- a/community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/internal/SqlAuditPersistence.java +++ b/community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/internal/SqlAuditPersistence.java @@ -17,9 +17,7 @@ import io.flamingock.internal.common.core.audit.AuditEntry; import io.flamingock.internal.common.core.context.RuntimeContext; -import io.flamingock.internal.common.core.journal.JournalEvent; import io.flamingock.internal.common.core.external.ExecutionWrapper; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; import io.flamingock.internal.core.context.BasicRuntimeContext; import io.flamingock.internal.core.external.store.audit.community.AbstractCommunityAuditPersistence; import io.flamingock.internal.core.journal.JournalEventSequencer; @@ -36,29 +34,30 @@ public class SqlAuditPersistence extends AbstractCommunityAuditPersistence { private final JournalEventSequencer journalEventSequencer; private final ExecutionWrapper txWrapper; private final boolean journalEventsEnabled; + private final SqlJournalHistoryAppender journalHistoryWriter; /** * Creates persistence over collaborators whose schema readiness belongs to the store and stage factory. * - * @param localConfiguration community configuration * @param auditRepository ready audit table writer/reader * @param journalEventStore ready relational Journal Event store * @param journalEventSequencer stage-scoped sequence allocator * @param txWrapper transaction wrapper shared with the SQL target system * @param journalEventsEnabled feature flag snapshot captured for this stage + * @param journalHistoryWriter explicitly configured journal history writer */ - public SqlAuditPersistence(CommunityConfigurable localConfiguration, - SqlAuditRepository auditRepository, + public SqlAuditPersistence(SqlAuditRepository auditRepository, SqlJournalEventStore journalEventStore, JournalEventSequencer journalEventSequencer, ExecutionWrapper txWrapper, - boolean journalEventsEnabled) { - super(localConfiguration); + boolean journalEventsEnabled, + SqlJournalHistoryAppender journalHistoryWriter) { this.auditRepository = auditRepository; this.journalEventStore = journalEventStore; this.journalEventSequencer = journalEventSequencer; this.txWrapper = txWrapper; this.journalEventsEnabled = journalEventsEnabled; + this.journalHistoryWriter = journalHistoryWriter; } @Override @@ -78,27 +77,33 @@ public List getAuditHistory() { return auditRepository.getAuditHistory(); } - // Keep the lock through transaction commit: save uses a caller-owned connection. @Override - public synchronized Result writeEntry(AuditEntry auditEntry) { + public Result writeEntry(AuditEntry auditEntry) { if (!journalEventsEnabled) { return auditRepository.append(auditEntry); } - RuntimeContext baseContext = new BasicRuntimeContext("write-changeState-" + auditEntry.getChangeId()); - Result result = txWrapper.wrapExecution(baseContext, runtimeContext -> { - Connection connection = runtimeContext.getContext().getRequiredDependencyValue(Connection.class); - JournalEvent journalEvent = journalEventSequencer.newEvent(auditEntry); - journalEventStore.append(connection, journalEvent); - Result currentStateResult = auditRepository.save(connection, auditEntry); - if (currentStateResult instanceof Result.Error) { - throw new IllegalStateException("Failed to replace local current audit state", - ((Result.Error) currentStateResult).getError()); + // The same stream lock protects independent history writes and audit writes through commit. + synchronized (journalEventSequencer) { + try { + RuntimeContext baseContext = new BasicRuntimeContext("write-changeState-" + auditEntry.getChangeId()); + Result result = txWrapper.wrapExecution(baseContext, runtimeContext -> { + Connection connection = runtimeContext.getContext().getRequiredDependencyValue(Connection.class); + journalHistoryWriter.append(connection, journalEventSequencer, auditEntry); + Result currentStateResult = auditRepository.save(connection, auditEntry); + if (currentStateResult instanceof Result.Error) { + throw new IllegalStateException("Failed to replace local current audit state", + ((Result.Error) currentStateResult).getError()); + } + return currentStateResult == null ? Result.OK() : currentStateResult; + }); + journalEventSequencer.confirm(); + return result; + } catch (RuntimeException | Error failure) { + journalEventSequencer.markWriteOutcomeUncertain(); + throw failure; } - return currentStateResult == null ? Result.OK() : currentStateResult; - }); - journalEventSequencer.confirm(); - return result; + } } } diff --git a/community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/internal/SqlJournalHistoryAppender.java b/community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/internal/SqlJournalHistoryAppender.java new file mode 100644 index 000000000..57e28f011 --- /dev/null +++ b/community/flamingock-sql-auditstore/src/main/java/io/flamingock/store/sql/internal/SqlJournalHistoryAppender.java @@ -0,0 +1,66 @@ +/* + * Copyright 2026 Flamingock (https://www.flamingock.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.flamingock.store.sql.internal; + +import io.flamingock.internal.common.core.audit.AuditEntry; +import io.flamingock.internal.common.core.audit.JournalHistoryAppender; +import io.flamingock.internal.common.core.external.ExecutionWrapper; +import io.flamingock.internal.common.core.journal.JournalEvent; +import io.flamingock.internal.core.context.BasicRuntimeContext; +import io.flamingock.internal.core.journal.JournalEventSequencer; +import io.flamingock.internal.core.journal.JournalEventSequencerFactory; +import io.flamingock.internal.util.Result; + +import java.sql.Connection; + +/** Writes journal events on either a caller's audit transaction or an independent audit transaction. */ +public class SqlJournalHistoryAppender implements JournalHistoryAppender { + private final SqlJournalEventStore journalStore; + private final JournalEventSequencerFactory sequencers; + private final ExecutionWrapper auditTxWrapper; + + public SqlJournalHistoryAppender(SqlJournalEventStore journalStore, + JournalEventSequencerFactory sequencers, + ExecutionWrapper auditTxWrapper) { + this.journalStore = journalStore; + this.sequencers = sequencers; + this.auditTxWrapper = auditTxWrapper; + } + + @Override + public Result append(String streamId, AuditEntry entry) { + JournalEventSequencer sequencer = sequencers.forStream(streamId); + synchronized (sequencer) { + try { + Result result = auditTxWrapper.wrapExecution(new BasicRuntimeContext("append-journal-" + streamId), + context -> append(context.getContext().getRequiredDependencyValue(Connection.class), + sequencer, entry)); + sequencer.confirm(); + return result; + } catch (RuntimeException | Error failure) { + sequencer.markWriteOutcomeUncertain(); + throw failure; + } + } + } + + /** Appends without committing; the caller confirms the sequence only after its transaction commits. */ + public Result append(Connection connection, JournalEventSequencer sequencer, AuditEntry entry) { + JournalEvent event = sequencer.newEvent(entry); + journalStore.append(connection, event); + return Result.OK(); + } +} diff --git a/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/SqlAuditStoreTest.java b/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/SqlAuditStoreTest.java index 75b6fb997..6583a04cc 100644 --- a/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/SqlAuditStoreTest.java +++ b/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/SqlAuditStoreTest.java @@ -30,6 +30,8 @@ import io.flamingock.internal.core.external.store.audit.community.CommunityAuditPersistence; import io.flamingock.internal.core.configuration.community.CommunityConfiguration; import io.flamingock.internal.core.context.SimpleContext; +import io.flamingock.internal.core.context.BasicRuntimeContext; +import io.flamingock.internal.common.core.error.DatabaseTransactionException; import io.flamingock.internal.util.FeatureFlag; import io.flamingock.internal.util.id.RunnerId; import io.flamingock.internal.common.sql.SqlDialect; @@ -443,6 +445,94 @@ void journalEnabledUsesStageScopedPersistenceAndIndependentReader(SqlDialect sql assertEquals(1, auditStore.getAuditReader().getAuditHistory().size()); assertEquals(1, countRows("flamingockAuditLog")); assertEquals(2, countRows("customJournalEvents")); + auditStore.getJournalHistoryAppender().append("history-stage", + auditEntry("independent-history", AuditEntry.Status.APPLIED)); + assertEquals(1, countRows("flamingockAuditLog")); + assertEquals(3, countRows("customJournalEvents")); + auditStore.getAuditHistoryAppender().append(auditEntry("legacy-history", AuditEntry.Status.APPLIED)); + assertEquals(2, countRows("flamingockAuditLog")); + } + + @Test + @DisplayName("audit and journal commit independently when the SQL target transaction rolls back") + void journalAndAuditSurviveTargetTransactionRollback() throws Exception { + FeatureFlag.enable(Features.JOURNAL_EVENTS); + context = setupTest(SqlDialect.H2, "h2"); + try (Connection connection = context.dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE rollback_probe (id INT PRIMARY KEY)"); + } + + SimpleContext baseContext = new SimpleContext(); + baseContext.addDependency(RunnerId.generate()); + baseContext.addDependency(new CommunityConfiguration()); + SqlTargetSystem targetSystem = new SqlTargetSystem("sql", context.dataSource); + targetSystem.initialize(baseContext); + SqlAuditStore auditStore = SqlAuditStore.from(targetSystem); + auditStore.initialize(baseContext); + CommunityAuditPersistence persistence = auditStore.getPersistenceFactory().get("rollback-stage"); + + DatabaseTransactionException failure = assertThrows(DatabaseTransactionException.class, + () -> targetSystem.getTxWrapper().wrapExecution(new BasicRuntimeContext("rollback-probe"), runtime -> { + Connection changeConnection = runtime.getContext().getRequiredDependencyValue(Connection.class); + try (Statement statement = changeConnection.createStatement()) { + statement.executeUpdate("INSERT INTO rollback_probe (id) VALUES (1)"); + } catch (SQLException exception) { + throw new IllegalStateException(exception); + } + persistence.writeEntry(auditEntry("rollback-change", AuditEntry.Status.APPLIED)); + throw new IllegalStateException("roll back change transaction"); + })); + + assertEquals("roll back change transaction", failure.getCause().getMessage()); + assertEquals(1, auditStore.getAuditReader().getAuditHistory().size()); + assertEquals(1, countRows("flamingockAuditLog")); + assertEquals(1, countRows("flamingockJournalEvents")); + assertEquals(0, countRows("rollback_probe")); + } + + @Test + void stageCreatedBeforeJournalOnlyAppendSharesStreamSequence() throws Exception { + FeatureFlag.enable(Features.JOURNAL_EVENTS); + context = setupTest(SqlDialect.H2, "h2"); + SimpleContext baseContext = new SimpleContext(); + baseContext.addDependency(RunnerId.generate()); + baseContext.addDependency(new CommunityConfiguration()); + SqlTargetSystem targetSystem = new SqlTargetSystem("sql", context.dataSource); + targetSystem.initialize(baseContext); + SqlAuditStore store = SqlAuditStore.from(targetSystem); + store.initialize(baseContext); + + CommunityAuditPersistence persistence = store.getPersistenceFactory().get("shared-stage"); + store.getJournalHistoryAppender().append("shared-stage", auditEntry("history", AuditEntry.Status.STARTED)); + persistence.writeEntry(auditEntry("first", AuditEntry.Status.APPLIED)); + persistence.writeEntry(auditEntry("second", AuditEntry.Status.APPLIED)); + + SqlJournalEventStore journal = new SqlJournalEventStore( + context.dataSource, "flamingockJournalEvents", targetSystem.getTxWrapper()); + journal.initialize(false); + assertEquals(3L, journal.getLastEventByStream("shared-stage").get().getStreamSequence()); + assertEquals(3, countRows("flamingockJournalEvents")); + assertEquals(2, countRows("flamingockAuditLog")); + } + + @Test + void journalFacadeRejectsFlagOffButHistoricalAppenderStillWorks() throws Exception { + context = setupTest(SqlDialect.H2, "h2"); + SimpleContext baseContext = new SimpleContext(); + baseContext.addDependency(RunnerId.generate()); + baseContext.addDependency(new CommunityConfiguration()); + SqlTargetSystem targetSystem = new SqlTargetSystem("sql", context.dataSource); + targetSystem.initialize(baseContext); + SqlAuditStore auditStore = SqlAuditStore.from(targetSystem); + auditStore.initialize(baseContext); + + assertThrows(IllegalStateException.class, () -> auditStore.getJournalHistoryAppender().append( + "history-stage", auditEntry("disabled", AuditEntry.Status.APPLIED))); + assertFalse(tableExists("flamingockJournalEvents")); + auditStore.getAuditHistoryAppender().append(auditEntry("legacy", AuditEntry.Status.STARTED)); + auditStore.getAuditHistoryAppender().append(auditEntry("legacy", AuditEntry.Status.APPLIED)); + assertEquals(2, countRows("flamingockAuditLog")); } @ParameterizedTest @@ -547,8 +637,9 @@ void autoCreateDisabledValidatesLockDuringStoreInitialization(SqlDialect sqlDial @ParameterizedTest @MethodSource("dialectProvider") - @DisplayName("a stage snapshots the journal flag once and uses the captured value") + @DisplayName("a journal-enabled store bootstraps storage before a stage writes") void stageSnapshotsJournalFlagOnce(SqlDialect sqlDialect, String dialectName) throws Exception { + FeatureFlag.enable(Features.JOURNAL_EVENTS); context = setupTest(sqlDialect, dialectName); SimpleContext baseContext = new SimpleContext(); baseContext.addDependency(RunnerId.generate()); @@ -557,16 +648,12 @@ void stageSnapshotsJournalFlagOnce(SqlDialect sqlDialect, String dialectName) th targetSystem.initialize(baseContext); SqlAuditStore auditStore = SqlAuditStore.from(targetSystem); auditStore.initialize(baseContext); + assertTrue(tableExists("flamingockJournalEvents")); - try (MockedStatic flags = org.mockito.Mockito.mockStatic(FeatureFlag.class)) { - flags.when(() -> FeatureFlag.isEnabled(Features.JOURNAL_EVENTS, false)) - .thenReturn(true, false); - - CommunityAuditPersistence persistence = auditStore.getPersistenceFactory().get("captured-stage"); - persistence.writeEntry(auditEntry("captured-flag", AuditEntry.Status.APPLIED)); + CommunityAuditPersistence persistence = auditStore.getPersistenceFactory().get("captured-stage"); + persistence.writeEntry(auditEntry("captured-flag", AuditEntry.Status.APPLIED)); - assertEquals(1, countRows("flamingockJournalEvents")); - } + assertEquals(1, countRows("flamingockJournalEvents")); } @ParameterizedTest diff --git a/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/internal/SqlAuditPersistenceJournalTest.java b/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/internal/SqlAuditPersistenceJournalTest.java index 7c7645c41..36dfccace 100644 --- a/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/internal/SqlAuditPersistenceJournalTest.java +++ b/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/internal/SqlAuditPersistenceJournalTest.java @@ -23,7 +23,6 @@ import io.flamingock.internal.common.core.feature.Features; import io.flamingock.internal.common.core.journal.JournalEvent; import io.flamingock.internal.common.core.journal.JournalEventType; -import io.flamingock.internal.core.configuration.community.CommunityConfiguration; import io.flamingock.internal.core.journal.JournalEventSequencer; import io.flamingock.internal.core.journal.JournalEventSequencerFactory; import io.flamingock.internal.core.transaction.TransactionManager; @@ -107,7 +106,8 @@ void persistenceInitializationConsumesReadyAuditWriter() throws Exception { SqlAuditRepository auditRepository = new SqlAuditRepository(dataSource, AUDIT_TABLE); auditRepository.initialize(true); SqlAuditPersistence persistence = new SqlAuditPersistence( - new CommunityConfiguration(), auditRepository, null, null, null, false); + auditRepository, null, null, null, false, + new SqlJournalHistoryAppender(null, null, txWrapper)); persistence.initialize(RunnerId.generate()); persistence.writeEntry(auditEntry("legacy-constructor", AuditEntry.Status.APPLIED)); @@ -122,7 +122,8 @@ void persistenceInitializationDoesNotPerformSchemaSetup() { SqlJournalEventStore journalStore = org.mockito.Mockito.mock(SqlJournalEventStore.class); JournalEventSequencer sequencer = org.mockito.Mockito.mock(JournalEventSequencer.class); SqlAuditPersistence persistence = new SqlAuditPersistence( - new CommunityConfiguration(), auditRepository, journalStore, sequencer, txWrapper, true); + auditRepository, journalStore, sequencer, txWrapper, true, + new SqlJournalHistoryAppender(journalStore, new JournalEventSequencerFactory(journalStore), txWrapper)); persistence.initialize(RunnerId.generate()); @@ -130,6 +131,48 @@ void persistenceInitializationDoesNotPerformSchemaSetup() { verify(journalStore, never()).initialize(ArgumentMatchers.anyBoolean()); } + @Test + @DisplayName("independent journal writer commits history without changing current audit state") + void independentJournalWriterDoesNotUpdateCurrentState() throws Exception { + SqlAuditRepository auditor = new SqlAuditRepository(dataSource, AUDIT_TABLE); + auditor.initialize(true); + SqlJournalEventStore journalStore = initializedJournalStore(); + SqlJournalHistoryAppender writer = new SqlJournalHistoryAppender(journalStore, + new JournalEventSequencerFactory(journalStore), txWrapper); + + writer.append(STREAM_ID, auditEntry("history-only", AuditEntry.Status.STARTED)); + writer.append(STREAM_ID, auditEntry("history-only", AuditEntry.Status.APPLIED)); + + assertTrue(auditor.getAuditHistory().isEmpty()); + assertEquals(2, journalStore.getUnacknowledgedEvents(10).size()); + assertEquals(2L, journalStore.getLastEventByStream(STREAM_ID).get().getStreamSequence()); + } + + @Test + @DisplayName("audit transaction commits independently of a rolled-back change transaction") + void auditWriteDoesNotEnlistInChangeTransaction() throws Exception { + SqlJournalEventStore journalStore = initializedJournalStore(); + SqlAuditPersistence persistence = persistenceFor(new SqlAuditRepository(dataSource, AUDIT_TABLE), + journalStore, newSequencer(journalStore), true); + try (Connection changeConnection = dataSource.getConnection()) { + changeConnection.createStatement().execute("CREATE TABLE change_work (id INTEGER)"); + changeConnection.setAutoCommit(false); + changeConnection.createStatement().execute("INSERT INTO change_work (id) VALUES (1)"); + + persistence.writeEntry(auditEntry("independent-audit", AuditEntry.Status.APPLIED)); + assertEquals(1, persistence.getAuditHistory().size()); + assertEquals(1, journalStore.getUnacknowledgedEvents(10).size()); + changeConnection.rollback(); + } + try (Connection verification = dataSource.getConnection(); + ResultSet rows = verification.createStatement().executeQuery("SELECT COUNT(*) FROM change_work")) { + assertTrue(rows.next()); + assertEquals(0, rows.getInt(1)); + } + assertEquals(1, persistence.getAuditHistory().size()); + assertEquals(1L, journalStore.getLastEventByStream(STREAM_ID).get().getStreamSequence()); + } + @Test @DisplayName("journal enabled retains event history while the audit table stores current state") void journalEnabledSplitsCurrentStateFromHistory() throws Exception { @@ -329,7 +372,8 @@ void journalDisabledWritesUseAppendOnlyRepositoryOperation() { SqlAuditRepository auditor = mock(SqlAuditRepository.class); when(auditor.append(ArgumentMatchers.any(AuditEntry.class))).thenReturn(Result.OK()); SqlAuditPersistence persistence = new SqlAuditPersistence( - new CommunityConfiguration(), auditor, null, null, null, false); + auditor, null, null, null, false, + new SqlJournalHistoryAppender(null, null, txWrapper)); Result result = persistence.writeEntry(auditEntry("append-only", AuditEntry.Status.APPLIED)); @@ -518,7 +562,8 @@ private SqlAuditPersistence persistenceFor(SqlAuditRepository auditor, boolean journalEventsEnabled) { auditor.initialize(true); SqlAuditPersistence persistence = new SqlAuditPersistence( - new CommunityConfiguration(), auditor, journalStore, sequencer, txWrapper, journalEventsEnabled); + auditor, journalStore, sequencer, txWrapper, journalEventsEnabled, + new SqlJournalHistoryAppender(journalStore, new JournalEventSequencerFactory(journalStore), txWrapper)); persistence.initialize(RunnerId.generate()); return persistence; } diff --git a/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/internal/SqlJournalDialectHelperTest.java b/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/internal/SqlJournalDialectHelperTest.java index 878c2a9b5..c38f1d597 100644 --- a/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/internal/SqlJournalDialectHelperTest.java +++ b/community/flamingock-sql-auditstore/src/test/java/io/flamingock/store/sql/internal/SqlJournalDialectHelperTest.java @@ -73,7 +73,7 @@ void generatesTypedSchemaAndIndexesForEveryDialect(SqlDialect dialect) { @Test @DisplayName("keeps Journal schema names separate from the ordered audit payload names") - void keepsMinimalNameOwnershipBoundaries() throws Exception { + void keepsMinimalNameOwnershipBoundaries() { List expectedAuditColumns = Arrays.asList( "execution_id", "stage_id", "change_id", "author", "created_at", "state", "invoked_class", "invoked_method", "source_file", "metadata", "execution_millis", @@ -92,8 +92,6 @@ void keepsMinimalNameOwnershipBoundaries() throws Exception { assertFalse(Modifier.isPublic(JournalEventConstants.class.getModifiers())); assertTrue(Modifier.isPublic(SqlJournalDialectHelper.class.getModifiers())); assertTrue(Modifier.isFinal(SqlJournalDialectHelper.class.getModifiers())); - assertClassIsAbsent("io.flamingock.store.sql.internal.SqlAuditColumnConstants"); - assertClassIsAbsent("io.flamingock.store.sql.internal.JournalEventPersistenceConstants"); } @ParameterizedTest(name = "{0} uses the exact journal type policy") @@ -178,10 +176,6 @@ void derivesDeterministicTableScopedIndexNames() { new SqlJournalDialectHelper(SqlDialect.H2).getIndexNames(shortTableName)); } - private static void assertClassIsAbsent(String className) { - assertThrows(ClassNotFoundException.class, () -> Class.forName(className)); - } - private static String varcharType(SqlDialect dialect, int size) { return (dialect == SqlDialect.ORACLE ? "VARCHAR2(" : "VARCHAR(") + size + ")"; } diff --git a/core/flamingock-core-commons/src/main/java/io/flamingock/internal/common/core/audit/AuditHistoryAppender.java b/core/flamingock-core-commons/src/main/java/io/flamingock/internal/common/core/audit/AuditHistoryAppender.java new file mode 100644 index 000000000..26794165f --- /dev/null +++ b/core/flamingock-core-commons/src/main/java/io/flamingock/internal/common/core/audit/AuditHistoryAppender.java @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Flamingock (https://www.flamingock.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.flamingock.internal.common.core.audit; + +import io.flamingock.internal.util.Result; + +/** Appends an entry to the motor-independent audit history. */ +public interface AuditHistoryAppender { + + /** Appends an audit entry and returns the write outcome. */ + Result append(AuditEntry entry); +} diff --git a/core/flamingock-core-commons/src/main/java/io/flamingock/internal/common/core/audit/JournalHistoryAppender.java b/core/flamingock-core-commons/src/main/java/io/flamingock/internal/common/core/audit/JournalHistoryAppender.java new file mode 100644 index 000000000..5df3cd8ca --- /dev/null +++ b/core/flamingock-core-commons/src/main/java/io/flamingock/internal/common/core/audit/JournalHistoryAppender.java @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Flamingock (https://www.flamingock.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.flamingock.internal.common.core.audit; + +import io.flamingock.internal.util.Result; + +/** Appends a journal entry to the specified history stream. */ +public interface JournalHistoryAppender { + + /** Appends an audit entry to a stream and returns the write outcome. */ + Result append(String streamId, AuditEntry entry); +} diff --git a/core/flamingock-core/src/main/java/io/flamingock/internal/core/builder/AbstractChangeRunnerBuilder.java b/core/flamingock-core/src/main/java/io/flamingock/internal/core/builder/AbstractChangeRunnerBuilder.java index 50e94609d..c6f4ae62c 100644 --- a/core/flamingock-core/src/main/java/io/flamingock/internal/core/builder/AbstractChangeRunnerBuilder.java +++ b/core/flamingock-core/src/main/java/io/flamingock/internal/core/builder/AbstractChangeRunnerBuilder.java @@ -16,6 +16,8 @@ package io.flamingock.internal.core.builder; import io.flamingock.api.external.TargetSystem; +import io.flamingock.internal.common.core.audit.AuditHistoryAppender; +import io.flamingock.internal.common.core.audit.JournalHistoryAppender; import io.flamingock.internal.common.core.audit.AuditPersistenceFactory; import io.flamingock.internal.common.core.audit.AuditWriter; import io.flamingock.internal.common.core.context.Context; @@ -32,6 +34,7 @@ import io.flamingock.internal.core.context.PriorityContextResolver; import io.flamingock.internal.core.context.SimpleContext; import io.flamingock.internal.core.external.store.AuditStore; +import io.flamingock.internal.core.external.store.HistoryAppenderProvider; import io.flamingock.internal.common.core.audit.AuditPersistence; import io.flamingock.internal.core.operation.OperationResolver; import io.flamingock.internal.core.plan.ExecutionPlanner; @@ -221,6 +224,8 @@ public final Runner build() { configureStoreAndTargetSystem(hierarchicalContext); + registerHistoryAppenders(hierarchicalContext, auditStore); + // Registered under the interface, not via new Dependency(instance): the factory is a lambda, // so the single-argument constructor would key it by its synthetic class and rely on the // resolver's assignable-scan fallback — and would raise a bare NullPointerException, instead @@ -257,6 +262,21 @@ public final Runner build() { } + static void registerHistoryAppenders(PriorityContext context, Object store) { + if (!(store instanceof HistoryAppenderProvider)) { + return; + } + HistoryAppenderProvider provider = (HistoryAppenderProvider) store; + AuditHistoryAppender appender = provider.getAuditHistoryAppender(); + if (appender != null) { + context.addDependency(new Dependency(AuditHistoryAppender.class, appender)); + } + JournalHistoryAppender journalWriter = provider.getJournalHistoryAppender(); + if (journalWriter != null) { + context.addDependency(new Dependency(JournalHistoryAppender.class, journalWriter)); + } + } + private LoadedPipeline loadPipeline(FlamingockMetadata flamingockMetadata) { List changeFiltersFromPlugins = pluginManager.getPlugins() .stream() diff --git a/core/flamingock-core/src/main/java/io/flamingock/internal/core/external/store/HistoryAppenderProvider.java b/core/flamingock-core/src/main/java/io/flamingock/internal/core/external/store/HistoryAppenderProvider.java new file mode 100644 index 000000000..6b0226d50 --- /dev/null +++ b/core/flamingock-core/src/main/java/io/flamingock/internal/core/external/store/HistoryAppenderProvider.java @@ -0,0 +1,29 @@ +/* + * Copyright 2026 Flamingock (https://www.flamingock.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.flamingock.internal.core.external.store; + +import io.flamingock.internal.common.core.audit.AuditHistoryAppender; +import io.flamingock.internal.common.core.audit.JournalHistoryAppender; + +/** Optional audit-store capability for independently injectable history writers. */ +public interface HistoryAppenderProvider { + + /** Returns the motor-independent audit history appender, if available. */ + AuditHistoryAppender getAuditHistoryAppender(); + + /** Returns the journal history writer, if available. */ + JournalHistoryAppender getJournalHistoryAppender(); +} diff --git a/core/flamingock-core/src/main/java/io/flamingock/internal/core/external/store/audit/community/AbstractCommunityAuditPersistence.java b/core/flamingock-core/src/main/java/io/flamingock/internal/core/external/store/audit/community/AbstractCommunityAuditPersistence.java index fc29a2f37..eecbc5b73 100644 --- a/core/flamingock-core/src/main/java/io/flamingock/internal/core/external/store/audit/community/AbstractCommunityAuditPersistence.java +++ b/core/flamingock-core/src/main/java/io/flamingock/internal/core/external/store/audit/community/AbstractCommunityAuditPersistence.java @@ -16,18 +16,12 @@ package io.flamingock.internal.core.external.store.audit.community; import io.flamingock.internal.util.id.RunnerId; -import io.flamingock.internal.core.configuration.community.CommunityConfigurable; public abstract class AbstractCommunityAuditPersistence implements CommunityAuditPersistence { - protected final CommunityConfigurable localConfiguration; - - protected AbstractCommunityAuditPersistence(CommunityConfigurable localConfiguration) { - this.localConfiguration = localConfiguration; + protected void doInitialize(RunnerId runnerId) { } - abstract protected void doInitialize(RunnerId runnerId); - public void initialize(RunnerId runnerId) { doInitialize(runnerId); } diff --git a/core/flamingock-core/src/main/java/io/flamingock/internal/core/journal/JournalEventSequencer.java b/core/flamingock-core/src/main/java/io/flamingock/internal/core/journal/JournalEventSequencer.java index 3d9dee8da..920e46f25 100644 --- a/core/flamingock-core/src/main/java/io/flamingock/internal/core/journal/JournalEventSequencer.java +++ b/core/flamingock-core/src/main/java/io/flamingock/internal/core/journal/JournalEventSequencer.java @@ -35,33 +35,61 @@ * beyond tidiness: a contiguous {@code streamSequence} is what lets a consumer reconstruct order and tell * "still in flight" from "lost", so gaps must never be produced by ordinary failure handling. *

- * If a caller forgets to confirm a write that did land, the reused position collides with the unique - * {@code (streamId, streamSequence)} index — a loud failure rather than a silent duplicate. + * If the write outcome is uncertain, the caller must mark it before the next allocation. The sequencer + * then reads the durable tail and will not allocate if that read fails. A caller that forgets both confirmation + * and uncertainty marking can still collide with the unique {@code (streamId, streamSequence)} index. */ public class JournalEventSequencer { private final String streamId; private long nextSequence; private boolean pendingConfirmation; + private boolean uncertainWriteOutcome; + private final JournalEventReader journalEventReader; JournalEventSequencer(String streamId, long initialSequence) { + this(streamId, initialSequence, null); + } + + JournalEventSequencer(String streamId, long initialSequence, JournalEventReader journalEventReader) { this.streamId = streamId; this.nextSequence = initialSequence; // seeded from outside + this.journalEventReader = journalEventReader; } /** * Builds the next event without spending its stream position; call {@link #confirm()} once the * event is durably written. */ - public JournalEvent newEvent(AuditEntry payload) { + public synchronized JournalEvent newEvent(AuditEntry payload) { + if (uncertainWriteOutcome) { + if (journalEventReader == null) { + throw new IllegalStateException("Cannot reconcile stream without a journal reader"); + } + long durableNext = journalEventReader.getLastEventByStream(streamId) + .map(event -> Math.addExact(event.getStreamSequence(), 1L)) + .orElse(1L); + nextSequence = Math.max(nextSequence, durableNext); + pendingConfirmation = false; + uncertainWriteOutcome = false; + } return getAuditEntryJournalEvent(payload, JournalEventType.CHANGE_STATE); } + /** + * Marks a write outcome as uncertain. The next allocation rereads the durable tail before issuing a position. + * Do not confirm an uncertain write: only the caller can confirm a known successful commit. + */ + public synchronized void markWriteOutcomeUncertain() { + uncertainWriteOutcome = true; + pendingConfirmation = false; + } + /** * Marks the position handed out by the last {@link #newEvent} as durably written, moving the stream on. * A no-op if nothing is outstanding. */ - public void confirm() { - if (pendingConfirmation) { + public synchronized void confirm() { + if (pendingConfirmation && !uncertainWriteOutcome) { nextSequence++; pendingConfirmation = false; } diff --git a/core/flamingock-core/src/main/java/io/flamingock/internal/core/journal/JournalEventSequencerFactory.java b/core/flamingock-core/src/main/java/io/flamingock/internal/core/journal/JournalEventSequencerFactory.java index 7f47700b1..55a863798 100644 --- a/core/flamingock-core/src/main/java/io/flamingock/internal/core/journal/JournalEventSequencerFactory.java +++ b/core/flamingock-core/src/main/java/io/flamingock/internal/core/journal/JournalEventSequencerFactory.java @@ -15,18 +15,24 @@ */ package io.flamingock.internal.core.journal; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + public class JournalEventSequencerFactory { private final JournalEventReader journalEventReader; + private final ConcurrentMap sequencers = new ConcurrentHashMap<>(); public JournalEventSequencerFactory(JournalEventReader journalEventReader) { this.journalEventReader = journalEventReader; } public JournalEventSequencer forStream(String streamId) { - long initialSequence = journalEventReader.getLastEventByStream(streamId) - .map(e -> e.getStreamSequence() + 1) - .orElse(1L); - return new JournalEventSequencer(streamId, initialSequence); + return sequencers.computeIfAbsent(streamId, id -> { + long initialSequence = journalEventReader.getLastEventByStream(id) + .map(e -> Math.addExact(e.getStreamSequence(), 1L)) + .orElse(1L); + return new JournalEventSequencer(id, initialSequence, journalEventReader); + }); } } diff --git a/core/flamingock-core/src/test/java/io/flamingock/internal/core/builder/HistoryAppenderProviderTest.java b/core/flamingock-core/src/test/java/io/flamingock/internal/core/builder/HistoryAppenderProviderTest.java new file mode 100644 index 000000000..4b313bf47 --- /dev/null +++ b/core/flamingock-core/src/test/java/io/flamingock/internal/core/builder/HistoryAppenderProviderTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2026 Flamingock (https://www.flamingock.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.flamingock.internal.core.builder; + +import io.flamingock.internal.common.core.audit.AuditHistoryAppender; +import io.flamingock.internal.common.core.audit.JournalHistoryAppender; +import io.flamingock.internal.core.context.PriorityContext; +import io.flamingock.internal.core.context.SimpleContext; +import io.flamingock.internal.core.external.store.HistoryAppenderProvider; +import io.flamingock.internal.core.external.store.AuditStore; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class HistoryAppenderProviderTest { + + @Test + void registersProvidedAppendersByTypeForContextResolution() { + HistoryAppenderProvider provider = mock(HistoryAppenderProvider.class); + AuditHistoryAppender appender = mock(AuditHistoryAppender.class); + JournalHistoryAppender journal = mock(JournalHistoryAppender.class); + when(provider.getAuditHistoryAppender()).thenReturn(appender); + when(provider.getJournalHistoryAppender()).thenReturn(journal); + PriorityContext context = context(); + + AbstractChangeRunnerBuilder.registerHistoryAppenders(context, provider); + + assertSame(appender, context.getDependency(AuditHistoryAppender.class).get().getInstance()); + assertSame(journal, context.getDependency(JournalHistoryAppender.class).get().getInstance()); + } + + @Test + void skipsUnavailableAppendersAndStoresWithoutTheCapability() { + PriorityContext context = context(); + HistoryAppenderProvider provider = mock(HistoryAppenderProvider.class); + AbstractChangeRunnerBuilder.registerHistoryAppenders(context, provider); + AbstractChangeRunnerBuilder.registerHistoryAppenders(context, mock(AuditStore.class)); + + assertFalse(context.getDependency(AuditHistoryAppender.class).isPresent()); + assertFalse(context.getDependency(JournalHistoryAppender.class).isPresent()); + } + + private PriorityContext context() { + return new PriorityContext(new SimpleContext()); + } +} diff --git a/core/flamingock-core/src/test/java/io/flamingock/internal/core/journal/JournalEventSequencerTest.java b/core/flamingock-core/src/test/java/io/flamingock/internal/core/journal/JournalEventSequencerTest.java index 66e0548e5..ffb450b21 100644 --- a/core/flamingock-core/src/test/java/io/flamingock/internal/core/journal/JournalEventSequencerTest.java +++ b/core/flamingock-core/src/test/java/io/flamingock/internal/core/journal/JournalEventSequencerTest.java @@ -23,10 +23,17 @@ import org.junit.jupiter.api.Test; import java.time.LocalDateTime; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; class JournalEventSequencerTest { @@ -54,6 +61,85 @@ void derivesStableIdempotencyKeyFromChangeStateIdentity() { assertNotEquals(first.getIdempotencyKey(), differentState.getIdempotencyKey()); } + @Test + void factorySharesSequencerPerStreamAndIsolatesOtherStreams() { + JournalEventReader reader = mock(JournalEventReader.class); + when(reader.getLastEventByStream("one")).thenReturn(Optional.empty()); + when(reader.getLastEventByStream("two")).thenReturn(Optional.empty()); + JournalEventSequencerFactory factory = new JournalEventSequencerFactory(reader); + JournalEventSequencer first = factory.forStream("one"); + JournalEventSequencer concurrent = CompletableFuture.supplyAsync(() -> factory.forStream("one")).join(); + JournalEventSequencer other = factory.forStream("two"); + + assertSame(first, concurrent); + assertNotEquals(first, other); + first.newEvent(auditEntry("execution-1", "change-1", AuditEntry.Status.APPLIED)); + first.confirm(); + assertEquals(2L, first.newEvent(auditEntry("execution-1", "change-2", AuditEntry.Status.APPLIED)) + .getStreamSequence()); + assertEquals(1L, other.newEvent(auditEntry("execution-1", "change-1", AuditEntry.Status.APPLIED)) + .getStreamSequence()); + verify(reader).getLastEventByStream("one"); + } + + @Test + void deterministicFailureReusesUnconfirmedPosition() { + JournalEventReader reader = mock(JournalEventReader.class); + when(reader.getLastEventByStream("one")).thenReturn(Optional.empty()); + JournalEventSequencer sequencer = new JournalEventSequencerFactory(reader).forStream("one"); + AuditEntry payload = auditEntry("execution-1", "change-1", AuditEntry.Status.APPLIED); + + JournalEvent failed = sequencer.newEvent(payload); + JournalEvent retry = sequencer.newEvent(payload); + assertEquals(failed.getStreamSequence(), retry.getStreamSequence()); + assertEquals(failed.getIdempotencyKey(), retry.getIdempotencyKey()); + assertNotEquals(failed.getEventId(), retry.getEventId()); + } + + @Test + void uncertainCommittedWriteReseedsFromDurableTail() { + JournalEventReader reader = mock(JournalEventReader.class); + JournalEvent durable = mock(JournalEvent.class); + when(durable.getStreamSequence()).thenReturn(3L); + when(reader.getLastEventByStream("one")) + .thenReturn(Optional.empty(), Optional.of(durable)); + JournalEventSequencer sequencer = new JournalEventSequencerFactory(reader).forStream("one"); + sequencer.newEvent(auditEntry("execution-1", "change-1", AuditEntry.Status.APPLIED)); + sequencer.markWriteOutcomeUncertain(); + sequencer.confirm(); + + assertEquals(4L, sequencer.newEvent(auditEntry("execution-1", "change-2", AuditEntry.Status.APPLIED)) + .getStreamSequence()); + } + + @Test + void uncertainRolledBackWriteReusesPosition() { + JournalEventReader reader = mock(JournalEventReader.class); + when(reader.getLastEventByStream("one")).thenReturn(Optional.empty()); + JournalEventSequencer sequencer = new JournalEventSequencerFactory(reader).forStream("one"); + sequencer.newEvent(auditEntry("execution-1", "change-1", AuditEntry.Status.APPLIED)); + sequencer.markWriteOutcomeUncertain(); + assertEquals(1L, sequencer.newEvent(auditEntry("execution-1", "change-2", AuditEntry.Status.APPLIED)) + .getStreamSequence()); + } + + @Test + void unreadableTailFailsClosedAndCanRetryReconciliation() { + JournalEventReader reader = mock(JournalEventReader.class); + when(reader.getLastEventByStream("one")) + .thenReturn(Optional.empty()) + .thenThrow(new IllegalStateException("tail unavailable")) + .thenReturn(Optional.empty()); + JournalEventSequencer sequencer = new JournalEventSequencerFactory(reader).forStream("one"); + sequencer.newEvent(auditEntry("execution-1", "change-1", AuditEntry.Status.APPLIED)); + sequencer.markWriteOutcomeUncertain(); + assertThrows(IllegalStateException.class, + () -> sequencer.newEvent(auditEntry("execution-1", "change-2", AuditEntry.Status.APPLIED))); + sequencer.confirm(); + assertEquals(1L, sequencer.newEvent(auditEntry("execution-1", "change-2", AuditEntry.Status.APPLIED)) + .getStreamSequence()); + } + private static AuditEntry auditEntry(String executionId, String changeId, AuditEntry.Status state) { return auditEntry(executionId, changeId, state, "author-1", LocalDateTime.of(2026, 1, 1, 0, 0)); }