diff --git a/connectors/mongo/conn.go b/connectors/mongo/conn.go index 4ecd8be7..5649a3b7 100644 --- a/connectors/mongo/conn.go +++ b/connectors/mongo/conn.go @@ -1139,6 +1139,11 @@ func (c *conn) buildIndexedBulkModels(updates []*adiomv1.Update) ([]indexedWrite if err != nil { return nil, false, err } + // Unknown update types and empty partial updates preserve the legacy + // successful-noop behavior and must not participate in deduplication. + if model == nil { + continue + } compiled = append(compiled, compiledUpdate{model: model, idKey: idKey, typ: update.GetType(), index: uint32(i)}) // #nosec G115 -- protobuf messages cannot contain more than uint32 items } @@ -1156,9 +1161,6 @@ func (c *conn) buildIndexedBulkModels(updates []*adiomv1.Update) ([]indexedWrite lastType = update.typ } - if update.model == nil { - continue - } indexed := indexedWriteModel{model: update.model, indices: []uint32{update.index}, idKey: update.idKey} if isNew { models = append(models, indexed) @@ -1355,6 +1357,15 @@ func maybeUnavailableError(err error) error { return connect.NewError(connect.CodeInternal, err) } +func mongoIDPart(id []*adiomv1.BsonValue) *adiomv1.BsonValue { + if len(id) == 0 { + return nil + } + // The BSON ID contract places MongoDB's _id last. Single-part legacy IDs + // are therefore handled by the same rule without relying on the name. + return id[len(id)-1] +} + // GetByIds implements adiomv1connect.ConnectorServiceHandler. func (c *conn) GetByIds(ctx context.Context, r *connect.Request[adiomv1.GetByIdsRequest]) (*connect.Response[adiomv1.GetByIdsResponse], error) { col, _, ok := GetCol(c.client, r.Msg.GetNamespace()) @@ -1371,7 +1382,7 @@ func (c *conn) GetByIds(ctx context.Context, r *connect.Request[adiomv1.GetByIds res[i] = &adiomv1.GetByIdsResponse_ResponseItem{} return nil } - bv := id.GetId()[0] + bv := mongoIDPart(id.GetId()) rawVal := bson.RawValue{ Type: bson.Type(bv.GetType()), Value: bv.GetData(), diff --git a/connectors/mongo/conn_unit_test.go b/connectors/mongo/conn_unit_test.go index 33f77cdd..7c55a293 100644 --- a/connectors/mongo/conn_unit_test.go +++ b/connectors/mongo/conn_unit_test.go @@ -34,6 +34,14 @@ func bsonID(t *testing.T, name string, v interface{}) *adiomv1.BsonValue { return &adiomv1.BsonValue{Name: name, Type: uint32(typ), Data: data} } +func TestMongoIDPartUsesLastPart(t *testing.T) { + shard := bsonID(t, "tenant", "acme") + id := bsonID(t, "_id", "item") + if got := mongoIDPart([]*adiomv1.BsonValue{shard, id}); got != id { + t.Fatalf("mongoIDPart() = %#v, want last part", got) + } +} + func TestPlanningFanoutLimit(t *testing.T) { assert.Equal(t, defaultNamespaceFanoutLimit, (&conn{flavor: FlavorMongoDB}).planningFanoutLimit()) assert.Equal(t, defaultNamespaceFanoutLimit, (&conn{flavor: FlavorDocumentDB}).planningFanoutLimit()) @@ -397,6 +405,20 @@ func TestBuildIndexedBulkModels_InvalidUpdateFailsBatch(t *testing.T) { assert.Error(t, err) } +func TestBuildIndexedBulkModels_UnknownUpdateDoesNotSuppressWrite(t *testing.T) { + c := &conn{} + unknown := insertU(t, "a", bson.M{"ignored": true}) + unknown.Type = adiomv1.UpdateType_UPDATE_TYPE_UNKNOWN + models, ordered, err := c.buildIndexedBulkModels([]*adiomv1.Update{ + insertU(t, "a", bson.M{"v": 1}), + unknown, + }) + require.NoError(t, err) + assert.False(t, ordered) + require.Len(t, models, 1) + assert.Equal(t, []uint32{0}, models[0].indices) +} + func TestFailuresFromBulkWrite_UnorderedMapsOriginalInputs(t *testing.T) { models := []indexedWriteModel{ {indices: []uint32{4, 1}}, diff --git a/connectors/postgres/bson_conv.go b/connectors/postgres/bson_conv.go index 971bb28d..ce5c63ea 100644 --- a/connectors/postgres/bson_conv.go +++ b/connectors/postgres/bson_conv.go @@ -2,30 +2,87 @@ package postgres import ( "bytes" + "fmt" + "net/netip" + "github.com/jackc/pgx/v5/pgtype" "go.mongodb.org/mongo-driver/v2/bson" ) -// bsonToMap converts BSON data to a map[string]interface{} excluding the _id field -func bsonToMap(reg *bson.Registry, data []byte) (map[string]interface{}, error) { +// bsonToMap converts BSON data to a map[string]interface{}. The _id field is +// removed when it is synthetic rather than a destination column. +func bsonToMap(reg *bson.Registry, data []byte, preserveID bool) (map[string]interface{}, error) { var m map[string]interface{} dec := bson.NewDecoder(bson.NewDocumentReader(bytes.NewReader(data))) dec.SetRegistry(reg) if err := dec.Decode(&m); err != nil { return nil, err } - // Remove _id as it's a synthetic field for postgres - delete(m, "_id") + if !preserveID { + delete(m, "_id") + } + for key, value := range m { + converted, err := bsonValueForPostgres(key, value) + if err != nil { + return nil, err + } + m[key] = converted + } return m, nil } +func bsonValueForPostgres(field string, value interface{}) (interface{}, error) { + binary, ok := value.(bson.Binary) + if !ok { + return value, nil + } + switch binary.Subtype { + case bson.TypeBinaryGeneric: + return binary.Data, nil + case bson.TypeBinaryUUIDOld: + return nil, fmt.Errorf("BSON legacy UUID subtype 3 is not supported for field %q", field) + case bson.TypeBinaryUUID: + if len(binary.Data) != 16 { + return nil, fmt.Errorf("BSON UUID field %q has %d bytes, expected 16", field, len(binary.Data)) + } + var uuid [16]byte + copy(uuid[:], binary.Data) + return pgtype.UUID{Bytes: uuid, Valid: true}, nil + default: + return value, nil + } +} + +func bsonSafeValue(value any) any { + switch value := value.(type) { + case netip.Addr: + return value.String() + case netip.Prefix: + return value.String() + case map[string]interface{}: + result := make(map[string]interface{}, len(value)) + for key, nested := range value { + result[key] = bsonSafeValue(nested) + } + return result + case []interface{}: + result := make([]interface{}, len(value)) + for i, nested := range value { + result[i] = bsonSafeValue(nested) + } + return result + default: + return value + } +} + // marshalWithRegistry marshals val using the given registry func marshalWithRegistry(reg *bson.Registry, val any) ([]byte, error) { var buf bytes.Buffer enc := bson.NewEncoder(bson.NewDocumentWriter(&buf)) enc.SetRegistry(reg) - if err := enc.Encode(val); err != nil { + if err := enc.Encode(bsonSafeValue(val)); err != nil { return nil, err } return buf.Bytes(), nil @@ -36,7 +93,7 @@ func marshalValueWithRegistry(reg *bson.Registry, val any) (bson.Type, []byte, e var buf bytes.Buffer enc := bson.NewEncoder(bson.NewDocumentWriter(&buf)) enc.SetRegistry(reg) - if err := enc.Encode(bson.D{{Key: "v", Value: val}}); err != nil { + if err := enc.Encode(bson.D{{Key: "v", Value: bsonSafeValue(val)}}); err != nil { return 0, nil, err } raw := bson.Raw(buf.Bytes()) diff --git a/connectors/postgres/bson_conv_test.go b/connectors/postgres/bson_conv_test.go new file mode 100644 index 00000000..a3c4c211 --- /dev/null +++ b/connectors/postgres/bson_conv_test.go @@ -0,0 +1,121 @@ +package postgres + +import ( + "net/netip" + "reflect" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + "go.mongodb.org/mongo-driver/v2/bson" +) + +func TestBSONToMapConvertsBinaryValues(t *testing.T) { + uuidBytes := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + doc, err := bson.Marshal(bson.D{ + {Key: "bytes", Value: bson.Binary{Subtype: bson.TypeBinaryGeneric, Data: []byte("data")}}, + {Key: "uuid", Value: bson.Binary{Subtype: bson.TypeBinaryUUID, Data: uuidBytes}}, + }) + if err != nil { + t.Fatal(err) + } + + got, err := bsonToMap(NewBSONRegistry(), doc, false) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got["bytes"], []byte("data")) { + t.Fatalf("bytes = %#v", got["bytes"]) + } + wantUUID := pgtype.UUID{Valid: true} + copy(wantUUID.Bytes[:], uuidBytes) + if !reflect.DeepEqual(got["uuid"], wantUUID) { + t.Fatalf("uuid = %#v, want %#v", got["uuid"], wantUUID) + } +} + +func TestBSONToMapRejectsLegacyUUID(t *testing.T) { + doc, err := bson.Marshal(bson.D{{ + Key: "uuid", + Value: bson.Binary{ + Subtype: bson.TypeBinaryUUIDOld, + Data: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + }, + }}) + if err != nil { + t.Fatal(err) + } + if _, err := bsonToMap(NewBSONRegistry(), doc, false); err == nil { + t.Fatal("expected legacy UUID subtype to be rejected") + } +} + +func TestBSONToMapPreservesIDForDestinationColumn(t *testing.T) { + doc, err := bson.Marshal(bson.D{{Key: "_id", Value: "source-id"}, {Key: "value", Value: "data"}}) + if err != nil { + t.Fatal(err) + } + + got, err := bsonToMap(NewBSONRegistry(), doc, true) + if err != nil { + t.Fatal(err) + } + if got["_id"] != "source-id" { + t.Fatalf("_id = %#v", got["_id"]) + } +} + +func TestMarshalWithRegistryNormalizesPostgresNetworkTypes(t *testing.T) { + prefix := netip.MustParsePrefix("192.0.2.0/24") + doc, err := marshalWithRegistry(NewBSONRegistry(), map[string]interface{}{"network": prefix}) + if err != nil { + t.Fatal(err) + } + var got map[string]interface{} + if err := bson.Unmarshal(doc, &got); err != nil { + t.Fatal(err) + } + if got["network"] != prefix.String() { + t.Fatalf("network = %#v, want %q", got["network"], prefix.String()) + } +} + +func TestMarshalValueWithRegistryNormalizesPostgresNetworkTypes(t *testing.T) { + tests := []struct { + name string + value any + want string + }{ + {name: "address", value: netip.MustParseAddr("192.0.2.1"), want: "192.0.2.1"}, + {name: "prefix", value: netip.MustParsePrefix("192.0.2.0/24"), want: "192.0.2.0/24"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + typ, data, err := marshalValueWithRegistry(NewBSONRegistry(), test.value) + if err != nil { + t.Fatal(err) + } + var got string + if err := bson.UnmarshalValue(typ, data, &got); err != nil { + t.Fatal(err) + } + if got != test.want { + t.Fatalf("network ID = %q, want %q", got, test.want) + } + }) + } +} + +func TestBSONRegistryEncodesPostgresUUIDAsUUID(t *testing.T) { + uuid := [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + typ, data, err := marshalValueWithRegistry(NewBSONRegistry(), uuid) + if err != nil { + t.Fatal(err) + } + var got bson.Binary + if err := bson.UnmarshalValue(typ, data, &got); err != nil { + t.Fatal(err) + } + if got.Subtype != bson.TypeBinaryUUID || !reflect.DeepEqual(got.Data, uuid[:]) { + t.Fatalf("binary = %#v", got) + } +} diff --git a/connectors/postgres/bson_registry.go b/connectors/postgres/bson_registry.go index c3d05d2a..6130b80c 100644 --- a/connectors/postgres/bson_registry.go +++ b/connectors/postgres/bson_registry.go @@ -12,8 +12,25 @@ import ( var ( tDecimal = reflect.TypeOf(decimal.Decimal{}) tTime = reflect.TypeOf(time.Time{}) + tUUID = reflect.TypeOf([16]byte{}) ) +type uuidCodec struct{} + +var _ bson.ValueEncoder = uuidCodec{} + +func (uuidCodec) EncodeValue(_ bson.EncodeContext, vw bson.ValueWriter, val reflect.Value) error { + if !val.IsValid() || val.Type() != tUUID { + return bson.ValueEncoderError{ + Name: "UUIDEncodeValue", + Types: []reflect.Type{tUUID}, + Received: val, + } + } + uuid := val.Interface().([16]byte) + return vw.WriteBinaryWithSubtype(uuid[:], bson.TypeBinaryUUID) +} + // decimalCodec handles encoding/decoding between decimal.Decimal and bson.Decimal128 type decimalCodec struct{} @@ -163,6 +180,7 @@ func NewBSONRegistry() *bson.Registry { reg.RegisterTypeMapEntry(bson.TypeDateTime, tTime) reg.RegisterTypeMapEntry(bson.TypeDecimal128, tDecimal) + reg.RegisterTypeEncoder(tUUID, uuidCodec{}) return reg } diff --git a/connectors/postgres/conn.go b/connectors/postgres/conn.go index 5b4ae91b..3e86471d 100644 --- a/connectors/postgres/conn.go +++ b/connectors/postgres/conn.go @@ -17,6 +17,7 @@ import ( "github.com/adiom-data/dsync/connectors/postgres/pglib" adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" "github.com/adiom-data/dsync/gen/adiom/v1/adiomv1connect" + "github.com/adiom-data/dsync/internal/gobregister" "github.com/cespare/xxhash" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" @@ -48,17 +49,63 @@ type PostgresSettings struct { EnableReplicaMode bool } +func init() { + gobregister.RegisterCursorTypes() +} + type conn struct { adiomv1connect.UnimplementedConnectorServiceHandler replicationUrl string id uint64 c *pgxpool.Pool - pkeys map[string][]string + pkeys map[string][]string + // sinkMetadata is a connection-time snapshot. Schemas are expected to stay + // unchanged for the lifetime of the connector. + sinkMetadata map[string]sinkTableMetadata + hasIDColumn map[string]bool + idAliases map[string]string settings PostgresSettings bsonRegistry *bson.Registry } +type batchSender interface { + SendBatch(context.Context, *pgx.Batch) pgx.BatchResults +} + +func (c *conn) sendSinkBatch(ctx context.Context, batch *pgx.Batch, count int, onError func(int, error)) error { + var sender batchSender = c.c + var tx pgx.Tx + if c.settings.EnableReplicaMode { + var err error + tx, err = c.c.Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(ctx) }() + if _, err := tx.Exec(ctx, "SET LOCAL session_replication_role = 'replica'"); err != nil { + return fmt.Errorf("failed to set session_replication_role: %w", err) + } + sender = tx + } + + results := sender.SendBatch(ctx, batch) + for i := 0; i < count; i++ { + if _, err := results.Exec(); err != nil { + onError(i, err) + _ = results.Close() + return err + } + } + if err := results.Close(); err != nil { + return err + } + if tx != nil { + return tx.Commit(ctx) + } + return nil +} + func (c *conn) Teardown() { c.c.Close() } @@ -124,6 +171,29 @@ type pkeyRow struct { ColumnName string } +type columnRow struct { + TableSchema string + TableName string + ColumnName string +} + +type sinkColumnMetadata struct { + TableSchema string + TableName string + Name string + DatabaseType string + Nullable bool + HasDefault bool + Generated bool + Identity bool + IdentityGeneration string +} + +type sinkTableMetadata struct { + Columns []sinkColumnMetadata + PrimaryKey []string +} + func getPrimaryKey(primaryKeys []string, data map[string]interface{}) []interface{} { var res []interface{} for _, col := range primaryKeys { @@ -133,7 +203,18 @@ func getPrimaryKey(primaryKeys []string, data map[string]interface{}) []interfac } func getTablePrimaryKeySchema(ctx context.Context, c *pgxpool.Pool) (map[string][]string, error) { - rows, err := c.Query(ctx, "SELECT t.table_schema, t.table_name, c.column_name FROM information_schema.key_column_usage AS c LEFT JOIN information_schema.table_constraints AS t ON t.constraint_name = c.constraint_name WHERE t.constraint_type = 'PRIMARY KEY' AND t.table_schema != 'pg_catalog' order by t.table_schema, t.table_name, c.ordinal_position;") + rows, err := c.Query(ctx, ` + SELECT t.table_schema, t.table_name, c.column_name + FROM information_schema.table_constraints AS t + JOIN information_schema.key_column_usage AS c + ON c.constraint_catalog = t.constraint_catalog + AND c.constraint_schema = t.constraint_schema + AND c.constraint_name = t.constraint_name + AND c.table_schema = t.table_schema + AND c.table_name = t.table_name + WHERE t.constraint_type = 'PRIMARY KEY' + AND t.table_schema != 'pg_catalog' + ORDER BY t.table_schema, t.table_name, c.ordinal_position`) if err != nil { return nil, err } @@ -149,6 +230,88 @@ func getTablePrimaryKeySchema(ctx context.Context, c *pgxpool.Pool) (map[string] return pkeys, nil } +func getTableColumns(ctx context.Context, c *pgxpool.Pool) (map[string]map[string]struct{}, error) { + rows, err := c.Query(ctx, ` + SELECT n.nspname AS table_schema, c.relname AS table_name, a.attname AS column_name + FROM pg_catalog.pg_attribute AS a + JOIN pg_catalog.pg_class AS c ON c.oid = a.attrelid + JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace + WHERE a.attnum > 0 + AND NOT a.attisdropped + AND c.relkind IN ('r', 'p') + AND n.nspname != 'pg_catalog'`) + if err != nil { + return nil, err + } + columnRows, err := pgx.CollectRows(rows, pgx.RowToStructByName[columnRow]) + if err != nil { + return nil, err + } + result := map[string]map[string]struct{}{} + for _, row := range columnRows { + namespace := row.TableSchema + "." + row.TableName + if result[namespace] == nil { + result[namespace] = map[string]struct{}{} + } + result[namespace][row.ColumnName] = struct{}{} + } + return result, nil +} + +func getSinkTableMetadata(ctx context.Context, c *pgxpool.Pool, pkeys map[string][]string) (map[string]sinkTableMetadata, error) { + rows, err := c.Query(ctx, ` + SELECT table_schema, + table_name, + column_name AS name, + udt_name AS database_type, + is_nullable = 'YES' AS nullable, + column_default IS NOT NULL AS has_default, + is_generated <> 'NEVER' AS generated, + is_identity = 'YES' AS identity, + COALESCE(identity_generation, '') AS identity_generation + FROM information_schema.columns + WHERE table_schema NOT IN ('pg_catalog', 'information_schema') + ORDER BY table_schema, table_name, ordinal_position`) + if err != nil { + return nil, err + } + columnRows, err := pgx.CollectRows(rows, pgx.RowToStructByName[sinkColumnMetadata]) + if err != nil { + return nil, err + } + result := map[string]sinkTableMetadata{} + for _, row := range columnRows { + namespace := row.TableSchema + "." + row.TableName + metadata := result[namespace] + metadata.Columns = append(metadata.Columns, row) + metadata.PrimaryKey = pkeys[namespace] + result[namespace] = metadata + } + return result, nil +} + +func idColumnMetadata(columns map[string]map[string]struct{}, pkeys map[string][]string) (map[string]bool, map[string]string) { + hasIDColumn := map[string]bool{} + idAliases := map[string]string{} + for namespace, tableColumns := range columns { + if _, exists := tableColumns["_id"]; !exists { + continue + } + hasIDColumn[namespace] = true + if slices.Contains(pkeys[namespace], "_id") { + continue + } + for suffix := 2; ; suffix++ { + alias := "_id" + strconv.Itoa(suffix) + if _, exists := tableColumns[alias]; !exists { + idAliases[namespace] = alias + break + } + } + } + return hasIDColumn, idAliases +} + func NewConn(ctx context.Context, settings PostgresSettings) (*conn, error) { url := settings.URL // Set some defaults @@ -171,14 +334,6 @@ func NewConn(ctx context.Context, settings PostgresSettings) (*conn, error) { afterConnect := func(ctx context.Context, conn *pgx.Conn) error { pgxdecimal.Register(conn.TypeMap()) - - // Enable replica mode for sink operations if requested - if settings.EnableReplicaMode { - if _, err := conn.Exec(ctx, "SET session_replication_role = 'replica'"); err != nil { - return fmt.Errorf("failed to set session_replication_role: %w", err) - } - } - return nil } config, err := pgxpool.ParseConfig(url) @@ -193,8 +348,20 @@ func NewConn(ctx context.Context, settings PostgresSettings) (*conn, error) { pkeys, err := getTablePrimaryKeySchema(ctx, c) if err != nil { + c.Close() return nil, err } + columns, err := getTableColumns(ctx, c) + if err != nil { + c.Close() + return nil, err + } + sinkMetadata, err := getSinkTableMetadata(ctx, c, pkeys) + if err != nil { + c.Close() + return nil, err + } + hasIDColumn, idAliases := idColumnMetadata(columns, pkeys) // Create BSON registry with decimal support bsonRegistry := NewBSONRegistry() @@ -204,6 +371,9 @@ func NewConn(ctx context.Context, settings PostgresSettings) (*conn, error) { replicationUrl: rUrl, c: c, pkeys: pkeys, + sinkMetadata: sinkMetadata, + hasIDColumn: hasIDColumn, + idAliases: idAliases, settings: settings, bsonRegistry: bsonRegistry, }, nil @@ -215,6 +385,11 @@ func (c *conn) GeneratePlan(ctx context.Context, r *connect.Request[adiomv1.Gene if len(namespaces) < 1 { return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("must specify at least one namespace")) } + for _, namespace := range namespaces { + if len(c.pkeys[namespace]) == 0 { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("no primary key found for namespace %s", namespace)) + } + } var partitions []*adiomv1.Partition var updatesPartitions []*adiomv1.UpdatesPartition @@ -371,6 +546,9 @@ func (c *conn) ListData(ctx context.Context, r *connect.Request[adiomv1.ListData partition := r.Msg.GetPartition() namespace := partition.GetNamespace() keys := c.pkeys[namespace] + if len(keys) == 0 { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("no primary key found for namespace %s", namespace)) + } rawCursor := r.Msg.GetCursor() if rawCursor == nil { rawCursor = partition.GetCursor() @@ -432,12 +610,9 @@ func (c *conn) ListData(ctx context.Context, r *connect.Request[adiomv1.ListData if i >= limit { break } - keys := getPrimaryKey(c.pkeys[namespace], m) - _, b, err := toMongoID(c.bsonRegistry, keys) - if err != nil { + if err := c.prepareSourceDocument(namespace, m); err != nil { return nil, connect.NewError(connect.CodeInternal, err) } - m["_id"] = b marshalled, err := marshalWithRegistry(c.bsonRegistry, m) if err != nil { return nil, connect.NewError(connect.CodeInternal, err) @@ -503,16 +678,68 @@ func toMongoID(reg *bson.Registry, in []interface{}) ([]*adiomv1.BsonValue, inte }}, b, nil } +func toNamedID(reg *bson.Registry, names []string, values map[string]interface{}) ([]*adiomv1.BsonValue, error) { + result := make([]*adiomv1.BsonValue, 0, len(names)) + for _, name := range names { + typ, data, err := marshalValueWithRegistry(reg, values[name]) + if err != nil { + return nil, err + } + result = append(result, &adiomv1.BsonValue{Name: name, Type: uint32(typ), Data: data}) + } + return result, nil +} + +func idLast(primaryKeys []string) []string { + result := make([]string, 0, len(primaryKeys)) + for _, key := range primaryKeys { + if key != "_id" { + result = append(result, key) + } + } + if slices.Contains(primaryKeys, "_id") { + result = append(result, "_id") + } + return result +} + +func (c *conn) sourceUpdateID(namespace string, values map[string]interface{}) ([]*adiomv1.BsonValue, error) { + primaryKeys := c.pkeys[namespace] + if c.hasIDColumn[namespace] && slices.Contains(primaryKeys, "_id") { + return toNamedID(c.bsonRegistry, idLast(primaryKeys), values) + } + keys, _, err := toMongoID(c.bsonRegistry, getPrimaryKey(primaryKeys, values)) + return keys, err +} + +func (c *conn) prepareSourceDocument(namespace string, values map[string]interface{}) error { + primaryKeys := c.pkeys[namespace] + if alias := c.idAliases[namespace]; alias != "" { + values[alias] = values["_id"] + delete(values, "_id") + } + if c.hasIDColumn[namespace] && slices.Contains(primaryKeys, "_id") { + return nil + } + _, id, err := toMongoID(c.bsonRegistry, getPrimaryKey(primaryKeys, values)) + if err != nil { + return err + } + values["_id"] = id + return nil +} + func (c *conn) toBsonAdiomUpdate(update pglib.Update) (*adiomv1.Update, error) { ns := update.Namespace + "." + update.TableName - primaryKeys := c.pkeys[ns] switch update.Type { case pglib.UpdateTypeInsert: - keys, b, err := toMongoID(c.bsonRegistry, getPrimaryKey(primaryKeys, update.New)) + keys, err := c.sourceUpdateID(ns, update.New) if err != nil { return nil, err } - update.New["_id"] = b + if err := c.prepareSourceDocument(ns, update.New); err != nil { + return nil, err + } marshalled, err := marshalWithRegistry(c.bsonRegistry, update.New) if err != nil { return nil, err @@ -523,13 +750,13 @@ func (c *conn) toBsonAdiomUpdate(update pglib.Update) (*adiomv1.Update, error) { Data: marshalled, }, nil case pglib.UpdateTypeUpdate: - // Actually we'll ignore "old" and just accept breaking behavior - // since in the mongo format you can't edit the _id - keys, b, err := toMongoID(c.bsonRegistry, getPrimaryKey(primaryKeys, update.New)) + keys, err := c.sourceUpdateID(ns, update.New) if err != nil { return nil, err } - update.New["_id"] = b + if err := c.prepareSourceDocument(ns, update.New); err != nil { + return nil, err + } marshalled, err := marshalWithRegistry(c.bsonRegistry, update.New) if err != nil { return nil, err @@ -540,7 +767,7 @@ func (c *conn) toBsonAdiomUpdate(update pglib.Update) (*adiomv1.Update, error) { Data: marshalled, }, nil case pglib.UpdateTypeDelete: - keys, _, err := toMongoID(c.bsonRegistry, getPrimaryKey(primaryKeys, update.Old)) + keys, err := c.sourceUpdateID(ns, update.Old) if err != nil { return nil, err } @@ -578,7 +805,7 @@ func (c *conn) StreamUpdates(ctx context.Context, req *connect.Request[adiomv1.S maxBatchSize := c.settings.StreamMaxBatchSize maxBatchWait := c.settings.StreamMaxBatchWait - ch := make(chan []pglib.Update, 10) + ch := make(chan pglib.UpdateBatch, 10) eg.Go(func() error { defer close(ch) return cs.Run(ctx, ch) @@ -615,11 +842,11 @@ func (c *conn) StreamUpdates(ctx context.Context, req *connect.Request[adiomv1.S select { case <-ctx.Done(): return nil - case updates, ok := <-ch: + case updateBatch, ok := <-ch: if !ok { return nil } - for _, update := range updates { + for _, update := range updateBatch.Updates { up, err := c.toBsonAdiomUpdate(update) if err != nil { return err @@ -633,7 +860,7 @@ func (c *conn) StreamUpdates(ctx context.Context, req *connect.Request[adiomv1.S } batch = append(batch, up) } - lastSafeLSN = updates[len(updates)-1].LSN + lastSafeLSN = max(lastSafeLSN, updateBatch.LSN) if maxBatchSize > 0 && len(batch) > maxBatchSize { if err := send(); err != nil { return err @@ -655,8 +882,107 @@ func (c *conn) StreamUpdates(ctx context.Context, req *connect.Request[adiomv1.S return nil } -// buildUpsertQuery creates an INSERT ... ON CONFLICT ... DO UPDATE query for postgres -func buildUpsertQuery(namespace string, pkeys []string, columns []string) string { +type postgresWriteOperation struct { + delete bool + overrideIdentity bool + columns []string + defaultColumns []string + values []interface{} + requestIndex int + data map[string]interface{} +} + +func (c *conn) sinkTable(namespace string) (sinkTableMetadata, error) { + metadata, ok := c.sinkMetadata[namespace] + if !ok || len(metadata.Columns) == 0 { + return sinkTableMetadata{}, fmt.Errorf("namespace %s was not found or has no visible columns", namespace) + } + if len(metadata.PrimaryKey) == 0 { + return sinkTableMetadata{}, fmt.Errorf("no primary key found for namespace %s", namespace) + } + return metadata, nil +} + +func planPostgresUpsert(metadata sinkTableMetadata, document map[string]interface{}, requestIndex int) (postgresWriteOperation, error) { + columnsByName := make(map[string]sinkColumnMetadata, len(metadata.Columns)) + for _, column := range metadata.Columns { + columnsByName[column.Name] = column + } + for name := range document { + if _, ok := columnsByName[name]; !ok { + return postgresWriteOperation{}, fmt.Errorf("column %q does not exist in destination table", name) + } + } + + columns := make([]string, 0, len(metadata.Columns)) + values := make([]interface{}, 0, len(metadata.Columns)) + defaultColumns := make([]string, 0, len(metadata.Columns)) + for _, key := range metadata.PrimaryKey { + column, ok := columnsByName[key] + if !ok { + return postgresWriteOperation{}, fmt.Errorf("primary key field %q is not present in table metadata", key) + } + if column.Generated && !column.Identity { + return postgresWriteOperation{}, fmt.Errorf("generated primary key field %q cannot be written explicitly", key) + } + value, exists := document[key] + if !exists || value == nil { + return postgresWriteOperation{}, fmt.Errorf("document is missing non-null key field %q", key) + } + columns = append(columns, key) + values = append(values, value) + } + + for _, column := range metadata.Columns { + if slices.Contains(metadata.PrimaryKey, column.Name) { + continue + } + if column.Generated || column.Identity { + if _, exists := document[column.Name]; exists { + return postgresWriteOperation{}, fmt.Errorf("column %q is generated by the destination and cannot be supplied", column.Name) + } + continue + } + value, exists := document[column.Name] + if exists { + if value == nil && !column.Nullable { + return postgresWriteOperation{}, fmt.Errorf("column %q is required and cannot be null", column.Name) + } + columns = append(columns, column.Name) + values = append(values, value) + continue + } + if column.HasDefault { + defaultColumns = append(defaultColumns, column.Name) + continue + } + if !column.Nullable { + return postgresWriteOperation{}, fmt.Errorf("document is missing required column %q", column.Name) + } + columns = append(columns, column.Name) + values = append(values, nil) + } + + overrideIdentity := false + for _, key := range metadata.PrimaryKey { + column := columnsByName[key] + if column.Identity && strings.EqualFold(column.IdentityGeneration, "ALWAYS") { + overrideIdentity = true + break + } + } + return postgresWriteOperation{ + overrideIdentity: overrideIdentity, + columns: columns, + defaultColumns: defaultColumns, + values: values, + requestIndex: requestIndex, + data: document, + }, nil +} + +// buildUpsertQuery creates an INSERT ... ON CONFLICT ... DO UPDATE query for postgres. +func buildUpsertQuery(namespace string, pkeys, columns, defaultColumns []string, overrideIdentity bool) string { sanitizedNamespace := SanitizeNamespace(namespace) // Build column list @@ -680,140 +1006,140 @@ func buildUpsertQuery(namespace string, pkeys []string, columns []string) string } conflictTarget := strings.Join(sanitizedPKeys, ", ") - // Build UPDATE SET clause (exclude primary keys) + // Build UPDATE SET clause (exclude primary keys). var updateSet []string for _, col := range columns { - // Skip primary key columns in the UPDATE clause if !slices.Contains(pkeys, col) { sanitizedCol := pgx.Identifier([]string{col}).Sanitize() updateSet = append(updateSet, fmt.Sprintf("%s = EXCLUDED.%s", sanitizedCol, sanitizedCol)) } } + for _, col := range defaultColumns { + updateSet = append(updateSet, fmt.Sprintf("%s = DEFAULT", pgx.Identifier([]string{col}).Sanitize())) + } + + identityClause := "" + if overrideIdentity { + identityClause = " OVERRIDING SYSTEM VALUE" + } - // If all columns are primary keys, just use DO NOTHING if len(updateSet) == 0 { return fmt.Sprintf( - "INSERT INTO %s (%s) VALUES %s ON CONFLICT (%s) DO NOTHING", - sanitizedNamespace, columnList, valuesPlaceholder, conflictTarget, + "INSERT INTO %s (%s)%s VALUES %s ON CONFLICT (%s) DO NOTHING", + sanitizedNamespace, columnList, identityClause, valuesPlaceholder, conflictTarget, ) } return fmt.Sprintf( - "INSERT INTO %s (%s) VALUES %s ON CONFLICT (%s) DO UPDATE SET %s", - sanitizedNamespace, columnList, valuesPlaceholder, conflictTarget, strings.Join(updateSet, ", "), + "INSERT INTO %s (%s)%s VALUES %s ON CONFLICT (%s) DO UPDATE SET %s", + sanitizedNamespace, columnList, identityClause, valuesPlaceholder, conflictTarget, strings.Join(updateSet, ", "), ) } -// WriteData implements adiomv1connect.ConnectorServiceHandler. -func (c *conn) WriteData(ctx context.Context, r *connect.Request[adiomv1.WriteDataRequest]) (*connect.Response[adiomv1.WriteDataResponse], error) { - namespace := r.Msg.GetNamespace() - pkeys := c.pkeys[namespace] - if len(pkeys) == 0 { - return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("no primary key found for namespace %s", namespace)) +func buildDeleteQuery(namespace string, pkeys []string) string { + whereClauses := make([]string, 0, len(pkeys)) + for i, pk := range pkeys { + whereClauses = append(whereClauses, fmt.Sprintf("%s = $%d", pgx.Identifier([]string{pk}).Sanitize(), i+1)) } + return fmt.Sprintf("DELETE FROM %s WHERE %s", SanitizeNamespace(namespace), strings.Join(whereClauses, " AND ")) +} - // Convert BSON documents to maps and collect all column names - var documents []map[string]interface{} - columnSet := make(map[string]bool) - - for _, data := range r.Msg.GetData() { - m, err := bsonToMap(c.bsonRegistry, data) - if err != nil { - if !errors.Is(err, context.Canceled) { - slog.Error(fmt.Sprintf("Failed to unmarshal BSON document: %v", err)) - } - return nil, connect.NewError(connect.CodeInternal, err) - } - documents = append(documents, m) - for col := range m { - columnSet[col] = true - } +func (c *conn) executePostgresWrites(ctx context.Context, namespace string, pkeys []string, operations []postgresWriteOperation) error { + if len(operations) == 0 { + return nil } - - // Build ordered column list (primary keys first for consistency) - //NOTE: for efficiency, we assume that all documents have the same set of columns - var columns []string - for _, pk := range pkeys { - if columnSet[pk] { - columns = append(columns, pk) + batch := &pgx.Batch{} + for _, operation := range operations { + if operation.delete { + batch.Queue(buildDeleteQuery(namespace, pkeys), operation.values...) + continue } + batch.Queue(buildUpsertQuery(namespace, pkeys, operation.columns, operation.defaultColumns, operation.overrideIdentity), operation.values...) } - for col := range columnSet { - if !slices.Contains(pkeys, col) { - columns = append(columns, col) + return c.sendSinkBatch(ctx, batch, len(operations), func(i int, err error) { + if errors.Is(err, context.Canceled) { + return } - } + operation := operations[i] + slog.Error("Failed to execute PostgreSQL sink operation", "index", operation.requestIndex, "error", err) + if operation.delete { + slog.Debug("Failed delete operation", "pkValues", operation.values) + return + } + for key, value := range operation.data { + slog.Debug("Operation field", "key", key, "value", value, "type", fmt.Sprintf("%T", value)) + } + }) +} - if len(columns) == 0 { - return connect.NewResponse(&adiomv1.WriteDataResponse{}), nil +// WriteData implements adiomv1connect.ConnectorServiceHandler. +func (c *conn) WriteData(ctx context.Context, r *connect.Request[adiomv1.WriteDataRequest]) (*connect.Response[adiomv1.WriteDataResponse], error) { + if r.Msg.GetPartialWriteResults() { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("partial write results are not supported by postgres")) } - - // Build upsert query - query := buildUpsertQuery(namespace, pkeys, columns) - - // Use pgx batch for efficient bulk inserts - batch := &pgx.Batch{} - for _, doc := range documents { - var values []interface{} - for _, col := range columns { - values = append(values, doc[col]) - } - batch.Queue(query, values...) + namespace := r.Msg.GetNamespace() + metadata, err := c.sinkTable(namespace) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) } - // Execute batch - br := c.c.SendBatch(ctx, batch) - defer br.Close() - - // Process results - for i := 0; i < len(documents); i++ { - _, err := br.Exec() + operations := make([]postgresWriteOperation, 0, len(r.Msg.GetData())) + for i, data := range r.Msg.GetData() { + document, err := bsonToMap(c.bsonRegistry, data, c.hasIDColumn[namespace]) if err != nil { if !errors.Is(err, context.Canceled) { - slog.Error(fmt.Sprintf("Failed to insert document %d: %v", i, err)) - for key, value := range documents[i] { - slog.Debug("Document field", "key", key, "value", value, "type", fmt.Sprintf("%T", value)) - } + slog.Error("Failed to unmarshal BSON document", "index", i, "error", err) } - return nil, connect.NewError(connect.CodeInternal, err) + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("decode document %d: %w", i, err)) } + operation, err := planPostgresUpsert(metadata, document, i) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("document %d: %w", i, err)) + } + operations = append(operations, operation) + } + if err := c.executePostgresWrites(ctx, namespace, metadata.PrimaryKey, operations); err != nil { + return nil, connect.NewError(connect.CodeInternal, err) } return connect.NewResponse(&adiomv1.WriteDataResponse{}), nil } -// dataIdIndex tracks the index of an update for deduplication -type dataIdIndex struct { - dataId []byte - index int -} - -// addToIdIndexMap adds or updates an entry in the deduplication map -func addToIdIndexMap(m map[int][]*dataIdIndex, update *adiomv1.Update) (*dataIdIndex, bool) { - hasher := xxhash.New() - _, _ = hasher.Write(update.GetId()[0].GetData()) - h := int(hasher.Sum64()) - items, found := m[h] - if found { - for _, item := range items { - if slices.Equal(item.dataId, update.GetId()[0].GetData()) { - return item, false +// fromMongoID extracts the original primary key values from BSON ID +func fromMongoID(reg *bson.Registry, id []*adiomv1.BsonValue, primaryKeys []string) ([]interface{}, error) { + if len(id) == 0 { + return nil, fmt.Errorf("expected at least one ID value") + } + if len(id) > 1 { + parts := make(map[string]*adiomv1.BsonValue, len(id)) + for _, part := range id { + if part.GetName() != "" { + parts[part.GetName()] = part } } - } - item := &dataIdIndex{update.GetId()[0].GetData(), -1} - m[h] = append(items, item) - return item, true -} - -// fromMongoID extracts the original primary key values from BSON ID -func fromMongoID(id []*adiomv1.BsonValue) ([]interface{}, error) { - if len(id) != 1 { - return nil, fmt.Errorf("expected exactly one ID value") + values := make([]interface{}, 0, len(primaryKeys)) + for _, primaryKey := range primaryKeys { + part, ok := parts[primaryKey] + if !ok { + return nil, fmt.Errorf("ID is missing primary key field %q", primaryKey) + } + var value interface{} + rawValue := bson.RawValue{Type: bson.Type(part.GetType()), Value: part.GetData()} + if err := rawValue.UnmarshalWithRegistry(reg, &value); err != nil { + return nil, err + } + value, err := bsonValueForPostgres(primaryKey, value) + if err != nil { + return nil, err + } + values = append(values, value) + } + return values, nil } var value interface{} - if err := bson.UnmarshalValue(bson.Type(id[0].GetType()), id[0].GetData(), &value); err != nil { + rawValue := bson.RawValue{Type: bson.Type(id[0].GetType()), Value: id[0].GetData()} + if err := rawValue.UnmarshalWithRegistry(reg, &value); err != nil { return nil, err } @@ -822,144 +1148,76 @@ func fromMongoID(id []*adiomv1.BsonValue) ([]interface{}, error) { var compositeKey []interface{} br := bytes.NewReader(b.Data) dec := gob.NewDecoder(br) - if err := dec.Decode(&compositeKey); err != nil { - return nil, err + if err := dec.Decode(&compositeKey); err == nil { + return compositeKey, nil } - return compositeKey, nil + // Generic BSON binary is also how a scalar bytea key is represented. + // Legacy composite IDs are gob-encoded, so only treat the value as a + // composite when gob decoding succeeds. + value = b.Data } // Single key + value, err := bsonValueForPostgres(primaryKeys[0], value) + if err != nil { + return nil, err + } return []interface{}{value}, nil } // WriteUpdates implements adiomv1connect.ConnectorServiceHandler. func (c *conn) WriteUpdates(ctx context.Context, r *connect.Request[adiomv1.WriteUpdatesRequest]) (*connect.Response[adiomv1.WriteUpdatesResponse], error) { - namespace := r.Msg.GetNamespace() - pkeys := c.pkeys[namespace] - if len(pkeys) == 0 { - return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("no primary key found for namespace %s", namespace)) + if r.Msg.GetPartialWriteResults() { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("partial write results are not supported by postgres")) } - - // Deduplication map to ensure each document is only updated once - hashToDataIdIndex := map[int][]*dataIdIndex{} - - // We'll store operations as either insert/update or delete - type operation struct { - isDelete bool - pkValues []interface{} - data map[string]interface{} + namespace := r.Msg.GetNamespace() + metadata, err := c.sinkTable(namespace) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) } - var operations []operation - - // Process updates with deduplication for _, update := range r.Msg.GetUpdates() { - dii, isNew := addToIdIndexMap(hashToDataIdIndex, update) + if update.GetType() == adiomv1.UpdateType_UPDATE_TYPE_PARTIAL_UPDATE { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("partial updates are not supported by postgres")) + } + } - pkValues, err := fromMongoID(update.GetId()) + operations := make([]postgresWriteOperation, 0, len(r.Msg.GetUpdates())) + for i, update := range r.Msg.GetUpdates() { + if update.GetType() == adiomv1.UpdateType_UPDATE_TYPE_UNKNOWN { + continue + } + pkValues, err := fromMongoID(c.bsonRegistry, update.GetId(), metadata.PrimaryKey) if err != nil { - return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("failed to extract primary key: %w", err)) + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("update %d: failed to extract primary key: %w", i, err)) } - - if len(pkValues) != len(pkeys) { - return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("primary key length mismatch: expected %d, got %d", len(pkeys), len(pkValues))) + if len(pkValues) != len(metadata.PrimaryKey) { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("update %d: primary key length mismatch: expected %d, got %d", i, len(metadata.PrimaryKey), len(pkValues))) } - - var op operation switch update.GetType() { case adiomv1.UpdateType_UPDATE_TYPE_INSERT, adiomv1.UpdateType_UPDATE_TYPE_UPDATE: - data, err := bsonToMap(c.bsonRegistry, update.GetData()) + document, err := bsonToMap(c.bsonRegistry, update.GetData(), c.hasIDColumn[namespace]) if err != nil { if !errors.Is(err, context.Canceled) { - slog.Error(fmt.Sprintf("Failed to unmarshal BSON document: %v", err)) + slog.Error("Failed to unmarshal BSON update", "index", i, "error", err) } - return nil, connect.NewError(connect.CodeInternal, err) + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("decode update %d: %w", i, err)) } - op = operation{ - isDelete: false, - pkValues: pkValues, - data: data, + for keyIndex, key := range metadata.PrimaryKey { + document[key] = pkValues[keyIndex] } - case adiomv1.UpdateType_UPDATE_TYPE_DELETE: - op = operation{ - isDelete: true, - pkValues: pkValues, + operation, err := planPostgresUpsert(metadata, document, i) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("update %d: %w", i, err)) } + operations = append(operations, operation) + case adiomv1.UpdateType_UPDATE_TYPE_DELETE: + operations = append(operations, postgresWriteOperation{delete: true, values: pkValues, requestIndex: i}) default: - return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("unsupported update type: %v", update.GetType())) - } - - if isNew { - dii.index = len(operations) - operations = append(operations, op) - } else { - // Replace the existing operation - operations[dii.index] = op + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("update %d: unsupported update type: %v", i, update.GetType())) } } - - // Build and execute batched operations - batch := &pgx.Batch{} - sanitizedNamespace := SanitizeNamespace(namespace) - - for _, op := range operations { - if op.isDelete { - // Build DELETE query - var whereClauses []string - for i, pk := range pkeys { - sanitizedPk := pgx.Identifier([]string{pk}).Sanitize() - whereClauses = append(whereClauses, fmt.Sprintf("%s = $%d", sanitizedPk, i+1)) - } - deleteQuery := fmt.Sprintf( - "DELETE FROM %s WHERE %s", - sanitizedNamespace, strings.Join(whereClauses, " AND "), - ) - batch.Queue(deleteQuery, op.pkValues...) - } else { - // Build column list and values for INSERT/UPDATE - var columns []string - var values []interface{} - - // Add primary keys first - for i, pk := range pkeys { - columns = append(columns, pk) - values = append(values, op.pkValues[i]) - } - - // Add other columns - for col, val := range op.data { - if !slices.Contains(pkeys, col) { - columns = append(columns, col) - values = append(values, val) - } - } - - // Build upsert query - upsertQuery := buildUpsertQuery(namespace, pkeys, columns) - batch.Queue(upsertQuery, values...) - } - } - - // Execute batch - br := c.c.SendBatch(ctx, batch) - defer br.Close() - - // Process results - for i := 0; i < len(operations); i++ { - _, err := br.Exec() - if err != nil { - if !errors.Is(err, context.Canceled) { - slog.Error(fmt.Sprintf("Failed to execute update %d: %v", i, err)) - if operations[i].isDelete { - slog.Debug("Failed delete operation", "pkValues", operations[i].pkValues) - } else { - slog.Debug("Failed insert/update operation") - for key, value := range operations[i].data { - slog.Debug("Operation field", "key", key, "value", value, "type", fmt.Sprintf("%T", value)) - } - } - } - return nil, connect.NewError(connect.CodeInternal, err) - } + if err := c.executePostgresWrites(ctx, namespace, metadata.PrimaryKey, operations); err != nil { + return nil, connect.NewError(connect.CodeInternal, err) } return connect.NewResponse(&adiomv1.WriteUpdatesResponse{}), nil diff --git a/connectors/postgres/cursor_test.go b/connectors/postgres/cursor_test.go new file mode 100644 index 00000000..ca11f519 --- /dev/null +++ b/connectors/postgres/cursor_test.go @@ -0,0 +1,62 @@ +package postgres + +import ( + "reflect" + "testing" + "time" + + "github.com/shopspring/decimal" +) + +func TestPartitionCursorRoundTripDatabaseTypes(t *testing.T) { + when := time.Date(2026, 10, 14, 12, 30, 0, 0, time.UTC) + uuid := [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + number := decimal.RequireFromString("1234567890.0123456789") + want := &partitionCursor{Low: []interface{}{uuid, when, number}} + + encoded, err := want.Encode() + if err != nil { + t.Fatal(err) + } + got, err := DecodePartitionCursor(encoded) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("cursor = %#v, want %#v", got, want) + } +} + +func TestCompositeMongoIDRoundTripDatabaseTypes(t *testing.T) { + uuid := [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + when := time.Date(2026, 10, 14, 12, 30, 0, 0, time.UTC) + number := decimal.RequireFromString("1234567890.0123456789") + want := []interface{}{uuid, when, number} + + id, _, err := toMongoID(NewBSONRegistry(), want) + if err != nil { + t.Fatal(err) + } + got, err := fromMongoID(NewBSONRegistry(), id, []string{"first", "second", "third"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("ID = %#v, want %#v", got, want) + } +} + +func TestScalarByteaMongoIDRoundTrip(t *testing.T) { + want := []byte("primary-key") + id, _, err := toMongoID(NewBSONRegistry(), []interface{}{want}) + if err != nil { + t.Fatal(err) + } + got, err := fromMongoID(NewBSONRegistry(), id, []string{"id"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, []interface{}{want}) { + t.Fatalf("ID = %#v, want %#v", got, []interface{}{want}) + } +} diff --git a/connectors/postgres/integration_test.go b/connectors/postgres/integration_test.go new file mode 100644 index 00000000..c005606f --- /dev/null +++ b/connectors/postgres/integration_test.go @@ -0,0 +1,419 @@ +package postgres + +import ( + "bytes" + "context" + "fmt" + "net/netip" + neturl "net/url" + "os" + "reflect" + "testing" + "time" + + "connectrpc.com/connect" + adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" + "github.com/jackc/pgx/v5" + "go.mongodb.org/mongo-driver/v2/bson" +) + +func livePostgresURL(t *testing.T) string { + t.Helper() + url := os.Getenv("POSTGRES_TEST_URL") + if url == "" { + t.Skip("POSTGRES_TEST_URL is not set") + } + return url +} + +func TestPostgresConnectorLiveRoundTrip(t *testing.T) { + ctx := context.Background() + url := livePostgresURL(t) + admin, err := pgx.Connect(ctx, url) + if err != nil { + t.Fatal(err) + } + defer admin.Close(ctx) + + for _, statement := range []string{ + `DROP SCHEMA IF EXISTS dsync_live CASCADE`, + `DROP SCHEMA IF EXISTS dsync_live_other CASCADE`, + `CREATE SCHEMA dsync_live`, + `CREATE SCHEMA dsync_live_other`, + `CREATE TABLE dsync_live.source_nonpk_id (pk integer PRIMARY KEY, _id text, _id2 text, value text)`, + `CREATE TABLE dsync_live_other.source_nonpk_id (code text PRIMARY KEY, value text)`, + `INSERT INTO dsync_live.source_nonpk_id VALUES (1, 'original-id', 'occupied', 'source')`, + `CREATE TABLE dsync_live.source_id_pk (tenant text, _id text, value text, PRIMARY KEY (tenant, _id))`, + `INSERT INTO dsync_live.source_id_pk VALUES ('tenant-a', 'id-a', 'source')`, + `CREATE TABLE dsync_live.source_network_pk (network inet PRIMARY KEY, value text)`, + `INSERT INTO dsync_live.source_network_pk VALUES ('192.0.2.0/24', 'source')`, + `CREATE TABLE dsync_live.sink_with_id (_id text PRIMARY KEY, value text, payload bytea, uid uuid, ip inet)`, + `CREATE TABLE dsync_live.sink_no_id (id integer PRIMARY KEY, value text)`, + `CREATE TABLE dsync_live.sink_composite (tenant text, _id text, value text, PRIMARY KEY (tenant, _id))`, + `CREATE TABLE dsync_live.sink_bytea (id bytea PRIMARY KEY, value text)`, + `CREATE TABLE dsync_live.sink_replace (id integer PRIMARY KEY, required text NOT NULL, status text NOT NULL DEFAULT 'fresh', note text, computed text GENERATED ALWAYS AS (required || '-computed') STORED)`, + `CREATE TABLE dsync_live.sink_identity (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, value text NOT NULL)`, + `CREATE TABLE dsync_live.sink_order (id integer PRIMARY KEY, value text NOT NULL CHECK (value <> 'bad'))`, + `CREATE TABLE dsync_live.sink_order_audit (sequence bigint GENERATED ALWAYS AS IDENTITY, action text NOT NULL)`, + `CREATE FUNCTION dsync_live.audit_sink_order() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN INSERT INTO dsync_live.sink_order_audit(action) VALUES (TG_OP); RETURN COALESCE(NEW, OLD); END $$`, + `CREATE TRIGGER audit_sink_order AFTER INSERT OR UPDATE OR DELETE ON dsync_live.sink_order FOR EACH ROW EXECUTE FUNCTION dsync_live.audit_sink_order()`, + `CREATE TABLE dsync_live.sink_restricted (id integer PRIMARY KEY, required text NOT NULL, status text NOT NULL DEFAULT 'fresh')`, + `CREATE TABLE dsync_live.sink_replica (id integer PRIMARY KEY, touched boolean NOT NULL DEFAULT false)`, + `CREATE FUNCTION dsync_live.mark_touched() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN NEW.touched = true; RETURN NEW; END $$`, + `CREATE TRIGGER mark_touched BEFORE INSERT ON dsync_live.sink_replica FOR EACH ROW EXECUTE FUNCTION dsync_live.mark_touched()`, + } { + if _, err := admin.Exec(ctx, statement); err != nil { + t.Fatalf("%s: %v", statement, err) + } + } + + c, err := NewConn(ctx, PostgresSettings{URL: url, Limit: 100}) + if err != nil { + t.Fatal(err) + } + defer c.Teardown() + if !reflect.DeepEqual(c.pkeys["dsync_live.source_nonpk_id"], []string{"pk"}) || + !reflect.DeepEqual(c.pkeys["dsync_live_other.source_nonpk_id"], []string{"code"}) { + t.Fatalf("primary keys leaked across schemas: %#v", c.pkeys) + } + + list := func(namespace string) bson.M { + t.Helper() + res, err := c.ListData(ctx, connect.NewRequest(&adiomv1.ListDataRequest{ + Partition: &adiomv1.Partition{Namespace: namespace}, + })) + if err != nil { + t.Fatal(err) + } + if len(res.Msg.GetData()) != 1 { + t.Fatalf("%s returned %d documents", namespace, len(res.Msg.GetData())) + } + document, err := bsonToMap(c.bsonRegistry, res.Msg.GetData()[0], true) + if err != nil { + t.Fatal(err) + } + return document + } + + nonPKID := list("dsync_live.source_nonpk_id") + if nonPKID["_id"] != int32(1) || nonPKID["_id2"] != "occupied" || nonPKID["_id3"] != "original-id" { + t.Fatalf("unexpected aliased source document: %#v", nonPKID) + } + idPK := list("dsync_live.source_id_pk") + if idPK["_id"] != "id-a" || idPK["tenant"] != "tenant-a" { + t.Fatalf("unexpected _id primary-key document: %#v", idPK) + } + networkPK := list("dsync_live.source_network_pk") + if networkPK["_id"] != "192.0.2.0/24" || networkPK["network"] != "192.0.2.0/24" { + t.Fatalf("unexpected network primary-key document: %#v", networkPK) + } + var network netip.Prefix + if err := admin.QueryRow(ctx, `SELECT network FROM dsync_live.source_network_pk`).Scan(&network); err != nil { + t.Fatal(err) + } + networkID, err := c.sourceUpdateID("dsync_live.source_network_pk", map[string]interface{}{"network": network}) + if err != nil { + t.Fatal(err) + } + var networkIDValue string + if err := bson.UnmarshalValue(bson.Type(networkID[0].GetType()), networkID[0].GetData(), &networkIDValue); err != nil { + t.Fatal(err) + } + if networkIDValue != "192.0.2.0/24" { + t.Fatalf("network update ID = %q", networkIDValue) + } + + uuidBytes := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + writeDocument := func(namespace string, document bson.M) { + t.Helper() + data, err := marshalWithRegistry(c.bsonRegistry, document) + if err != nil { + t.Fatal(err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{Namespace: namespace, Data: [][]byte{data}})); err != nil { + t.Fatal(err) + } + } + writeDocument("dsync_live.sink_with_id", bson.M{ + "_id": "kept", "value": "written", + "payload": bson.Binary{Subtype: bson.TypeBinaryGeneric, Data: []byte("bytes")}, + "uid": bson.Binary{Subtype: bson.TypeBinaryUUID, Data: uuidBytes}, + "ip": "192.0.2.1", + }) + var gotID, gotValue, gotUUID, gotIP string + var gotPayload []byte + if err := admin.QueryRow(ctx, `SELECT _id, value, payload, uid::text, host(ip) FROM dsync_live.sink_with_id`).Scan(&gotID, &gotValue, &gotPayload, &gotUUID, &gotIP); err != nil { + t.Fatal(err) + } + if gotID != "kept" || gotValue != "written" || !bytes.Equal(gotPayload, []byte("bytes")) || gotUUID != "00010203-0405-0607-0809-0a0b0c0d0e0f" || gotIP != "192.0.2.1" { + t.Fatalf("unexpected typed sink row: id=%q value=%q payload=%q uuid=%q ip=%q", gotID, gotValue, gotPayload, gotUUID, gotIP) + } + + writeDocument("dsync_live.sink_no_id", bson.M{"_id": "synthetic", "id": int32(7), "value": "written"}) + var noIDValue string + if err := admin.QueryRow(ctx, `SELECT value FROM dsync_live.sink_no_id WHERE id = 7`).Scan(&noIDValue); err != nil || noIDValue != "written" { + t.Fatalf("sink without _id: value=%q err=%v", noIDValue, err) + } + + replacementDocuments := []bson.M{ + {"id": int32(1), "required": "first", "status": "custom", "note": "present"}, + {"id": int32(2), "required": "second"}, + } + rejectedGenerated, err := marshalWithRegistry(c.bsonRegistry, bson.M{"id": int32(3), "required": "third", "computed": "source-value"}) + if err != nil { + t.Fatal(err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dsync_live.sink_replace", + Data: [][]byte{rejectedGenerated}, + })); err == nil { + t.Fatal("expected supplied generated column to be rejected") + } + var replacementData [][]byte + for _, document := range replacementDocuments { + data, err := marshalWithRegistry(c.bsonRegistry, document) + if err != nil { + t.Fatal(err) + } + replacementData = append(replacementData, data) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dsync_live.sink_replace", + Data: replacementData, + })); err != nil { + t.Fatal(err) + } + var status string + var note *string + var computed string + if err := admin.QueryRow(ctx, `SELECT status, note, computed FROM dsync_live.sink_replace WHERE id = 2`).Scan(&status, ¬e, &computed); err != nil { + t.Fatal(err) + } + if status != "fresh" || note != nil || computed != "second-computed" { + t.Fatalf("per-document defaults: status=%q note=%v computed=%q", status, note, computed) + } + + replacementID, _, err := toMongoID(c.bsonRegistry, []interface{}{int32(1)}) + if err != nil { + t.Fatal(err) + } + replacementUpdate, err := marshalWithRegistry(c.bsonRegistry, bson.M{"required": "replaced"}) + if err != nil { + t.Fatal(err) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "dsync_live.sink_replace", + Updates: []*adiomv1.Update{{ + Id: replacementID, Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, Data: replacementUpdate, + }}, + })); err != nil { + t.Fatal(err) + } + if err := admin.QueryRow(ctx, `SELECT status, note FROM dsync_live.sink_replace WHERE id = 1`).Scan(&status, ¬e); err != nil { + t.Fatal(err) + } + if status != "fresh" || note != nil { + t.Fatalf("full replacement did not reset omitted columns: status=%q note=%v", status, note) + } + + writeDocument("dsync_live.sink_identity", bson.M{"id": int64(99), "value": "explicit"}) + var identityValue string + if err := admin.QueryRow(ctx, `SELECT value FROM dsync_live.sink_identity WHERE id = 99`).Scan(&identityValue); err != nil || identityValue != "explicit" { + t.Fatalf("identity override: value=%q err=%v", identityValue, err) + } + + orderedID, _, err := toMongoID(c.bsonRegistry, []interface{}{int32(1)}) + if err != nil { + t.Fatal(err) + } + orderedFirst, err := marshalWithRegistry(c.bsonRegistry, bson.M{"value": "first"}) + if err != nil { + t.Fatal(err) + } + orderedLast, err := marshalWithRegistry(c.bsonRegistry, bson.M{"value": "last"}) + if err != nil { + t.Fatal(err) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "dsync_live.sink_order", + Updates: []*adiomv1.Update{ + {Id: orderedID, Type: adiomv1.UpdateType_UPDATE_TYPE_INSERT, Data: orderedFirst}, + {Id: orderedID, Type: adiomv1.UpdateType_UPDATE_TYPE_DELETE}, + {Id: orderedID, Type: adiomv1.UpdateType_UPDATE_TYPE_INSERT, Data: orderedLast}, + }, + })); err != nil { + t.Fatal(err) + } + var actions []string + if err := admin.QueryRow(ctx, `SELECT array_agg(action ORDER BY sequence) FROM dsync_live.sink_order_audit`).Scan(&actions); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(actions, []string{"INSERT", "DELETE", "INSERT"}) { + t.Fatalf("ordered update actions = %#v", actions) + } + + atomicGoodID, _, err := toMongoID(c.bsonRegistry, []interface{}{int32(2)}) + if err != nil { + t.Fatal(err) + } + atomicBadID, _, err := toMongoID(c.bsonRegistry, []interface{}{int32(3)}) + if err != nil { + t.Fatal(err) + } + atomicGood, err := marshalWithRegistry(c.bsonRegistry, bson.M{"value": "good"}) + if err != nil { + t.Fatal(err) + } + atomicBad, err := marshalWithRegistry(c.bsonRegistry, bson.M{"value": "bad"}) + if err != nil { + t.Fatal(err) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "dsync_live.sink_order", + Updates: []*adiomv1.Update{ + {Id: atomicGoodID, Type: adiomv1.UpdateType_UPDATE_TYPE_INSERT, Data: atomicGood}, + {Id: atomicBadID, Type: adiomv1.UpdateType_UPDATE_TYPE_INSERT, Data: atomicBad}, + }, + })); err == nil { + t.Fatal("expected batched constraint error") + } + var atomicCount int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM dsync_live.sink_order WHERE id IN (2, 3)`).Scan(&atomicCount); err != nil { + t.Fatal(err) + } + if atomicCount != 0 { + t.Fatalf("failed batch committed %d rows", atomicCount) + } + + compositeID, err := toNamedID(c.bsonRegistry, []string{"tenant", "_id"}, map[string]interface{}{"tenant": "tenant-a", "_id": "id-a"}) + if err != nil { + t.Fatal(err) + } + updateData, err := marshalWithRegistry(c.bsonRegistry, bson.M{"value": "updated"}) + if err != nil { + t.Fatal(err) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "dsync_live.sink_composite", + Updates: []*adiomv1.Update{{Id: compositeID, Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, Data: updateData}}, + })); err != nil { + t.Fatal(err) + } + var compositeValue string + if err := admin.QueryRow(ctx, `SELECT value FROM dsync_live.sink_composite WHERE tenant = 'tenant-a' AND _id = 'id-a'`).Scan(&compositeValue); err != nil || compositeValue != "updated" { + t.Fatalf("multipart update: value=%q err=%v", compositeValue, err) + } + + byteaKey := []byte{0xde, 0xad, 0xbe, 0xef} + byteaID, _, err := toMongoID(c.bsonRegistry, []interface{}{byteaKey}) + if err != nil { + t.Fatal(err) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "dsync_live.sink_bytea", + Updates: []*adiomv1.Update{{Id: byteaID, Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, Data: updateData}}, + })); err != nil { + t.Fatal(err) + } + var gotByteaKey []byte + if err := admin.QueryRow(ctx, `SELECT id FROM dsync_live.sink_bytea`).Scan(&gotByteaKey); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(gotByteaKey, byteaKey) { + t.Fatalf("bytea key = %x, want %x", gotByteaKey, byteaKey) + } + + sinkRole := fmt.Sprintf("dsync_sink_live_%d", time.Now().UnixNano()) + sanitizedSinkRole := pgx.Identifier{sinkRole}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE ROLE "+sanitizedSinkRole+" LOGIN PASSWORD 'dsync-sink-password'"); err != nil { + t.Fatal(err) + } + defer func() { + _, _ = admin.Exec(ctx, "DROP OWNED BY "+sanitizedSinkRole) + _, _ = admin.Exec(ctx, "DROP ROLE "+sanitizedSinkRole) + }() + for _, statement := range []string{ + "GRANT USAGE ON SCHEMA dsync_live TO " + sanitizedSinkRole, + "GRANT SELECT, INSERT, UPDATE, DELETE ON dsync_live.sink_restricted TO " + sanitizedSinkRole, + } { + if _, err := admin.Exec(ctx, statement); err != nil { + t.Fatal(err) + } + } + var isSuperuser bool + if err := admin.QueryRow(ctx, `SELECT rolsuper FROM pg_roles WHERE rolname = $1`, sinkRole).Scan(&isSuperuser); err != nil { + t.Fatal(err) + } + if isSuperuser { + t.Fatal("restricted sink role unexpectedly has superuser privileges") + } + var databaseName string + if err := admin.QueryRow(ctx, `SELECT current_database()`).Scan(&databaseName); err != nil { + t.Fatal(err) + } + if _, err := admin.Exec(ctx, "GRANT CONNECT ON DATABASE "+pgx.Identifier{databaseName}.Sanitize()+" TO "+sanitizedSinkRole); err != nil { + t.Fatal(err) + } + sinkURL, err := neturl.Parse(url) + if err != nil { + t.Fatal(err) + } + sinkURL.User = neturl.UserPassword(sinkRole, "dsync-sink-password") + restrictedConn, err := NewConn(ctx, PostgresSettings{URL: sinkURL.String(), Limit: 100}) + if err != nil { + t.Fatalf("connect as restricted sink role: %v", err) + } + defer restrictedConn.Teardown() + writeRestricted := func(document bson.M) { + t.Helper() + data, err := marshalWithRegistry(restrictedConn.bsonRegistry, document) + if err != nil { + t.Fatal(err) + } + if _, err := restrictedConn.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dsync_live.sink_restricted", + Data: [][]byte{data}, + })); err != nil { + t.Fatalf("write as restricted sink role: %v", err) + } + } + writeRestricted(bson.M{"id": int32(1), "required": "inserted", "status": "custom"}) + writeRestricted(bson.M{"id": int32(1), "required": "replaced"}) + restrictedID, _, err := toMongoID(restrictedConn.bsonRegistry, []interface{}{int32(1)}) + if err != nil { + t.Fatal(err) + } + if _, err := restrictedConn.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "dsync_live.sink_restricted", + Updates: []*adiomv1.Update{{Id: restrictedID, Type: adiomv1.UpdateType_UPDATE_TYPE_DELETE}}, + })); err != nil { + t.Fatalf("delete as restricted sink role: %v", err) + } + var restrictedCount int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM dsync_live.sink_restricted`).Scan(&restrictedCount); err != nil { + t.Fatal(err) + } + if restrictedCount != 0 { + t.Fatalf("restricted sink left %d rows after delete", restrictedCount) + } + + replicaConn, err := NewConn(ctx, PostgresSettings{URL: url, Limit: 100, EnableReplicaMode: true}) + if err != nil { + t.Fatal(err) + } + defer replicaConn.Teardown() + replicaData, err := marshalWithRegistry(replicaConn.bsonRegistry, bson.M{"_id": "synthetic", "id": int32(9)}) + if err != nil { + t.Fatal(err) + } + if _, err := replicaConn.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dsync_live.sink_replica", + Data: [][]byte{replicaData}, + })); err != nil { + t.Fatal(err) + } + var touched bool + if err := admin.QueryRow(ctx, `SELECT touched FROM dsync_live.sink_replica WHERE id = 9`).Scan(&touched); err != nil || touched { + t.Fatalf("replica-mode trigger suppression: touched=%v err=%v", touched, err) + } +} diff --git a/connectors/postgres/namespace_test.go b/connectors/postgres/namespace_test.go new file mode 100644 index 00000000..8cc9b739 --- /dev/null +++ b/connectors/postgres/namespace_test.go @@ -0,0 +1,80 @@ +package postgres + +import ( + "reflect" + "testing" +) + +func TestIDColumnMetadataChoosesUnusedAliasForNonPrimaryID(t *testing.T) { + columns := map[string]map[string]struct{}{ + "public.items": {"item_id": {}, "_id": {}, "_id2": {}}, + } + hasID, aliases := idColumnMetadata(columns, map[string][]string{"public.items": {"item_id"}}) + if !hasID["public.items"] { + t.Fatal("expected _id column") + } + if aliases["public.items"] != "_id3" { + t.Fatalf("alias = %q, want _id3", aliases["public.items"]) + } +} + +func TestIDColumnMetadataDoesNotAliasPrimaryID(t *testing.T) { + columns := map[string]map[string]struct{}{ + "public.items": {"tenant": {}, "_id": {}}, + } + hasID, aliases := idColumnMetadata(columns, map[string][]string{"public.items": {"tenant", "_id"}}) + if !hasID["public.items"] { + t.Fatal("expected _id column") + } + if aliases["public.items"] != "" { + t.Fatalf("unexpected alias %q", aliases["public.items"]) + } +} + +func TestSourceIDUsesNamedPrimaryKeyPartsWithIDLast(t *testing.T) { + c := &conn{ + pkeys: map[string][]string{"public.items": {"_id", "tenant"}}, + hasIDColumn: map[string]bool{"public.items": true}, + bsonRegistry: NewBSONRegistry(), + } + id, err := c.sourceUpdateID("public.items", map[string]interface{}{"_id": "item", "tenant": "acme"}) + if err != nil { + t.Fatal(err) + } + if got := []string{id[0].GetName(), id[1].GetName()}; !reflect.DeepEqual(got, []string{"tenant", "_id"}) { + t.Fatalf("ID order = %v", got) + } + values, err := fromMongoID(NewBSONRegistry(), id, []string{"_id", "tenant"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(values, []interface{}{"item", "acme"}) { + t.Fatalf("values = %#v", values) + } + document := map[string]interface{}{"_id": "item", "tenant": "acme"} + if err := c.prepareSourceDocument("public.items", document); err != nil { + t.Fatal(err) + } + if document["_id"] != "item" { + t.Fatalf("document _id = %#v", document["_id"]) + } +} + +func TestPrepareSourceDocumentAliasesNonPrimaryID(t *testing.T) { + c := &conn{ + pkeys: map[string][]string{"public.items": {"item_id"}}, + hasIDColumn: map[string]bool{"public.items": true}, + idAliases: map[string]string{"public.items": "_id2"}, + bsonRegistry: NewBSONRegistry(), + } + document := map[string]interface{}{"item_id": int32(7), "_id": "original"} + if err := c.prepareSourceDocument("public.items", document); err != nil { + t.Fatal(err) + } + if document["_id2"] != "original" { + t.Fatalf("alias = %#v", document["_id2"]) + } + if document["_id"] != int32(7) { + t.Fatalf("synthetic _id = %#v", document["_id"]) + } +} diff --git a/connectors/postgres/pglib/integration_test.go b/connectors/postgres/pglib/integration_test.go new file mode 100644 index 00000000..96d435ab --- /dev/null +++ b/connectors/postgres/pglib/integration_test.go @@ -0,0 +1,138 @@ +package pglib + +import ( + "context" + "net/url" + "os" + "testing" + "time" + + "github.com/jackc/pglogrepl" + "github.com/jackc/pgx/v5" +) + +func TestChangeStreamLiveToastAndTruncate(t *testing.T) { + connectionURL := os.Getenv("POSTGRES_TEST_URL") + if connectionURL == "" { + t.Skip("POSTGRES_TEST_URL is not set") + } + ctx := context.Background() + admin, err := pgx.Connect(ctx, connectionURL) + if err != nil { + t.Fatal(err) + } + + const publication = "dsync_live_publication" + const slot = "dsync_live_slot" + var cancelStream context.CancelFunc + var streamDone <-chan struct{} + t.Cleanup(func() { + if cancelStream != nil { + cancelStream() + select { + case <-streamDone: + case <-time.After(5 * time.Second): + t.Error("timed out waiting for replication stream to stop") + } + } + + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for _, statement := range []string{ + `SELECT pg_drop_replication_slot('dsync_live_slot') FROM pg_replication_slots WHERE slot_name = 'dsync_live_slot'`, + `DROP PUBLICATION IF EXISTS dsync_live_publication`, + `DROP SCHEMA IF EXISTS dsync_stream_live CASCADE`, + } { + if _, err := admin.Exec(cleanupCtx, statement); err != nil { + t.Errorf("cleanup %s: %v", statement, err) + } + } + if err := admin.Close(cleanupCtx); err != nil { + t.Errorf("close admin connection: %v", err) + } + }) + + for _, statement := range []string{ + `DROP PUBLICATION IF EXISTS dsync_live_publication`, + `SELECT pg_drop_replication_slot('dsync_live_slot') FROM pg_replication_slots WHERE slot_name = 'dsync_live_slot'`, + `DROP SCHEMA IF EXISTS dsync_stream_live CASCADE`, + `CREATE SCHEMA dsync_stream_live`, + `CREATE TABLE dsync_stream_live.events (id integer PRIMARY KEY, payload text, marker text)`, + `ALTER TABLE dsync_stream_live.events REPLICA IDENTITY FULL`, + `INSERT INTO dsync_stream_live.events SELECT 1, string_agg(md5(i::text), ''), 'before' FROM generate_series(1, 2000) AS i`, + `CREATE PUBLICATION dsync_live_publication FOR TABLE dsync_stream_live.events`, + } { + if _, err := admin.Exec(ctx, statement); err != nil { + t.Fatalf("%s: %v", statement, err) + } + } + + var startText string + if err := admin.QueryRow(ctx, `SELECT lsn::text FROM pg_create_logical_replication_slot($1, 'pgoutput')`, slot).Scan(&startText); err != nil { + t.Fatal(err) + } + startLSN, err := pglogrepl.ParseLSN(startText) + if err != nil { + t.Fatal(err) + } + replicationURL, err := url.Parse(connectionURL) + if err != nil { + t.Fatal(err) + } + query := replicationURL.Query() + query.Set("replication", "database") + replicationURL.RawQuery = query.Encode() + + streamCtx, cancel := context.WithCancel(ctx) + cancelStream = cancel + done := make(chan struct{}) + streamDone = done + updates := make(chan UpdateBatch, 4) + streamErrors := make(chan error, 1) + go func() { + defer close(done) + streamErrors <- (&ChangeStream{ + ReplicationURL: replicationURL.String(), + ReplicationSlotName: slot, + PublicationName: publication, + StartLSN: uint64(startLSN), + MinFlushDelay: time.Millisecond, + StandbyMessageTimeout: 100 * time.Millisecond, + }).Run(streamCtx, updates) + }() + + if _, err := admin.Exec(ctx, `UPDATE dsync_stream_live.events SET marker = 'after' WHERE id = 1`); err != nil { + t.Fatal(err) + } + if _, err := admin.Exec(ctx, `TRUNCATE dsync_stream_live.events`); err != nil { + t.Fatal(err) + } + + nextBatch := func() UpdateBatch { + t.Helper() + select { + case batch := <-updates: + return batch + case err := <-streamErrors: + t.Fatalf("change stream failed: %v", err) + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for change stream") + } + return UpdateBatch{} + } + + updateBatch := nextBatch() + if len(updateBatch.Updates) != 1 { + t.Fatalf("update batch has %d updates", len(updateBatch.Updates)) + } + update := updateBatch.Updates[0] + payload, ok := update.New["payload"].(string) + if !ok || len(payload) != 64000 || update.New["marker"] != "after" { + t.Fatalf("TOAST update was not reconstructed: payload type=%T length=%d marker=%v", update.New["payload"], len(payload), update.New["marker"]) + } + + truncateBatch := nextBatch() + if len(truncateBatch.Updates) != 0 || truncateBatch.LSN == 0 { + t.Fatalf("TRUNCATE batch = %#v", truncateBatch) + } +} diff --git a/connectors/postgres/pglib/stream.go b/connectors/postgres/pglib/stream.go index b99c0ccc..e4a2f35e 100644 --- a/connectors/postgres/pglib/stream.go +++ b/connectors/postgres/pglib/stream.go @@ -3,6 +3,7 @@ package pglib import ( "context" "fmt" + "sort" "time" "github.com/jackc/pglogrepl" @@ -90,6 +91,16 @@ type Update struct { LSN uint64 } +// UpdateBatch contains all row changes committed by a transaction and the LSN +// that is safe to resume from. Updates may be empty (for example, TRUNCATE is +// not representable as a row update), but the commit LSN is still meaningful. +type UpdateBatch struct { + Updates []Update + LSN uint64 +} + +type unchangedToastValue struct{} + type ChangeStream struct { ReplicationURL string ReplicationSlotName string @@ -110,6 +121,8 @@ func decodeToMap(state *replState, relationID uint32, columns []*pglogrepl.Tuple switch col.DataType { case 'n': values[colName] = nil + case 'u': + values[colName] = unchangedToastValue{} case 't': val, err := decodeTextColumnData(state.typeMap, col.Data, relation.Columns[i].DataType) if err != nil { @@ -128,7 +141,31 @@ func decodeTextColumnData(mi *pgtype.Map, data []byte, dataType uint32) (interfa return string(data), nil } -func (c *ChangeStream) Run(ctx context.Context, ch chan<- []Update) error { +func fillUnchangedValues(newValues, oldValues map[string]interface{}) { + for column, newValue := range newValues { + if _, unchanged := newValue.(unchangedToastValue); !unchanged { + continue + } + if oldValue, present := oldValues[column]; present { + if _, unchanged := oldValue.(unchangedToastValue); !unchanged { + newValues[column] = oldValue + } + } + } +} + +func unchangedToastColumns(values map[string]interface{}) []string { + var columns []string + for column, value := range values { + if _, unchanged := value.(unchangedToastValue); unchanged { + columns = append(columns, column) + } + } + sort.Strings(columns) + return columns +} + +func (c *ChangeStream) Run(ctx context.Context, ch chan<- UpdateBatch) error { standbyMessageTimeout := time.Second * 10 minFlushDelay := c.MinFlushDelay if c.StandbyMessageTimeout > 0 { @@ -146,7 +183,6 @@ func (c *ChangeStream) Run(ctx context.Context, ch chan<- []Update) error { "proto_version '2'", fmt.Sprintf("publication_names '%v'", c.PublicationName), "messages 'true'", - "streaming 'true'", } if err := pglogrepl.StartReplication(ctx, pgConn, c.ReplicationSlotName, pglogrepl.LSN(startLSN+1), pglogrepl.StartReplicationOptions{PluginArgs: pluginArguments}); err != nil { @@ -240,7 +276,7 @@ func (c *ChangeStream) Run(ctx context.Context, ch chan<- []Update) error { state.currentLSN = logicalMsg.FinalLSN case *pglogrepl.CommitMessage: select { - case ch <- state.updates: + case ch <- UpdateBatch{Updates: state.updates, LSN: uint64(state.currentLSN)}: case <-ctx.Done(): return nil } @@ -269,6 +305,12 @@ func (c *ChangeStream) Run(ctx context.Context, ch chan<- []Update) error { if err != nil { return err } + // With REPLICA IDENTITY FULL, the old tuple supplies values + // represented as unchanged TOAST markers in the new tuple. + fillUnchangedValues(values, oldValues) + } + if columns := unchangedToastColumns(values); len(columns) > 0 { + return fmt.Errorf("update to %s.%s contains unchanged TOAST columns %v; set REPLICA IDENTITY FULL", ns, table, columns) } state.updates = append(state.updates, Update{ Namespace: ns, diff --git a/connectors/postgres/pglib/stream_test.go b/connectors/postgres/pglib/stream_test.go new file mode 100644 index 00000000..14156964 --- /dev/null +++ b/connectors/postgres/pglib/stream_test.go @@ -0,0 +1,27 @@ +package pglib + +import ( + "reflect" + "testing" +) + +func TestFillUnchangedValues(t *testing.T) { + newValues := map[string]interface{}{"id": int32(1), "name": "new", "toasted": unchangedToastValue{}} + oldValues := map[string]interface{}{"id": int32(1), "name": "old", "toasted": "large value"} + + fillUnchangedValues(newValues, oldValues) + + want := map[string]interface{}{"id": int32(1), "name": "new", "toasted": "large value"} + if !reflect.DeepEqual(newValues, want) { + t.Fatalf("values = %#v, want %#v", newValues, want) + } +} + +func TestUnchangedToastColumns(t *testing.T) { + values := map[string]interface{}{"id": int32(1), "toasted": unchangedToastValue{}} + got := unchangedToastColumns(values) + want := []string{"toasted"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("columns = %#v, want %#v", got, want) + } +} diff --git a/connectors/postgres/sink_test.go b/connectors/postgres/sink_test.go new file mode 100644 index 00000000..3720646d --- /dev/null +++ b/connectors/postgres/sink_test.go @@ -0,0 +1,176 @@ +package postgres + +import ( + "context" + "fmt" + "reflect" + "strings" + "testing" + + "connectrpc.com/connect" + adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" +) + +func TestPlanPostgresUpsertUsesReplacementSemantics(t *testing.T) { + metadata := sinkTableMetadata{ + Columns: []sinkColumnMetadata{ + {Name: "id", Identity: true, IdentityGeneration: "ALWAYS"}, + {Name: "name"}, + {Name: "status", HasDefault: true}, + {Name: "note", Nullable: true}, + {Name: "computed", Generated: true}, + }, + PrimaryKey: []string{"id"}, + } + document := map[string]interface{}{ + "id": int32(7), + "name": "example", + } + operation, err := planPostgresUpsert(metadata, document, 3) + if err != nil { + t.Fatal(err) + } + if !operation.overrideIdentity { + t.Fatal("expected GENERATED ALWAYS identity override") + } + if !reflect.DeepEqual(operation.columns, []string{"id", "name", "note"}) { + t.Fatalf("columns = %#v", operation.columns) + } + if !reflect.DeepEqual(operation.defaultColumns, []string{"status"}) { + t.Fatalf("default columns = %#v", operation.defaultColumns) + } + if !reflect.DeepEqual(operation.values, []interface{}{int32(7), "example", nil}) { + t.Fatalf("values = %#v", operation.values) + } + query := buildUpsertQuery("public.items", metadata.PrimaryKey, operation.columns, operation.defaultColumns, operation.overrideIdentity) + for _, fragment := range []string{ + `INSERT INTO "public"."items" ("id", "name", "note") OVERRIDING SYSTEM VALUE`, + `"name" = EXCLUDED."name"`, + `"note" = EXCLUDED."note"`, + `"status" = DEFAULT`, + } { + if !strings.Contains(query, fragment) { + t.Fatalf("query %q does not contain %q", query, fragment) + } + } +} + +func TestPlanPostgresUpsertRejectsInvalidDocuments(t *testing.T) { + metadata := sinkTableMetadata{ + Columns: []sinkColumnMetadata{{Name: "id"}, {Name: "required"}}, + PrimaryKey: []string{"id"}, + } + tests := []struct { + name string + document map[string]interface{} + contains string + }{ + {name: "missing key", document: map[string]interface{}{"required": "x"}, contains: `key field "id"`}, + {name: "missing required", document: map[string]interface{}{"id": 1}, contains: `required column "required"`}, + {name: "null required", document: map[string]interface{}{"id": 1, "required": nil}, contains: `cannot be null`}, + {name: "unknown column", document: map[string]interface{}{"id": 1, "required": "x", "extra": true}, contains: `column "extra" does not exist`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := planPostgresUpsert(metadata, test.document, 0) + if err == nil || !strings.Contains(err.Error(), test.contains) { + t.Fatalf("error = %v, want containing %q", err, test.contains) + } + }) + } +} + +func TestPlanPostgresUpsertRejectsGeneratedPrimaryKey(t *testing.T) { + _, err := planPostgresUpsert(sinkTableMetadata{ + Columns: []sinkColumnMetadata{{Name: "id", Generated: true}}, + PrimaryKey: []string{"id"}, + }, map[string]interface{}{"id": 1}, 0) + if err == nil || !strings.Contains(err.Error(), "generated primary key") { + t.Fatalf("error = %v", err) + } +} + +func TestPlanPostgresUpsertRejectsSuppliedGeneratedNonKey(t *testing.T) { + for _, column := range []sinkColumnMetadata{ + {Name: "computed", Generated: true}, + {Name: "generated_id", Identity: true}, + } { + metadata := sinkTableMetadata{ + Columns: []sinkColumnMetadata{{Name: "id"}, column}, + PrimaryKey: []string{"id"}, + } + _, err := planPostgresUpsert(metadata, map[string]interface{}{ + "id": int64(1), + column.Name: "source value", + }, 0) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf(`column %q is generated by the destination`, column.Name)) { + t.Fatalf("expected generated-column error for %q, got %v", column.Name, err) + } + } +} + +func TestPlanPostgresUpsertAllowsOmittedGeneratedNonKey(t *testing.T) { + metadata := sinkTableMetadata{ + Columns: []sinkColumnMetadata{ + {Name: "id"}, + {Name: "generated_id", Identity: true}, + }, + PrimaryKey: []string{"id"}, + } + if _, err := planPostgresUpsert(metadata, map[string]interface{}{"id": int64(1)}, 0); err != nil { + t.Fatalf("unexpected error for omitted generated column: %v", err) + } +} + +func TestWriteUpdatesUnknownIsNoop(t *testing.T) { + c := &conn{sinkMetadata: map[string]sinkTableMetadata{ + "public.items": {Columns: []sinkColumnMetadata{{Name: "id"}}, PrimaryKey: []string{"id"}}, + }} + _, err := c.WriteUpdates(context.Background(), connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "public.items", + Updates: []*adiomv1.Update{{Type: adiomv1.UpdateType_UPDATE_TYPE_UNKNOWN}}, + })) + if err != nil { + t.Fatal(err) + } +} + +func TestPostgresSinkRejectsPartialResultsAndUpdates(t *testing.T) { + c := &conn{sinkMetadata: map[string]sinkTableMetadata{ + "public.items": {Columns: []sinkColumnMetadata{{Name: "id"}}, PrimaryKey: []string{"id"}}, + }} + _, err := c.WriteData(context.Background(), connect.NewRequest(&adiomv1.WriteDataRequest{PartialWriteResults: true})) + if connect.CodeOf(err) != connect.CodeInvalidArgument { + t.Fatalf("WriteData error = %v", err) + } + _, err = c.WriteUpdates(context.Background(), connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "public.items", + Updates: []*adiomv1.Update{{Type: adiomv1.UpdateType_UPDATE_TYPE_PARTIAL_UPDATE}}, + })) + if connect.CodeOf(err) != connect.CodeInvalidArgument { + t.Fatalf("WriteUpdates error = %v", err) + } +} + +func TestPostgresSinkReportsMalformedInputAsInvalidArgument(t *testing.T) { + c := &conn{ + sinkMetadata: map[string]sinkTableMetadata{ + "public.items": {Columns: []sinkColumnMetadata{{Name: "id"}}, PrimaryKey: []string{"id"}}, + }, + bsonRegistry: NewBSONRegistry(), + } + _, err := c.WriteData(context.Background(), connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "public.items", + Data: [][]byte{{0x01, 0x02}}, + })) + if connect.CodeOf(err) != connect.CodeInvalidArgument { + t.Fatalf("WriteData error = %v", err) + } + _, err = c.WriteUpdates(context.Background(), connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "public.items", + Updates: []*adiomv1.Update{{Type: adiomv1.UpdateType_UPDATE_TYPE_DELETE}}, + })) + if connect.CodeOf(err) != connect.CodeInvalidArgument { + t.Fatalf("WriteUpdates error = %v", err) + } +} diff --git a/connectors/sqlbatch/config_test.go b/connectors/sqlbatch/config_test.go new file mode 100644 index 00000000..983389c5 --- /dev/null +++ b/connectors/sqlbatch/config_test.go @@ -0,0 +1,67 @@ +package sqlbatch + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestSourceMappingRequiresLimit(t *testing.T) { + c := &conn{config: Config{Mappings: map[string]Mapping{ + "public.items": {Namespace: "public.items", Query: "SELECT * FROM public.items"}, + }}} + _, err := c.sourceMapping("public.items") + if err == nil || !strings.Contains(err.Error(), "invalid or unspecified limit") { + t.Fatalf("expected invalid limit error, got %v", err) + } +} + +func TestParseConfigAllowsSinkWithoutSourceMappings(t *testing.T) { + path := filepath.Join(t.TempDir(), "sqlbatch.yml") + contents := []byte("driver: pgx\nconnectionstring: postgres://example\nsink:\n writebatchsize: 25\n tables:\n public.large_items:\n writebatchsize: 5\n") + if err := os.WriteFile(path, contents, 0o600); err != nil { + t.Fatal(err) + } + config, err := parseConfigFromFile(path) + if err != nil { + t.Fatal(err) + } + if config.Sink.WriteBatchSize != 25 { + t.Fatalf("unexpected sink config: %#v", config.Sink) + } + if got := config.Sink.writeBatchSize("public.large_items"); got != 5 { + t.Fatalf("table batch size = %d, want 5", got) + } + if got := config.Sink.writeBatchSize("public.items"); got != 25 { + t.Fatalf("global batch size = %d, want 25", got) + } +} + +func TestParseConfigRejectsSourceMappingWithoutLimit(t *testing.T) { + path := filepath.Join(t.TempDir(), "sqlbatch.yml") + contents := []byte("driver: pgx\nconnectionstring: postgres://example\nmappings:\n - namespace: public.items\n query: SELECT * FROM public.items\n") + if err := os.WriteFile(path, contents, 0o600); err != nil { + t.Fatal(err) + } + _, err := parseConfigFromFile(path) + if err == nil || !strings.Contains(err.Error(), "invalid or unspecified limit") { + t.Fatalf("expected invalid limit error, got %v", err) + } +} + +func TestParseConfigRejectsInvalidSinkBatchSizes(t *testing.T) { + tests := []string{ + "driver: pgx\nconnectionstring: postgres://example\nsink:\n writebatchsize: -1\n", + "driver: pgx\nconnectionstring: postgres://example\nsink:\n tables:\n public.items:\n writebatchsize: 0\n", + } + for i, contents := range tests { + path := filepath.Join(t.TempDir(), "sqlbatch.yml") + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + if _, err := parseConfigFromFile(path); err == nil { + t.Fatalf("case %d: expected invalid sink batch size error", i) + } + } +} diff --git a/connectors/sqlbatch/conn.go b/connectors/sqlbatch/conn.go index 9fbde784..31e36d9f 100644 --- a/connectors/sqlbatch/conn.go +++ b/connectors/sqlbatch/conn.go @@ -12,18 +12,21 @@ import ( "os" "slices" "strings" + "sync" "time" "connectrpc.com/connect" "github.com/adiom-data/dsync/connectors/sqlbatch/dialects" adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" "github.com/adiom-data/dsync/gen/adiom/v1/adiomv1connect" + "github.com/adiom-data/dsync/internal/gobregister" + "github.com/microsoft/go-mssqldb/msdsn" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" ) func init() { - gob.Register(time.Time{}) + gobregister.RegisterCursorTypes() } type ChangeMapping struct { @@ -47,11 +50,31 @@ type Mapping struct { Fetchers int } +type SinkConfig struct { + // WriteBatchSize is the connector-wide maximum number of compatible writes + // combined into one statement. Values less than one use the built-in default. + WriteBatchSize int + // Tables optionally overrides WriteBatchSize for individual namespaces. + Tables map[string]SinkTableConfig +} + +type SinkTableConfig struct { + WriteBatchSize int +} + +func (c SinkConfig) writeBatchSize(namespace string) int { + if table, ok := c.Tables[namespace]; ok && table.WriteBatchSize > 0 { + return table.WriteBatchSize + } + return c.WriteBatchSize +} + type Config struct { ID string Driver string ConnectionString string Mappings map[string]Mapping + Sink SinkConfig } type ConfigYml struct { @@ -59,6 +82,7 @@ type ConfigYml struct { Driver string ConnectionString string Mappings []Mapping + Sink SinkConfig } type SQLDialect interface { @@ -67,6 +91,13 @@ type SQLDialect interface { FetchTemplate(cols []string, query string) string Placeholder(i int) string KeySub(cols []string, count int) string + InspectTable(context.Context, *sql.DB, string) (dialects.TableMetadata, error) + ConvertJSONValue(dialects.ColumnMetadata, any) (any, error) + ConvertDriverValue(dialects.ColumnMetadata, any) (any, error) + UpsertQuery(namespace string, keys, columns, columnTypes, defaultColumns, identityColumns []string, count int, overrideIdentity bool) (string, error) + DeleteQuery(namespace string, keys []string, count int) string + MaxQueryParameters() int + SupportsIdentityOverride() bool } func parseConfigFromFile(file string) (Config, error) { @@ -78,10 +109,21 @@ func parseConfigFromFile(file string) (Config, error) { if err := yaml.Unmarshal(f, &m); err != nil { return Config{}, fmt.Errorf("err parsing yml %w", err) } + if m.Sink.WriteBatchSize < 0 { + return Config{}, fmt.Errorf("sink write batch size cannot be negative: %v", m.Sink.WriteBatchSize) + } + for namespace, table := range m.Sink.Tables { + if namespace == "" { + return Config{}, fmt.Errorf("sink table namespace cannot be empty") + } + if table.WriteBatchSize < 1 { + return Config{}, fmt.Errorf("sink table %v has invalid write batch size: %v", namespace, table.WriteBatchSize) + } + } res := map[string]Mapping{} for _, mapping := range m.Mappings { if mapping.Limit < 1 { - return Config{}, fmt.Errorf("invalid or unspecified limit: %v", mapping.Limit) + return Config{}, fmt.Errorf("source mapping for %v has invalid or unspecified limit: %v", mapping.Namespace, mapping.Limit) } if len(mapping.DecodeJSON) > 0 { mapping.decodeJSON = map[string]struct{}{} @@ -99,9 +141,21 @@ func parseConfigFromFile(file string) (Config, error) { Driver: m.Driver, ConnectionString: m.ConnectionString, Mappings: res, + Sink: m.Sink, }, nil } +func (c *conn) sourceMapping(namespace string) (Mapping, error) { + mapping, ok := c.config.Mappings[namespace] + if !ok { + return Mapping{}, fmt.Errorf("no source mapping for %v", namespace) + } + if mapping.Limit < 1 { + return Mapping{}, fmt.Errorf("source mapping for %v has invalid or unspecified limit: %v", namespace, mapping.Limit) + } + return mapping, nil +} + type updateCursor struct { Cursor []any } @@ -157,9 +211,11 @@ func DecodePartitionCursor(in []byte) (*partitionCursor, error) { } type conn struct { - c *sql.DB - config Config - dialect SQLDialect + c *sql.DB + config Config + dialect SQLDialect + metadataMu sync.Mutex + sinkMetadata map[string]dialects.TableMetadata } // GetByIds implements [adiomv1connect.ConnectorServiceHandler]. @@ -179,9 +235,37 @@ func ScanRow(rows *sql.Rows, numCols int) ([]any, error) { return values, nil } -func ParseRow(cols []string, rows *sql.Rows, mapping *Mapping) (map[string]any, error) { +func convertDriverValues(cols []string, columnTypes []*sql.ColumnType, values []any, dialect SQLDialect) error { + if len(columnTypes) != len(cols) || len(values) != len(cols) { + return fmt.Errorf("column metadata has %d entries and row has %d values for %d columns", len(columnTypes), len(values), len(cols)) + } + for i, col := range cols { + converted, err := dialect.ConvertDriverValue(dialects.ColumnMetadata{ + Name: col, + DatabaseType: columnTypes[i].DatabaseTypeName(), + }, values[i]) + if err != nil { + return fmt.Errorf("convert column %q: %w", col, err) + } + values[i] = converted + } + return nil +} + +func scanRowWithTypes(cols []string, columnTypes []*sql.ColumnType, rows *sql.Rows, dialect SQLDialect) ([]any, error) { + values, err := ScanRow(rows, len(cols)) + if err != nil { + return nil, err + } + if err := convertDriverValues(cols, columnTypes, values, dialect); err != nil { + return nil, err + } + return values, nil +} + +func ParseRow(cols []string, columnTypes []*sql.ColumnType, rows *sql.Rows, mapping *Mapping, dialect SQLDialect) (map[string]any, error) { m := make(map[string]any, len(cols)) - parsedRow, err := ScanRow(rows, len(cols)) + parsedRow, err := scanRowWithTypes(cols, columnTypes, rows, dialect) if err != nil { return nil, fmt.Errorf("err in scan row: %w", err) } @@ -220,8 +304,8 @@ func (c *conn) GeneratePlan(ctx context.Context, r *connect.Request[adiomv1.Gene namespaces := r.Msg.GetNamespaces() if len(namespaces) == 0 { namespaces = []string{} - for k := range c.config.Mappings { - namespaces = append(namespaces, k) + for namespace := range c.config.Mappings { + namespaces = append(namespaces, namespace) } } if r.Msg.InitialSync { @@ -232,9 +316,9 @@ func (c *conn) GeneratePlan(ctx context.Context, r *connect.Request[adiomv1.Gene return nil, err } estimatedCount := metadataResp.Msg.GetCount() - mapping, ok := c.config.Mappings[namespace] - if !ok { - return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("no mapping for %v", namespace)) + mapping, err := c.sourceMapping(namespace) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) } rows, err := c.c.QueryContext(ctx, mapping.PartitionQuery) if err != nil { @@ -245,9 +329,14 @@ func (c *conn) GeneratePlan(ctx context.Context, r *connect.Request[adiomv1.Gene _ = rows.Close() return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("err columns: %w", err)) } + columnTypes, err := rows.ColumnTypes() + if err != nil { + _ = rows.Close() + return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("err column types: %w", err)) + } var low []any for rows.Next() { - parsedRow, err := ScanRow(rows, len(cols)) + parsedRow, err := scanRowWithTypes(cols, columnTypes, rows, c.dialect) if err != nil { _ = rows.Close() return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("err scanning row: %w", err)) @@ -323,14 +412,14 @@ func (c *conn) GeneratePlan(ctx context.Context, r *connect.Request[adiomv1.Gene var updatePartitions []*adiomv1.UpdatesPartition if r.Msg.Updates { for _, namespace := range namespaces { - mapping, ok := c.config.Mappings[namespace] - if !ok { - return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("no mapping for %v", namespace)) + mapping, err := c.sourceMapping(namespace) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) } if len(mapping.Changes) == 0 { continue } - updateCursor, err := GetUpdateCursor(ctx, c.c, &mapping) + updateCursor, err := GetUpdateCursor(ctx, c.c, &mapping, c.dialect) if err != nil { return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("unable to get update cursor for %v", namespace)) } @@ -369,15 +458,18 @@ func (c *conn) GetInfo(context.Context, *connect.Request[adiomv1.GetInfoRequest] MultiNamespacePlan: true, DefaultPlan: true, }, + Sink: &adiomv1.Capabilities_Sink{ + SupportedDataTypes: []adiomv1.DataType{adiomv1.DataType_DATA_TYPE_JSON_ID}, + }, }, }), nil } // GetNamespaceMetadata implements adiomv1connect.ConnectorServiceHandler. func (c *conn) GetNamespaceMetadata(ctx context.Context, r *connect.Request[adiomv1.GetNamespaceMetadataRequest]) (*connect.Response[adiomv1.GetNamespaceMetadataResponse], error) { - mapping, ok := c.config.Mappings[r.Msg.GetNamespace()] - if !ok { - return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("no mapping for %v", r.Msg.GetNamespace())) + mapping, err := c.sourceMapping(r.Msg.GetNamespace()) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) } if mapping.NoCount { @@ -412,9 +504,9 @@ func (c *conn) ListData(ctx context.Context, r *connect.Request[adiomv1.ListData if err != nil { return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("error decoding cursor")) } - mapping, ok := c.config.Mappings[namespace] - if !ok { - return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("no mapping for %v", namespace)) + mapping, err := c.sourceMapping(namespace) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) } template := c.dialect.ListDataTemplate(cursor.Cols, mapping.Query) @@ -430,12 +522,16 @@ func (c *conn) ListData(ctx context.Context, r *connect.Request[adiomv1.ListData if err != nil { return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("err getting columns: %w", err)) } + columnTypes, err := rows.ColumnTypes() + if err != nil { + return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("err getting column types: %w", err)) + } var data [][]byte var count int var nextLow []any for rows.Next() { count += 1 - m, err := ParseRow(cols, rows, &mapping) + m, err := ParseRow(cols, columnTypes, rows, &mapping, c.dialect) if err != nil { return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("err in parse row: %w", err)) } @@ -489,9 +585,9 @@ func (c *conn) StreamUpdates(ctx context.Context, r *connect.Request[adiomv1.Str if err != nil { return connect.NewError(connect.CodeInternal, fmt.Errorf("err decoding cursor: %w", err)) } - mapping, ok := c.config.Mappings[r.Msg.GetNamespaces()[0]] - if !ok { - return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("no mapping for %v", r.Msg.GetNamespaces()[0])) + mapping, err := c.sourceMapping(r.Msg.GetNamespaces()[0]) + if err != nil { + return connect.NewError(connect.CodeInvalidArgument, err) } ch := make(chan *adiomv1.StreamUpdatesResponse) @@ -527,16 +623,6 @@ func (c *conn) StreamUpdates(ctx context.Context, r *connect.Request[adiomv1.Str return nil } -// WriteData implements adiomv1connect.ConnectorServiceHandler. -func (c *conn) WriteData(ctx context.Context, r *connect.Request[adiomv1.WriteDataRequest]) (*connect.Response[adiomv1.WriteDataResponse], error) { - panic("unimplemented") -} - -// WriteUpdates implements adiomv1connect.ConnectorServiceHandler. -func (c *conn) WriteUpdates(context.Context, *connect.Request[adiomv1.WriteUpdatesRequest]) (*connect.Response[adiomv1.WriteUpdatesResponse], error) { - panic("unimplemented") -} - func (c *conn) Teardown() { _ = c.c.Close() } @@ -549,7 +635,11 @@ func NewConn(cfgFile string) (*conn, error) { var dialect SQLDialect switch config.Driver { case "sqlserver": - dialect = dialects.SQLServer{} + dsn, err := msdsn.Parse(config.ConnectionString) + if err != nil { + return nil, fmt.Errorf("parse SQL Server connection string: %w", err) + } + dialect = dialects.SQLServer{GuidConversion: dsn.Encoding.GuidConversion} case "postgres", "pgx": config.Driver = "pgx" dialect = dialects.Postgres{} diff --git a/connectors/sqlbatch/cs.go b/connectors/sqlbatch/cs.go index 8e301564..ba3db1c8 100644 --- a/connectors/sqlbatch/cs.go +++ b/connectors/sqlbatch/cs.go @@ -15,7 +15,7 @@ import ( "golang.org/x/sync/errgroup" ) -func GetUpdateCursor(ctx context.Context, c *sql.DB, mapping *Mapping) (updateCursor, error) { +func GetUpdateCursor(ctx context.Context, c *sql.DB, mapping *Mapping, dialect SQLDialect) (updateCursor, error) { eg, ctx := errgroup.WithContext(ctx) newUpdateCursor := updateCursor{ Cursor: make([]any, len(mapping.Changes)), @@ -35,9 +35,13 @@ func GetUpdateCursor(ctx context.Context, c *sql.DB, mapping *Mapping) (updateCu if len(cols) != 1 { return fmt.Errorf("error expected exactly 1 column for cursor") } + columnTypes, err := rows.ColumnTypes() + if err != nil { + return fmt.Errorf("error getting column types: %w", err) + } // Only want the first item if rows.Next() { - res, err := ScanRow(rows, len(cols)) + res, err := scanRowWithTypes(cols, columnTypes, rows, dialect) if err != nil { return fmt.Errorf("error scanning row: %w", err) } @@ -162,11 +166,16 @@ func Poller(ctx context.Context, cursor updateCursor, mapping *Mapping, c *sql.D _ = rows.Close() return fmt.Errorf("error expected at least 3 columns for changes") } + columnTypes, err := rows.ColumnTypes() + if err != nil { + _ = rows.Close() + return fmt.Errorf("error getting column types: %w", err) + } var count int for rows.Next() { count += 1 - res, err := ScanRow(rows, len(cols)) + res, err := scanRowWithTypes(cols, columnTypes, rows, dialect) if err != nil { _ = rows.Close() return fmt.Errorf("error scanning row: %w", err) @@ -404,9 +413,14 @@ func Poller(ctx context.Context, cursor updateCursor, mapping *Mapping, c *sql.D _ = rows.Close() return fmt.Errorf("err getting columns: %w", err) } + columnTypes, err := rows.ColumnTypes() + if err != nil { + _ = rows.Close() + return fmt.Errorf("err getting column types: %w", err) + } results := make([]*adiomv1.Update, 0, len(request.Keys)) for rows.Next() { - m, err := ParseRow(cols, rows, mapping) + m, err := ParseRow(cols, columnTypes, rows, mapping, dialect) if err != nil { _ = rows.Close() return fmt.Errorf("err in parse row: %w", err) diff --git a/connectors/sqlbatch/dialects/common.go b/connectors/sqlbatch/dialects/common.go index c8050fe0..9543f037 100644 --- a/connectors/sqlbatch/dialects/common.go +++ b/connectors/sqlbatch/dialects/common.go @@ -1,9 +1,297 @@ package dialects import ( + "database/sql" + "encoding/base64" + "encoding/json" + "fmt" + "sort" "strings" + "time" ) +type ColumnMetadata struct { + Name string + DatabaseType string + Nullable bool + HasDefault bool + Generated bool + Identity bool + IdentityGeneration string +} + +type TableMetadata struct { + Columns []ColumnMetadata + PrimaryKey []string +} + +func ConvertDriverValue(_ ColumnMetadata, value any) (any, error) { + return value, nil +} + +func SplitNamespace(namespace string) (schema, table string, err error) { + parts := strings.Split(namespace, ".") + switch len(parts) { + case 1: + table = parts[0] + case 2: + schema, table = parts[0], parts[1] + default: + return "", "", fmt.Errorf("expected table or schema.table namespace, got %q", namespace) + } + if table == "" || (len(parts) == 2 && schema == "") { + return "", "", fmt.Errorf("invalid table namespace %q", namespace) + } + return schema, table, nil +} + +func ScanTableMetadata(rows *sql.Rows) (TableMetadata, error) { + var metadata TableMetadata + primaryKeys := map[int]string{} + for rows.Next() { + var column ColumnMetadata + var nullable, hasDefault, generated, identity, identityGeneration string + var primaryKeyOrdinal int + if err := rows.Scan( + &column.Name, + &column.DatabaseType, + &nullable, + &hasDefault, + &generated, + &identity, + &identityGeneration, + &primaryKeyOrdinal, + ); err != nil { + return TableMetadata{}, err + } + column.Nullable = strings.EqualFold(nullable, "YES") + column.HasDefault = strings.EqualFold(hasDefault, "YES") + column.Generated = strings.EqualFold(generated, "YES") + column.Identity = strings.EqualFold(identity, "YES") + column.IdentityGeneration = identityGeneration + metadata.Columns = append(metadata.Columns, column) + if primaryKeyOrdinal > 0 { + primaryKeys[primaryKeyOrdinal] = column.Name + } + } + if err := rows.Err(); err != nil { + return TableMetadata{}, err + } + if len(metadata.Columns) == 0 { + return TableMetadata{}, fmt.Errorf("table was not found or has no visible columns") + } + ordinals := make([]int, 0, len(primaryKeys)) + for ordinal := range primaryKeys { + ordinals = append(ordinals, ordinal) + } + sort.Ints(ordinals) + for _, ordinal := range ordinals { + metadata.PrimaryKey = append(metadata.PrimaryKey, primaryKeys[ordinal]) + } + return metadata, nil +} + +func ConvertJSONValue(column ColumnMetadata, value any, binaryTypes ...string) (any, error) { + return convertJSONValue(column, value, binaryTypes, nil) +} + +func convertJSONValue(column ColumnMetadata, value any, binaryTypes, numericTextTypes []string) (any, error) { + switch value := value.(type) { + case json.Number: + for _, numericType := range numericTextTypes { + if strings.EqualFold(column.DatabaseType, numericType) { + // Keep every value for a decimal-capable column in the same exact + // representation so a batched row source does not have to reconcile + // integer and string bind types for one column. + return value.String(), nil + } + } + if integer, err := value.Int64(); err == nil { + return integer, nil + } + return value.String(), nil + case map[string]any, []any: + encoded, err := json.Marshal(value) + if err != nil { + return nil, err + } + return string(encoded), nil + case string: + for _, binaryType := range binaryTypes { + if strings.EqualFold(column.DatabaseType, binaryType) { + decoded, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return nil, fmt.Errorf("decode base64 value for binary column %q: %w", column.Name, err) + } + return decoded, nil + } + } + } + return value, nil +} + +// ConvertRFC3339Value converts the connector's temporal wire representation +// to time.Time before it reaches a database driver. SQLBatch emits time.Time +// values as RFC3339 JSON through encoding/json, so sinks use the same format. +func ConvertRFC3339Value(column ColumnMetadata, value any, temporalTypes ...string) (any, error) { + text, ok := value.(string) + if !ok { + return value, nil + } + for _, temporalType := range temporalTypes { + if !strings.EqualFold(column.DatabaseType, temporalType) { + continue + } + parsed, err := time.Parse(time.RFC3339Nano, text) + if err != nil { + return nil, fmt.Errorf("parse RFC3339 value for temporal column %q: %w", column.Name, err) + } + return parsed, nil + } + return value, nil +} + +type temporalValueKind int + +const ( + temporalTimestamp temporalValueKind = iota + temporalDate + temporalOracleDate + temporalTime +) + +func convertTemporalValue(column ColumnMetadata, value any, kind temporalValueKind, preserveOffset bool) (any, error) { + if value == nil { + return nil, nil + } + var parsed time.Time + switch value := value.(type) { + case time.Time: + parsed = value + case string: + var err error + switch { + case (kind == temporalDate || kind == temporalOracleDate) && len(value) == len(time.DateOnly): + parsed, err = time.Parse(time.DateOnly, value) + case kind == temporalTime: + for _, layout := range []string{"15:04:05Z07:00", "15:04:05"} { + parsed, err = time.Parse(layout, value) + if err == nil { + break + } + } + if err != nil { + parsed, err = time.Parse(time.RFC3339Nano, value) + } + default: + parsed, err = time.Parse(time.RFC3339Nano, value) + } + if err != nil { + return nil, fmt.Errorf("parse temporal column %q of type %s: %w", column.Name, column.DatabaseType, err) + } + default: + return nil, fmt.Errorf("temporal column %q of type %s requires an RFC3339 string, date-only string, or BSON datetime; got %T", column.Name, column.DatabaseType, value) + } + if !preserveOffset { + parsed = parsed.UTC() + } + if kind == temporalDate { + return time.Date(parsed.Year(), parsed.Month(), parsed.Day(), 0, 0, 0, 0, time.UTC), nil + } + if kind == temporalTime { + return time.Date(1, time.January, 1, parsed.Hour(), parsed.Minute(), parsed.Second(), parsed.Nanosecond(), time.UTC), nil + } + return parsed, nil +} + +func QuoteIdentifier(identifier string) string { + return `"` + strings.ReplaceAll(identifier, `"`, `""`) + `"` +} + +func QuoteNamespace(namespace string) string { + parts := strings.Split(namespace, ".") + for i, part := range parts { + parts[i] = QuoteIdentifier(part) + } + return strings.Join(parts, ".") +} + +func QuotedIdentifiers(identifiers []string) string { + quoted := make([]string, len(identifiers)) + for i, identifier := range identifiers { + quoted[i] = QuoteIdentifier(identifier) + } + return strings.Join(quoted, ", ") +} + +func Placeholders(count int, placeholder func(int) string) string { + return PlaceholdersFrom(0, count, placeholder) +} + +func PlaceholdersFrom(start, count int, placeholder func(int) string) string { + values := make([]string, count) + for i := range count { + values[i] = placeholder(start + i) + } + return strings.Join(values, ", ") +} + +func ValueRows(columns, count int, placeholder func(int) string) string { + rows := make([]string, count) + for i := range count { + rows[i] = "(" + PlaceholdersFrom(i*columns, columns, placeholder) + ")" + } + return strings.Join(rows, ", ") +} + +func DeleteQuery(namespace string, keys []string, count int, placeholder func(int) string) string { + rows := make([]string, count) + for row := range count { + predicates := make([]string, len(keys)) + for i, key := range keys { + predicates[i] = QuoteIdentifier(key) + " = " + placeholder(row*len(keys)+i) + } + rows[row] = "(" + strings.Join(predicates, " AND ") + ")" + } + return "DELETE FROM " + QuoteNamespace(namespace) + " WHERE " + strings.Join(rows, " OR ") +} + +func SourceColumns(columns []string, qualifier string) string { + values := make([]string, len(columns)) + for i, column := range columns { + values[i] = qualifier + "." + QuoteIdentifier(column) + } + return strings.Join(values, ", ") +} + +func Assignments(columns []string, target, source string) string { + values := make([]string, len(columns)) + for i, column := range columns { + values[i] = target + "." + QuoteIdentifier(column) + " = " + source + "." + QuoteIdentifier(column) + } + return strings.Join(values, ", ") +} + +func DefaultAssignments(columns []string, target string) string { + values := make([]string, len(columns)) + for i, column := range columns { + prefix := "" + if target != "" { + prefix = target + "." + } + values[i] = prefix + QuoteIdentifier(column) + " = DEFAULT" + } + return strings.Join(values, ", ") +} + +func JoinPredicate(keys []string, target, source string) string { + values := make([]string, len(keys)) + for i, key := range keys { + values[i] = target + "." + QuoteIdentifier(key) + " = " + source + "." + QuoteIdentifier(key) + } + return strings.Join(values, " AND ") +} + func BuildQuery(cols []string, placeholder func(int) string, placeholderStart int, compare string, lastCompare string) string { var buf strings.Builder var inner strings.Builder diff --git a/connectors/sqlbatch/dialects/common_test.go b/connectors/sqlbatch/dialects/common_test.go index 4bd9fe88..587a7050 100644 --- a/connectors/sqlbatch/dialects/common_test.go +++ b/connectors/sqlbatch/dialects/common_test.go @@ -1,13 +1,178 @@ package dialects_test import ( + "bytes" + "encoding/json" "fmt" "testing" + "time" "github.com/adiom-data/dsync/connectors/sqlbatch/dialects" + mssql "github.com/microsoft/go-mssqldb" "github.com/stretchr/testify/assert" ) +func TestTemporalValuesUseRFC3339(t *testing.T) { + want := time.Date(2026, 10, 15, 12, 34, 56, 123456000, time.FixedZone("-0700", -7*60*60)) + tests := []struct { + name string + dialect interface { + ConvertJSONValue(dialects.ColumnMetadata, any) (any, error) + } + databaseType string + }{ + {name: "postgres", dialect: dialects.Postgres{}, databaseType: "timestamptz"}, + {name: "sqlserver", dialect: dialects.SQLServer{}, databaseType: "datetimeoffset"}, + {name: "oracle", dialect: dialects.Oracle{}, databaseType: "TIMESTAMP WITH TIME ZONE"}, + {name: "oracle precision", dialect: dialects.Oracle{}, databaseType: "TIMESTAMP(6) WITH TIME ZONE"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.dialect.ConvertJSONValue(dialects.ColumnMetadata{Name: "occurred_at", DatabaseType: test.databaseType}, "2026-10-15T12:34:56.123456-07:00") + if err != nil { + t.Fatal(err) + } + if !got.(time.Time).Equal(want) { + t.Fatalf("time = %v, want %v", got, want) + } + }) + } +} + +func TestTemporalValuesRejectNonRFC3339(t *testing.T) { + _, err := (dialects.Oracle{}).ConvertJSONValue( + dialects.ColumnMetadata{Name: "occurred_at", DatabaseType: "DATE"}, + "15-OCT-26", + ) + if err == nil { + t.Fatal("expected invalid RFC3339 value to fail") + } +} + +func TestSQLServerUniqueIdentifierFromBinaryValue(t *testing.T) { + data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + for _, test := range []struct { + name string + guidConversion bool + want mssql.UniqueIdentifier + }{ + {name: "disabled", want: mssql.UniqueIdentifier{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}, + {name: "enabled", guidConversion: true, want: mssql.UniqueIdentifier{3, 2, 1, 0, 5, 4, 7, 6, 8, 9, 10, 11, 12, 13, 14, 15}}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := (dialects.SQLServer{GuidConversion: test.guidConversion}).ConvertJSONValue( + dialects.ColumnMetadata{Name: "id", DatabaseType: "uniqueidentifier"}, + data, + ) + if err != nil { + t.Fatal(err) + } + if got != test.want { + t.Fatalf("uniqueidentifier = %#v, want %#v", got, test.want) + } + }) + } +} + +func TestSQLServerDriverValuesUseUnambiguousJSONTypes(t *testing.T) { + column := dialects.ColumnMetadata{Name: "id", DatabaseType: "uniqueidentifier"} + rawTDS := []byte{3, 2, 1, 0, 5, 4, 7, 6, 8, 9, 10, 11, 12, 13, 14, 15} + got, err := (dialects.SQLServer{}).ConvertDriverValue(column, rawTDS) + if err != nil { + t.Fatal(err) + } + const want = "00010203-0405-0607-0809-0A0B0C0D0E0F" + if got != want { + t.Fatalf("uniqueidentifier = %#v, want %q", got, want) + } + + canonical := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + got, err = (dialects.SQLServer{GuidConversion: true}).ConvertDriverValue(column, canonical) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("converted uniqueidentifier = %#v, want %q", got, want) + } + + got, err = (dialects.SQLServer{}).ConvertDriverValue( + dialects.ColumnMetadata{Name: "amount", DatabaseType: "decimal"}, + []byte("1234567890.0123456789"), + ) + if err != nil { + t.Fatal(err) + } + if got != "1234567890.0123456789" { + t.Fatalf("decimal = %#v", got) + } + + binary := []byte{0, 1, 254, 255} + got, err = (dialects.SQLServer{}).ConvertDriverValue( + dialects.ColumnMetadata{Name: "payload", DatabaseType: "varbinary"}, + binary, + ) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got.([]byte), binary) { + t.Fatalf("binary = %#v", got) + } +} + +func TestSQLServerExactNumericValuesExpandExponentNotation(t *testing.T) { + dialect := dialects.SQLServer{} + for _, test := range []struct { + name string + value any + want string + }{ + {name: "positive exponent", value: json.Number("1e3"), want: "1000"}, + {name: "fractional mantissa", value: json.Number("1.25E+2"), want: "125"}, + {name: "negative exponent", value: json.Number("1e-3"), want: "0.001"}, + {name: "negative value", value: json.Number("-1.20e-2"), want: "-0.012"}, + {name: "numeric string", value: "2.5e1", want: "25"}, + {name: "zero", value: json.Number("0e999999999"), want: "0"}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := dialect.ConvertJSONValue( + dialects.ColumnMetadata{Name: "amount", DatabaseType: "decimal"}, + test.value, + ) + assert.NoError(t, err) + assert.Equal(t, test.want, got) + }) + } + + got, err := dialect.ConvertJSONValue( + dialects.ColumnMetadata{Name: "measurement", DatabaseType: "float"}, + json.Number("1e3"), + ) + assert.NoError(t, err) + assert.Equal(t, "1e3", got) + + _, err = dialect.ConvertJSONValue( + dialects.ColumnMetadata{Name: "amount", DatabaseType: "numeric"}, + json.Number("1e999999999"), + ) + assert.ErrorContains(t, err, "too large to expand") + + _, err = dialect.ConvertJSONValue( + dialects.ColumnMetadata{Name: "amount", DatabaseType: "numeric"}, + "1einvalid", + ) + assert.ErrorContains(t, err, "parse exponent-form number") +} + +func upsertQuery(t *testing.T, dialect interface { + UpsertQuery(string, []string, []string, []string, []string, []string, int, bool) (string, error) +}, namespace string, keys, columns, defaultColumns []string, count int, overrideIdentity bool) string { + t.Helper() + columnTypes := make([]string, len(columns)) + query, err := dialect.UpsertQuery(namespace, keys, columns, columnTypes, defaultColumns, nil, count, overrideIdentity) + assert.NoError(t, err) + return query +} + func TestKeySub(t *testing.T) { p := func(i int) string { return fmt.Sprintf("X%v", i) } assert.Equal(t, "(X0)", dialects.KeySub(1, 1, p)) @@ -15,3 +180,244 @@ func TestKeySub(t *testing.T) { assert.Equal(t, "(X0, X1, X2), (X3, X4, X5)", dialects.KeySub(3, 2, p)) assert.Equal(t, "(X0, X1, X2)", dialects.KeySub(3, 1, p)) } + +func TestPostgresWriteQueries(t *testing.T) { + d := dialects.Postgres{} + assert.Equal(t, + `INSERT INTO "public"."users" ("tenant", "id", "name") VALUES ($1, $2, $3) ON CONFLICT ("tenant", "id") DO UPDATE SET "name" = EXCLUDED."name"`, + upsertQuery(t, d, "public.users", []string{"tenant", "id"}, []string{"tenant", "id", "name"}, nil, 1, false), + ) + assert.Equal(t, + `DELETE FROM "public"."users" WHERE ("tenant" = $1 AND "id" = $2)`, + d.DeleteQuery("public.users", []string{"tenant", "id"}, 1), + ) +} + +func TestPostgresKeyOnlyUpsert(t *testing.T) { + d := dialects.Postgres{} + assert.Equal(t, + `INSERT INTO "items" ("id") VALUES ($1) ON CONFLICT ("id") DO NOTHING`, + upsertQuery(t, d, "items", []string{"id"}, []string{"id"}, nil, 1, false), + ) +} + +func TestSQLServerUpsertQuery(t *testing.T) { + d := dialects.SQLServer{} + assert.Equal(t, + `MERGE INTO "dbo"."users" AS target USING (VALUES (@p1, @p2)) AS source ("id", "name") ON target."id" = source."id" WHEN MATCHED THEN UPDATE SET target."name" = source."name" WHEN NOT MATCHED THEN INSERT ("id", "name") VALUES (source."id", source."name");`, + upsertQuery(t, d, "dbo.users", []string{"id"}, []string{"id", "name"}, nil, 1, false), + ) +} + +func TestOracleUpsertQuery(t *testing.T) { + d := dialects.Oracle{} + assert.Equal(t, + `MERGE INTO "APP"."USERS" target USING (SELECT :1 AS "ID", :2 AS "NAME" FROM DUAL) source ON (target."ID" = source."ID") WHEN MATCHED THEN UPDATE SET target."NAME" = source."NAME" WHEN NOT MATCHED THEN INSERT ("ID", "NAME") VALUES (source."ID", source."NAME")`, + upsertQuery(t, d, "APP.USERS", []string{"ID"}, []string{"ID", "NAME"}, nil, 1, false), + ) +} + +func TestMultiRowWriteQueries(t *testing.T) { + assert.Equal(t, + `INSERT INTO "items" ("id", "name") VALUES ($1, $2), ($3, $4) ON CONFLICT ("id") DO UPDATE SET "name" = EXCLUDED."name"`, + upsertQuery(t, dialects.Postgres{}, "items", []string{"id"}, []string{"id", "name"}, nil, 2, false), + ) + assert.Equal(t, + `DELETE FROM "items" WHERE ("id" = $1) OR ("id" = $2)`, + (dialects.Postgres{}).DeleteQuery("items", []string{"id"}, 2), + ) + assert.Contains(t, + upsertQuery(t, dialects.SQLServer{}, "items", []string{"id"}, []string{"id", "name"}, nil, 2, false), + `VALUES (@p1, @p2), (@p3, @p4)`, + ) + assert.Contains(t, + upsertQuery(t, dialects.Oracle{}, "ITEMS", []string{"ID"}, []string{"ID", "NAME"}, nil, 2, false), + `SELECT :1 AS "ID", :2 AS "NAME" FROM DUAL UNION ALL SELECT :3 AS "ID", :4 AS "NAME" FROM DUAL`, + ) +} + +func TestWriteQueriesResetOmittedColumnsToDefault(t *testing.T) { + assert.Equal(t, + `INSERT INTO "items" ("id", "name") VALUES ($1, $2) ON CONFLICT ("id") DO UPDATE SET "name" = EXCLUDED."name", "status" = DEFAULT`, + upsertQuery(t, dialects.Postgres{}, "items", []string{"id"}, []string{"id", "name"}, []string{"status"}, 1, false), + ) + assert.Contains(t, + upsertQuery(t, dialects.SQLServer{}, "items", []string{"id"}, []string{"id", "name"}, []string{"status"}, 1, false), + `target."name" = source."name", target."status" = DEFAULT`, + ) + assert.Contains(t, + upsertQuery(t, dialects.Oracle{}, "ITEMS", []string{"ID"}, []string{"ID", "NAME"}, []string{"STATUS"}, 1, false), + `target."NAME" = source."NAME", target."STATUS" = DEFAULT`, + ) +} + +func TestGeneratedIdentityWriteQueries(t *testing.T) { + assert.Contains(t, + upsertQuery(t, dialects.Postgres{}, "items", []string{"id"}, []string{"id", "name"}, nil, 1, true), + `) OVERRIDING SYSTEM VALUE VALUES (`, + ) + assert.Contains(t, + upsertQuery(t, dialects.SQLServer{}, "items", []string{"id"}, []string{"id", "name"}, nil, 1, true), + `SET IDENTITY_INSERT "items" ON; BEGIN TRY MERGE`, + ) + _, err := (dialects.Oracle{}).UpsertQuery("ITEMS", []string{"ID"}, []string{"ID"}, []string{"NUMBER"}, nil, []string{"ID"}, 1, true) + assert.ErrorContains(t, err, "cannot be written explicitly") +} + +func TestIdentityColumnsAreInsertOnly(t *testing.T) { + for _, test := range []struct { + name string + dialect interface { + UpsertQuery(string, []string, []string, []string, []string, []string, int, bool) (string, error) + } + }{ + {name: "Postgres", dialect: dialects.Postgres{}}, + {name: "SQL Server", dialect: dialects.SQLServer{}}, + } { + t.Run(test.name, func(t *testing.T) { + query, err := test.dialect.UpsertQuery( + "items", + []string{"id"}, + []string{"id", "sequence", "name"}, + []string{"bigint", "bigint", "text"}, + nil, + []string{"sequence"}, + 1, + true, + ) + assert.NoError(t, err) + assert.NotContains(t, query, `target."sequence" = source."sequence"`) + assert.NotContains(t, query, `"sequence" = EXCLUDED."sequence"`) + assert.Contains(t, query, `"sequence"`) + }) + } +} + +func TestSQLServerCastsBinaryParameters(t *testing.T) { + query, err := (dialects.SQLServer{}).UpsertQuery( + "items", + []string{"id"}, + []string{"id", "payload"}, + []string{"bigint", "varbinary"}, + nil, + nil, + 2, + false, + ) + assert.NoError(t, err) + assert.Contains(t, query, `VALUES (@p1, CAST(@p2 AS varbinary(max))), (@p3, CAST(@p4 AS varbinary(max)))`) +} + +func TestOracleConvertsBlobParameters(t *testing.T) { + query, err := (dialects.Oracle{}).UpsertQuery( + "ITEMS", + []string{"ID"}, + []string{"ID", "PAYLOAD"}, + []string{"NUMBER", "BLOB"}, + nil, + nil, + 2, + false, + ) + assert.NoError(t, err) + assert.Contains(t, query, `SELECT CAST(:1 AS NUMBER) AS "ID", TO_BLOB(:2) AS "PAYLOAD" FROM DUAL`) + assert.Contains(t, query, `SELECT CAST(:3 AS NUMBER) AS "ID", TO_BLOB(:4) AS "PAYLOAD" FROM DUAL`) +} + +func TestOracleCastsNumericParameters(t *testing.T) { + query, err := (dialects.Oracle{}).UpsertQuery( + "ITEMS", + []string{"ID"}, + []string{"ID", "AMOUNT"}, + []string{"NUMBER", "BINARY_DOUBLE"}, + nil, + nil, + 2, + false, + ) + assert.NoError(t, err) + assert.Contains(t, query, `SELECT CAST(:1 AS NUMBER) AS "ID", CAST(:2 AS BINARY_DOUBLE) AS "AMOUNT" FROM DUAL`) + assert.Contains(t, query, `SELECT CAST(:3 AS NUMBER) AS "ID", CAST(:4 AS BINARY_DOUBLE) AS "AMOUNT" FROM DUAL`) +} + +func TestOracleTemporalConversionAndDeferredTypes(t *testing.T) { + dialect := dialects.Oracle{} + value, err := dialect.ConvertJSONValue(dialects.ColumnMetadata{Name: "PAYLOAD", DatabaseType: "RAW"}, nil) + assert.NoError(t, err) + assert.IsType(t, []byte(nil), value) + + value, err = dialect.ConvertJSONValue(dialects.ColumnMetadata{Name: "CREATED_AT", DatabaseType: "TIMESTAMP(6)"}, "2026-10-19T12:34:56+05:30") + assert.NoError(t, err) + assert.Equal(t, time.Date(2026, 10, 19, 7, 4, 56, 0, time.UTC), value) + value, err = dialect.ConvertJSONValue(dialects.ColumnMetadata{Name: "EVENT_AT", DatabaseType: "DATE"}, "2026-10-19T12:34:56+05:30") + assert.NoError(t, err) + assert.Equal(t, time.Date(2026, 10, 19, 7, 4, 56, 0, time.UTC), value) + value, err = dialect.ConvertJSONValue(dialects.ColumnMetadata{Name: "EVENT_AT", DatabaseType: "DATE"}, "2026-10-19") + assert.NoError(t, err) + assert.Equal(t, time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC), value) + value, err = dialect.ConvertJSONValue(dialects.ColumnMetadata{Name: "CREATED_AT", DatabaseType: "TIMESTAMP(6) WITH TIME ZONE"}, "2026-10-19T12:34:56+05:30") + assert.NoError(t, err) + assert.Equal(t, 5*60+30, offsetMinutes(value.(time.Time))) + _, err = dialect.ConvertJSONValue(dialects.ColumnMetadata{Name: "LEGACY", DatabaseType: "LONG RAW"}, "AAE=") + assert.ErrorContains(t, err, "does not support supplied values") +} + +func TestOraclePreservesTimestampPrecisionInCasts(t *testing.T) { + query, err := (dialects.Oracle{}).UpsertQuery( + "ITEMS", + []string{"ID"}, + []string{"ID", "PLAIN", "ZONED", "LOCAL_ZONED"}, + []string{"NUMBER", "TIMESTAMP(9)", "TIMESTAMP(9) WITH TIME ZONE", "TIMESTAMP(9) WITH LOCAL TIME ZONE"}, + nil, + nil, + 1, + false, + ) + assert.NoError(t, err) + assert.Contains(t, query, `CAST(:2 AS TIMESTAMP(9)) AS "PLAIN"`) + assert.Contains(t, query, `CAST(:3 AS TIMESTAMP(9) WITH TIME ZONE) AS "ZONED"`) + assert.Contains(t, query, `CAST(:4 AS TIMESTAMP(9) WITH LOCAL TIME ZONE) AS "LOCAL_ZONED"`) +} + +func offsetMinutes(value time.Time) int { + _, offset := value.Zone() + return offset / 60 +} + +func TestSQLServerTemporalConversion(t *testing.T) { + dialect := dialects.SQLServer{} + value, err := dialect.ConvertJSONValue(dialects.ColumnMetadata{Name: "created_at", DatabaseType: "datetime2"}, "2026-10-19T12:34:56+05:30") + assert.NoError(t, err) + assert.Equal(t, time.Date(2026, 10, 19, 7, 4, 56, 0, time.UTC), value) + + query, err := dialect.UpsertQuery("items", []string{"id"}, []string{"id", "created_at"}, []string{"bigint", "datetime2"}, nil, nil, 2, false) + assert.NoError(t, err) + assert.Contains(t, query, `VALUES (@p1, CAST(@p2 AS datetime2)), (@p3, CAST(@p4 AS datetime2))`) +} + +func TestQuoteNamespaceEscapesIdentifiers(t *testing.T) { + assert.Equal(t, `"odd""schema"."table"`, dialects.QuoteNamespace(`odd"schema.table`)) +} + +func TestPostgresJSONBinaryConversion(t *testing.T) { + d := dialects.Postgres{} + got, err := d.ConvertJSONValue(dialects.ColumnMetadata{Name: "payload", DatabaseType: "bytea"}, "AAH+/w==") + assert.NoError(t, err) + assert.True(t, bytes.Equal([]byte{0, 1, 0xfe, 0xff}, got.([]byte))) + + got, err = d.ConvertJSONValue(dialects.ColumnMetadata{Name: "payload", DatabaseType: "text"}, "AAH+/w==") + assert.NoError(t, err) + assert.Equal(t, "AAH+/w==", got) +} + +func TestSplitNamespace(t *testing.T) { + schema, table, err := dialects.SplitNamespace("public.items") + assert.NoError(t, err) + assert.Equal(t, "public", schema) + assert.Equal(t, "items", table) + + schema, table, err = dialects.SplitNamespace("items") + assert.NoError(t, err) + assert.Empty(t, schema) + assert.Equal(t, "items", table) +} diff --git a/connectors/sqlbatch/dialects/oracle.go b/connectors/sqlbatch/dialects/oracle.go index 2e46dc82..a1ce3d10 100644 --- a/connectors/sqlbatch/dialects/oracle.go +++ b/connectors/sqlbatch/dialects/oracle.go @@ -1,15 +1,185 @@ package dialects import ( + "context" + "database/sql" + "encoding/json" "fmt" + "regexp" + "slices" "strconv" "strings" - _ "github.com/sijms/go-ora/v2" + go_ora "github.com/sijms/go-ora/v2" ) type Oracle struct{} +func (Oracle) ConvertDriverValue(column ColumnMetadata, value any) (any, error) { + return ConvertDriverValue(column, value) +} + +var oracleTimestampTypePattern = regexp.MustCompile(`(?i)^timestamp(?:\([0-9]+\))?(?: with (?:local )?time zone)?$`) + +func (Oracle) MaxQueryParameters() int { + return 65535 +} + +func (Oracle) SupportsIdentityOverride() bool { + return false +} + +func (d Oracle) InspectTable(ctx context.Context, db *sql.DB, namespace string) (TableMetadata, error) { + schema, table, err := SplitNamespace(namespace) + if err != nil { + return TableMetadata{}, err + } + rows, err := db.QueryContext(ctx, ` + SELECT c.column_name, + c.data_type, + CASE WHEN c.nullable = 'Y' THEN 'YES' ELSE 'NO' END, + CASE WHEN c.default_length IS NULL OR c.default_length = 0 THEN 'NO' ELSE 'YES' END, + CASE WHEN c.virtual_column = 'YES' THEN 'YES' ELSE 'NO' END, + CASE WHEN c.identity_column = 'YES' THEN 'YES' ELSE 'NO' END, + NVL(identity.generation_type, 'NONE'), + NVL(pk.position, 0) + FROM all_tab_cols c + LEFT JOIN all_tab_identity_cols identity + ON identity.owner = c.owner + AND identity.table_name = c.table_name + AND identity.column_name = c.column_name + LEFT JOIN ( + SELECT cc.owner, cc.table_name, cc.column_name, cc.position + FROM all_constraints con + JOIN all_cons_columns cc + ON cc.owner = con.owner + AND cc.constraint_name = con.constraint_name + WHERE con.constraint_type = 'P' + ) pk + ON pk.owner = c.owner + AND pk.table_name = c.table_name + AND pk.column_name = c.column_name + WHERE c.owner = COALESCE(NULLIF(:1, ''), SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')) + AND c.table_name = :2 + AND c.hidden_column = 'NO' + ORDER BY c.column_id`, schema, table) + if err != nil { + return TableMetadata{}, err + } + defer rows.Close() + return ScanTableMetadata(rows) +} + +func (Oracle) ConvertJSONValue(column ColumnMetadata, value any) (any, error) { + databaseType := strings.ToLower(column.DatabaseType) + if temporalType, preserveOffset := oracleTemporalType(column.DatabaseType); temporalType != "" { + kind := temporalTimestamp + if temporalType == "DATE" { + // Unlike a SQL DATE, Oracle DATE also stores hours, minutes, and seconds. + // Accept a date-only value as midnight, but preserve a supplied timestamp. + kind = temporalOracleDate + } + return convertTemporalValue(column, value, kind, preserveOffset) + } + if value != nil && strings.HasPrefix(databaseType, "interval") { + return nil, fmt.Errorf("Oracle sink does not yet support supplied values for interval column %q of type %s", column.Name, column.DatabaseType) + } + if value != nil && (databaseType == "long" || databaseType == "long raw") { + return nil, fmt.Errorf("Oracle sink does not support supplied values for column %q of type %s", column.Name, column.DatabaseType) + } + if value == nil && databaseType == "raw" { + // go-ora otherwise describes an untyped nil as NCHAR, which cannot be + // combined with non-null RAW binds in the UNION ALL row source. + return []byte(nil), nil + } + if number, ok := value.(json.Number); ok && oracleNumericType(column.DatabaseType) != "" { + return go_ora.NewNumberFromString(number.String()) + } + return ConvertJSONValue(column, value, "raw", "long raw", "blob") +} + +func oracleTemporalType(databaseType string) (string, bool) { + databaseType = strings.TrimSpace(databaseType) + lowerType := strings.ToLower(databaseType) + switch { + case lowerType == "date": + return "DATE", false + case oracleTimestampTypePattern.MatchString(databaseType): + // DATA_TYPE comes from Oracle metadata. Validate its shape before using it + // as a cast target, and retain any declared fractional-second precision. + return strings.ToUpper(databaseType), strings.Contains(lowerType, "with time zone") + default: + return "", false + } +} + +func oracleNumericType(databaseType string) string { + switch strings.ToLower(databaseType) { + case "number", "decimal", "numeric": + return "NUMBER" + case "float": + return "FLOAT" + case "binary_float": + return "BINARY_FLOAT" + case "binary_double": + return "BINARY_DOUBLE" + default: + return "" + } +} + +func (d Oracle) UpsertQuery(namespace string, keys, columns, columnTypes []string, defaultColumns, identityColumns []string, count int, overrideIdentity bool) (string, error) { + if overrideIdentity { + return "", fmt.Errorf("Oracle GENERATED ALWAYS identity columns cannot be written explicitly") + } + nonKeys := make([]string, 0, len(columns)) + for _, column := range columns { + if !slices.Contains(keys, column) && !slices.Contains(identityColumns, column) { + nonKeys = append(nonKeys, column) + } + } + rows := make([]string, count) + for row := range count { + selects := make([]string, len(columns)) + for i, column := range columns { + value := d.Placeholder(row*len(columns) + i) + if i < len(columnTypes) { + if temporalType, _ := oracleTemporalType(columnTypes[i]); temporalType != "" { + value = "CAST(" + value + " AS " + temporalType + ")" + } else if numericType := oracleNumericType(columnTypes[i]); numericType != "" { + // Besides making every UNION ALL branch agree, the cast gives an + // untyped nil bind the destination's numeric family. + value = "CAST(" + value + " AS " + numericType + ")" + } else if strings.EqualFold(columnTypes[i], "blob") { + // Oracle otherwise infers a NULL BLOB bind as a character value, + // making batched UNION ALL sources fail with ORA-01790. + value = "TO_BLOB(" + value + ")" + } + } + selects[i] = value + " AS " + QuoteIdentifier(column) + } + rows[row] = "SELECT " + strings.Join(selects, ", ") + " FROM DUAL" + } + var query strings.Builder + query.WriteString("MERGE INTO " + QuoteNamespace(namespace) + " target USING (" + strings.Join(rows, " UNION ALL ") + ") source ON (" + JoinPredicate(keys, "target", "source") + ")") + if len(nonKeys) > 0 || len(defaultColumns) > 0 { + assignments := make([]string, 0, 2) + if len(nonKeys) > 0 { + assignments = append(assignments, Assignments(nonKeys, "target", "source")) + } + if len(defaultColumns) > 0 { + assignments = append(assignments, DefaultAssignments(defaultColumns, "target")) + } + query.WriteString(" WHEN MATCHED THEN UPDATE SET " + strings.Join(assignments, ", ")) + } + query.WriteString(" WHEN NOT MATCHED THEN INSERT (" + QuotedIdentifiers(columns) + ") VALUES (" + SourceColumns(columns, "source") + ")") + return query.String(), nil +} + +func (d Oracle) DeleteQuery(namespace string, keys []string, count int) string { + return DeleteQuery(namespace, keys, count, d.Placeholder) +} + // KeySub implements sqlbatch.SQLDialect. func (d Oracle) KeySub(cols []string, count int) string { numCols := len(cols) diff --git a/connectors/sqlbatch/dialects/postgres.go b/connectors/sqlbatch/dialects/postgres.go index 2a0f171f..bf71b23c 100644 --- a/connectors/sqlbatch/dialects/postgres.go +++ b/connectors/sqlbatch/dialects/postgres.go @@ -1,15 +1,117 @@ package dialects import ( + "context" + "database/sql" "fmt" + "slices" "strconv" "strings" + "github.com/jackc/pgx/v5/pgtype" _ "github.com/jackc/pgx/v5/stdlib" ) type Postgres struct{} +func (Postgres) ConvertDriverValue(column ColumnMetadata, value any) (any, error) { + return ConvertDriverValue(column, value) +} + +func (Postgres) MaxQueryParameters() int { + return 65535 +} + +func (Postgres) SupportsIdentityOverride() bool { + return true +} + +func (d Postgres) InspectTable(ctx context.Context, db *sql.DB, namespace string) (TableMetadata, error) { + schema, table, err := SplitNamespace(namespace) + if err != nil { + return TableMetadata{}, err + } + rows, err := db.QueryContext(ctx, ` + SELECT c.column_name, + c.udt_name, + c.is_nullable, + CASE WHEN c.column_default IS NULL THEN 'NO' ELSE 'YES' END, + CASE WHEN c.is_generated = 'NEVER' THEN 'NO' ELSE 'YES' END, + c.is_identity, + COALESCE(c.identity_generation, ''), + COALESCE(pk.ordinal_position, 0) + FROM information_schema.columns c + LEFT JOIN ( + SELECT kcu.table_schema, kcu.table_name, kcu.column_name, kcu.ordinal_position + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON kcu.constraint_catalog = tc.constraint_catalog + AND kcu.constraint_schema = tc.constraint_schema + AND kcu.constraint_name = tc.constraint_name + AND kcu.table_name = tc.table_name + WHERE tc.constraint_type = 'PRIMARY KEY' + ) pk + ON pk.table_schema = c.table_schema + AND pk.table_name = c.table_name + AND pk.column_name = c.column_name + WHERE c.table_schema = COALESCE(NULLIF($1, ''), current_schema()) + AND c.table_name = $2 + ORDER BY c.ordinal_position`, schema, table) + if err != nil { + return TableMetadata{}, err + } + defer rows.Close() + return ScanTableMetadata(rows) +} + +func (Postgres) ConvertJSONValue(column ColumnMetadata, value any) (any, error) { + value, err := ConvertJSONValue(column, value, "bytea") + if err != nil { + return nil, err + } + if strings.EqualFold(column.DatabaseType, "uuid") { + if data, ok := value.([]byte); ok { + if len(data) != 16 { + return nil, fmt.Errorf("UUID column %q has %d bytes, expected 16", column.Name, len(data)) + } + var uuid [16]byte + copy(uuid[:], data) + value = pgtype.UUID{Bytes: uuid, Valid: true} + } + } + return ConvertRFC3339Value(column, value, "date", "timestamp", "timestamptz") +} + +func (d Postgres) UpsertQuery(namespace string, keys, columns, _ []string, defaultColumns, identityColumns []string, count int, overrideIdentity bool) (string, error) { + nonKeys := make([]string, 0, len(columns)) + for _, column := range columns { + if !slices.Contains(keys, column) && !slices.Contains(identityColumns, column) { + nonKeys = append(nonKeys, column) + } + } + query := "INSERT INTO " + QuoteNamespace(namespace) + " (" + QuotedIdentifiers(columns) + ")" + if overrideIdentity { + query += " OVERRIDING SYSTEM VALUE" + } + query += " VALUES " + ValueRows(len(columns), count, d.Placeholder) + " ON CONFLICT (" + QuotedIdentifiers(keys) + ") " + if len(nonKeys) == 0 && len(defaultColumns) == 0 { + return query + "DO NOTHING", nil + } + assignments := make([]string, 0, len(nonKeys)+len(defaultColumns)) + for _, column := range nonKeys { + quoted := QuoteIdentifier(column) + assignments = append(assignments, quoted+" = EXCLUDED."+quoted) + } + if len(defaultColumns) > 0 { + assignments = append(assignments, DefaultAssignments(defaultColumns, "")) + } + return query + "DO UPDATE SET " + strings.Join(assignments, ", "), nil +} + +func (d Postgres) DeleteQuery(namespace string, keys []string, count int) string { + return DeleteQuery(namespace, keys, count, d.Placeholder) +} + // KeySub implements sqlbatch.SQLDialect. func (d Postgres) KeySub(cols []string, count int) string { return KeySub(len(cols), count, d.Placeholder) diff --git a/connectors/sqlbatch/dialects/sqlserver.go b/connectors/sqlbatch/dialects/sqlserver.go index 8a0ba594..71c2042c 100644 --- a/connectors/sqlbatch/dialects/sqlserver.go +++ b/connectors/sqlbatch/dialects/sqlserver.go @@ -1,14 +1,278 @@ package dialects import ( + "context" + "database/sql" + "encoding/json" "fmt" + "slices" "strconv" "strings" - _ "github.com/microsoft/go-mssqldb" + mssql "github.com/microsoft/go-mssqldb" + "github.com/shopspring/decimal" ) -type SQLServer struct{} +type SQLServer struct { + // GuidConversion mirrors go-mssqldb's connection setting. Without it the + // driver exposes uniqueidentifier values in TDS byte order. + GuidConversion bool +} + +func (d SQLServer) ConvertDriverValue(column ColumnMetadata, value any) (any, error) { + data, ok := value.([]byte) + if !ok { + return value, nil + } + switch strings.ToLower(column.DatabaseType) { + case "decimal", "numeric", "money", "smallmoney": + return string(data), nil + case "uniqueidentifier": + if len(data) != 16 { + return nil, fmt.Errorf("uniqueidentifier column %q has %d bytes, expected 16", column.Name, len(data)) + } + var identifier mssql.UniqueIdentifier + if d.GuidConversion { + copy(identifier[:], data) + } else if err := identifier.Scan(data); err != nil { + return nil, fmt.Errorf("convert uniqueidentifier column %q: %w", column.Name, err) + } + return identifier.String(), nil + default: + return value, nil + } +} + +func (SQLServer) MaxQueryParameters() int { + // Leave room below SQL Server's 2,100 parameter limit for driver and + // statement-level parameters. + return 2000 +} + +func (SQLServer) SupportsIdentityOverride() bool { + return true +} + +func (d SQLServer) InspectTable(ctx context.Context, db *sql.DB, namespace string) (TableMetadata, error) { + schema, table, err := SplitNamespace(namespace) + if err != nil { + return TableMetadata{}, err + } + rows, err := db.QueryContext(ctx, ` + SELECT c.name, + bt.name, + CASE WHEN c.is_nullable = 1 THEN 'YES' ELSE 'NO' END, + CASE WHEN dc.object_id IS NULL THEN 'NO' ELSE 'YES' END, + CASE WHEN c.is_computed = 1 OR c.generated_always_type <> 0 OR bt.name IN ('timestamp', 'rowversion') THEN 'YES' ELSE 'NO' END, + CASE WHEN c.is_identity = 1 THEN 'YES' ELSE 'NO' END, + CASE WHEN c.is_identity = 1 THEN 'ALWAYS' ELSE '' END, + COALESCE(pk.key_ordinal, 0) + FROM sys.tables t + JOIN sys.schemas s ON s.schema_id = t.schema_id + JOIN sys.columns c ON c.object_id = t.object_id + JOIN sys.types bt + ON bt.system_type_id = c.system_type_id + AND bt.user_type_id = bt.system_type_id + LEFT JOIN sys.default_constraints dc ON dc.object_id = c.default_object_id + LEFT JOIN ( + SELECT ic.object_id, ic.column_id, ic.key_ordinal + FROM sys.indexes i + JOIN sys.index_columns ic + ON ic.object_id = i.object_id + AND ic.index_id = i.index_id + WHERE i.is_primary_key = 1 + ) pk + ON pk.object_id = c.object_id + AND pk.column_id = c.column_id + WHERE s.name = COALESCE(NULLIF(@p1, ''), SCHEMA_NAME()) + AND t.name = @p2 + ORDER BY c.column_id`, schema, table) + if err != nil { + return TableMetadata{}, err + } + defer rows.Close() + return ScanTableMetadata(rows) +} + +func (d SQLServer) ConvertJSONValue(column ColumnMetadata, value any) (any, error) { + if sqlServerExactNumericType(column.DatabaseType) { + var err error + value, err = normalizeExactNumericExponent(value) + if err != nil { + return nil, fmt.Errorf("normalize exact numeric column %q: %w", column.Name, err) + } + } + value, err := convertJSONValue( + column, + value, + []string{"binary", "varbinary", "image", "timestamp", "rowversion"}, + []string{"decimal", "numeric", "money", "smallmoney", "float", "real"}, + ) + if err != nil { + return nil, err + } + if strings.EqualFold(column.DatabaseType, "uniqueidentifier") { + if data, ok := value.([]byte); ok { + if len(data) != 16 { + return nil, fmt.Errorf("uniqueidentifier column %q has %d bytes, expected 16", column.Name, len(data)) + } + var identifier mssql.UniqueIdentifier + if d.GuidConversion { + // UniqueIdentifier.Value converts canonical bytes to TDS byte order, + // while the driver's GUID conversion converts them once more. Scan + // here so those two conversions cancel and the server receives the + // same UUID represented by the BSON subtype 4 bytes. + if err := identifier.Scan(data); err != nil { + return nil, fmt.Errorf("convert uniqueidentifier column %q: %w", column.Name, err) + } + } else { + copy(identifier[:], data) + } + value = identifier + } + } + if temporalType, dateOnly, timeOnly, preserveOffset := sqlServerTemporalType(column.DatabaseType); temporalType != "" { + kind := temporalTimestamp + switch { + case dateOnly: + kind = temporalDate + case timeOnly: + kind = temporalTime + } + return convertTemporalValue(column, value, kind, preserveOffset) + } + return value, nil +} + +func sqlServerExactNumericType(databaseType string) bool { + switch strings.ToLower(databaseType) { + case "decimal", "numeric", "money", "smallmoney": + return true + default: + return false + } +} + +func normalizeExactNumericExponent(value any) (any, error) { + var text string + switch value := value.(type) { + case json.Number: + text = value.String() + case string: + text = value + default: + return value, nil + } + if !strings.ContainsAny(text, "eE") { + return value, nil + } + + parsed, err := decimal.NewFromString(text) + if err != nil { + return nil, fmt.Errorf("parse exponent-form number %q: %w", text, err) + } + if parsed.IsZero() { + return json.Number("0"), nil + } + // Decimal.String materializes exponent padding, so bound the rendered form + // before calling it. SQL Server exact numerics are at most 38 digits; this + // leaves ample room while preventing a tiny input from causing a huge allocation. + const maxExpandedDecimalLength = 4096 + if decimalExpansionLength(parsed) > maxExpandedDecimalLength { + return nil, fmt.Errorf("exponent-form number %q is too large to expand", text) + } + return json.Number(parsed.String()), nil +} + +func decimalExpansionLength(value decimal.Decimal) int64 { + coefficient := value.Coefficient() + negative := coefficient.Sign() < 0 + if negative { + coefficient.Neg(coefficient) + } + digits := int64(len(coefficient.String())) + exponent := int64(value.Exponent()) + + length := digits + if exponent >= 0 { + length += exponent + } else if scale := -exponent; digits > scale { + length++ // decimal point + } else { + length = scale + 2 // "0.", leading fractional zeros, and digits + } + if negative { + length++ + } + return length +} + +func sqlServerTemporalType(databaseType string) (sqlType string, dateOnly, timeOnly, preserveOffset bool) { + switch strings.ToLower(databaseType) { + case "date": + return "date", true, false, false + case "time": + return "time", false, true, false + case "datetime", "datetime2", "smalldatetime": + return strings.ToLower(databaseType), false, false, false + case "datetimeoffset": + return "datetimeoffset", false, false, true + default: + return "", false, false, false + } +} + +func (d SQLServer) UpsertQuery(namespace string, keys, columns, columnTypes, defaultColumns, identityColumns []string, count int, overrideIdentity bool) (string, error) { + nonKeys := make([]string, 0, len(columns)) + for _, column := range columns { + if !slices.Contains(keys, column) && !slices.Contains(identityColumns, column) { + nonKeys = append(nonKeys, column) + } + } + var query strings.Builder + query.WriteString("MERGE INTO " + QuoteNamespace(namespace) + " AS target USING (VALUES " + d.valueRows(columnTypes, count) + ") AS source (" + QuotedIdentifiers(columns) + ") ON " + JoinPredicate(keys, "target", "source")) + if len(nonKeys) > 0 || len(defaultColumns) > 0 { + assignments := make([]string, 0, 2) + if len(nonKeys) > 0 { + assignments = append(assignments, Assignments(nonKeys, "target", "source")) + } + if len(defaultColumns) > 0 { + assignments = append(assignments, DefaultAssignments(defaultColumns, "target")) + } + query.WriteString(" WHEN MATCHED THEN UPDATE SET " + strings.Join(assignments, ", ")) + } + query.WriteString(" WHEN NOT MATCHED THEN INSERT (" + QuotedIdentifiers(columns) + ") VALUES (" + SourceColumns(columns, "source") + ");") + if !overrideIdentity { + return query.String(), nil + } + table := QuoteNamespace(namespace) + return "SET IDENTITY_INSERT " + table + " ON; BEGIN TRY " + query.String() + " SET IDENTITY_INSERT " + table + " OFF; END TRY BEGIN CATCH SET IDENTITY_INSERT " + table + " OFF; THROW; END CATCH", nil +} + +func (d SQLServer) valueRows(columnTypes []string, count int) string { + rows := make([]string, count) + for row := range count { + values := make([]string, len(columnTypes)) + for column, databaseType := range columnTypes { + value := d.Placeholder(row*len(columnTypes) + column) + if temporalType, _, _, _ := sqlServerTemporalType(databaseType); temporalType != "" { + value = "CAST(" + value + " AS " + temporalType + ")" + } else { + switch strings.ToLower(databaseType) { + case "binary", "varbinary", "image": + value = "CAST(" + value + " AS varbinary(max))" + } + } + values[column] = value + } + rows[row] = "(" + strings.Join(values, ", ") + ")" + } + return strings.Join(rows, ", ") +} + +func (d SQLServer) DeleteQuery(namespace string, keys []string, count int) string { + return DeleteQuery(namespace, keys, count, d.Placeholder) +} // KeySub implements sqlbatch.SQLDialect. func (d SQLServer) KeySub(cols []string, count int) string { diff --git a/connectors/sqlbatch/sink.go b/connectors/sqlbatch/sink.go new file mode 100644 index 00000000..eed0ffc7 --- /dev/null +++ b/connectors/sqlbatch/sink.go @@ -0,0 +1,523 @@ +package sqlbatch + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math/big" + "slices" + "strconv" + "strings" + "time" + + "connectrpc.com/connect" + "github.com/adiom-data/dsync/connectors/sqlbatch/dialects" + adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" + "go.mongodb.org/mongo-driver/v2/bson" +) + +const DefaultWriteBatchSize = 100 + +func decodeJSONDocument(data []byte, columns map[string]dialects.ColumnMetadata, dialect SQLDialect) (map[string]any, error) { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + var document map[string]any + if err := decoder.Decode(&document); err != nil { + return nil, err + } + if document == nil { + return nil, fmt.Errorf("expected a JSON object") + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + if err == nil { + return nil, fmt.Errorf("expected one JSON object") + } + return nil, err + } + for key, value := range document { + column, ok := columns[key] + if !ok { + return nil, fmt.Errorf("destination table has no column %q", key) + } + converted, err := dialect.ConvertJSONValue(column, value) + if err != nil { + return nil, err + } + document[key] = converted + } + for key, column := range columns { + if _, exists := document[key]; exists || !column.Nullable || column.HasDefault || column.Generated || column.Identity { + continue + } + converted, err := dialect.ConvertJSONValue(column, nil) + if err != nil { + return nil, err + } + document[key] = converted + } + return document, nil +} + +func sqlValue(value any) any { + switch value := value.(type) { + case json.Number: + if integer, err := value.Int64(); err == nil { + return integer + } + return value.String() + case bson.Binary: + return value.Data + case bson.Decimal128: + return value.String() + case bson.DateTime: + return time.UnixMilli(int64(value)) + case bson.ObjectID: + return value.Hex() + case map[string]any, []any: + encoded, err := json.Marshal(value) + if err == nil { + return string(encoded) + } + } + return value +} + +func numericJSONValue(value any) (json.Number, bool) { + switch value := value.(type) { + case int32: + return json.Number(strconv.FormatInt(int64(value), 10)), true + case int64: + return json.Number(strconv.FormatInt(value, 10)), true + case float64: + return json.Number(strconv.FormatFloat(value, 'g', -1, 64)), true + case bson.Decimal128: + return json.Number(value.String()), true + default: + return "", false + } +} + +func documentValues(document map[string]any, columns []string) []any { + values := make([]any, len(columns)) + for i, column := range columns { + values[i] = document[column] + } + return values +} + +func updateKeyValues(update *adiomv1.Update, metadata dialects.TableMetadata, dialect SQLDialect) ([]any, error) { + keys := metadata.PrimaryKey + columns := columnMetadataByName(metadata) + parts := make(map[string]*adiomv1.BsonValue, len(update.GetId())) + for _, part := range update.GetId() { + if part.GetName() != "" { + parts[part.GetName()] = part + } + } + values := make([]any, len(keys)) + for i, key := range keys { + column, exists := columns[key] + if !exists { + return nil, fmt.Errorf("primary key field %q is not present in table metadata", key) + } + part, exists := parts[key] + if !exists { + return nil, fmt.Errorf("update ID is missing key field %q", key) + } + var value any + if err := bson.UnmarshalValue(bson.Type(part.GetType()), part.GetData(), &value); err != nil { + return nil, fmt.Errorf("decode key field %q: %w", key, err) + } + if binary, ok := value.(bson.Binary); ok && binary.Subtype == bson.TypeBinaryUUIDOld && + (strings.EqualFold(column.DatabaseType, "uuid") || strings.EqualFold(column.DatabaseType, "uniqueidentifier")) { + return nil, fmt.Errorf("legacy BSON UUID subtype 3 is not supported for key field %q", key) + } + if number, ok := numericJSONValue(value); ok { + converted, err := dialect.ConvertJSONValue(column, number) + if err != nil { + return nil, fmt.Errorf("convert key field %q: %w", key, err) + } + value = converted + } else { + value = sqlValue(value) + converted, err := dialect.ConvertJSONValue(column, value) + if err != nil { + return nil, fmt.Errorf("convert key field %q: %w", key, err) + } + value = converted + } + if value == nil { + return nil, fmt.Errorf("update ID has null key field %q", key) + } + values[i] = value + } + return values, nil +} + +func (c *conn) tableMetadata(ctx context.Context, namespace string) (dialects.TableMetadata, error) { + c.metadataMu.Lock() + defer c.metadataMu.Unlock() + if metadata, ok := c.sinkMetadata[namespace]; ok { + return metadata, nil + } + metadata, err := c.dialect.InspectTable(ctx, c.c, namespace) + if err != nil { + return dialects.TableMetadata{}, fmt.Errorf("inspect destination table %q: %w", namespace, err) + } + if len(metadata.PrimaryKey) == 0 { + return dialects.TableMetadata{}, fmt.Errorf("destination table %q has no primary key", namespace) + } + if c.sinkMetadata == nil { + c.sinkMetadata = map[string]dialects.TableMetadata{} + } + c.sinkMetadata[namespace] = metadata + return metadata, nil +} + +func columnMetadataByName(metadata dialects.TableMetadata) map[string]dialects.ColumnMetadata { + columns := make(map[string]dialects.ColumnMetadata, len(metadata.Columns)) + for _, column := range metadata.Columns { + columns[column.Name] = column + } + return columns +} + +type writeOperation struct { + delete bool + overrideIdentity bool + columns []string + columnTypes []string + defaultColumns []string + identityColumns []string + values []any + key string +} + +func numericDatabaseType(databaseType string) bool { + switch strings.ToLower(databaseType) { + case "tinyint", "smallint", "int", "integer", "bigint", + "int2", "int4", "int8", "numeric", "decimal", "money", "smallmoney", + "real", "float", "float4", "float8", "double precision", + "number", "binary_float", "binary_double": + return true + default: + return false + } +} + +func canonicalNumericKey(value any) (string, bool) { + var text string + switch value := value.(type) { + case string: + text = value + case int: + text = strconv.FormatInt(int64(value), 10) + case int32: + text = strconv.FormatInt(int64(value), 10) + case int64: + text = strconv.FormatInt(value, 10) + case float32: + text = strconv.FormatFloat(float64(value), 'g', -1, 32) + case float64: + text = strconv.FormatFloat(value, 'g', -1, 64) + case json.Number: + text = value.String() + case interface{ String() (string, error) }: + var err error + text, err = value.String() + if err != nil { + return "", false + } + default: + return "", false + } + if number, ok := new(big.Rat).SetString(text); ok { + return number.RatString(), true + } + return "", false +} + +func operationKey(values []any, columns []dialects.ColumnMetadata) string { + normalized := make([]any, len(values)) + for i, value := range values { + normalized[i] = value + if i < len(columns) && numericDatabaseType(columns[i].DatabaseType) { + if number, ok := canonicalNumericKey(value); ok { + normalized[i] = []string{"numeric", number} + } + } + } + encoded, err := json.Marshal(normalized) + if err != nil { + return fmt.Sprint(normalized) + } + return string(encoded) +} + +func upsertOperation(metadata dialects.TableMetadata, document map[string]any) (writeOperation, error) { + keys := metadata.PrimaryKey + columns := make([]string, 0, len(metadata.Columns)) + columnTypes := make([]string, 0, len(metadata.Columns)) + defaultColumns := make([]string, 0, len(metadata.Columns)) + identityColumns := make([]string, 0, len(metadata.Columns)) + keyValues := make([]any, len(keys)) + keyColumns := make([]dialects.ColumnMetadata, len(keys)) + overrideIdentity := false + for i, key := range keys { + var keyMetadata *dialects.ColumnMetadata + for columnIndex := range metadata.Columns { + if metadata.Columns[columnIndex].Name == key { + keyMetadata = &metadata.Columns[columnIndex] + break + } + } + if keyMetadata == nil { + return writeOperation{}, fmt.Errorf("primary key field %q is not present in table metadata", key) + } + if keyMetadata.Generated && !keyMetadata.Identity { + return writeOperation{}, fmt.Errorf("generated primary key field %q cannot be written explicitly", key) + } + value, exists := document[key] + if !exists || value == nil { + return writeOperation{}, fmt.Errorf("document is missing non-null key field %q", key) + } + columns = append(columns, key) + columnTypes = append(columnTypes, keyMetadata.DatabaseType) + keyValues[i] = value + keyColumns[i] = *keyMetadata + if keyMetadata.Identity { + identityColumns = append(identityColumns, key) + overrideIdentity = overrideIdentity || strings.EqualFold(keyMetadata.IdentityGeneration, "ALWAYS") + } + } + for _, column := range metadata.Columns { + if slices.Contains(keys, column.Name) { + continue + } + value, exists := document[column.Name] + if column.Generated && !column.Identity { + if exists { + return writeOperation{}, fmt.Errorf("generated column %q cannot be written explicitly", column.Name) + } + continue + } + if column.Identity { + if !exists { + continue + } + if value == nil { + return writeOperation{}, fmt.Errorf("identity column %q cannot be null", column.Name) + } + columns = append(columns, column.Name) + columnTypes = append(columnTypes, column.DatabaseType) + identityColumns = append(identityColumns, column.Name) + overrideIdentity = overrideIdentity || strings.EqualFold(column.IdentityGeneration, "ALWAYS") + continue + } + if exists { + if value == nil && !column.Nullable { + return writeOperation{}, fmt.Errorf("column %q is required and cannot be null", column.Name) + } + columns = append(columns, column.Name) + columnTypes = append(columnTypes, column.DatabaseType) + continue + } + if column.HasDefault { + defaultColumns = append(defaultColumns, column.Name) + continue + } + if !column.Nullable { + return writeOperation{}, fmt.Errorf("document is missing required column %q", column.Name) + } + document[column.Name] = nil + columns = append(columns, column.Name) + columnTypes = append(columnTypes, column.DatabaseType) + } + return writeOperation{ + overrideIdentity: overrideIdentity, + columns: columns, + columnTypes: columnTypes, + defaultColumns: defaultColumns, + identityColumns: identityColumns, + values: documentValues(document, columns), + key: operationKey(keyValues, keyColumns), + }, nil +} + +func validateIdentityOverride(dialect SQLDialect, operation writeOperation) error { + if operation.overrideIdentity && !dialect.SupportsIdentityOverride() { + return fmt.Errorf("destination does not support explicitly writing GENERATED ALWAYS identity columns %v", operation.identityColumns) + } + return nil +} + +func writeBatchSize(configured, maxParameters, parametersPerRow int) (int, error) { + if configured < 0 { + return 0, fmt.Errorf("configured write batch size cannot be negative: %d", configured) + } + batchSize := configured + usingDefault := batchSize == 0 + if usingDefault { + batchSize = DefaultWriteBatchSize + } + if maxParameters < 1 || parametersPerRow < 1 { + return batchSize, nil + } + maxRows := maxParameters / parametersPerRow + if maxRows < 1 { + return 0, fmt.Errorf("write requires %d parameters per row, exceeding the destination limit of %d", parametersPerRow, maxParameters) + } + if batchSize > maxRows { + if !usingDefault { + return 0, fmt.Errorf("configured write batch size %d exceeds the destination maximum of %d rows for writes with %d parameters per row", batchSize, maxRows, parametersPerRow) + } + batchSize = maxRows + } + return batchSize, nil +} + +func (c *conn) executeWriteOperations(ctx context.Context, namespace string, keys []string, operations []writeOperation) error { + // Flushes are independent database statements. If a later flush fails, an + // error is returned but earlier flushes may already have committed. + var pending []writeOperation + seen := map[string]struct{}{} + flush := func() error { + if len(pending) == 0 { + return nil + } + var query string + if pending[0].delete { + query = c.dialect.DeleteQuery(namespace, keys, len(pending)) + } else { + var err error + query, err = c.dialect.UpsertQuery(namespace, keys, pending[0].columns, pending[0].columnTypes, pending[0].defaultColumns, pending[0].identityColumns, len(pending), pending[0].overrideIdentity) + if err != nil { + return err + } + } + var values []any + for _, operation := range pending { + values = append(values, operation.values...) + } + if _, err := c.c.ExecContext(ctx, query, values...); err != nil { + return err + } + pending = nil + seen = map[string]struct{}{} + return nil + } + for _, operation := range operations { + batchSize, err := writeBatchSize(c.config.Sink.writeBatchSize(namespace), c.dialect.MaxQueryParameters(), len(operation.values)) + if err != nil { + return err + } + _, duplicate := seen[operation.key] + incompatible := len(pending) > 0 && (pending[0].delete != operation.delete || + pending[0].overrideIdentity != operation.overrideIdentity || + !slices.Equal(pending[0].columns, operation.columns) || + !slices.Equal(pending[0].columnTypes, operation.columnTypes) || + !slices.Equal(pending[0].defaultColumns, operation.defaultColumns) || + !slices.Equal(pending[0].identityColumns, operation.identityColumns)) + if len(pending) >= batchSize || incompatible || duplicate { + if err := flush(); err != nil { + return err + } + } + pending = append(pending, operation) + seen[operation.key] = struct{}{} + } + return flush() +} + +// WriteData implements adiomv1connect.ConnectorServiceHandler. +func (c *conn) WriteData(ctx context.Context, r *connect.Request[adiomv1.WriteDataRequest]) (*connect.Response[adiomv1.WriteDataResponse], error) { + if r.Msg.GetPartialWriteResults() { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("partial write results are not supported by sqlbatch")) + } + namespace := r.Msg.GetNamespace() + metadata, err := c.tableMetadata(ctx, namespace) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) + } + columns := columnMetadataByName(metadata) + operations := make([]writeOperation, 0, len(r.Msg.GetData())) + for i, data := range r.Msg.GetData() { + document, err := decodeJSONDocument(data, columns, c.dialect) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("decode document %d: %w", i, err)) + } + operation, err := upsertOperation(metadata, document) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("document %d: %w", i, err)) + } + if err := validateIdentityOverride(c.dialect, operation); err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("document %d: %w", i, err)) + } + operations = append(operations, operation) + } + if err := c.executeWriteOperations(ctx, namespace, metadata.PrimaryKey, operations); err != nil { + return nil, connect.NewError(connect.CodeInternal, err) + } + return connect.NewResponse(&adiomv1.WriteDataResponse{}), nil +} + +// WriteUpdates implements adiomv1connect.ConnectorServiceHandler. +func (c *conn) WriteUpdates(ctx context.Context, r *connect.Request[adiomv1.WriteUpdatesRequest]) (*connect.Response[adiomv1.WriteUpdatesResponse], error) { + if r.Msg.GetPartialWriteResults() { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("partial write results are not supported by sqlbatch")) + } + namespace := r.Msg.GetNamespace() + metadata, err := c.tableMetadata(ctx, namespace) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) + } + columns := columnMetadataByName(metadata) + keyColumns := make([]dialects.ColumnMetadata, len(metadata.PrimaryKey)) + for i, key := range metadata.PrimaryKey { + keyColumns[i] = columns[key] + } + for _, update := range r.Msg.GetUpdates() { + if update.GetType() == adiomv1.UpdateType_UPDATE_TYPE_PARTIAL_UPDATE { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("partial updates are not supported by sqlbatch")) + } + } + + operations := make([]writeOperation, 0, len(r.Msg.GetUpdates())) + for i, update := range r.Msg.GetUpdates() { + if update.GetType() == adiomv1.UpdateType_UPDATE_TYPE_UNKNOWN { + continue + } + keyValues, err := updateKeyValues(update, metadata, c.dialect) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("update %d: %w", i, err)) + } + switch update.GetType() { + case adiomv1.UpdateType_UPDATE_TYPE_INSERT, adiomv1.UpdateType_UPDATE_TYPE_UPDATE: + document, err := decodeJSONDocument(update.GetData(), columns, c.dialect) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("decode update %d: %w", i, err)) + } + for keyIndex, key := range metadata.PrimaryKey { + document[key] = keyValues[keyIndex] + } + operation, err := upsertOperation(metadata, document) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("update %d: %w", i, err)) + } + if err := validateIdentityOverride(c.dialect, operation); err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("update %d: %w", i, err)) + } + operations = append(operations, operation) + case adiomv1.UpdateType_UPDATE_TYPE_DELETE: + operations = append(operations, writeOperation{delete: true, values: keyValues, key: operationKey(keyValues, keyColumns)}) + default: + return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("unsupported update type %v", update.GetType())) + } + } + if err := c.executeWriteOperations(ctx, namespace, metadata.PrimaryKey, operations); err != nil { + return nil, connect.NewError(connect.CodeInternal, err) + } + return connect.NewResponse(&adiomv1.WriteUpdatesResponse{}), nil +} diff --git a/connectors/sqlbatch/sink_integration_test.go b/connectors/sqlbatch/sink_integration_test.go new file mode 100644 index 00000000..0c05f1cd --- /dev/null +++ b/connectors/sqlbatch/sink_integration_test.go @@ -0,0 +1,795 @@ +package sqlbatch + +import ( + "context" + "database/sql" + "encoding/json" + "os" + "testing" + "time" + + "connectrpc.com/connect" + "github.com/adiom-data/dsync/connectors/sqlbatch/dialects" + adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" + "github.com/microsoft/go-mssqldb/msdsn" + "go.mongodb.org/mongo-driver/v2/bson" +) + +func assertRFC3339SourceRow(t *testing.T, ctx context.Context, db *sql.DB, dialect SQLDialect, query string, temporalColumns ...string) { + t.Helper() + rows, err := db.QueryContext(ctx, query) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + columns, err := rows.Columns() + if err != nil { + t.Fatal(err) + } + columnTypes, err := rows.ColumnTypes() + if err != nil { + t.Fatal(err) + } + if !rows.Next() { + t.Fatalf("query returned no temporal row: %s", query) + } + row, err := ParseRow(columns, columnTypes, rows, &Mapping{}, dialect) + if err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(row) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := json.Unmarshal(encoded, &document); err != nil { + t.Fatal(err) + } + for _, column := range temporalColumns { + value, ok := document[column].(string) + if !ok { + t.Fatalf("source column %q encoded as %T, want RFC3339 string; JSON=%s", column, document[column], encoded) + } + if _, err := time.Parse(time.RFC3339Nano, value); err != nil { + t.Fatalf("source column %q value %q is not RFC3339: %v", column, value, err) + } + } +} + +func TestPostgresSinkLive(t *testing.T) { + connectionURL := os.Getenv("POSTGRES_TEST_URL") + if connectionURL == "" { + t.Skip("POSTGRES_TEST_URL is not set") + } + ctx := context.Background() + db, err := sql.Open("pgx", connectionURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, statement := range []string{ + `DROP SCHEMA IF EXISTS sqlbatch_live CASCADE`, + `CREATE SCHEMA sqlbatch_live`, + `CREATE TABLE sqlbatch_live.items (tenant text, id bigint, name text NOT NULL CHECK (name <> 'forbidden'), amount numeric DEFAULT 42, payload bytea, PRIMARY KEY (tenant, id))`, + `CREATE TABLE sqlbatch_live.identity_items (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, name text NOT NULL)`, + `CREATE TABLE sqlbatch_live.uuid_items (id uuid PRIMARY KEY, occurred_at timestamptz NOT NULL)`, + } { + if _, err := db.ExecContext(ctx, statement); err != nil { + t.Fatal(err) + } + } + + c := &conn{ + c: db, + dialect: dialects.Postgres{}, + } + + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "sqlbatch_live.items", + Data: [][]byte{[]byte(`{"tenant":"a","id":1,"name":"first","amount":1234567890.0123456789,"payload":"AAH+/w=="}`)}, + })); err != nil { + t.Fatal(err) + } + var bytesMatch bool + if err := db.QueryRowContext(ctx, `SELECT payload = decode('0001feff', 'hex') FROM sqlbatch_live.items WHERE tenant = 'a' AND id = 1`).Scan(&bytesMatch); err != nil || !bytesMatch { + t.Fatalf("bytea round trip: match=%v err=%v", bytesMatch, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "sqlbatch_live.items", + Data: [][]byte{[]byte(`{"tenant":"a","id":1,"name":"second"}`)}, + })); err != nil { + t.Fatal(err) + } + var name, amount string + if err := db.QueryRowContext(ctx, `SELECT name, amount::text FROM sqlbatch_live.items WHERE tenant = 'a' AND id = 1`).Scan(&name, &amount); err != nil { + t.Fatal(err) + } + if name != "second" || amount != "42" { + t.Fatalf("initial upsert row: name=%q amount=%q", name, amount) + } + + id, _, err := ToID([]string{"tenant", "id"}, []any{"a", int64(1)}) + if err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, `UPDATE sqlbatch_live.items SET amount = 99 WHERE tenant = 'a' AND id = 1`); err != nil { + t.Fatal(err) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "sqlbatch_live.items", + Updates: []*adiomv1.Update{{ + Id: id, Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, + Data: []byte(`{"tenant":"wrong","id":99,"name":"updated"}`), + }}, + })); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, `SELECT name, amount::text FROM sqlbatch_live.items WHERE tenant = 'a' AND id = 1`).Scan(&name, &amount); err != nil || name != "updated" || amount != "42" { + t.Fatalf("full update row: name=%q amount=%q err=%v", name, amount, err) + } + + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "sqlbatch_live.items", + Data: [][]byte{ + []byte(`{"tenant":"b","id":2,"name":"batched-1"}`), + []byte(`{"tenant":"c","id":3,"name":"continued"}`), + }, + })); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, `SELECT name FROM sqlbatch_live.items WHERE tenant = 'c' AND id = 3`).Scan(&name); err != nil || name != "continued" { + t.Fatalf("continued row: name=%q err=%v", name, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "sqlbatch_live.items", + Data: [][]byte{ + []byte(`{"tenant":"f","id":6,"name":"duplicate-first"}`), + []byte(`{"tenant":"f","id":6,"name":"duplicate-last"}`), + }, + })); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, `SELECT name FROM sqlbatch_live.items WHERE tenant = 'f' AND id = 6`).Scan(&name); err != nil || name != "duplicate-last" { + t.Fatalf("duplicate-key ordering: name=%q err=%v", name, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "sqlbatch_live.items", + PartialWriteResults: true, + Data: [][]byte{[]byte(`{"tenant":"d","id":4,"name":"unsupported"}`)}, + })); err == nil { + t.Fatal("expected partial-result request to be rejected") + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "sqlbatch_live.items", + Data: [][]byte{ + []byte(`{"tenant":"d","id":4,"name":"forbidden"}`), + []byte(`{"tenant":"e","id":5,"name":"same-batch"}`), + }, + })); err == nil { + t.Fatal("expected whole-batch database error") + } + var count int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlbatch_live.items WHERE tenant IN ('d', 'e')`).Scan(&count); err != nil || count != 0 { + t.Fatalf("failed batch count=%d err=%v", count, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "sqlbatch_live.items", + Data: [][]byte{ + []byte(`{"tenant":"g","id":7,"name":"would-be-valid"}`), + []byte(`{"tenant":"h","id":8}`), + }, + })); err == nil { + t.Fatal("expected missing required column error") + } + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlbatch_live.items WHERE tenant IN ('g', 'h')`).Scan(&count); err != nil || count != 0 { + t.Fatalf("prevalidated request count=%d err=%v", count, err) + } + + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "sqlbatch_live.items", + Updates: []*adiomv1.Update{{Id: id, Type: adiomv1.UpdateType_UPDATE_TYPE_DELETE}}, + })); err != nil { + t.Fatal(err) + } + count = 0 + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlbatch_live.items WHERE tenant = 'a' AND id = 1`).Scan(&count); err != nil || count != 0 { + t.Fatalf("delete count=%d err=%v", count, err) + } + + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "sqlbatch_live.identity_items", + Data: [][]byte{[]byte(`{"id":42,"name":"preserved"}`)}, + })); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, `SELECT name FROM sqlbatch_live.identity_items WHERE id = 42`).Scan(&name); err != nil || name != "preserved" { + t.Fatalf("generated identity row: name=%q err=%v", name, err) + } + + const uuidText = "00010203-0405-0607-0809-0a0b0c0d0e0f" + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "sqlbatch_live.uuid_items", + Data: [][]byte{[]byte(`{"id":"` + uuidText + `","occurred_at":"2026-10-15T12:34:56.123456-07:00"}`)}, + })); err != nil { + t.Fatal(err) + } + uuidType, uuidData, err := bson.MarshalValue(bson.Binary{ + Subtype: bson.TypeBinaryUUID, + Data: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "sqlbatch_live.uuid_items", + Updates: []*adiomv1.Update{{ + Id: []*adiomv1.BsonValue{{Name: "id", Type: uint32(uuidType), Data: uuidData}}, + Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, + Data: []byte(`{"occurred_at":"2026-10-16T01:02:03.654321Z"}`), + }}, + })); err != nil { + t.Fatal(err) + } + var occurredAt time.Time + if err := db.QueryRowContext(ctx, `SELECT occurred_at FROM sqlbatch_live.uuid_items WHERE id = $1`, uuidText).Scan(&occurredAt); err != nil { + t.Fatal(err) + } + if want := time.Date(2026, 10, 16, 1, 2, 3, 654321000, time.UTC); !occurredAt.Equal(want) { + t.Fatalf("PostgreSQL timestamp = %v, want %v", occurredAt, want) + } + assertRFC3339SourceRow(t, ctx, db, c.dialect, `SELECT occurred_at FROM sqlbatch_live.uuid_items WHERE id = '`+uuidText+`'`, "occurred_at") +} + +func TestSQLServerSinkLive(t *testing.T) { + connectionURL := os.Getenv("SQLSERVER_TEST_URL") + if connectionURL == "" { + t.Skip("SQLSERVER_TEST_URL is not set") + } + dsn, err := msdsn.Parse(connectionURL) + if err != nil { + t.Fatal(err) + } + dialect := dialects.SQLServer{GuidConversion: dsn.Encoding.GuidConversion} + ctx := context.Background() + db, err := sql.Open("sqlserver", connectionURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, statement := range []string{ + `DROP TABLE IF EXISTS dbo.sqlbatch_live_items`, + `DROP TABLE IF EXISTS dbo.sqlbatch_live_identity_items`, + `DROP TABLE IF EXISTS dbo.sqlbatch_live_guid_items`, + `DROP TABLE IF EXISTS dbo.sqlbatch_live_nonkey_identity`, + `DROP TABLE IF EXISTS dbo.sqlbatch_live_decimal_keys`, + `DROP TABLE IF EXISTS dbo.sqlbatch_live_temporal`, + `CREATE TABLE dbo.sqlbatch_live_items (tenant nvarchar(20) NOT NULL, id bigint NOT NULL, name nvarchar(100) NOT NULL CHECK (name <> 'forbidden'), amount decimal(20,4) CONSTRAINT sqlbatch_live_amount_default DEFAULT 42, payload varbinary(max), CONSTRAINT sqlbatch_live_items_pk PRIMARY KEY (tenant, id))`, + `CREATE TABLE dbo.sqlbatch_live_identity_items (id bigint IDENTITY(1,1) PRIMARY KEY, name nvarchar(100) NOT NULL)`, + `CREATE TABLE dbo.sqlbatch_live_nonkey_identity (key_id bigint PRIMARY KEY, generated_id bigint IDENTITY(1,1) NOT NULL, name nvarchar(100) NOT NULL, computed AS (key_id + generated_id))`, + `CREATE TABLE dbo.sqlbatch_live_decimal_keys (id decimal(20,4) PRIMARY KEY, name nvarchar(100) NOT NULL)`, + `CREATE TABLE dbo.sqlbatch_live_temporal (id bigint PRIMARY KEY, occurred_at datetime2, event_date date, zoned_at datetimeoffset, at_time time)`, + `CREATE TABLE dbo.sqlbatch_live_guid_items (id uniqueidentifier PRIMARY KEY, occurred_at datetimeoffset(7) NOT NULL, amount decimal(20,10) NOT NULL, payload varbinary(max))`, + } { + if _, err := db.ExecContext(ctx, statement); err != nil { + t.Fatal(err) + } + } + + c := &conn{c: db, dialect: dialect} + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dbo.sqlbatch_live_items", + Data: [][]byte{ + []byte(`{"tenant":"a","id":1,"name":"first","payload":"AAH+/w=="}`), + []byte(`{"tenant":"b","id":2,"name":"second"}`), + }, + })); err != nil { + t.Fatal(err) + } + var count int + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dbo.sqlbatch_live_items WHERE amount = 42`).Scan(&count); err != nil || count != 2 { + t.Fatalf("defaulted row count=%d err=%v", count, err) + } + var bytesMatch bool + if err := db.QueryRowContext(ctx, `SELECT CASE WHEN payload = 0x0001FEFF THEN 1 ELSE 0 END FROM dbo.sqlbatch_live_items WHERE tenant = 'a' AND id = 1`).Scan(&bytesMatch); err != nil || !bytesMatch { + t.Fatalf("varbinary round trip: match=%v err=%v", bytesMatch, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dbo.sqlbatch_live_items", + Data: [][]byte{ + []byte(`{"tenant":"c","id":3,"name":"whole","amount":1}`), + []byte(`{"tenant":"d","id":4,"name":"fractional","amount":1.5}`), + []byte(`{"tenant":"exponent","id":5,"name":"exponent","amount":1.25e2}`), + }, + })); err != nil { + t.Fatalf("mixed decimal insert batch: %v", err) + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dbo.sqlbatch_live_items WHERE (tenant = 'c' AND amount = 1) OR (tenant = 'd' AND amount = 1.5) OR (tenant = 'exponent' AND amount = 125)`).Scan(&count); err != nil || count != 3 { + t.Fatalf("mixed decimal insert count=%d err=%v", count, err) + } + + id, _, err := ToID([]string{"tenant", "id"}, []any{"a", int64(1)}) + if err != nil { + t.Fatal(err) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "dbo.sqlbatch_live_items", + Updates: []*adiomv1.Update{{ + Id: id, Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, + Data: []byte(`{"name":"updated"}`), + }}, + })); err != nil { + t.Fatal(err) + } + var name string + if err := db.QueryRowContext(ctx, `SELECT name FROM dbo.sqlbatch_live_items WHERE tenant = 'a' AND id = 1`).Scan(&name); err != nil || name != "updated" { + t.Fatalf("updated row: name=%q err=%v", name, err) + } + mixedUpdates := make([]*adiomv1.Update, 0, 2) + for _, item := range []struct { + tenant string + id int64 + data string + }{ + {tenant: "c", id: 3, data: `{"name":"whole-updated","amount":2}`}, + {tenant: "d", id: 4, data: `{"name":"fractional-updated","amount":2.5}`}, + } { + itemID, _, err := ToID([]string{"tenant", "id"}, []any{item.tenant, item.id}) + if err != nil { + t.Fatal(err) + } + mixedUpdates = append(mixedUpdates, &adiomv1.Update{Id: itemID, Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, Data: []byte(item.data)}) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "dbo.sqlbatch_live_items", + Updates: mixedUpdates, + })); err != nil { + t.Fatalf("mixed decimal update batch: %v", err) + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dbo.sqlbatch_live_items WHERE (tenant = 'c' AND amount = 2) OR (tenant = 'd' AND amount = 2.5)`).Scan(&count); err != nil || count != 2 { + t.Fatalf("mixed decimal update count=%d err=%v", count, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dbo.sqlbatch_live_decimal_keys", + Data: [][]byte{ + []byte(`{"id":1,"name":"whole"}`), + []byte(`{"id":1.5,"name":"fractional"}`), + }, + })); err != nil { + t.Fatalf("mixed decimal key insert batch: %v", err) + } + decimalID, err := bson.ParseDecimal128("1.5") + if err != nil { + t.Fatal(err) + } + decimalKeyUpdates := make([]*adiomv1.Update, 0, 2) + for _, item := range []struct { + id any + name string + }{ + {id: int64(1), name: "whole-updated"}, + {id: decimalID, name: "fractional-updated"}, + } { + itemID, _, err := ToID([]string{"id"}, []any{item.id}) + if err != nil { + t.Fatal(err) + } + decimalKeyUpdates = append(decimalKeyUpdates, &adiomv1.Update{Id: itemID, Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, Data: []byte(`{"name":"` + item.name + `"}`)}) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "dbo.sqlbatch_live_decimal_keys", + Updates: decimalKeyUpdates, + })); err != nil { + t.Fatalf("mixed decimal key update batch: %v", err) + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dbo.sqlbatch_live_decimal_keys WHERE name LIKE '%-updated'`).Scan(&count); err != nil || count != 2 { + t.Fatalf("mixed decimal key update count=%d err=%v", count, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dbo.sqlbatch_live_decimal_keys", + Data: [][]byte{ + []byte(`{"id":1,"name":"equivalent-first"}`), + []byte(`{"id":1.0,"name":"equivalent-last"}`), + }, + })); err != nil { + t.Fatalf("equivalent decimal keys: %v", err) + } + if err := db.QueryRowContext(ctx, `SELECT name FROM dbo.sqlbatch_live_decimal_keys WHERE id = 1`).Scan(&name); err != nil || name != "equivalent-last" { + t.Fatalf("equivalent decimal key ordering: name=%q err=%v", name, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dbo.sqlbatch_live_temporal", + Data: [][]byte{ + []byte(`{"id":1,"occurred_at":"2026-10-19T12:34:56+05:30","event_date":"2026-10-19","zoned_at":"2026-10-19T12:34:56+05:30","at_time":"12:34:56+05:30"}`), + []byte(`{"id":2}`), + }, + })); err != nil { + t.Fatalf("temporal insert batch: %v", err) + } + var occurredAt, eventDate, zonedAt, atTime string + var zonedOffset int + if err := db.QueryRowContext(ctx, `SELECT CONVERT(varchar(33), occurred_at, 126), CONVERT(varchar(10), event_date, 23), CONVERT(varchar(19), zoned_at, 126), DATEPART(TZOFFSET, zoned_at), CONVERT(varchar(8), at_time, 108) FROM dbo.sqlbatch_live_temporal WHERE id = 1`).Scan(&occurredAt, &eventDate, &zonedAt, &zonedOffset, &atTime); err != nil { + t.Fatal(err) + } + if occurredAt != "2026-10-19T07:04:56" || eventDate != "2026-10-19" || zonedAt != "2026-10-19T12:34:56" || zonedOffset != 330 || atTime != "07:04:56" { + t.Fatalf("temporal values: occurred=%q date=%q zoned=%q offset=%d time=%q", occurredAt, eventDate, zonedAt, zonedOffset, atTime) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dbo.sqlbatch_live_temporal", + Data: [][]byte{ + []byte(`{"id":3}`), + []byte(`{"id":4,"occurred_at":"not-a-timestamp"}`), + }, + })); err == nil { + t.Fatal("expected invalid SQL Server temporal value to be rejected") + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dbo.sqlbatch_live_temporal`).Scan(&count); err != nil || count != 2 { + t.Fatalf("prevalidated temporal request count=%d err=%v", count, err) + } + + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dbo.sqlbatch_live_identity_items", + Data: [][]byte{[]byte(`{"id":42,"name":"preserved"}`)}, + })); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, `SELECT name FROM dbo.sqlbatch_live_identity_items WHERE id = 42`).Scan(&name); err != nil || name != "preserved" { + t.Fatalf("identity row: name=%q err=%v", name, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dbo.sqlbatch_live_nonkey_identity", + Data: [][]byte{[]byte(`{"key_id":1,"generated_id":42,"name":"rejected","computed":43}`)}, + })); err == nil { + t.Fatal("expected supplied computed column to be rejected") + } + for _, data := range []string{ + `{"key_id":1,"generated_id":42,"name":"inserted"}`, + `{"key_id":1,"generated_id":42,"name":"updated"}`, + } { + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dbo.sqlbatch_live_nonkey_identity", + Data: [][]byte{[]byte(data)}, + })); err != nil { + t.Fatalf("non-key identity write: %v", err) + } + } + var generatedID, computed int64 + if err := db.QueryRowContext(ctx, `SELECT generated_id, computed, name FROM dbo.sqlbatch_live_nonkey_identity WHERE key_id = 1`).Scan(&generatedID, &computed, &name); err != nil || generatedID != 42 || computed != 43 || name != "updated" { + t.Fatalf("non-key identity row: generated=%d computed=%d name=%q err=%v", generatedID, computed, name, err) + } + + const guidText = "00010203-0405-0607-0809-0a0b0c0d0e0f" + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "dbo.sqlbatch_live_guid_items", + Data: [][]byte{[]byte(`{"id":"` + guidText + `","occurred_at":"2026-10-15T12:34:56.123456-07:00","amount":1234567890.0123456789,"payload":"AAH+/w=="}`)}, + })); err != nil { + t.Fatal(err) + } + rows, err := db.QueryContext(ctx, `SELECT id, amount, payload FROM dbo.sqlbatch_live_guid_items WHERE id = '`+guidText+`'`) + if err != nil { + t.Fatal(err) + } + columns, err := rows.Columns() + if err != nil { + rows.Close() + t.Fatal(err) + } + columnTypes, err := rows.ColumnTypes() + if err != nil { + rows.Close() + t.Fatal(err) + } + if !rows.Next() { + rows.Close() + t.Fatal("SQL Server source query returned no row") + } + sourceRow, err := ParseRow(columns, columnTypes, rows, &Mapping{}, c.dialect) + rows.Close() + if err != nil { + t.Fatal(err) + } + if sourceRow["id"] != "00010203-0405-0607-0809-0A0B0C0D0E0F" { + t.Fatalf("source uniqueidentifier = %#v", sourceRow["id"]) + } + if sourceRow["amount"] != "1234567890.0123456789" { + t.Fatalf("source decimal = %#v", sourceRow["amount"]) + } + sourceJSON, err := json.Marshal(sourceRow) + if err != nil { + t.Fatal(err) + } + var sourceDocument map[string]any + if err := json.Unmarshal(sourceJSON, &sourceDocument); err != nil { + t.Fatal(err) + } + if sourceDocument["payload"] != "AAH+/w==" { + t.Fatalf("source binary = %#v", sourceDocument["payload"]) + } + c.config.Mappings = map[string]Mapping{ + "dbo.sqlbatch_live_guid_items": { + Namespace: "dbo.sqlbatch_live_guid_items", + CountQuery: `SELECT COUNT(*) FROM dbo.sqlbatch_live_guid_items`, + Query: `SELECT id, occurred_at, amount, payload FROM dbo.sqlbatch_live_guid_items`, + PartitionQuery: `SELECT id FROM dbo.sqlbatch_live_guid_items`, + Limit: 100, + Cols: []string{"id"}, + Changes: []ChangeMapping{{ + InitialCursorQuery: `SELECT TOP (1) id FROM dbo.sqlbatch_live_guid_items ORDER BY id DESC`, + }}, + }, + } + plan, err := c.GeneratePlan(ctx, connect.NewRequest(&adiomv1.GeneratePlanRequest{ + Namespaces: []string{"dbo.sqlbatch_live_guid_items"}, + InitialSync: true, + Updates: true, + })) + if err != nil { + t.Fatal(err) + } + if len(plan.Msg.GetPartitions()) == 0 { + t.Fatal("SQL Server plan returned no initial-sync partitions") + } + partition, err := DecodePartitionCursor(plan.Msg.GetPartitions()[0].GetCursor()) + if err != nil { + t.Fatal(err) + } + if len(partition.High) != 1 || partition.High[0] != "00010203-0405-0607-0809-0A0B0C0D0E0F" { + t.Fatalf("partition GUID boundary = %#v", partition.High) + } + if len(plan.Msg.GetUpdatesPartitions()) != 1 { + t.Fatalf("update partitions = %d", len(plan.Msg.GetUpdatesPartitions())) + } + updateCursor, err := DecodeUpdateCursor(plan.Msg.GetUpdatesPartitions()[0].GetCursor()) + if err != nil { + t.Fatal(err) + } + if len(updateCursor.Cursor) != 1 || updateCursor.Cursor[0] != "00010203-0405-0607-0809-0A0B0C0D0E0F" { + t.Fatalf("update GUID cursor = %#v", updateCursor.Cursor) + } + guidType, guidData, err := bson.MarshalValue(bson.Binary{ + Subtype: bson.TypeBinaryUUID, + Data: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "dbo.sqlbatch_live_guid_items", + Updates: []*adiomv1.Update{{ + Id: []*adiomv1.BsonValue{{Name: "id", Type: uint32(guidType), Data: guidData}}, + Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, + Data: []byte(`{"occurred_at":"2026-10-16T01:02:03.654321Z","amount":1234567890.0123456789,"payload":"AAH+/w=="}`), + }}, + })); err != nil { + t.Fatal(err) + } + assertRFC3339SourceRow(t, ctx, db, c.dialect, `SELECT occurred_at FROM dbo.sqlbatch_live_guid_items WHERE id = '`+guidText+`'`, "occurred_at") + + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "dbo.sqlbatch_live_items", + Updates: []*adiomv1.Update{{Id: id, Type: adiomv1.UpdateType_UPDATE_TYPE_DELETE}}, + })); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dbo.sqlbatch_live_items WHERE tenant = 'a' AND id = 1`).Scan(&count); err != nil || count != 0 { + t.Fatalf("delete count=%d err=%v", count, err) + } +} + +func TestOracleSinkLive(t *testing.T) { + connectionURL := os.Getenv("ORACLE_TEST_URL") + if connectionURL == "" { + t.Skip("ORACLE_TEST_URL is not set") + } + ctx := context.Background() + db, err := sql.Open("oracle", connectionURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, table := range []string{"SQLBATCH_LIVE_ITEMS", "SQLBATCH_LIVE_IDENTITY", "SQLBATCH_LIVE_ALWAYS", "SQLBATCH_LIVE_DECIMAL_KEYS", "SQLBATCH_LIVE_TEMPORAL"} { + statement := `BEGIN EXECUTE IMMEDIATE 'DROP TABLE ` + table + ` PURGE'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN RAISE; END IF; END;` + if _, err := db.ExecContext(ctx, statement); err != nil { + t.Fatal(err) + } + } + for _, statement := range []string{ + `CREATE TABLE SQLBATCH_LIVE_ITEMS (TENANT VARCHAR2(20) NOT NULL, ID NUMBER(19) NOT NULL, NAME VARCHAR2(100) NOT NULL CHECK (NAME <> 'forbidden'), AMOUNT NUMBER(20,4) DEFAULT 42, PAYLOAD BLOB, RAWPAYLOAD RAW(16), CONSTRAINT SQLBATCH_LIVE_ITEMS_PK PRIMARY KEY (TENANT, ID))`, + `CREATE TABLE SQLBATCH_LIVE_IDENTITY (ID NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, NAME VARCHAR2(100) NOT NULL)`, + `CREATE TABLE SQLBATCH_LIVE_ALWAYS (ID NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, NAME VARCHAR2(100) NOT NULL)`, + `CREATE TABLE SQLBATCH_LIVE_DECIMAL_KEYS (ID NUMBER(20,4) PRIMARY KEY, NAME VARCHAR2(100) NOT NULL)`, + `CREATE TABLE SQLBATCH_LIVE_TEMPORAL (ID NUMBER PRIMARY KEY, OCCURRED_AT TIMESTAMP(9), EVENT_DATE DATE, ZONED_AT TIMESTAMP(9) WITH TIME ZONE, LOCAL_ZONED_AT TIMESTAMP(9) WITH LOCAL TIME ZONE)`, + } { + if _, err := db.ExecContext(ctx, statement); err != nil { + t.Fatal(err) + } + } + + c := &conn{c: db, dialect: dialects.Oracle{}} + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "SQLBATCH_LIVE_ITEMS", + Data: [][]byte{ + []byte(`{"TENANT":"a","ID":1,"NAME":"first","PAYLOAD":"AAH+/w==","RAWPAYLOAD":"AAE="}`), + []byte(`{"TENANT":"b","ID":2,"NAME":"second"}`), + }, + })); err != nil { + t.Fatal(err) + } + var count int + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM SQLBATCH_LIVE_ITEMS WHERE AMOUNT = 42`).Scan(&count); err != nil || count != 2 { + t.Fatalf("defaulted row count=%d err=%v", count, err) + } + var payloadHex string + if err := db.QueryRowContext(ctx, `SELECT RAWTOHEX(DBMS_LOB.SUBSTR(PAYLOAD, 4, 1)) FROM SQLBATCH_LIVE_ITEMS WHERE TENANT = 'a' AND ID = 1`).Scan(&payloadHex); err != nil || payloadHex != "0001FEFF" { + t.Fatalf("blob round trip: value=%q err=%v", payloadHex, err) + } + if err := db.QueryRowContext(ctx, `SELECT RAWTOHEX(RAWPAYLOAD) FROM SQLBATCH_LIVE_ITEMS WHERE TENANT = 'a' AND ID = 1`).Scan(&payloadHex); err != nil || payloadHex != "0001" { + t.Fatalf("raw round trip: value=%q err=%v", payloadHex, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "SQLBATCH_LIVE_ITEMS", + Data: [][]byte{ + []byte(`{"TENANT":"c","ID":3,"NAME":"whole","AMOUNT":1}`), + []byte(`{"TENANT":"d","ID":4,"NAME":"fractional","AMOUNT":1.5}`), + }, + })); err != nil { + t.Fatalf("mixed decimal insert batch: %v", err) + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM SQLBATCH_LIVE_ITEMS WHERE (TENANT = 'c' AND AMOUNT = 1) OR (TENANT = 'd' AND AMOUNT = 1.5)`).Scan(&count); err != nil || count != 2 { + t.Fatalf("mixed decimal insert count=%d err=%v", count, err) + } + + id, _, err := ToID([]string{"TENANT", "ID"}, []any{"a", int64(1)}) + if err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, `UPDATE SQLBATCH_LIVE_ITEMS SET AMOUNT = 99 WHERE TENANT = 'a' AND ID = 1`); err != nil { + t.Fatal(err) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "SQLBATCH_LIVE_ITEMS", + Updates: []*adiomv1.Update{{ + Id: id, Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, + Data: []byte(`{"NAME":"updated"}`), + }}, + })); err != nil { + t.Fatal(err) + } + var name, amount string + if err := db.QueryRowContext(ctx, `SELECT NAME, TO_CHAR(AMOUNT, 'FM9999999999999990D0000', 'NLS_NUMERIC_CHARACTERS=''.,''') FROM SQLBATCH_LIVE_ITEMS WHERE TENANT = 'a' AND ID = 1`).Scan(&name, &amount); err != nil || name != "updated" || amount != "42.0000" { + t.Fatalf("updated row: name=%q amount=%q err=%v", name, amount, err) + } + mixedUpdates := make([]*adiomv1.Update, 0, 2) + for _, item := range []struct { + tenant string + id int64 + data string + }{ + {tenant: "c", id: 3, data: `{"NAME":"whole-updated","AMOUNT":2}`}, + {tenant: "d", id: 4, data: `{"NAME":"fractional-updated","AMOUNT":2.5}`}, + } { + itemID, _, err := ToID([]string{"TENANT", "ID"}, []any{item.tenant, item.id}) + if err != nil { + t.Fatal(err) + } + mixedUpdates = append(mixedUpdates, &adiomv1.Update{Id: itemID, Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, Data: []byte(item.data)}) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "SQLBATCH_LIVE_ITEMS", + Updates: mixedUpdates, + })); err != nil { + t.Fatalf("mixed decimal update batch: %v", err) + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM SQLBATCH_LIVE_ITEMS WHERE (TENANT = 'c' AND AMOUNT = 2) OR (TENANT = 'd' AND AMOUNT = 2.5)`).Scan(&count); err != nil || count != 2 { + t.Fatalf("mixed decimal update count=%d err=%v", count, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "SQLBATCH_LIVE_DECIMAL_KEYS", + Data: [][]byte{ + []byte(`{"ID":1,"NAME":"whole"}`), + []byte(`{"ID":1.5,"NAME":"fractional"}`), + }, + })); err != nil { + t.Fatalf("mixed decimal key insert batch: %v", err) + } + decimalID, err := bson.ParseDecimal128("1.5") + if err != nil { + t.Fatal(err) + } + decimalKeyUpdates := make([]*adiomv1.Update, 0, 2) + for _, item := range []struct { + id any + name string + }{ + {id: int64(1), name: "whole-updated"}, + {id: decimalID, name: "fractional-updated"}, + } { + itemID, _, err := ToID([]string{"ID"}, []any{item.id}) + if err != nil { + t.Fatal(err) + } + decimalKeyUpdates = append(decimalKeyUpdates, &adiomv1.Update{Id: itemID, Type: adiomv1.UpdateType_UPDATE_TYPE_UPDATE, Data: []byte(`{"NAME":"` + item.name + `"}`)}) + } + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "SQLBATCH_LIVE_DECIMAL_KEYS", + Updates: decimalKeyUpdates, + })); err != nil { + t.Fatalf("mixed decimal key update batch: %v", err) + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM SQLBATCH_LIVE_DECIMAL_KEYS WHERE NAME LIKE '%-updated'`).Scan(&count); err != nil || count != 2 { + t.Fatalf("mixed decimal key update count=%d err=%v", count, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "SQLBATCH_LIVE_TEMPORAL", + Data: [][]byte{ + []byte(`{"ID":1,"OCCURRED_AT":"2026-10-19T12:34:56.123456789+05:30","EVENT_DATE":"2026-10-19T12:34:56+05:30","ZONED_AT":"2026-10-19T12:34:56.123456789+05:30","LOCAL_ZONED_AT":"2026-10-19T12:34:56.123456789+05:30"}`), + []byte(`{"ID":2}`), + }, + })); err != nil { + t.Fatalf("temporal insert batch: %v", err) + } + var occurredAt, eventDate, zonedOffset, localZonedUTC string + if err := db.QueryRowContext(ctx, `SELECT TO_CHAR(OCCURRED_AT, 'YYYY-MM-DD HH24:MI:SS.FF9'), TO_CHAR(EVENT_DATE, 'YYYY-MM-DD HH24:MI:SS'), TO_CHAR(ZONED_AT, 'TZH:TZM'), TO_CHAR(SYS_EXTRACT_UTC(LOCAL_ZONED_AT), 'YYYY-MM-DD HH24:MI:SS.FF9') FROM SQLBATCH_LIVE_TEMPORAL WHERE ID = 1`).Scan(&occurredAt, &eventDate, &zonedOffset, &localZonedUTC); err != nil { + t.Fatal(err) + } + if occurredAt != "2026-10-19 07:04:56.123456789" || eventDate != "2026-10-19 07:04:56" || zonedOffset != "+05:30" || localZonedUTC != "2026-10-19 07:04:56.123456789" { + t.Fatalf("temporal values: occurred=%q date=%q zoned offset=%q local zoned UTC=%q", occurredAt, eventDate, zonedOffset, localZonedUTC) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "SQLBATCH_LIVE_TEMPORAL", + Data: [][]byte{ + []byte(`{"ID":3}`), + []byte(`{"ID":4,"OCCURRED_AT":"not-a-timestamp"}`), + }, + })); err == nil { + t.Fatal("expected invalid Oracle temporal value to be rejected") + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM SQLBATCH_LIVE_TEMPORAL`).Scan(&count); err != nil || count != 2 { + t.Fatalf("prevalidated temporal request count=%d err=%v", count, err) + } + + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "SQLBATCH_LIVE_IDENTITY", + Data: [][]byte{[]byte(`{"ID":42,"NAME":"preserved"}`)}, + })); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, `SELECT NAME FROM SQLBATCH_LIVE_IDENTITY WHERE ID = 42`).Scan(&name); err != nil || name != "preserved" { + t.Fatalf("identity row: name=%q err=%v", name, err) + } + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "SQLBATCH_LIVE_ALWAYS", + Data: [][]byte{[]byte(`{"ID":42,"NAME":"rejected"}`)}, + })); err == nil || connect.CodeOf(err) != connect.CodeInvalidArgument { + t.Fatalf("expected GENERATED ALWAYS identity write to fail as invalid argument, got %v", err) + } + + if _, err := c.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: "SQLBATCH_LIVE_TEMPORAL", + Data: [][]byte{[]byte( + `{"ID":1,"OCCURRED_AT":"2026-10-15T12:34:56.123456-07:00","EVENT_DATE":"2026-10-15T00:00:00Z"}`, + )}, + })); err != nil { + t.Fatal(err) + } + var sourceOccurredAt, sourceEventDate time.Time + if err := db.QueryRowContext(ctx, `SELECT OCCURRED_AT, EVENT_DATE FROM SQLBATCH_LIVE_TEMPORAL WHERE ID = 1`).Scan(&sourceOccurredAt, &sourceEventDate); err != nil { + t.Fatal(err) + } + if want := time.Date(2026, 10, 15, 19, 34, 56, 123456000, time.UTC); !sourceOccurredAt.Equal(want) { + t.Fatalf("Oracle timestamp = %v, want instant %v", sourceOccurredAt, want) + } + if sourceEventDate.Year() != 2026 || sourceEventDate.Month() != time.October || sourceEventDate.Day() != 15 { + t.Fatalf("Oracle date = %v", sourceEventDate) + } + assertRFC3339SourceRow(t, ctx, db, c.dialect, `SELECT OCCURRED_AT, EVENT_DATE FROM SQLBATCH_LIVE_TEMPORAL WHERE ID = 1`, "OCCURRED_AT", "EVENT_DATE") + + if _, err := c.WriteUpdates(ctx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: "SQLBATCH_LIVE_ITEMS", + Updates: []*adiomv1.Update{{Id: id, Type: adiomv1.UpdateType_UPDATE_TYPE_DELETE}}, + })); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM SQLBATCH_LIVE_ITEMS WHERE TENANT = 'a' AND ID = 1`).Scan(&count); err != nil || count != 0 { + t.Fatalf("delete count=%d err=%v", count, err) + } +} diff --git a/connectors/sqlbatch/sink_test.go b/connectors/sqlbatch/sink_test.go new file mode 100644 index 00000000..2fced84c --- /dev/null +++ b/connectors/sqlbatch/sink_test.go @@ -0,0 +1,415 @@ +package sqlbatch + +import ( + "encoding/json" + "reflect" + "slices" + "testing" + + "github.com/adiom-data/dsync/connectors/sqlbatch/dialects" + adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" + "github.com/jackc/pgx/v5/pgtype" + "go.mongodb.org/mongo-driver/v2/bson" +) + +func TestDecodeJSONDocumentPreservesIntegerAndDecimalText(t *testing.T) { + columns := map[string]dialects.ColumnMetadata{ + "id": {Name: "id", DatabaseType: "int8"}, + "amount": {Name: "amount", DatabaseType: "numeric"}, + "nested": {Name: "nested", DatabaseType: "jsonb"}, + } + document, err := decodeJSONDocument([]byte(`{"id":9007199254740993,"amount":1234567890.0123456789,"nested":{"ok":true}}`), columns, dialects.Postgres{}) + if err != nil { + t.Fatal(err) + } + if document["id"] != int64(9007199254740993) { + t.Fatalf("id = %#v", document["id"]) + } + if document["amount"] != "1234567890.0123456789" { + t.Fatalf("amount = %#v", document["amount"]) + } + if document["nested"] != `{"ok":true}` { + t.Fatalf("nested = %#v", document["nested"]) + } +} + +func TestDecodeJSONDocumentNormalizesBatchedDecimalValues(t *testing.T) { + tests := []struct { + name string + dialect SQLDialect + columnType string + }{ + {name: "SQL Server", dialect: dialects.SQLServer{}, columnType: "decimal"}, + {name: "Oracle", dialect: dialects.Oracle{}, columnType: "NUMBER"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + columns := map[string]dialects.ColumnMetadata{ + "amount": {Name: "amount", DatabaseType: test.columnType}, + } + var valueType reflect.Type + for _, item := range []struct { + input string + want string + }{ + {input: `{"amount":1}`, want: "1"}, + {input: `{"amount":1.5}`, want: "1.5"}, + } { + document, err := decodeJSONDocument([]byte(item.input), columns, test.dialect) + if err != nil { + t.Fatal(err) + } + value := document["amount"] + if valueType == nil { + valueType = reflect.TypeOf(value) + } else if reflect.TypeOf(value) != valueType { + t.Fatalf("amount type = %T, want %v", value, valueType) + } + var text string + switch value := value.(type) { + case string: + text = value + case interface{ String() (string, error) }: + text, err = value.String() + if err != nil { + t.Fatal(err) + } + default: + t.Fatalf("amount = %#v (%T), want exact decimal representation", value, value) + } + if text != item.want { + t.Fatalf("amount = %q, want %q", text, item.want) + } + } + }) + } +} + +func TestDecodeJSONDocumentDoesNotSynthesizeGeneratedOrIdentityColumns(t *testing.T) { + columns := map[string]dialects.ColumnMetadata{ + "id": {Name: "id", DatabaseType: "bigint"}, + "computed": {Name: "computed", DatabaseType: "bigint", Nullable: true, Generated: true}, + "sequence": {Name: "sequence", DatabaseType: "bigint", Identity: true}, + } + document, err := decodeJSONDocument([]byte(`{"id":1}`), columns, dialects.SQLServer{}) + if err != nil { + t.Fatal(err) + } + if _, exists := document["computed"]; exists { + t.Fatal("omitted generated column was synthesized") + } + if _, exists := document["sequence"]; exists { + t.Fatal("omitted identity column was synthesized") + } +} + +func TestUpsertOperationUsesKeysThenPhysicalColumnOrder(t *testing.T) { + document := map[string]any{"name": "value", "id": int64(2), "tenant": "a", "active": true} + metadata := dialects.TableMetadata{ + Columns: []dialects.ColumnMetadata{ + {Name: "id"}, + {Name: "tenant"}, + {Name: "name", Nullable: true}, + {Name: "active", Nullable: true}, + }, + PrimaryKey: []string{"tenant", "id"}, + } + operation, err := upsertOperation(metadata, document) + if err != nil { + t.Fatal(err) + } + want := []string{"tenant", "id", "name", "active"} + if !reflect.DeepEqual(operation.columns, want) { + t.Fatalf("columns = %#v, want %#v", operation.columns, want) + } +} + +func TestUpsertOperationAppliesDefaultAndNullForOmittedColumns(t *testing.T) { + metadata := dialects.TableMetadata{ + Columns: []dialects.ColumnMetadata{ + {Name: "id"}, + {Name: "status", HasDefault: true}, + {Name: "note", Nullable: true}, + }, + PrimaryKey: []string{"id"}, + } + operation, err := upsertOperation(metadata, map[string]any{"id": int64(1)}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(operation.columns, []string{"id", "note"}) { + t.Fatalf("columns = %#v", operation.columns) + } + if !reflect.DeepEqual(operation.defaultColumns, []string{"status"}) { + t.Fatalf("default columns = %#v", operation.defaultColumns) + } + if !reflect.DeepEqual(operation.values, []any{int64(1), nil}) { + t.Fatalf("values = %#v", operation.values) + } +} + +func TestUpdateKeyValuesUsesConfiguredOrder(t *testing.T) { + value := func(name string, value any) *adiomv1.BsonValue { + t.Helper() + typ, data, err := bson.MarshalValue(value) + if err != nil { + t.Fatal(err) + } + return &adiomv1.BsonValue{Name: name, Type: uint32(typ), Data: data} + } + update := &adiomv1.Update{Id: []*adiomv1.BsonValue{value("id", int32(7)), value("tenant", "a")}} + metadata := dialects.TableMetadata{ + Columns: []dialects.ColumnMetadata{ + {Name: "tenant", DatabaseType: "text"}, + {Name: "id", DatabaseType: "int8"}, + }, + PrimaryKey: []string{"tenant", "id"}, + } + values, err := updateKeyValues(update, metadata, dialects.Postgres{}) + if err != nil { + t.Fatal(err) + } + want := []any{"a", int64(7)} + if !reflect.DeepEqual(values, want) { + t.Fatalf("values = %#v, want %#v", values, want) + } +} + +func TestUpdateKeyValuesRejectsMissingKey(t *testing.T) { + metadata := dialects.TableMetadata{Columns: []dialects.ColumnMetadata{{Name: "id", DatabaseType: "int8"}}, PrimaryKey: []string{"id"}} + _, err := updateKeyValues(&adiomv1.Update{}, metadata, dialects.Postgres{}) + if err == nil { + t.Fatal("expected missing-key error") + } +} + +func TestUpdateKeyValuesConvertsPostgresUUID(t *testing.T) { + data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + typ, encoded, err := bson.MarshalValue(bson.Binary{Subtype: bson.TypeBinaryUUID, Data: data}) + if err != nil { + t.Fatal(err) + } + metadata := dialects.TableMetadata{ + Columns: []dialects.ColumnMetadata{{Name: "id", DatabaseType: "uuid"}}, + PrimaryKey: []string{"id"}, + } + values, err := updateKeyValues(&adiomv1.Update{Id: []*adiomv1.BsonValue{{ + Name: "id", Type: uint32(typ), Data: encoded, + }}}, metadata, dialects.Postgres{}) + if err != nil { + t.Fatal(err) + } + want := pgtype.UUID{Bytes: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, Valid: true} + if !reflect.DeepEqual(values, []any{want}) { + t.Fatalf("values = %#v, want %#v", values, []any{want}) + } +} + +func TestUpdateKeyValuesRejectsLegacyUUID(t *testing.T) { + typ, encoded, err := bson.MarshalValue(bson.Binary{ + Subtype: bson.TypeBinaryUUIDOld, + Data: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + }) + if err != nil { + t.Fatal(err) + } + metadata := dialects.TableMetadata{ + Columns: []dialects.ColumnMetadata{{Name: "id", DatabaseType: "uuid"}}, + PrimaryKey: []string{"id"}, + } + _, err = updateKeyValues(&adiomv1.Update{Id: []*adiomv1.BsonValue{{ + Name: "id", Type: uint32(typ), Data: encoded, + }}}, metadata, dialects.Postgres{}) + if err == nil { + t.Fatal("expected legacy UUID subtype to be rejected") + } +} + +func TestUpdateKeyValuesNormalizesNumericRepresentations(t *testing.T) { + bsonValue := func(value any) *adiomv1.BsonValue { + t.Helper() + typ, data, err := bson.MarshalValue(value) + if err != nil { + t.Fatal(err) + } + return &adiomv1.BsonValue{Name: "id", Type: uint32(typ), Data: data} + } + decimalValue, err := bson.ParseDecimal128("1.5") + if err != nil { + t.Fatal(err) + } + updates := []*adiomv1.Update{ + {Id: []*adiomv1.BsonValue{bsonValue(int64(1))}}, + {Id: []*adiomv1.BsonValue{bsonValue(decimalValue)}}, + } + tests := []struct { + name string + dialect SQLDialect + columnType string + }{ + {name: "SQL Server", dialect: dialects.SQLServer{}, columnType: "decimal"}, + {name: "Oracle", dialect: dialects.Oracle{}, columnType: "NUMBER"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + metadata := dialects.TableMetadata{Columns: []dialects.ColumnMetadata{{Name: "id", DatabaseType: test.columnType}}, PrimaryKey: []string{"id"}} + var valueType reflect.Type + for _, update := range updates { + values, err := updateKeyValues(update, metadata, test.dialect) + if err != nil { + t.Fatal(err) + } + if valueType == nil { + valueType = reflect.TypeOf(values[0]) + } else if reflect.TypeOf(values[0]) != valueType { + t.Fatalf("key type = %T, want %v", values[0], valueType) + } + } + }) + } +} + +func TestOperationKeyDistinguishesOracleNumbers(t *testing.T) { + dialect := dialects.Oracle{} + column := dialects.ColumnMetadata{Name: "id", DatabaseType: "NUMBER"} + one, err := dialect.ConvertJSONValue(column, json.Number("1")) + if err != nil { + t.Fatal(err) + } + two, err := dialect.ConvertJSONValue(column, json.Number("2")) + if err != nil { + t.Fatal(err) + } + columns := []dialects.ColumnMetadata{{Name: "id", DatabaseType: "NUMBER"}} + if operationKey([]any{one}, columns) == operationKey([]any{two}, columns) { + t.Fatal("distinct Oracle numbers produced the same operation key") + } +} + +func TestOperationKeyCanonicalizesNumericRepresentationsOnly(t *testing.T) { + numeric := []dialects.ColumnMetadata{{Name: "id", DatabaseType: "decimal"}} + if operationKey([]any{"1"}, numeric) != operationKey([]any{"1.0"}, numeric) { + t.Fatal("equivalent decimal keys should produce the same operation key") + } + if operationKey([]any{"1"}, numeric) == operationKey([]any{"2"}, numeric) { + t.Fatal("distinct decimal keys should produce different operation keys") + } + text := []dialects.ColumnMetadata{{Name: "id", DatabaseType: "nvarchar"}} + if operationKey([]any{"A"}, text) == operationKey([]any{"a"}, text) { + t.Fatal("text keys should retain exact connector-side comparison") + } +} + +func TestUpsertOperationRejectsSuppliedGeneratedColumn(t *testing.T) { + metadata := dialects.TableMetadata{ + Columns: []dialects.ColumnMetadata{ + {Name: "id"}, + {Name: "computed", Generated: true}, + }, + PrimaryKey: []string{"id"}, + } + if _, err := upsertOperation(metadata, map[string]any{"id": int64(1), "computed": int64(2)}); err == nil { + t.Fatal("expected supplied generated column to be rejected") + } + operation, err := upsertOperation(metadata, map[string]any{"id": int64(1)}) + if err != nil { + t.Fatal(err) + } + if slices.Contains(operation.columns, "computed") { + t.Fatal("omitted generated column should not be written") + } +} + +func TestUpsertOperationIncludesSuppliedNonKeyIdentity(t *testing.T) { + metadata := dialects.TableMetadata{ + Columns: []dialects.ColumnMetadata{ + {Name: "id"}, + {Name: "sequence", Identity: true, IdentityGeneration: "ALWAYS"}, + }, + PrimaryKey: []string{"id"}, + } + operation, err := upsertOperation(metadata, map[string]any{"id": int64(1), "sequence": int64(42)}) + if err != nil { + t.Fatal(err) + } + if !operation.overrideIdentity || !reflect.DeepEqual(operation.identityColumns, []string{"sequence"}) { + t.Fatalf("identity operation = %#v", operation) + } + if !reflect.DeepEqual(operation.columns, []string{"id", "sequence"}) { + t.Fatalf("columns = %#v", operation.columns) + } +} + +func TestValidateIdentityOverride(t *testing.T) { + operation := writeOperation{overrideIdentity: true, identityColumns: []string{"id"}} + if err := validateIdentityOverride(dialects.Oracle{}, operation); err == nil { + t.Fatal("expected Oracle identity override to be rejected") + } + if err := validateIdentityOverride(dialects.Postgres{}, operation); err != nil { + t.Fatalf("Postgres identity override: %v", err) + } + if err := validateIdentityOverride(dialects.SQLServer{}, operation); err != nil { + t.Fatalf("SQL Server identity override: %v", err) + } +} + +func TestUpsertOperationRequiresCompleteGeneratedKey(t *testing.T) { + metadata := dialects.TableMetadata{ + Columns: []dialects.ColumnMetadata{{ + Name: "id", + Identity: true, + IdentityGeneration: "ALWAYS", + }}, + PrimaryKey: []string{"id"}, + } + _, err := upsertOperation(metadata, map[string]any{}) + if err == nil { + t.Fatal("expected missing-key error") + } + operation, err := upsertOperation(metadata, map[string]any{"id": int64(7)}) + if err != nil { + t.Fatal(err) + } + if !operation.overrideIdentity { + t.Fatal("expected identity override") + } +} + +func TestUpsertOperationRejectsComputedPrimaryKey(t *testing.T) { + metadata := dialects.TableMetadata{ + Columns: []dialects.ColumnMetadata{{Name: "id", Generated: true}}, + PrimaryKey: []string{"id"}, + } + _, err := upsertOperation(metadata, map[string]any{"id": int64(7)}) + if err == nil { + t.Fatal("expected generated-key error") + } +} + +func TestWriteBatchSizeUsesDefaultAndDestinationCap(t *testing.T) { + tests := []struct { + name string + configured int + maxParameters int + parametersPerRow int + want int + }{ + {name: "default", maxParameters: 65535, parametersPerRow: 3, want: DefaultWriteBatchSize}, + {name: "configured", configured: 25, maxParameters: 65535, parametersPerRow: 3, want: 25}, + {name: "default destination cap", maxParameters: 2000, parametersPerRow: 30, want: 66}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := writeBatchSize(test.configured, test.maxParameters, test.parametersPerRow) + if err != nil { + t.Fatal(err) + } + if got != test.want { + t.Fatalf("batch size = %d, want %d", got, test.want) + } + }) + } + if _, err := writeBatchSize(100, 2000, 30); err == nil { + t.Fatal("expected configured batch size above destination limit to fail") + } +} diff --git a/internal/gobregister/gobregister.go b/internal/gobregister/gobregister.go new file mode 100644 index 00000000..be63be97 --- /dev/null +++ b/internal/gobregister/gobregister.go @@ -0,0 +1,25 @@ +package gobregister + +import ( + "encoding/gob" + "net/netip" + "sync" + "time" + + "github.com/shopspring/decimal" +) + +var cursorTypesOnce sync.Once + +// RegisterCursorTypes registers concrete database types stored inside +// interface-valued gob cursors and composite IDs. gob's registry is global, so +// all connectors must share this once guard. +func RegisterCursorTypes() { + cursorTypesOnce.Do(func() { + gob.Register(time.Time{}) + gob.Register([16]byte{}) + gob.Register(decimal.Decimal{}) + gob.Register(netip.Addr{}) + gob.Register(netip.Prefix{}) + }) +}