From c38a37cc60deeef46d5ed8651e030160e60a7cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 9 Nov 2026 06:42:59 +0100 Subject: [PATCH 1/4] feat(sql): quoted arguments and implicit comparisons read as DuckDB does (SCHEMA-5b) Story SCHEMA-5b. Reviewed: r0:must-fix, r1:must-fix, r2:must-fix, r3:must-fix, r4:clean. Co-Authored-By: Claude Opus 5.5 --- documentation/sql/functions_conditional.md | 189 ++- documentation/sql/operators.md | 56 +- documentation/sql/type_conversion.md | 18 + ...tClientQuotedArgumentConditionalSpec.scala | 26 + ...lClientQuotedArgumentConditionalSpec.scala | 26 + ...lClientQuotedArgumentConditionalSpec.scala | 26 + .../arrow/JavaClientSchemaConsumerSpec.scala | 44 +- ...aClientQuotedArgumentConditionalSpec.scala | 26 + ...aClientQuotedArgumentConditionalSpec.scala | 26 + .../elastic/sql/function/cond/package.scala | 785 +++++++++--- .../sql/function/convert/package.scala | 27 +- .../elastic/sql/function/string/package.scala | 85 +- .../operator/math/ArithmeticExpression.scala | 14 +- .../app/softnetwork/elastic/sql/package.scala | 121 +- .../softnetwork/elastic/sql/query/Where.scala | 228 +++- .../elastic/sql/type/SQLTypeUtils.scala | 465 +++++++- .../elastic/schema/FieldCapsSpec.scala | 12 +- .../function/cond/MixedArmsRefusalSpec.scala | 8 +- .../function/cond/QuotedArmReadingSpec.scala | 1047 +++++++++++++++++ .../query/ImplicitComparisonCastSpec.scala | 396 +++++++ .../sql/query/NamelessKeyFilterSpec.scala | 15 +- .../elastic/sql/query/NullIfOperandSpec.scala | 89 +- .../sql/query/TypeCheckResolutionSpec.scala | 13 +- .../elastic/client/DateArithmeticSpec.scala | 30 +- .../QuotedArgumentConditionalSpec.scala | 821 +++++++++++++ 25 files changed, 4329 insertions(+), 264 deletions(-) create mode 100644 es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientQuotedArgumentConditionalSpec.scala create mode 100644 es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientQuotedArgumentConditionalSpec.scala create mode 100644 es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientQuotedArgumentConditionalSpec.scala create mode 100644 es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientQuotedArgumentConditionalSpec.scala create mode 100644 es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientQuotedArgumentConditionalSpec.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/QuotedArmReadingSpec.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/query/ImplicitComparisonCastSpec.scala create mode 100644 testkit/src/main/scala/app/softnetwork/elastic/client/QuotedArgumentConditionalSpec.scala diff --git a/documentation/sql/functions_conditional.md b/documentation/sql/functions_conditional.md index 14919e7e..497c37e8 100644 --- a/documentation/sql/functions_conditional.md +++ b/documentation/sql/functions_conditional.md @@ -22,8 +22,186 @@ column no mapping types — beside a `DATE` makes a `TIMESTAMP`. - **A `NULL` literal argument is folded**, as DuckDB and PostgreSQL do: `NULLIF(x, NULL)` is `x`, `NULLIF(NULL, x)` is `NULL`, `COALESCE` drops a `NULL` argument, and `GREATEST` / `LEAST` skip one. With no other argument left, the result is `NULL`. +- *Since `0.25.0`.* **A quoted argument is read as the type of the arguments beside it**, as DuckDB + reads it, in `COALESCE`, `GREATEST`, `LEAST` and the results of a `CASE`: beside a number it is + the number it spells (beside a whole type, the whole number it rounds to, half away from zero), + beside a `DATE` or a `TIMESTAMP` the date or the timestamp it names, beside a `BOOLEAN` the + boolean it spells. `GREATEST(n, '1.5')` over an `INT` column answers `2`, an `INT` (it was + refused); `COALESCE(n, '3')` answers `3`, an `INT`, where it answered the text `'3'` on every row; + `COALESCE(d, '2024-01-15')` is a `DATE` (it was text). The literal takes the column's type — + `GREATEST(t, '1.5')` over a `TINYINT` is a `TINYINT` — and a whole number past that type's range + makes a `BIGINT` (`GREATEST(t, '200')` is `200`). Every reader of the type follows: `DESCRIBE`, + JDBC and Arrow metadata, a `CREATE TABLE … AS SELECT` mapping, `ORDER BY`, a `GROUP BY` key, a + comparison or a cast of the result. A quoted argument that does not spell a value of that type + (`'abc'` beside a number, `'3'` beside a date) leaves the whole expression as it was: + `COALESCE(n, 'abc')` answers text and `GREATEST(n, 'abc')` is refused. +- *Since `0.25.0`.* **A `BOOLEAN` beside whole numbers answers `1` or `0`**, their type, as DuckDB + reads it: `COALESCE(b, 1)` answers `1` / `0` in a query (it answered `true` / `false` in an + integer column), and `GREATEST(b, 1)` / `LEAST(b, 1)` answer in a query and in a computed column + (they were refused in a query; a computed column stored no value). Beside a fractional number a + `BOOLEAN` is left as it was, and `GREATEST` over booleans alone is refused. - In a computed column (`SCRIPT AS`) and in a materialized view's computed field, a conditional is - rendered as before `0.24.0`, so a stored pipeline does not change. + rendered as before `0.24.0`, so a stored pipeline does not change — except where a quoted argument + is now read as a value (`COALESCE(n, '3')` stores the number `3`, and `'1.5'` beside an integer + column stores `2` where Elasticsearch truncated the text to `1`), for `GREATEST` / `LEAST` + over a `BOOLEAN` beside whole numbers, and for a `CASE` that compares a `BOOLEAN` with a number + (below): `ALTER TABLE` re-creates such a pipeline. A quoted argument read as a date or a + timestamp keeps the expression's `0.24.0` text type there (in a pipeline and in a transform + alike), so a function over it parses it as before — `DATE_ADD(COALESCE(d, '2024-01-15'), INTERVAL + 1 DAY)` stores `2024-01-16` where `d` is missing — and a comparison, an `IN`, a `BETWEEN`, a + `NULLIF` or a `CASE … WHEN` compares that text as before (`CASE WHEN COALESCE(d, '2024-01-15') = + '2024-01-15' THEN 1 ELSE 0 END` keeps its pipeline). `CREATE TABLE` and `ALTER TABLE` judge the + column on that text type too, as before: over a `KEYWORD` `k` and a `DATE` `d2`, + `NULLIF(COALESCE(d, '2024-01-15'), k)` is accepted and `NULLIF(COALESCE(d, '2024-01-15'), d2)` + is refused; `NULLIF(COALESCE(d, '2024-01-15'), k) - d2` and + `COALESCE(NULLIF(COALESCE(d, '2024-01-15'), k), d2) - d2` are accepted (they were refused; the + pipeline subtracts the dates as `COALESCE(d, '2024-01-15') - d2` does). A quoted argument read as + a number makes the conditional a number there too, so `NULLIF(COALESCE(n, '3'), k)`, + `NULLIF(k, COALESCE(n, '3'))`, `NULLIF(COALESCE(n, '3'), '2024-01-15')`, + `COALESCE(NULLIF(COALESCE(n, '3'), k), k)` and `COALESCE(NULLIF(COALESCE(n, '3'), k), g)` are + refused by `CREATE TABLE` and `ALTER TABLE` — including any `ALTER TABLE` of a table that holds + one — where the pipeline compared the number with the text and stored no value on every + document holding both. Beside a `DATE` it is stored as its date + (`'2024-01-15 10:30:00'` stores `'2024-01-15'`, which the `date` field reads; it stored the text + as written), beside a `TIMESTAMP` as written. In `CREATE TABLE` and `ALTER TABLE` this also holds + under a cast: `CAST(CASE WHEN … THEN d ELSE '2024-01-15 10:30:00' END AS VARCHAR)` stores + `'2024-01-15'` (it stored the text as written); its `AS DATE` and `AS TIMESTAMP` twins are refused + by Elasticsearch when the pipeline is created, as before. +- *Since `0.25.0`.* **A `CASE` that compares a `BOOLEAN` with a number in a computed column** + compares the boolean as `1` / `0`, as a query does (see + [Comparing values of different types](operators.md#operator-)). The pipeline compared the boolean + itself with the number and stored a wrong value or none (`b = 1` was never true, `b < 1` stored no + value, `b BETWEEN 1 AND 1` failed every document, `b <> 1` stored `1` for a `"true"` source). + With `b` a `BOOLEAN`, `n` an `INT`, `l` a `BIGINT` and `x` a `DOUBLE` column, these `SCRIPT AS` + shapes render a new pipeline, the same list in `CREATE TABLE` and in `ALTER TABLE` (which + re-creates it): + - `CASE WHEN b op n` and `CASE WHEN n op b`, and the same two over `l`, for every `op` of `=`, + `<>`, `!=`, `<`, `<=`, `>`, `>=`; + - `CASE WHEN b op x` and `CASE WHEN x op b` for `=`, `<>`, `!=` (an ordering against a `DOUBLE` + keeps its pipeline); + - `CASE WHEN b op 1` for every `op` of `=`, `<>`, `!=`, `<`, `<=`, `>`, `>=`, and + `CASE WHEN b IN (1)`, `CASE WHEN b NOT IN (1)`, `CASE WHEN b BETWEEN 1 AND 1`, + `CASE WHEN b NOT BETWEEN 1 AND 1`; the same eleven with `0`; + - `CASE WHEN b op 1.5` for `=`, `<>`, `!=`, and `CASE WHEN b IN (1.5)`, + `CASE WHEN b NOT IN (1.5)`; + - `CASE b WHEN v` for `v` of `n`, `l`, `x`, `1`, `0`, `1.5`, and `CASE v WHEN b` for `v` of `n`, + `l`, `x`; + - `GREATEST(b, v)` and `LEAST(b, v)` for `v` of `n`, `l`, `1`, `0`, and `GREATEST(v, b)`, + `LEAST(v, b)` for `v` of `n`, `l` (above); + - `NULLIF(b, v)` for `v` of `n`, `l`, `x`, `1`, `0`, `1.5`, and `NULLIF(v, b)` for `v` of `n`, + `l`, `x`, which were refused and are accepted. +- *Since `0.25.0`.* **A string function over a number, a boolean or a conditional in a computed + column** reads the value's text, as a query does (see + [Type conversion](type_conversion.md)): `UPPER(n)`, `LENGTH(n)`, `SUBSTRING(n, 1, 1)`, + `REVERSE(b)`, `UPPER(COALESCE(n, '3'))`, and `UPPER`, `LOWER`, `TRIM` or `LENGTH` over a `CASE`, + or over a `COALESCE` of a number, a date or a timestamp, render a new pipeline, which + `ALTER TABLE` re-creates. `UPPER(n)` stored no value; + `LOWER(CASE WHEN … THEN 'A' ELSE 'B' END)` stored `'A'` unchanged, lowering only the `ELSE`. +- *Since `0.25.0`.* The new types reach every consumer: a `UNION ALL` branch (`COALESCE(n, '3')` + is an `INT` slot, it was `VARCHAR`), a window function's argument, a nested conditional + (`COALESCE(b, NULL, 'true')` takes `COALESCE(b, NULL, TRUE)`'s type), a cast + (`CAST(CASE WHEN … THEN ti ELSE '1.5' END AS BIGINT)` converts the number `2` where it parsed the + text `'1.5'` and failed), an arithmetic (`COALESCE(n, '1.5') + 1` answers; it was refused), and a + string function, which reads the value's text (`UPPER(CASE WHEN … THEN n ELSE '3' END)`). Over an + `ARRAY` column the conditional reads its element in a query: `GREATEST(an, '1.5')` answers + (it was refused), as `GREATEST(an, 2)` does. A consumer that cannot take the new type is refused + as its unquoted twin is: `COALESCE(n, '3') LIKE '1%'`, `CAST(COALESCE(n, '3') AS DATE)`, + `COALESCE(n, '3') IN ('3', 'abc')` and, over a `KEYWORD` `k`, `COALESCE(d, '2024-01-15') = k`, + `COALESCE(n, '3') > k` and `CASE WHEN … THEN n ELSE '3' END > k` are refused (they failed on + every row, or answered wrong rows, before: `COALESCE(n, '3') > k` compared a number with a text, + which Painless refuses). So are, over a `KEYWORD` `k` and a `DATE` `d2`, a conditional holding a + quoted date or timestamp compared with `k` — `NULLIF(F, k)`, `NULLIF(k, F)`, + `CASE F WHEN k THEN …`, `F = k`, and the expressions over them, such as + `CASE WHEN NULLIF(F, k) = '2024-01-15' THEN …`, for `F` one of `COALESCE(d, '2024-01-15')`, + `COALESCE(d, '2024-01-15 10:30:00')`, `COALESCE(ts, '2024-01-15')`, + `COALESCE(ts, '2024-01-15 10:30:00')`, `COALESCE(d, ts, '2024-01-15 10:30:00')`, + `CASE WHEN … THEN d ELSE '2024-01-15' END`, `CASE n WHEN … THEN d ELSE '2024-01-15' END`, + `CASE WHEN … THEN ts ELSE '2024-01-15 10:30:00' END`, `NULLIF(COALESCE(d, '2024-01-15'), k)`, + `NULLIF(k, COALESCE(d, '2024-01-15'))` and `NULLIF(COALESCE(d, '2024-01-15'), '2024-01-15')`, + and the literal-first forms with a time part or beside a timestamp, + `COALESCE('2024-01-15 10:30:00', d)`, `COALESCE('2024-01-15', ts)` and + `COALESCE('2024-01-15 10:30:00', ts)` — and, over a number read so, + `CASE WHEN COALESCE(n, '3') = '2024-01-15' THEN …`, + `CASE WHEN COALESCE(n, '3') > '2024-01-10' THEN …`, + `CASE WHEN COALESCE(n, '3') BETWEEN d2 AND '2024-12-31' THEN …`, + `NULLIF(COALESCE(n, '3'), '2024-01-15')` and + `CASE WHEN NULLIF(COALESCE(n, '3'), k) = '2024-01-15' THEN …`: each compared a date or a number + with a text before. Over the three `CASE` operands and the literal-first forms with a time part + or beside a timestamp that answered wrong values on rows that tell the answers apart; over the + other operands it failed. Two forms answer instead, as the next two items say: a literal-first + `COALESCE('2024-01-15', d)` compared for equality with a text, and + `COALESCE(NULLIF(F, k), k)`. +- *Since `0.25.0`.* **A literal-first `COALESCE` whose literal is a date as `yyyy-MM-dd` compares + that literal's text for equality with a text**: `COALESCE('2024-01-15', d)` can only answer its + literal, so beside a text operand that is no literal (a `KEYWORD` `k`, `UPPER(k)`), in a query, + `=`, `<>`, `!=`, `NULLIF` and `CASE … WHEN` compare the text `'2024-01-15'`, and `NULLIF` + answers it, a `VARCHAR`: `CASE WHEN COALESCE('2024-01-15', d) = k THEN 1 ELSE 0 END`, + `NULLIF(COALESCE('2024-01-15', d), k)`, `NULLIF(k, COALESCE('2024-01-15', d))`, + `CASE COALESCE('2024-01-15', d) WHEN k THEN …`, `CASE k WHEN COALESCE('2024-01-15', d) THEN …`, + `COALESCE(NULLIF(COALESCE('2024-01-15', d), k), k)` and `WHERE COALESCE('2024-01-15', d) = k` + answer as before, the same values (a `NULL` argument before the literal, or a second literal + after it, too). DuckDB refuses them. A text that spells the same date differently is compared + as text, as before: `k = '2024-01-15T00:00:00Z'` or `'3'` does not equal it. Ordering + (`COALESCE('2024-01-15', d) < k`, `>= k`) stays refused: it failed. `BETWEEN` is the exception: + between two text bounds, `CASE WHEN COALESCE('2024-01-15', d) BETWEEN k AND g THEN 1 ELSE 0 END` + is still accepted, and still fails when the query runs, as before. Outside a query, beside a + number or a date, or where a `CASE` compares its operand with values that are not all text, the + conditional keeps its `DATE`. +- *Since `0.25.0`.* **`COALESCE(NULLIF(F, k), k)` answers `COALESCE(F, k)`**, in a query, where + `F` is a conditional holding a quoted date or timestamp and `k` a text: `F`'s text where `F` is + not `NULL`, else `k` — `COALESCE(NULLIF(CASE WHEN … THEN d ELSE '2024-01-15' END, k), k)` (and + over `ts`, and `CASE n WHEN …`), `COALESCE(NULLIF(COALESCE(d, '2024-01-15'), k), k)` (and over + `ts`, a time part, or `COALESCE(d, ts, …)`), `COALESCE(NULLIF(NULLIF(COALESCE(d, '2024-01-15'), + '2024-01-15'), k), k)`, `COALESCE(NULLIF(k, F), F)`, with more arguments after or before + (`COALESCE(NULLIF(F, k), k, g)`, `COALESCE(g, NULLIF(F, k), k)`) and under a string function. + Those over the three `CASE` operands answered the right dates before, in another text: a date + now answers `'2024-01-15'` where it answered `'2024-01-15T00:00Z'`, and the literal of a + timestamp `CASE` answers `'2024-01-15T10:30Z'` where it answered `'2024-01-15 10:30:00'`; the + others failed. Where `k` spells `F`'s date or instant in another text + (`k = '2024-01-15T00:00:00Z'` beside a date), the answer is `F`'s text, the same date. A `NULLIF` + that is not followed by its second argument, or whose argument is a column or a literal, keeps + its refusal (`COALESCE(NULLIF(F, k), g)`, `COALESCE(NULLIF(d, k), k)`). +- *Since `0.25.0`.* **`COALESCE` over a `GREATEST` or `LEAST` holding a literal answers**: such an + argument is never `NULL`, so `COALESCE` ends there, in a query, a computed column + (`CREATE TABLE`, `ALTER TABLE`, `UPDATE … SET`) and a transform. `COALESCE(GREATEST(n, 3), l)`, + `COALESCE(LEAST(x, 1.5), x)`, `COALESCE(GREATEST(n, 3), n, 7)` and their computed columns failed + to compile (`Cannot cast from [long] to [java.lang.Object]`), as did a computed column + `c VARCHAR SCRIPT AS (COALESCE(GREATEST(n, '3'), k))` (`Cannot cast from [java.lang.String] to + [double]`); the quoted forms, refused before — `COALESCE(GREATEST(n, '3'), n)`, + `COALESCE(LEAST(x, '1.5'), x)` over a `TINYINT`, an `INT` or a `DOUBLE` with `'3'`, `'1.5'` or + `'200'`, and `COALESCE(GREATEST(n, '2'), k)` — answer too. Per group the same rule renders, + but `SELECT g, COALESCE(GREATEST(MAX(n), 3), 0) … GROUP BY g` is refused before it runs, as + before. A `GREATEST` with no literal (`COALESCE(GREATEST(n, l), x)`) and a nested + `COALESCE(COALESCE(GREATEST(n, 3), l), x)` render as before. +- *Since `0.25.0`.* **Nested beside a text argument, in a query, a conditional that reads a quoted + argument answers text, as before**: over a `KEYWORD` `k`, `COALESCE(COALESCE(n, '3'), k)`, + `COALESCE(k, COALESCE(n, '3'))`, `CASE WHEN … THEN COALESCE(n, '3') ELSE k END`, + `COALESCE(CASE WHEN … THEN n ELSE '3' END, k)`, `COALESCE(COALESCE(d, '2024-01-15'), k)`, + `COALESCE(COALESCE(n, '3'), CAST(n AS VARCHAR))` or + `COALESCE(COALESCE(n, '3'), COALESCE(d, '2024-01-15'))` are a `VARCHAR`, each nested value + answered as the text of the value it is read as: `COALESCE(COALESCE(n, '1.5'), k)` answers `'2'` + where `n` is missing (it answered `'1.5'`), and a date its ISO date. A nested `GREATEST`, `LEAST` + or `NULLIF` value reading a quoted argument answers its text too, which also makes answered, over + a `DATE` `d2`, statements that were refused because the nested expression mixed a text with a + number or a date: `COALESCE(GREATEST(n, '2'), k)`, `COALESCE(GREATEST(d, '2024-01-15'), k)`, + `COALESCE(COALESCE(GREATEST(d, '2024-01-15'), d2), k)`, + `COALESCE(COALESCE(COALESCE(d, '2024-01-15'), d2), k)`, + `COALESCE(COALESCE(d2, COALESCE(d, '2024-01-15')), k)`, + `COALESCE(CASE WHEN … THEN COALESCE(d, '2024-01-15') ELSE d2 END, k)` and + `COALESCE(NULLIF(COALESCE(d, '2024-01-15'), d2), k)`. Beside a number or a date the nested value + keeps the type it is read as (`COALESCE(COALESCE(n, '3'), x)` is a `DOUBLE`), and arguments that + mix either way stay refused (`COALESCE(COALESCE(n, '3'), d)`). A computed column stores the + value the nested conditional reads (`COALESCE(COALESCE(n, '1.5'), k)` stores `2` where `n` is + missing). +- *Since `0.25.0`.* **`COALESCE` over a `CASE` tests the whole `CASE` for `NULL`**: + `COALESCE(CASE WHEN c THEN a ELSE b END, k)` answers `k` where `c` holds and `a` is `NULL`. It + tested the `ELSE` alone and answered `NULL` there, and a `NULL` `ELSE` + (`CASE WHEN c THEN a ELSE NULL END`) compared `null` with `null`. Every `COALESCE` with a `CASE` + argument before its last one renders new bytes — in a query, in a computed column (`CREATE TABLE`, + `ALTER TABLE`, which re-creates the pipeline, and `UPDATE … SET`) and in a transform — for + instance `COALESCE(CASE WHEN … THEN d ELSE '2024-01-15' END, k)`, + `COALESCE(CASE n WHEN 1 THEN d ELSE '2024-01-15' END, d2)` and + `COALESCE(CASE WHEN … THEN n ELSE l END, 7)`. --- @@ -346,6 +524,15 @@ FROM products A cast can *create* the mismatch as well as cure it: `NULLIF(CAST(qty AS KEYWORD), 0)` renders text against a number and is refused, even though `qty` is numeric. + *Since `0.25.0`*, a **query** compares a text with a number or a boolean, and a boolean with a + number, as DuckDB casts them (see + [Comparing values of different types](operators.md#operator-)): `NULLIF(status, 0)` reads + `status` as a `BIGINT` and answers `status`, NULL where it reads as `0`; a `status` that spells + no number fails the query. `NULLIF(flag, 0)` over a `BOOLEAN` compares it as `1` / `0`, in a query + and in a computed column. A **computed column** keeps the refusal above for a text against a + number or a boolean: an ingest pipeline would silently drop the document whose text converts to + nothing. + The same rule refuses a **date against a number** (`NULLIF(created_at, 0)`) and a **boolean against anything but a boolean** (`NULLIF(is_active, 0)`). Before **0.24.0** these were accepted and compared with Painless `==`, which answers `false` for a date against a number without ever diff --git a/documentation/sql/operators.md b/documentation/sql/operators.md index a07ce7a2..6f09cc7f 100644 --- a/documentation/sql/operators.md +++ b/documentation/sql/operators.md @@ -634,13 +634,17 @@ SELECT * FROM emp WHERE department = 'IT'; SELECT * FROM orders WHERE customer_id = shipping_customer_id; --- Columns of incompatible types (id KEYWORD, n INT) are refused by name once the column types --- are known, rather than answering no rows +-- Since 0.25.0, a text column against a number column (id KEYWORD, n INT) is compared as DuckDB +-- casts it: the text is read as a number (see "Comparing values of different types" below) SELECT * FROM t WHERE id = n; --- Error: Type mismatch: 'KEYWORD' is not compatible with 'INT' in expression: id = n + +-- Columns whose types no comparison reads (a KEYWORD against a DATE) are refused by name once the +-- column types are known, rather than answering no rows +SELECT * FROM t WHERE id = d; +-- Error: Type mismatch: 'KEYWORD' is not compatible with 'DATE' in expression: id = d -- Cast one side to the other's type -SELECT * FROM t WHERE id = CAST(n AS VARCHAR); +SELECT * FROM t WHERE id = CAST(d AS VARCHAR); ``` **String Comparison:** @@ -681,6 +685,50 @@ SELECT * FROM emp WHERE manager = NULL; -- Returns no rows! SELECT * FROM emp WHERE manager IS NULL; ``` +**Comparing values of different types** *(since `0.25.0`)*: + +A comparison between a text and a number or a boolean, and between a boolean and a number, answers +for the operators DuckDB binds, as DuckDB casts it: + +| Left / right | Operators | Compared as | +|--------------------------------------|---------------------------------------------|--------------------| +| text and `TINYINT` … `BIGINT` | `=`, `<>`, `!=`, `IN`, `NULLIF`, `CASE x WHEN` | `BIGINT` | +| text and `DOUBLE`, `REAL`, `NUMERIC` | `=`, `<>`, `!=`, `IN`, `NULLIF`, `CASE x WHEN` | `DOUBLE` | +| text and `BOOLEAN` | `=`, `<>`, `!=`, `IN`, `NULLIF`, `CASE x WHEN` | `BOOLEAN` | +| `BOOLEAN` and `TINYINT` … `BIGINT` | every comparison, `BETWEEN`, `IN` | the whole type (`1` / `0`) | +| `BOOLEAN` and `DOUBLE`, `REAL` | `=`, `<>`, `!=`, `IN`, `NULLIF`, `CASE x WHEN` | `DOUBLE` (`1.0` / `0.0`) | + +```sql +-- k KEYWORD holds '1', ' 1 ', '1.5'; n INT holds 1, 1, 2 +SELECT * FROM t WHERE k = n; -- the three rows: '1.5' is read as 2 +SELECT CASE WHEN k = 1 THEN 'one' ELSE 'other' END AS v FROM t; +SELECT NULLIF(k, 0) AS v FROM t; -- k itself, NULL where k reads as 0 +SELECT * FROM t WHERE b = n; -- b BOOLEAN read as 1 / 0 +``` + +- A text is read as a whole number by rounding half away from zero (`'1.5'` is `2`, `'-2.5'` is + `-3`), as a `DOUBLE` by the number it spells, as a `BOOLEAN` by the spellings a boolean literal + reads (`t`, `true`, `y`, `yes`, `on`, `1` / `f`, `false`, `n`, `no`, `off`, `0`, any case, + surrounding spaces ignored). A stored text that spells no such value (`'abc'`, `''`, `'1,5'`, + `'NaN'`, `'2'` beside a boolean) **fails the query**, as DuckDB fails it. `'on'`, `'off'` and + `' true '` beside a boolean answer, where DuckDB fails. +- Ordering a text against a number (`k < n`, `k BETWEEN 1 AND 2`) stays refused, as DuckDB refuses + it; so does a text against a `DATE` or a `TIMESTAMP` column. +- Over an `ARRAY` column, a query compares the element it reads. +- A `HAVING` group filter reads a boolean metric as `1.0` / `0.0`: `HAVING MAX(b) > 0`. +- A bare column compared with a literal in a `WHERE` clause (`WHERE k = 1`) is still a query on the + column's mapping, which Elasticsearch coerces as it always has; the rule above applies to a + scripted comparison (`CASE WHEN k = 1`, a function of the column, two columns). +- Outside a query — a computed column (`SCRIPT AS`, including the ones a `CREATE TABLE … AS + SELECT` copies), an ingest pipeline, a materialized view's transform — a text side keeps + `0.24.0`'s verdict: `NULLIF(k, 0)` is refused there, because a pipeline would silently drop a + document whose text converts to nothing. The boolean rows apply everywhere. A conditional that + reads a quoted argument as a number is that number there too (see + [Conditional functions](functions_conditional.md)), so `NULLIF(COALESCE(n, '3'), k)`, + `NULLIF(k, COALESCE(n, '3'))` and `NULLIF(COALESCE(n, '3'), '2024-01-15')` are refused in a + computed column, where `0.24.0` accepted them over a pipeline that stored no value on every + document holding `n` and `k`. + --- ### Operator: `<>`, `!=` diff --git a/documentation/sql/type_conversion.md b/documentation/sql/type_conversion.md index 130862c0..9764fefd 100644 --- a/documentation/sql/type_conversion.md +++ b/documentation/sql/type_conversion.md @@ -72,6 +72,24 @@ SELECT '2025-09-11'::DATE AS d, '125'::BIGINT AS b FROM logs; - `TRY_CAST` (`SAFE_CAST`) returns `NULL` instead of failing. - `::` is syntactic sugar, easier to read in queries. - Type inference relies on `baseType`, and explicit `CAST`/`TRY_CAST`/`::` updates the type context for following functions. +- *Since `0.25.0`.* A `BOOLEAN` cannot be cast to a `DATE`, a `TIME` or a `TIMESTAMP` — in a query, + a computed column and a pipeline alike, with `CAST`, `TRY_CAST` or `::` — as DuckDB refuses it: + `CAST(b AS DATE)` used to answer the boolean itself in a date column. +- *Since `0.25.0`.* A comparison between values of different types reads them as DuckDB casts them + (see [Comparing values of different types](operators.md#operator-)): a text compared with a + number is read as one, rounding half away from zero for a whole type. `CAST(k AS BIGINT)` keeps + its strict parse (`'1.5'` fails the cast). +- *Since `0.25.0`.* A string function (`UPPER`, `LOWER`, `TRIM`, `LENGTH`, `SUBSTRING`, `LEFT`, + `RIGHT`, `REPLACE`, `REVERSE`) over a number or a boolean reads its text: `UPPER(n)` and + `LENGTH(n)` answer (they failed the query), and over a `CASE` they read the whole `CASE`'s value. + The same holds in a computed column (`SCRIPT AS`), whose pipeline `ALTER TABLE` re-creates (see + [Conditional functions](functions_conditional.md)). +- *Since `0.25.0`.* A text that spells a number past the `DOUBLE` range (`'1e400'`, `'-1e400'`) is + read in two ways, as `0.24.0`'s literal rule and DuckDB read it. As a quoted literal it reads as no + number, like `'NaN'` and `'Infinity'`: `COALESCE(x, '1e400')` answers the text and + `GREATEST(x, '1e400')` is refused (DuckDB answers `1.5` and `inf`). As a stored text compared with + a `DOUBLE` column it is `+Infinity` or `-Infinity`, DuckDB's value: `k = x` is false and `k <> x` + true. A stored `'NaN'` or `'Infinity'` fails the query. --- diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientQuotedArgumentConditionalSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientQuotedArgumentConditionalSpec.scala new file mode 100644 index 00000000..5f73cc2e --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientQuotedArgumentConditionalSpec.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JestClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JestClientQuotedArgumentConditionalSpec + extends QuotedArgumentConditionalSpec + with ElasticDockerTestKit { + override lazy val client: GatewayApi = new JestClientSpi().client(elasticConfig) +} diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientQuotedArgumentConditionalSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientQuotedArgumentConditionalSpec.scala new file mode 100644 index 00000000..0b0a8502 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientQuotedArgumentConditionalSpec.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientQuotedArgumentConditionalSpec + extends QuotedArgumentConditionalSpec + with ElasticDockerTestKit { + override lazy val client: GatewayApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientQuotedArgumentConditionalSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientQuotedArgumentConditionalSpec.scala new file mode 100644 index 00000000..0b0a8502 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientQuotedArgumentConditionalSpec.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientQuotedArgumentConditionalSpec + extends QuotedArgumentConditionalSpec + with ElasticDockerTestKit { + override lazy val client: GatewayApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala index b389fc62..16cd8fe3 100644 --- a/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala +++ b/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala @@ -95,6 +95,21 @@ class JavaClientSchemaConsumerSpec extends AnyFlatSpecLike with ElasticDockerTes "s2" -> """{"id":"s2","k":"b","sf":2.5,"hf":1.5,"u":8,"dn":"2024-02-01T00:00:00Z","ip":"10.0.0.2"}""" ) + // SCHEMA-5b (D14): a quoted argument read as the type beside it, a BOOLEAN beside a whole number, + // a text compared with a number + private val s5 = "s5c_t" + + private val s5Mapping: String = + """{"properties": { + | "id": {"type": "keyword"}, "n": {"type": "integer"}, "b": {"type": "boolean"}, + | "k": {"type": "keyword"} + |}}""".stripMargin + + private val s5Docs: Seq[(String, String)] = Seq( + "q1" -> """{"id":"q1","n":1,"b":true,"k":"5"}""", + "q2" -> """{"id":"q2","b":false,"k":"0"}""" + ) + private def load(index: String, mappingJson: String, documents: Seq[(String, String)]): Unit = { Try(client.deleteIndex(index)) client @@ -112,10 +127,11 @@ class JavaClientSchemaConsumerSpec extends AnyFlatSpecLike with ElasticDockerTes load(s7, s7Mapping, s7Docs) load("s7c_p1", """{"properties": {"k": {"type": "keyword"}}}""", Seq("p1" -> """{"k":"007"}""")) load("s7c_p2", """{"properties": {"k": {"type": "long"}}}""", Seq("p2" -> """{"k":7}""")) + load(s5, s5Mapping, s5Docs) } override def afterAll(): Unit = { - Seq(table, s7, "s7c_p1", "s7c_p2").foreach(index => Try(client.deleteIndex(index))) + Seq(table, s7, "s7c_p1", "s7c_p2", s5).foreach(index => Try(client.deleteIndex(index))) super.afterAll() } @@ -484,4 +500,30 @@ class JavaClientSchemaConsumerSpec extends AnyFlatSpecLike with ElasticDockerTes rows.map(_("ck")).distinct shouldBe Seq("C") rowsAndSchema(run(s"SELECT k FROM $s7Pattern"))._1.map(_("k")).toSet shouldBe Set("007", "7") } + + "Arrow" should "read SCHEMA-5b's conditionals by the types core reports (D14)" in { + val sql = s"SELECT id, COALESCE(n, '3') AS c, GREATEST(b, 1) AS g, NULLIF(k, 0) AS z FROM $s5" + val result = run(sql) + val (rows, schema) = rowsAndSchema(result) + extractRows(result).map(_.toMap) shouldBe rows.map(_.toMap) + // COALESCE(n, '3') was VARCHAR, GREATEST(b, 1) and NULLIF(k, 0) refused, before SCHEMA-5b + schema.map(_.columns.map(c => c.name -> c.sqlType.typeId)) shouldBe Some( + Seq("id" -> "VARCHAR", "c" -> "INT", "g" -> "BIGINT", "z" -> "VARCHAR") + ) + rows + .map(r => String.valueOf(r("id")) -> ((r("c"), r("g"), r.getOrElse("z", null)))) + .toMap shouldBe Map( + "q1" -> ((Integer.valueOf(1), java.lang.Long.valueOf(1L), "5")), + "q2" -> ((Integer.valueOf(3), java.lang.Long.valueOf(1L), null)) + ) + } + + "The JDBC driver" should "read SCHEMA-5b's conditionals by its match (D14)" in { + val rows = extractRows(run(s"SELECT id, COALESCE(n, '3') AS c, GREATEST(b, 1) AS g FROM $s5")) + rows + .map(r => ValueCoercion.coerceToString(r("id")) -> ValueCoercion.coerceToString(r("c"))) + .toMap shouldBe + Map("q1" -> "1", "q2" -> "3") + rows.map(r => ValueCoercion.coerceToString(r("g"))).toSet shouldBe Set("1") + } } diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientQuotedArgumentConditionalSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientQuotedArgumentConditionalSpec.scala new file mode 100644 index 00000000..69395cbf --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientQuotedArgumentConditionalSpec.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientQuotedArgumentConditionalSpec + extends QuotedArgumentConditionalSpec + with ElasticDockerTestKit { + override lazy val client: GatewayApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientQuotedArgumentConditionalSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientQuotedArgumentConditionalSpec.scala new file mode 100644 index 00000000..69395cbf --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientQuotedArgumentConditionalSpec.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientQuotedArgumentConditionalSpec + extends QuotedArgumentConditionalSpec + with ElasticDockerTestKit { + override lazy val client: GatewayApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 8ed3aab4..322263d9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -165,6 +165,94 @@ package object cond { */ trait AnswersArms extends Function { def answeredArms: List[PainlessScript] + + private[this] val _armReadings = new SQLTypeUtils.PerScope[Option[SQLTypeUtils.ArmReadings]] + private[this] val _armTypes = new SQLTypeUtils.PerScope[List[SQLType]] + + /** What the string-literal arms are READ as (SCHEMA-5b D1, `SQLTypeUtils.armReadings`), + * computed ONCE per node and per scope (`SQLTypeUtils.PerScope`, A-45 (2), A-95): the type + * rule, the reported type and the rendering all read it. Attaching the schema copies the node, + * so a resolved statement reads its literals against the resolved types. A computed column's + * validation has its own slot: an `ARRAY` arm is read whole there and as its element in a + * query. + */ + final def armReadings: Option[SQLTypeUtils.ArmReadings] = + _armReadings(SQLTypeUtils.armReadings(answeredArms)) + + /** The type each arm is read as (SCHEMA-5b D2 / D5, `SQLTypeUtils.armTypesOf`), per scope as + * [[armReadings]]. In a query, a nested conditional holding a read quoted argument is the + * VARCHAR 0.24.0 typed it where only that keeps the arms from mixing (A-97). + */ + final def armTypes: List[SQLType] = + _armTypes(SQLTypeUtils.armTypesOf(answeredArms, armReadings)) + + /** The reading of the arm at `index`, if it is a string literal read beside the others. */ + final def readingAt(index: Int): Option[ComparisonRule.Reading] = + armReadings.flatMap(_.readings.get(index)) + + /** The position of `arm` among [[answeredArms]], by reference. */ + final def armIndex(arm: PainlessScript): Int = answeredArms.indexWhere(_ eq arm) + + /** The rendering of the READ literal at `index` (SCHEMA-5b D3 (4)), if it is one. + * + * Outside a query (A-92) a TIMESTAMP reading is the literal as written -- the arm's own + * rendering, 0.24.0's bytes, which every parser of a consumer reads -- and a DATE reading its + * ISO date (`SQLTypeUtils.readPainless`). + */ + final def readArmPainless(index: Int, context: Option[PainlessContext]): Option[String] = + for { + r <- armReadings + reading <- r.readings.get(index) + } yield reading match { + case t: ComparisonRule.TemporalReading + if t.sqlType != SQLTypes.Date && SQLTypeUtils.rendersOutsideQuery => + answeredArms(index).painless(context) + case _ => SQLTypeUtils.readPainless(reading, r.literalType, context) + } + + /** 🔴 Is the arm at `index` a literal read as a DATE or a TIMESTAMP where a script reads whole + * values (`SQLTypeUtils.readsWholeValues`): a processor, a transform, and the validation of + * the computed column that installs such a processor? Such an arm renders TEXT in the + * processor (its ISO date, or the literal as written, [[readArmPainless]]: `ctx` holds JSON), + * so it keeps its own type, text, wherever a type of the node decides between a text and a + * temporal value (A-92, #292): the legacy type ([[legacyArmTypes]]: `baseType`, `out`) and the + * type a comparison reads the node with (`SQLTypeUtils.textTypeOutsideQuery`). + * + * The validation reads it too (A-96): a verdict judges the processor it installs. Typed DATE + * there, `NULLIF(COALESCE(d, '2024-01-15'), k)` was refused on CREATE and on every later ALTER + * of a table holding it ("compares DATE with KEYWORD"), where 0.24.0 compares the two texts, + * and `YEAR(NULLIF(COALESCE(d, '2024-01-15'), d2))` was accepted over a processor calling + * `.get(ChronoField.YEAR)` on a String, where 0.24.0 refuses it. + */ + final def readsAsTextOutsideQuery(index: Int): Boolean = + SQLTypeUtils.readsWholeValues && + readingAt(index).exists(_.isInstanceOf[ComparisonRule.TemporalReading]) + + /** Is the arm at `index` RENDERED as text ([[readsAsTextOutsideQuery]]), in a script rendered + * outside a query (`SQLTypeUtils.rendersOutsideQuery`)? The rendering's switch: the arm a + * COALESCE cuts at, and a CASE result's conversion (`Case.armOf`). + */ + final def rendersAsTextOutsideQuery(index: Int): Boolean = + SQLTypeUtils.rendersOutsideQuery && readsAsTextOutsideQuery(index) + + /** The types the legacy derivations read (an ingest processor's, arms with no common type): + * each arm's `out`, a read literal its reading's type (SCHEMA-5b D3 (5)). + * + * 🔴 Outside a query, and while a computed column is validated (A-96), a literal read as a + * DATE or a TIMESTAMP keeps its own type, text ([[readsAsTextOutsideQuery]], A-92), so the + * node keeps 0.24.0's type and every consumer keeps 0.24.0's parse of it. Typed DATE over that + * text, `DATE_ADD(COALESCE(d, '2024-01-15'), …)` called `.plus` on a String: every document + * failed, and `ignore_failure` dropped the column silently. + */ + final def legacyArmTypes: List[SQLType] = + answeredArms.zipWithIndex.map { case (a, i) => + if (readsAsTextOutsideQuery(i)) a.out + else + readingAt(i) match { + case Some(_) => armReadings.map(_.literalType).getOrElse(a.out) + case None => a.out + } + } } /** 🔴 DuckDB 1.5.5's refusal of a conditional whose arms have NO common type, asked once the @@ -173,47 +261,27 @@ package object cond { * INTEGER and DATE in CASE expression - an explicit cast is required`, measured). Rendered, such * a node answered two classes in one column, or failed in Painless. * - * Read on what each arm REPORTS. A BOOLEAN beside a number is accepted, as DuckDB types it an - * INTEGER. Left out, as DuckDB reads them by the other arms: a NULL, a string literal (read as - * the type beside it), an arm of a type still unknown (no schema, `date_nanos`); and an - * aggregate arm, whose per-group value Elasticsearch computes. `expression` names the node as - * DuckDB does (`CASE expression`, `COALESCE operator`); DuckDB names the newer arm first in a - * CASE and the earlier one first in a COALESCE. + * Read on the type each arm is READ as (`AnswersArms.armTypes`), by the ONE family rule + * (`SQLTypeUtils.mixedArmTypes`, SCHEMA-5b A-97) that the arm types' own text rule reads. A + * BOOLEAN beside a number is accepted, as DuckDB types it an INTEGER. Left out, as DuckDB reads + * them by the other arms: a NULL, a string literal (read as the type beside it), an arm of a + * type still unknown (no schema, `date_nanos`); and an aggregate arm, whose per-group value + * Elasticsearch computes. In a query, a conditional holding a read quoted argument nested beside + * text is the VARCHAR 0.24.0 typed it ([[AnswersArms.armTypes]]), so it mixes with nothing it + * did not mix with then. `expression` names the node as DuckDB does (`CASE expression`, + * `COALESCE operator`); DuckDB names the newer arm first in a CASE and the earlier one first in + * a COALESCE. */ private[cond] def mixedArmsError( - arms: List[PainlessScript], + node: AnswersArms, expression: String, newerFirst: Boolean - ): Option[String] = { - def family(t: SQLType): Int = - if (t.isNumber) 0 - else if (t == SQLTypes.Boolean) 1 - else if ( - t == SQLTypes.Date || t == SQLTypes.DateTime || t == SQLTypes.Timestamp || - t == SQLTypes.Temporal - ) 2 - else if (t == SQLTypes.Time) 3 - else if (t.isText) 4 - else -1 - // a BOOLEAN and a number have a common type, INTEGER - def common(a: Int, b: Int): Boolean = a == b || Set(a, b) == Set(0, 1) - if (arms.exists(_.hasAggregation)) None - else { - val typed = arms - .filterNot(a => ComparisonRule.stringLiteral(a).isDefined) - // an `ARRAY` arm is one X element (SCHEMA-6 D7) - .map(a => SQLTypeUtils.operandType(a.reportedType)) - .filterNot(t => t == SQLTypes.Null || t.isUnknown) - .filter(family(_) >= 0) - typed.headOption.flatMap { first => - typed.tail.find(t => !common(family(first), family(t))).map { other => - val (a, b) = if (newerFirst) (other, first) else (first, other) - s"Cannot mix values of type ${a.typeId} and ${b.typeId} in $expression - an explicit " + - "cast is required" - } - } + ): Option[String] = + SQLTypeUtils.mixedArmTypes(node.answeredArms, node.armTypes).map { case (first, other) => + val (a, b) = if (newerFirst) (other, first) else (first, other) + s"Cannot mix values of type ${a.typeId} and ${b.typeId} in $expression - an explicit " + + "cast is required" } - } sealed trait ConditionalFunction[In <: SQLType] extends TransformFunction[In, SQLBool] @@ -276,13 +344,44 @@ package object cond { extends TransformFunction[SQLAny, SQLType] with FunctionWithIdentifier with AnswersArms { - override def answeredArms: List[PainlessScript] = values + + /** The arms COALESCE answers (SCHEMA-5b A-98 β, R-115): its arguments, but in a QUERY a + * `NULLIF(a, b)` arm directly followed by `b` that its temporal-against-text rule refuses over + * a read-moved conditional (`NullIf.answeredByFirst`) is `a` -- `COALESCE(NULLIF(a, b), b)` is + * `COALESCE(a, b)` -- where the arms then mix nothing (`SQLTypeUtils.mixedArmTypes`, A-97 + * typing a read-moved arm VARCHAR beside text). `COALESCE(NULLIF(CASE WHEN n > 0 THEN d ELSE + * '2024-01-15' END, k), k)` answers, per row, that CASE's text or `k`. One pass over the + * arguments; computed once per scope. The type, the refusal and the rendering read it (the + * replaced NULLIF is not judged: `functionsOf` walks [[args]]); `sql` and `update` keep + * `values`. + */ + override def answeredArms: List[PainlessScript] = _answeredArms(nullIfsAnsweredByFirst) + + private[this] val _answeredArms = new SQLTypeUtils.PerScope[List[PainlessScript]] + + private[this] def nullIfsAnsweredByFirst: List[PainlessScript] = + if (SQLTypeUtils.readsWholeValues) values + else { + val arms = values.zipWithIndex.map { case (arm, i) => + SQLTypeUtils.conditionalNode(arm) match { + case n: NullIf + if values.lift(i + 1).exists(_.sql == n.expr2.sql) && n.answeredByFirst => + n.expr1 + case _ => arm + } + } + if (arms.corresponds(values)(_ eq _)) values + else { + val types = SQLTypeUtils.armTypesOf(arms, SQLTypeUtils.armReadings(arms)) + if (SQLTypeUtils.mixedArmTypes(arms, types).isEmpty) arms else values + } + } def operator: ConditionalOp = Coalesce override def fun: Option[ConditionalOp] = Some(operator) - override def args: List[PainlessScript] = values + override def args: List[PainlessScript] = answeredArms override def outputType: SQLType = baseType //SQLTypeUtils.leastCommonSuperType(args.map(_.baseType)) @@ -299,13 +398,17 @@ package object cond { * `COALESCE(CAST(n AS DOUBLE), 2)` the double of `n` or the integer 2, `COALESCE(CAST(d AS * DATE), ts)` a `LocalDate` or a `ZonedDateTime`. Arms with no common type keep the least * common supertype of their `out` and render as they did (`SQLTypeUtils.armsType`). + * + * The legacy type also where the node is text (`SQLTypeUtils.textTypeOutsideQuery`: a literal + * read as a date, the validation of a computed column, A-96), as its processor types it. */ override def baseType: SQLType = - if (SQLTypeUtils.rendersOutsideQuery) SQLTypeUtils.leastCommonSuperType(argTypes) + if (SQLTypeUtils.rendersOutsideQuery || SQLTypeUtils.textTypeOutsideQuery(this).isDefined) + SQLTypeUtils.leastCommonSuperType(legacyArmTypes) else SQLTypeUtils - .armsType(answeredArms, reportedType) - .getOrElse(SQLTypeUtils.leastCommonSuperType(argTypes)) + .armsType(this, reportedType) + .getOrElse(SQLTypeUtils.leastCommonSuperType(legacyArmTypes)) /** `COALESCE` returns whichever arm is first non-null, so the type it REPORTS is the common * super type of the arms — the same derivation as [[baseType]], over the arms' own reported @@ -314,7 +417,15 @@ package object cond { * `TIMESTAMP` column is `TIMESTAMP`. An arm of a still-unknown type (a `date_nanos` column) * beside a DATE makes it a TIMESTAMP (`SQLTypeUtils.armsReportedType`). */ - override def reportedType: SQLType = SQLTypeUtils.armsReportedType(reportedArgTypes) + override def reportedType: SQLType = SQLTypeUtils.armsReportedType(armTypes) + + /** A string literal read beside the other arms renders the value it is read as (SCHEMA-5b D3 + * (4)): `COALESCE(n, '3')` answers the number 3, not the text. + */ + override protected def ownArgumentPainless( + index: Int, + context: Option[PainlessContext] + ): Option[String] = readArmPainless(index, context) override def validate(): Either[String, Unit] = { if (values.isEmpty) Left("COALESCE requires at least one argument") @@ -323,7 +434,7 @@ package object cond { /** Arms with no common type are refused, as DuckDB refuses them ([[mixedArmsError]]). */ override def typeError: Option[String] = - mixedArmsError(answeredArms, "COALESCE operator", newerFirst = false) + mixedArmsError(this, "COALESCE operator", newerFirst = false) override def checkIfNullable: Boolean = false @@ -337,10 +448,10 @@ package object cond { // `.sql` and an ingest processor's bytes are unchanged. case _ if SQLTypeUtils.convertsArms(context) && - values.exists(ArithmeticExpression.nullLiteral) => + answeredArms.exists(ArithmeticExpression.nullLiteral) => callArgs .map(_.trim) - .zip(values) + .zip(answeredArms) .filterNot(a => ArithmeticExpression.nullLiteral(a._2)) match { case Nil => "null" case (single, value) :: Nil => arm(single, value, last = true, context) @@ -351,23 +462,52 @@ package object cond { // opened nowhere: `String.valueOf( : param1))`, which Painless cannot parse. Degenerate SQL, // legal SQL, and UNREACHABLE until issue #367 stopped predicates from dropping the operand's // transform — `WHERE COALESCE(d) = 'x'` compared the raw column before and emits this now. - case single :: Nil => arm(single.trim, values.head, last = true, context) + case single :: Nil => arm(single.trim, answeredArms.head, last = true, context) // 🔴 A RIGHT fold, because the old one opened `values.length - 1` parentheses and closed // exactly ONE: at two arguments that balances by luck, and at three or more it emitted // `(a != null ? a : (b != null ? b : c)` — Painless cannot parse it, so `COALESCE(a, b, c)` // was a shard failure in EVERY venue, SELECT included, and no test had ever looked. Found by // the delimiter check in `PredicateTransformSurvivalSpec`. Byte-identical at two arguments, // which is what every existing fixture pins. - case _ => folded(callArgs.map(_.trim).zip(values), context) + case _ => folded(callArgs.map(_.trim).zip(answeredArms), context) } private[this] def folded( args: List[(String, PainlessScript)], context: Option[PainlessContext] - ): String = - args.init.foldRight(arm(args.last._1, args.last._2, last = true, context)) { + ): String = { + // a READ literal is never NULL (SCHEMA-5b): COALESCE ends there, and the arms after it can + // never be answered -- `3 != null` would not even compile. Outside a query a read date or + // timestamp is text, typed and rendered as 0.24.0 renders it (A-92), so it is tested as then. + // Nor is a GREATEST / LEAST holding a literal (A-99, R-116: `NumericReducer.neverNull`), in + // every scope: tested, its primitive cast failed to compile (`((long) …) != null`) + val cut = args.indexWhere { case (_, value) => + val index = armIndex(value) + (readingAt(index).isDefined && !rendersAsTextOutsideQuery(index)) || + (SQLTypeUtils.conditionalNode(value) match { + case reducer: NumericReducer => reducer.neverNull + case _ => false + }) + } + val kept = if (cut >= 0) args.take(cut + 1) else args + kept.init.foldRight(arm(kept.last._1, kept.last._2, last = true, context)) { case ((rendered, value), rest) => - s"($rendered != null ? ${arm(rendered, value, last = false, context)} : $rest)" + val tested = nullTested(rendered, value) + s"($tested != null ? ${arm(rendered, value, last = false, context)} : $rest)" + } + } + + /** 🔴 An arm as its NULL test reads it (SCHEMA-5b R-113): a `CASE` arm renders an unenclosed + * conditional (`c ? a : b`), and `c ? a : b != null ? … : k` tests `b` alone -- Painless's + * conditional binds last -- so `COALESCE(CASE WHEN c THEN a ELSE b END, k)` answered `a`, + * NULL, where `c` held and `a` was NULL, instead of `k`, and a NULL `ELSE` compared `null != + * null`. The arm is tested enclosed, as ONE value (`SQLTypeUtils.asOneValue`); every other + * rendering keeps its bytes. In every scope: the processor tested `b` alone too. + */ + private[this] def nullTested(rendered: String, value: PainlessScript): String = + SQLTypeUtils.conditionalNode(value) match { + case _: Case => SQLTypeUtils.asOneValue(rendered) + case _ => rendered } /** An arm as COALESCE answers it: in a query, the type COALESCE reports, converted once from @@ -383,17 +523,30 @@ package object cond { ): String = if ( !SQLTypeUtils.convertsArms(context) || - SQLTypeUtils.armsType(answeredArms, reportedType).isEmpty + SQLTypeUtils.armsType(this, reportedType).isEmpty ) rendered else { + val index = armIndex(value) val converted = - SQLTypeUtils.convertArm( - value, - rendered, - baseType, - last && SQLTypeUtils.mayRenderNull(value, context), - context - ) + armReadings.filter(_.readings.contains(index)) match { + // a read literal is converted FROM the type it is read as (SCHEMA-5b D3 (4)) + case Some(r) => + SQLTypeUtils.convertRendered( + rendered, + r.literalType, + baseType, + nullable = false, + context + ) + case None => + SQLTypeUtils.convertArm( + value, + rendered, + baseType, + last && SQLTypeUtils.mayRenderNull(value, context), + context + ) + } // a primitive (`((double) x)`) opposite a branch that can be NULL is boxed, as every // conversion behind a NULL guard is (`SQLTypeUtils.coerce`) if (last || converted == rendered || !baseType.isNumber) converted @@ -408,7 +561,7 @@ package object cond { */ override def nullable: Boolean = if (SQLTypeUtils.rendersOutsideQuery) values.forall(_.nullable) - else values.filterNot(ArithmeticExpression.nullLiteral).forall(_.nullable) + else answeredArms.filterNot(ArithmeticExpression.nullLiteral).forall(_.nullable) override def update(request: query.SingleSearch): Coalesce = { this.copy(values = values.map { @@ -436,11 +589,53 @@ package object cond { private[cond] case object WhenAsResult extends WhenComparison private[cond] case object WhenInstants extends WhenComparison private[cond] case object WhenNumbers extends WhenComparison + private[cond] case object WhenImplicit extends WhenComparison + + /** A side of the implicit-cast table (`ComparisonRule.implicitTarget`, SCHEMA-5b D6): neither a + * string literal -- READ by what it is compared with -- nor NULL. + */ + private[cond] def tableSide(arg: PainlessScript): Boolean = + ComparisonRule.stringLiteral(arg).isEmpty && !ArithmeticExpression.nullLiteral(arg) + + /** The implicit target of `left` against `right` (SCHEMA-5b D6) for a pair main's rule refuses + * (`mainRefuses`), with the table's sides only, and no text side beside an aggregate (a group + * filter converts nothing); `None` otherwise. + */ + private[cond] def implicitTargetOf( + leftArg: PainlessScript, + rightArg: PainlessScript, + left: SQLType, + right: SQLType, + equality: Boolean, + mainRefuses: Boolean + ): Option[SQLType] = + if (!mainRefuses || !tableSide(leftArg) || !tableSide(rightArg)) None + else { + val text = + ComparisonRule.textType(SQLTypeUtils.operandType(left)) || + ComparisonRule.textType(SQLTypeUtils.operandType(right)) + if (text && (leftArg.hasAggregation || rightArg.hasAggregation)) None + else ComparisonRule.implicitTarget(left, right, equality) + } case class NullIf(expr1: PainlessScript, expr2: PainlessScript) extends ConditionalFunction[SQLAny] with AnswersArms { - override def answeredArms: List[PainlessScript] = List(expr1) + + /** The first argument as NULLIF compares and answers it: the literal a literal-first `COALESCE` + * is read in place of beside a text second argument (SCHEMA-5b A-98 α, + * `ComparisonRule.literalInPlaceOf`), else `expr1`. `NULLIF(COALESCE('2024-01-15', d), k)` + * compares and answers `'2024-01-15'`, VARCHAR, as 0.24.0 did (right on every row, RUN). Every + * reader of an argument's type, reading or rendering reads [[first]] and [[second]]. + */ + private[cond] def first: PainlessScript = + ComparisonRule.literalInPlaceOf(expr1, expr2).getOrElse(expr1) + + /** The second argument as NULLIF compares it ([[first]]). */ + private[cond] def second: PainlessScript = + ComparisonRule.literalInPlaceOf(expr2, expr1).getOrElse(expr2) + + override def answeredArms: List[PainlessScript] = List(first) override def conditionalOp: ConditionalOp = NullIf @@ -454,11 +649,15 @@ package object cond { * that argument is converted to in a query where it is the answer ([[toPainlessCall]]), as a * `CASE` converts its results. `NULLIF(d, x)` over a DATE `d` answered `d`'s doc value, a * `ZonedDateTime`, in a DATE column. A first argument READ beside the second (`firstReading`) - * renders that reading already, and keeps [[comparedType]]. + * renders that reading already, and keeps [[comparedType]], as does a first argument that is + * text where its processor runs (`SQLTypeUtils.textTypeOutsideQuery`, A-96). */ override def baseType: SQLType = - (if (firstReading.isDefined || SQLTypeUtils.rendersOutsideQuery) None - else SQLTypeUtils.armsType(answeredArms, reportedType)).getOrElse(comparedType) + (if ( + firstReading.isDefined || SQLTypeUtils.rendersOutsideQuery || + SQLTypeUtils.textTypeOutsideQuery(this).isDefined + ) None + else SQLTypeUtils.armsType(this, reportedType)).getOrElse(comparedType) /** The type the two arguments are COMPARED in -- what [[toPainlessCall]] switches on to pick a * comparison, unless a cast over the `NULLIF` sets one: the least common supertype of what @@ -533,13 +732,14 @@ package object cond { * Computed ONCE per `NULLIF` and per scope, and memoised (`SQLTypeUtils.PerScope`: its type, * its rendering and the comparison it sits in all ask it; `ComparisonRule` says why with no * lock); attaching the schema copies the `NULLIF` (`update`), so a resolved statement's - * reading is read against the resolved types. + * reading is read against the resolved types. A computed column's validation has its own slot + * (A-95): the declared type it reads may differ there. */ private[cond] def secondReading: Option[ComparisonRule.Reading] = - _secondReading(ComparisonRule.readingOf(expr2, ComparisonRule.declaredTypeOf(expr1))) + _secondReading(ComparisonRule.readingOf(second, ComparisonRule.declaredTypeOf(first))) private[cond] def firstReading: Option[ComparisonRule.Reading] = - _firstReading(ComparisonRule.readingOf(expr1, ComparisonRule.declaredTypeOf(expr2))) + _firstReading(ComparisonRule.readingOf(first, ComparisonRule.declaredTypeOf(second))) private[this] val _secondReading = new SQLTypeUtils.PerScope[Option[ComparisonRule.Reading]] private[this] val _firstReading = new SQLTypeUtils.PerScope[Option[ComparisonRule.Reading]] @@ -564,8 +764,8 @@ package object cond { /** A literal argument counts as what it is read as: `NULLIF(n, '3')` compares two numbers. */ override def argTypes: List[SQLType] = List( - firstReading.map(_.sqlType).getOrElse(argTypeOf(expr1)), - secondReading.map(_.sqlType).getOrElse(argTypeOf(expr2)) + firstReading.map(_.sqlType).getOrElse(argTypeOf(first)), + secondReading.map(_.sqlType).getOrElse(argTypeOf(second)) ) /** 🔴 `NULLIF` REPORTS THE TYPE OF ITS FIRST ARGUMENT, not the super type of both. @@ -596,7 +796,13 @@ package object cond { * which a `CREATE TABLE … AS SELECT` declared, storing text. */ override def reportedType: SQLType = - firstReading.map(_.sqlType).getOrElse(reportedArgTypes.headOption.getOrElse(baseType)) + firstReading + .map(_.sqlType) + .getOrElse( + // a first argument read in place of its literal (A-98 α) answers that literal's text + if (first ne expr1) first.reportedType + else reportedArgTypes.headOption.getOrElse(baseType) + ) /** The rejection this `NULLIF` earns when its two arguments RENDER types SQL does not compare * (issue #382, the LEAD RULING of 2026-09-23). @@ -683,26 +889,61 @@ package object cond { private def temporalAgainstText(left: SQLType, right: SQLType): Boolean = (left.isTemporal && right.isText) || (left.isText && right.isTemporal) + /** The type the two arguments are compared IN by the implicit-cast table (SCHEMA-5b D6), for a + * pair main refuses ([[comparable]] over what they render, or the comparison rule over what + * they declare): `NULLIF(k, 0)` over a KEYWORD `k` compares `k` read as a BIGINT with 0, and + * answers `k`; `NULLIF(b, 0)` compares `b` as 1 / 0. A literal READ beside the other argument + * keeps its reading. Asked LIVE: the table's text rows hold in a query only (A-82), so a + * computed column keeps #382's refusal. + */ + private[cond] def implicitTarget: Option[SQLType] = + if (firstReading.isDefined || secondReading.isDefined) None + else { + val left = ComparisonRule.comparedType(None, first) + val right = ComparisonRule.comparedType(None, second) + val mainRefuses = + !comparable(argTypeOf(first), argTypeOf(second)) || + ComparisonRule.mismatch(left, right).isDefined + implicitTargetOf(first, second, left, right, equality = true, mainRefuses) + } + + /** Is this NULLIF refused by its temporal-against-text rule ([[typeMismatchError]])? */ + private[cond] def temporalAgainstTextRefused: Boolean = + secondReading.isEmpty && firstReading.isEmpty && + temporalAgainstText(argTypeOf(first), argTypeOf(second)) + + /** 🔴 Is this NULLIF, an arm of a `COALESCE` directly followed by its second argument, answered + * there by its FIRST (SCHEMA-5b A-98 β, read by `Coalesce.answeredArms`)? Where it is refused + * by its temporal-against-text rule ([[temporalAgainstTextRefused]]), neither argument is a + * string literal and one is a read-moved conditional (`SQLTypeUtils.readMoved`): then + * `COALESCE(NULLIF(a, b), b)` is `COALESCE(a, b)` -- `a` where `a` is not `b`, and `b` where + * it is, or where `a` is NULL. The `COALESCE` asks it in a query only. + */ + private[cond] def answeredByFirst: Boolean = + temporalAgainstTextRefused && + ComparisonRule.stringLiteral(expr1).isEmpty && ComparisonRule.stringLiteral(expr2).isEmpty && + (SQLTypeUtils.readMoved(expr1) || SQLTypeUtils.readMoved(expr2)) + def typeMismatchError: Option[String] = { - val left = argTypeOf(expr1) - val right = argTypeOf(expr2) + val left = argTypeOf(first) + val right = argTypeOf(second) // A literal the other argument READS (`secondReading`) is a value of that argument's kind: // comparable by construction, and rendered as that value. Asked first, by the query seam // and the DDL one alike, so a computed column and a query read `NULLIF(n, '3')` the same. if (secondReading.isDefined || firstReading.isDefined) None - else if (temporalAgainstText(left, right)) + else if (temporalAgainstTextRefused) // The resolver goes first so a MALFORMED literal is named as such (`'x'` is not a date for // this field's format) rather than reported as a bare type mismatch; a WELL-FORMED one // falls through to a message that says what is actually wrong -- the engine, not the SQL. - temporalLiteralError(expr1, left, expr2, right) - .orElse(temporalLiteralError(expr2, right, expr1, left)) + temporalLiteralError(first, left, second, right) + .orElse(temporalLiteralError(second, right, first, left)) .orElse( Some( s"$sql compares ${left.typeId} with ${right.typeId}: comparing a temporal value " + "with a string is not supported yet, so cast the string to the column's type" ) ) - else if (!comparable(left, right)) + else if (!comparable(left, right) && implicitTarget.isEmpty) Some( s"$sql compares ${left.typeId} with ${right.typeId}: NULLIF requires two arguments of " + "comparable types, so cast one of them" @@ -864,7 +1105,12 @@ package object cond { case List(_, _) if SQLTypeUtils.convertsArms(context) && ArithmeticExpression.nullLiteral(expr1) => bound("null", context) - case List(rendered0, rendered1) => + case List(own0, own1) => + // an argument read in place of its literal (A-98 α) renders that literal's quoted text + def inPlace(arg: PainlessScript, own: PainlessScript, rendered: String): String = + if (arg eq own) rendered else ComparisonRule.quotedText(arg).getOrElse(rendered) + val rendered0 = inPlace(first, expr1, own0) + val rendered1 = inPlace(second, expr2, own1) // A string literal argument is the value the other argument READS it as -- the number // or the boolean it spells, rendered as one (`secondReading` / `firstReading`). def scalar(reading: Option[ComparisonRule.Reading], rendered: String): String = @@ -872,15 +1118,15 @@ package object cond { val arg0 = operand(scalar(firstReading, rendered0), context) val typedAnswer = SQLTypeUtils.convertsArms(context) && firstReading.isEmpty && - SQLTypeUtils.armsType(answeredArms, reportedType).isDefined - if (SQLTypeUtils.convertsArms(context) && ArithmeticExpression.nullLiteral(expr2)) + SQLTypeUtils.armsType(this, reportedType).isDefined + if (SQLTypeUtils.convertsArms(context) && ArithmeticExpression.nullLiteral(second)) return bound( if (typedAnswer) SQLTypeUtils.convertArm( - expr1, + first, arg0, baseType, - SQLTypeUtils.mayRenderNull(expr1, context), + SQLTypeUtils.mayRenderNull(first, context), context ) else arg0, @@ -906,7 +1152,7 @@ package object cond { // there. An ingest processor keeps its bytes. val answer = if (typedAnswer) - SQLTypeUtils.convertArm(expr1, arg0, baseType, nullable = false, context) + SQLTypeUtils.convertArm(first, arg0, baseType, nullable = false, context) else arg0 def nullIf(comparison: String): String = s"$arg0 == null || ($arg1 != null && $comparison) ? null : $answer" @@ -927,16 +1173,36 @@ package object cond { // `processorInstant` arm below -- and its source is persisted: it keeps its bytes.) val mixedTemporals = !context.exists(_.isProcessor) && !receiverIsTime && - dateCarrying(ComparisonRule.declaredTypeOf(expr1)) && - dateCarrying(ComparisonRule.declaredTypeOf(expr2)) && { - val (java1, java2) = (argTypeOf(expr1), argTypeOf(expr2)) + dateCarrying(ComparisonRule.declaredTypeOf(first)) && + dateCarrying(ComparisonRule.declaredTypeOf(second)) && { + val (java1, java2) = (argTypeOf(first), argTypeOf(second)) !java1.isUnknown && !java2.isUnknown && java1 != java2 } lazy val instants = - s"${ArithmeticExpression.comparedMillis(expr1, firstReading, arg0, context)} == " + - s"${ArithmeticExpression.comparedMillis(expr2, secondReading, arg1, context)}" + s"${ArithmeticExpression.comparedMillis(first, firstReading, arg0, context)} == " + + s"${ArithmeticExpression.comparedMillis(second, secondReading, arg1, context)}" val expr = - if (secondReading.exists(_.isInstanceOf[ComparisonRule.TemporalReading])) + if (implicitTarget.isDefined) { + // SCHEMA-5b D6 / D7: both arguments read as the type the table compares them in; + // NULLIF still answers its FIRST argument as it is + val target = implicitTarget.get + val left = + SQLTypeUtils.implicitCast(arg0, argTypeOf(first), target, nullable = false, context) + val right = + SQLTypeUtils.implicitCast( + arg1, + argTypeOf(second), + target, + nullable = false, + context + ) + // a literal second argument is never NULL, and `0 != null` does not compile + if ( + NumericReducer.numberLiteral(second) || + NumericReducer.booleanLiteral(second).isDefined + ) s"$arg0 == null || $left == $right ? null : $answer" + else nullIf(s"$left == $right") + } else if (secondReading.exists(_.isInstanceOf[ComparisonRule.TemporalReading])) s"$arg0 == null || ($instants) ? null : $answer" else if (temporalLiteral) s"$arg0 == null || ($arg1 != null && $instants) ? null : $answer" @@ -981,7 +1247,7 @@ package object cond { // `==` is null-safe in Painless, so a numeric / boolean pair needs no guard -- // but a CONVERTED answer does: `NULLIF(d, 2)` rendered `p1 == 2 ? null : // p1.toInstant()…`, which ran on the NULL of every document missing `d`. - case _ if answer != arg0 && SQLTypeUtils.mayRenderNull(expr1, context) => + case _ if answer != arg0 && SQLTypeUtils.mayRenderNull(first, context) => s"$arg0 == null || $arg0 == $arg1 ? null : $answer" case _ => s"$arg0 == $arg1 ? null : $answer" } @@ -1012,15 +1278,18 @@ package object cond { */ override def typeError: Option[String] = typeMismatchError.orElse { - // judged on the readings this NULLIF already holds (`firstReading`, `secondReading`) - ComparisonRule - .mismatch( - ComparisonRule.comparedType(firstReading, expr1), - ComparisonRule.comparedType(secondReading, expr2) - ) - .map { case (left, right) => - s"Type mismatch: output '${left.typeId}' is not compatible with input '${right.typeId}'" - } + // judged on the readings this NULLIF already holds (`firstReading`, `secondReading`); a + // pair the implicit-cast table binds is compared (SCHEMA-5b D6) + if (implicitTarget.isDefined) None + else + ComparisonRule + .mismatch( + ComparisonRule.comparedType(firstReading, first), + ComparisonRule.comparedType(secondReading, second) + ) + .map { case (left, right) => + s"Type mismatch: output '${left.typeId}' is not compatible with input '${right.typeId}'" + } } override def update(request: query.SingleSearch): NullIf = { @@ -1079,18 +1348,21 @@ package object cond { * type for the class the CASE answers: `DATE_TRUNC(CASE … END, MONTH)` called `truncatedTo` on * the `LocalDate` rows. * - * Arms with no common type keep the arms' `out` ([[legacyType]]), and render as they did. + * Arms with no common type keep the arms' `out` ([[legacyType]]), and render as they did; so + * does a CASE that is text where its processor runs (`SQLTypeUtils.textTypeOutsideQuery`, a + * literal read as a date, the validation of a computed column, A-96). */ override def baseType: SQLType = - if (SQLTypeUtils.rendersOutsideQuery) legacyType - else SQLTypeUtils.armsType(arms, reportedType).getOrElse(legacyType) + if (SQLTypeUtils.rendersOutsideQuery || SQLTypeUtils.textTypeOutsideQuery(this).isDefined) + legacyType + else SQLTypeUtils.armsType(this, reportedType).getOrElse(legacyType) /** The type of the CASE's value where its arms have no common type ([[SQLTypeUtils.armsType]]) * and in an ingest processor, whose persisted bytes do not move: the least common supertype of * the arms' `out`, which each arm is converted to from its own `baseType`. */ private[this] def legacyType: SQLType = - SQLTypeUtils.leastCommonSuperType(arms.map(_.out)) + SQLTypeUtils.leastCommonSuperType(legacyArmTypes) /** The type a cast over the CASE sets (`CAST(CASE … END AS DATE)`): a simple CASE compares in * it, and the arms are converted to it where the CASE keeps its old conversion @@ -1119,12 +1391,34 @@ package object cond { * the cast's target were converted a second time -- `.toInstant()` on the `LocalDate` they had * become. */ - private[this] def armOf(arm: PainlessScript, context: Option[PainlessContext]): String = - if ( - SQLTypeUtils.convertsArms(context) && !SQLTypeUtils.armTypeUnknown(arm) && - SQLTypeUtils.armsType(arms, reportedType).isDefined - ) SQLTypeUtils.convertArm(arm, baseType, context) - else SQLTypeUtils.coerce(arm, arm.baseType, legacyOut, context) + private[this] def armOf(arm: PainlessScript, context: Option[PainlessContext]): String = { + val index = armIndex(arm) + // a READ literal (SCHEMA-5b D4) renders the value it is read as, and is converted FROM the + // type it is read as: in a processor too, where `coerce` would parse its text per document + (armReadings.filter(_.readings.contains(index)), readArmPainless(index, context)) match { + case (Some(r), Some(read)) => + if ( + SQLTypeUtils + .convertsArms(context) && SQLTypeUtils.armsType(this, reportedType).isDefined + ) + SQLTypeUtils.convertRendered(read, r.literalType, baseType, nullable = false, context) + else if (rendersAsTextOutsideQuery(index)) + // A-89 / A-92: outside a query a read date / timestamp is TEXT (`ctx` holds JSON, which + // a `date` field and the consumers' parsers read; `legacyArmTypes` keeps its text type) + // and that text never feeds a java.time conversion -- `"2024-01-15".atStartOfDay(…)` + // is refused by Painless when the pipeline is created. Without a cast, the reading's + // text; under a cast over the CASE, the arm as 0.24.0 renders it. + if (castType.isEmpty) read + else SQLTypeUtils.coerce(arm, arm.baseType, legacyOut, context) + else SQLTypeUtils.coerce(read, r.literalType, legacyOut, nullable = false, context) + case _ => + if ( + SQLTypeUtils.convertsArms(context) && !SQLTypeUtils.armTypeUnknown(arm) && + SQLTypeUtils.armsType(this, reportedType).isDefined + ) SQLTypeUtils.convertArm(arm, baseType, context) + else SQLTypeUtils.coerce(arm, arm.baseType, legacyOut, context) + } + } /** The type a simple CASE compares its operand and each `WHEN` value in: the common supertype * of the operand and the results -- what [[baseType]] answered while it included the operand @@ -1132,7 +1426,49 @@ package object cond { * Only the VALUE's type moved ([[baseType]]). */ private[this] def comparisonType: SQLType = - castType.filter(_ != legacyType).getOrElse(SQLTypeUtils.leastCommonSuperType(argTypes)) + castType + .filter(_ != legacyType) + .getOrElse(SQLTypeUtils.leastCommonSuperType(comparedArgTypes)) + + /** [[argTypes]] with the operand as its `WHEN` values compare it: a literal read in place of + * the operand ([[whenOperand]], A-98 α) is its text. + */ + private[this] def comparedArgTypes: List[SQLType] = + expression match { + case Some(e) if whenOperand(e) ne e => SQLTypes.Varchar :: argTypes.tail + case _ => argTypes + } + + /** 🔴 The CASE operand as its `WHEN` values compare it (SCHEMA-5b A-98 α, R-119): the literal a + * literal-first `COALESCE` is read in place of (`ComparisonRule.literalInPlaceOf`) where EVERY + * value reads it so -- the operand is rendered once for all of them -- else the operand. `CASE + * COALESCE('2024-01-15', d) WHEN k …` compares `'2024-01-15'` with `k`, as 0.24.0 did. + */ + private[this] def whenOperand(expr: PainlessScript): PainlessScript = { + val inPlace = conditions.map { case (cond, _) => ComparisonRule.literalInPlaceOf(expr, cond) } + if (inPlace.nonEmpty && inPlace.forall(_.isDefined)) inPlace.head.get else expr + } + + /** A `WHEN` value as it is compared with the operand `expr` (A-98 α): `CASE k WHEN + * COALESCE('2024-01-15', d) …` compares `k` with `'2024-01-15'`. + */ + private[this] def whenValue(expr: PainlessScript, cond: PainlessScript): PainlessScript = + ComparisonRule.literalInPlaceOf(cond, expr).getOrElse(cond) + + /** An operand or a value as the generic `WHEN` comparison renders it, converted to `compared`: + * one read in place of its literal (A-98 α) is that literal's quoted text. + */ + private[this] def whenSide( + side: PainlessScript, + own: PainlessScript, + compared: SQLType, + context: Option[PainlessContext] + ): String = + ComparisonRule.quotedText(side).filter(_ => side ne own) match { + case Some(text) => + SQLTypeUtils.coerce(text, SQLTypes.Varchar, compared, nullable = false, context) + case None => SQLTypeUtils.coerce(own, own.baseType, compared, context) + } override def validate(): Either[String, Unit] = { // Story 22.2 (AD-6) — a CASE-WHEN condition is parsed by `case_condition`, which shares @@ -1160,7 +1496,7 @@ package object cond { * type that is still unknown is accepted: Elasticsearch answers. */ override def typeError: Option[String] = - conditionsTypeError.orElse(mixedArmsError(answeredArms, "CASE expression", newerFirst = true)) + conditionsTypeError.orElse(mixedArmsError(this, "CASE expression", newerFirst = true)) private[this] def conditionsTypeError: Option[String] = expression match { @@ -1173,10 +1509,15 @@ package object cond { // on their DECLARED types, a DATE comparable with a TIMESTAMP, a string literal read by // what it is compared with -- `CASE n WHEN '3'`, `CASE d WHEN CAST('2024-01-31' AS DATE)` // -- judged on the readings this CASE already holds (`whenReadings`). - case Some(e) if { + // a value the implicit-cast table binds is compared (SCHEMA-5b D6) + case Some(e) if whenImplicitTarget(e).isDefined => None + // the operand and each value as the equality compares them (A-98 α, [[whenOperand]]) + case Some(operand) if { + val e = whenOperand(operand) val (declared, readings) = whenReadings(e) val literalExpression = ComparisonRule.stringLiteral(e).isDefined - conditions.zip(readings).exists { case ((cond, _), reading) => + conditions.zip(readings).exists { case ((value, _), reading) => + val cond = whenValue(operand, value) val left = if (!literalExpression) declared else @@ -1199,9 +1540,9 @@ package object cond { * `date_nanos` column) is beside a DATE one (`SQLTypeUtils.armsReportedType`). */ override def reportedType: SQLType = - SQLTypeUtils.armsReportedType( - conditions.map { case (_, result) => result.reportedType } ++ default.map(_.reportedType) - ) + // the THEN results and the ELSE are [[answeredArms]]; a read literal and a BOOLEAN beside + // whole numbers are the type they are read as (SCHEMA-5b, `armTypes`) + SQLTypeUtils.armsReportedType(armTypes) /** Whether the CASE EXPRESSION -- the receiver of every `WHEN` comparison -- renders a * `java.time.LocalTime`. `out` reports the CASE's result type and cannot say (issue #373). @@ -1315,6 +1656,24 @@ package object cond { } else WhenAsResult } + /** The type `CASE WHEN …` compares its expression and EVERY value in by the + * implicit-cast table (SCHEMA-5b D6, an equality): `CASE k WHEN 1` over a KEYWORD `k` reads + * `k` as a BIGINT, `CASE b WHEN 1` reads `b` as 1 / 0. Only when every value needs the table + * (no string literal, which the expression reads; none main compares as it stands); `None` + * otherwise, and the CASE keeps main's rule. Asked LIVE (A-82). + */ + private[this] def whenImplicitTarget(expr: PainlessScript): Option[SQLType] = { + val declared = ComparisonRule.declaredTypeOf(expr) + val targets = conditions.map { case (cond, _) => + val right = ComparisonRule.comparedType(None, cond) + val mainRefuses = ComparisonRule.mismatch(declared, right).isDefined + implicitTargetOf(expr, cond, declared, right, equality = true, mainRefuses) + } + if (targets.nonEmpty && targets.forall(_.isDefined)) + Some(SQLTypeUtils.leastCommonSuperType(targets.flatten)) + else None + } + /** The CASE expression's DECLARED type and what each `WHEN` value is read as beside it * (`ComparisonRule.readingOf`), computed ONCE per CASE and per scope, and memoised * (`SQLTypeUtils.PerScope`; `ComparisonRule` says why with no lock): its type rule, the @@ -1417,12 +1776,30 @@ package object cond { s"($ref != null ? (def)(${ArithmeticExpression.comparedMillis(operand, None, ref, context)}) : null)" ) val e = bind(expr.painless(context)) - val key = if (kind == WhenInstants) instant(expr, e) else e + // SCHEMA-5b D6 / D7: the expression and each value read as the type the table compares in + val target = if (kind == WhenImplicit) whenImplicitTarget(expr) else None + def implicitly(operand: PainlessScript, ref: String): String = + bind( + SQLTypeUtils.implicitCast( + ref, + ComparisonRule.declaredTypeOf(operand), + target.get, + nullable = true, + context + ) + ) + val key = + if (kind == WhenInstants) instant(expr, e) + else if (target.isDefined) implicitly(expr, e) + else e conditions .zip(readings) .map { case ((cond, res), reading) => val r = whenResult(cond, res, context) val test = reading match { + case _ if target.isDefined => + val value = implicitly(cond, bind(cond.painless(context))) + s"$value != null && $key == $value" case Some(t: ComparisonRule.TemporalReading) => s"$key == ${t.epochMillis}L" case Some(scalar) => s"$key == ${ComparisonRule.scalarPainless(scalar).getOrElse(cond.painless(context))}" @@ -1449,19 +1826,25 @@ package object cond { case Some(ctx) => var cases = expression match { - case Some(expr) if whenComparison(expr, context) != WhenAsResult => + case Some(expr) if whenImplicitTarget(expr).isDefined => + comparedWhens(expr, WhenImplicit, ctx, context) + // an operand read in place of its literal (A-98 α) is text: compared as a result + case Some(expr) if whenComparison(whenOperand(expr), context) != WhenAsResult => comparedWhens(expr, whenComparison(expr, context), ctx, context) - case Some(expr) => // case with expression to evaluate + case Some(operand) => // case with expression to evaluate // the operand and each value are converted to the type they are COMPARED in; the - // results, below, to the CASE's own (`out`) + // results, below, to the CASE's own (`out`); each as the equality compares it + // (A-98 α) + val expr = whenOperand(operand) val compared = comparisonType - val e = SQLTypeUtils.coerce(expr, expr.baseType, compared, context) + val e = whenSide(expr, operand, compared, context) val expParam = ctx.addParam( LiteralParam(e) ) conditions .zip(whenReadings(expr)._2) - .map { case ((cond, res), whenReading) => + .map { case ((value, res), whenReading) => + val cond = whenValue(operand, value) val name = cond match { case e: Expression => @@ -1494,7 +1877,7 @@ package object cond { .coerce(_, reading.sqlType, compared, nullable = false, context) ) ) - .getOrElse(SQLTypeUtils.coerce(cond, cond.baseType, compared, context)) + .getOrElse(whenSide(cond, value, compared, context)) ) val r = boxWhenNoDefault(res match { @@ -1694,14 +2077,17 @@ package object cond { * * Computed ONCE per reducer and per scope (`SQLTypeUtils.PerScope`): a declared type may * depend on the scope, so a query and an ingest processor rendering the same node each read - * their own. + * their own, and so does a computed column's validation (A-95): it reads [[armTypes]]. */ - private[cond] def numericType: Option[SQLNumeric] = _numericType(numericTypeOfValues) + private[cond] def numericType: Option[SQLNumeric] = + _numericType(numericTypeOfValues) private[this] val _numericType = new SQLTypeUtils.PerScope[Option[SQLNumeric]] private[this] def numericTypeOfValues: Option[SQLNumeric] = { - val declared = values.map(ComparisonRule.declaredTypeOf).filterNot(_ == SQLTypes.Null) + // each argument's declared type (`ComparisonRule.declaredTypeOf`, its reported type), a read + // literal and a BOOLEAN beside whole numbers the type they are read as (SCHEMA-5b `armTypes`) + val declared = armTypes.filterNot(_ == SQLTypes.Null) val fractional = declared.filter(t => t.isNumber && !NumericReducer.wholeNumber(t)) if (declared.isEmpty || declared.exists(t => !t.isNumber && !t.isUnknown)) None else if (declared.forall(NumericReducer.wholeNumber)) @@ -1725,10 +2111,10 @@ package object cond { override def baseType: SQLType = if (overDates) (if (SQLTypeUtils.rendersOutsideQuery) None - else SQLTypeUtils.armsType(answeredArms, reportedType)).getOrElse(SQLTypes.Timestamp) + else SQLTypeUtils.armsType(this, reportedType)).getOrElse(SQLTypes.Timestamp) else numericType.getOrElse(foldedArgument.getOrElse { - SQLTypeUtils.leastCommonSuperType(argTypes) match { + SQLTypeUtils.leastCommonSuperType(legacyArmTypes) match { case n: SQLNumeric => n case _ => outputType } @@ -1768,8 +2154,15 @@ package object cond { * boolean, a TIME), or numbers mixed with dates, is refused by name. */ override def typeError: Option[String] = { - val known = values - .map(v => (v: PainlessScript) -> renderedTypeOf(v)) + // a read literal and a BOOLEAN beside whole numbers count as the type they are read as + // (SCHEMA-5b D3 (3) / D5): neither is an offender + val read = armTypes + val known = values.zipWithIndex + .map { case (v, i) => + (v: PainlessScript) -> + (if (readingAt(i).isDefined || SQLTypeUtils.booleanBesideWhole(this, i)) read(i) + else renderedTypeOf(v)) + } .filterNot(_._2.isInstanceOf[SQLAny]) val numbers = known.filter(_._2.isInstanceOf[SQLNumeric]) val dates = known.filter(a => NumericReducer.dateCarrying(a._2)) @@ -1799,7 +2192,24 @@ package object cond { * the arguments' REPORTED types, as `COALESCE` reports (`SQLTypeUtils.armsReportedType`). */ override def reportedType: SQLType = - numericType.getOrElse(SQLTypeUtils.armsReportedType(reportedArgTypes)) + numericType.getOrElse(SQLTypeUtils.armsReportedType(armTypes)) + + /** A string literal read beside the other arguments renders the value it is read as (SCHEMA-5b + * D3 (4)): `GREATEST(n, '1.5')` compares `n` with the number 2. + */ + override protected def ownArgumentPainless( + index: Int, + context: Option[PainlessContext] + ): Option[String] = readArmPainless(index, context) + + /** 🔴 Is this GREATEST / LEAST never NULL (SCHEMA-5b A-99, R-116)? A number literal or a READ + * literal is among its arguments: it is never NULL, and NULLs are skipped, so neither is the + * result -- in any scope and on any major. ONE predicate (#292) for the primitive cast of the + * per-document value ([[typedResult]]) and for the arm a `COALESCE` ends at + * (`Coalesce.folded`), which tested `((long) …) != null`: Painless refuses to compile it. + */ + private[cond] def neverNull: Boolean = + values.indices.exists(i => NumericReducer.numberLiteral(values(i)) || readingAt(i).isDefined) /** NULL where every argument is; a NULL literal argument is NULL, and folded away in a query * ([[toPainlessCall]]): `LEAST(NULL, x)` is `x`. An ingest processor renders it as before. @@ -1916,7 +2326,7 @@ package object cond { case Some((ctx, t)) => val cast = SQLTypeUtils.painlessType(t) val v = value.trim - if (values.exists(NumericReducer.numberLiteral)) + if (neverNull) s"(($cast) ${if (NumericReducer.oneValue(v)) v else s"($v)"})" else { val isName = @@ -1943,9 +2353,36 @@ package object cond { private def typedArgument( rendered: String, argument: PainlessScript, + index: Int, context: Option[PainlessContext] ): String = perDocumentType(context) match { + // 🔴 A BOOLEAN beside whole numbers is the number DuckDB reads it as, 1 or 0 (SCHEMA-5b D5, + // R-107), converted by `coerce`'s BOOLEAN -> NUMERIC arm, per document in a query and in an + // ingest processor -- where `Math.max` over a `Boolean` threw and `ignore_failure` stored + // nothing. A processor reads the raw `_source` value as Elasticsearch reads a boolean: + // `true` / `"true"` are true, `false` / `"false"` / `""` false. A NULL stays NULL (skipped). + // a boolean CONSTANT is its numeral (`1L`): `(true ? 1L : 0L)` is a conditional over a + // constant, which Elasticsearch 6.8's Painless refuses to compile + case Some((_, t)) + if SQLTypeUtils.booleanBesideWhole(this, index) && + NumericReducer.booleanLiteral(argument).isDefined => + SQLTypeUtils + .booleanConstant(NumericReducer.booleanLiteral(argument).get, t) + .getOrElse(rendered) + case Some((ctx, t)) if SQLTypeUtils.booleanBesideWhole(this, index) => + val raw = rendered.trim + val name = + if (ArithmeticExpression.isBareName(raw) && (raw.head.isLetter || raw.head == '_')) raw + else ctx.bindLocal(raw, "lv") + val operand = + if (ctx.isProcessor) SQLTypeUtils.processorBoolean(name) + else name + val converted = + SQLTypeUtils.coerce(operand, SQLTypes.Boolean, t, nullable = false, Some(ctx)) + if (nullableArgument(argument, context)) + ctx.bindLocal(s"($name == null ? null : (def)($converted))", "lv") + else converted case Some((_, SQLTypes.BigInt)) if NumericReducer.numberLiteral(argument) => val r = rendered.trim if ( @@ -1966,14 +2403,15 @@ package object cond { // and PostgreSQL skip a NULL, and rendered it was `Math.min(null, param1)`, which Painless // refuses to compile. With none left the reducer is NULL. An ingest processor keeps its // bytes. - val arguments = - callArgs.zip(values).filterNot { case (_, argument) => + val indexed = + callArgs.zip(values).zipWithIndex.filterNot { case ((_, argument), _) => SQLTypeUtils.convertsArms(context) && ArithmeticExpression.nullLiteral(argument) } + val arguments = indexed.map(_._1) if (arguments.isEmpty && callArgs.nonEmpty) return "null" val typedArgs = - arguments.map { case (rendered, argument) => - typedArgument(rendered, argument, context) + indexed.map { case ((rendered, argument), index) => + typedArgument(rendered, argument, index, context) } typedResult( typedArgs match { @@ -2008,33 +2446,45 @@ package object cond { if (callArgs.isEmpty) throw new IllegalArgumentException(s"$operator requires at least one argument") val perDocument = context.filterNot(_.isTransform) - val millis = callArgs.zip(values).filterNot(a => ArithmeticExpression.nullLiteral(a._2)).map { - case (ref, argument) => - val nullable = nullableArgument(argument, context) - perDocument match { - case Some(ctx) => - val name = - if (ArithmeticExpression.isBareName(ref.trim)) ref.trim - else ctx.bindLocal(ref, "lv") - val converted = ArithmeticExpression.instantMillis(argument, name, context) - val guarded = - if (nullable) s"($name == null ? null : (def)($converted))" else converted - (ctx.bindLocal(guarded, "lv"), nullable) - case None if argument.isAggregation => - val metric = ref.trim - val converted = ArithmeticExpression.instantMillis(argument, metric, context) - // A NULL metric (a group filter's, over a group with none of its values) is handed - // back as it is -- the same NULL, which the fold skips. The group filter's gate - // (`MetricSelectorScript.disqualifyRendering`) refuses a rendering that spells - // `? null :` or reads `def`, so neither is written: the conditional is a `def` by its - // first branch, the metric itself. - if (converted == metric) (metric, nullable) - else if (nullable) (s"($metric == null ? $metric : $converted)", nullable) - else (converted, nullable) - case None => - (ArithmeticExpression.instantMillis(argument, ref.trim, context), nullable) - } - } + val millis = callArgs + .zip(values) + .zipWithIndex + .filterNot(a => ArithmeticExpression.nullLiteral(a._1._2)) + .map { + // a string literal READ as a date or a timestamp is the instant it denotes, a constant + // (SCHEMA-5b D3 (4)): never NULL, never parsed per document + case ((_, _), index) + if readingAt(index).exists(_.isInstanceOf[ComparisonRule.TemporalReading]) => + ( + s"${readingAt(index).get.asInstanceOf[ComparisonRule.TemporalReading].epochMillis}L", + false + ) + case ((ref, argument), _) => + val nullable = nullableArgument(argument, context) + perDocument match { + case Some(ctx) => + val name = + if (ArithmeticExpression.isBareName(ref.trim)) ref.trim + else ctx.bindLocal(ref, "lv") + val converted = ArithmeticExpression.instantMillis(argument, name, context) + val guarded = + if (nullable) s"($name == null ? null : (def)($converted))" else converted + (ctx.bindLocal(guarded, "lv"), nullable) + case None if argument.isAggregation => + val metric = ref.trim + val converted = ArithmeticExpression.instantMillis(argument, metric, context) + // A NULL metric (a group filter's, over a group with none of its values) is handed + // back as it is -- the same NULL, which the fold skips. The group filter's gate + // (`MetricSelectorScript.disqualifyRendering`) refuses a rendering that spells + // `? null :` or reads `def`, so neither is written: the conditional is a `def` by its + // first branch, the metric itself. + if (converted == metric) (metric, nullable) + else if (nullable) (s"($metric == null ? $metric : $converted)", nullable) + else (converted, nullable) + case None => + (ArithmeticExpression.instantMillis(argument, ref.trim, context), nullable) + } + } if (millis.isEmpty) return if (perDocument.isEmpty) "((def) null)" else "null" val (winner, nullable) = fold(millis, integral = false) perDocument match { @@ -2094,6 +2544,19 @@ package object cond { case _ => false } + /** The value of a boolean literal (`TRUE` / `FALSE`), bare or as the one function of the + * nameless identifier the parser wraps an operand in. + */ + private[cond] def booleanLiteral(v: PainlessScript): Option[Boolean] = v match { + case b: BooleanValue => Some(b.value) + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (b: BooleanValue) :: Nil => Some(b.value) + case _ => None + } + case _ => None + } + /** A number literal -- bare, or the one function of the nameless identifier the parser wraps an * operand in -- which is never NULL. */ diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala index 71ba057f..2ea329d7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala @@ -330,17 +330,30 @@ package object convert { * column is a date. Asked once the column types are known, like every type rule; an operand * whose type is still unknown is accepted. A number cast to a TIMESTAMP keeps core's * epoch-millisecond conversion. + * + * A BOOLEAN converted to a DATE, a TIME or a TIMESTAMP is refused too, in every scope, as + * DuckDB refuses it (lead ruling R-110, SCHEMA-5b): no conversion makes a boolean a temporal, + * and the value used to pass through unconverted under the temporal's label (`CAST(b AS DATE)` + * answered `true`), or to fail on every row once its text was parsed (`CAST(COALESCE(b, + * 'true') AS DATE)`). */ override def typeError: Option[String] = - if (targetType != SQLTypes.Date) None - else { - val source = convertedType - if (!source.isNumber) None - else + targetType match { + case SQLTypes.Date | SQLTypes.Time | SQLTypes.Timestamp | SQLTypes.DateTime + if convertedType == SQLTypes.Boolean => Some( - s"Type mismatch: cannot cast ${source.typeId} to DATE in expression: $sql; a number " + - "is not a date: add a number of days to a date instead" + s"Type mismatch: cannot cast BOOLEAN to ${targetType.typeId} in expression: $sql; a " + + "boolean is not a date or a time" ) + case SQLTypes.Date => + val source = convertedType + if (!source.isNumber) None + else + Some( + s"Type mismatch: cannot cast ${source.typeId} to DATE in expression: $sql; a number " + + "is not a date: add a number of days to a date instead" + ) + case _ => None } /** The type of the value converted, as it renders: its outermost function's own type -- never diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index 43527630..6522ae99 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -31,6 +31,7 @@ import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.`type`.{ SQLBigInt, SQLBool, + SQLTemporal, SQLType, SQLTypeUtils, SQLTypes, @@ -143,10 +144,59 @@ package object string { context: Option[PainlessContext] ): String = { callArgs match { - case List(str) => s"$str${stringOp.painless(context)}" + case List(str) => overReceiver(str, context)(r => s"$r${stringOp.painless(context)}") case _ => throw new IllegalArgumentException(s"${stringOp.sql} requires 1 argument") } } + + /** 🔴 The call `call` over this function's RECEIVER -- its first argument, rendered `rendered` + * -- read as a text (lead ruling R-109, SCHEMA-5b): + * - a receiver of a known type that is no text (a number, a boolean; a date or a timestamp a + * conditional answers, which a query converts to its `java.time` value) is its text, + * `String.valueOf(…)`: `UPPER(n)`, `LENGTH(n)` called `toUpperCase` / `length` on a `Long` + * and failed the query (MEASURED on Elasticsearch 8.18.3). A bare date column keeps its + * bytes: its doc value would print as a `ZonedDateTime`, not as the date; + * - a CONDITIONAL receiver (a `CASE`, a `COALESCE`, a `NULLIF`, a `GREATEST` / `LEAST`) is + * ONE value, in parentheses: `param2 ? param1 : "3"` with `.toUpperCase()` appended called + * it on the last arm alone. A conditional that can answer NULL is bound once and guarded, + * so its NULL is NULL, never the text `"null"` nor a NullPointerException. + * + * Every other receiver -- a text, a type still unknown (no schema), a function over a text -- + * keeps its bytes. + */ + protected final def overReceiver(rendered: String, context: Option[PainlessContext])( + call: String => String + ): String = + args.headOption match { + case Some(receiver) => + val conditional = SQLTypeUtils.conditional(receiver) + val nonText = + Receivers.nonText(SQLTypeUtils.operandType(receiver.reportedType), conditional) + val one = SQLTypeUtils.asOneValue(rendered) + def text(r: String): String = if (nonText) s"String.valueOf($r)" else r + if (!nonText && (!conditional || one == rendered)) call(rendered) + else + context match { + case Some(ctx) if conditional && SQLTypeUtils.mayRenderNull(receiver, context) => + val value = ctx.bindLocal(rendered, "lv") + s"($value == null ? null : (def)(${call(text(value))}))" + // a call over a non-text receiver is typed `def`, as main's doc-value receiver typed + // it: `LENGTH(n)`'s `int` beside its NULL guard's `null` does not compile in Painless + // ("Cannot cast from [int] to [java.lang.Object]", MEASURED on Elasticsearch 8.18.3) + case _ if nonText => s"((def)(${call(text(one))}))" + case _ => call(one) + } + case None => call(rendered) + } + } + + private[string] object Receivers { + + /** Is `t` a KNOWN type whose value is no text: a number, a boolean -- or a temporal, when a + * conditional answers it? + */ + def nonText(t: SQLType, conditional: Boolean): Boolean = + t.isNumber || t == SQLTypes.Boolean || (conditional && t.isInstanceOf[SQLTemporal]) } case class StringFunctionWithOp(str: PainlessScript, stringOp: StringOp) @@ -177,12 +227,16 @@ package object string { ): String = { callArgs match { // SUBSTRING(expr, start, length) - case List(arg0, _, _) => - s"((String)$arg0).substring(${start - 1}, (int)Math.min(${start - 1 + length.get}, ((String)$arg0).length()))" + case List(str, _, _) => + overReceiver(str, context) { arg0 => + s"((String)$arg0).substring(${start - 1}, (int)Math.min(${start - 1 + length.get}, ((String)$arg0).length()))" + } // SUBSTRING(expr, start) - case List(arg0, arg1) => - s"((String)$arg0).substring((int)Math.min(${start - 1}, ((String)$arg0).length() - 1))" + case List(str, _) => + overReceiver(str, context) { arg0 => + s"((String)$arg0).substring((int)Math.min(${start - 1}, ((String)$arg0).length() - 1))" + } case _ => throw new IllegalArgumentException("SUBSTRING requires 2 or 3 arguments") } @@ -319,8 +373,10 @@ package object string { context: Option[PainlessContext] ): String = { callArgs match { - case List(arg0, arg1) => - s"((String)$arg0).substring(0, (int)Math.min($arg1, ((String)$arg0).length()))" + case List(str, arg1) => + overReceiver(str, context) { arg0 => + s"((String)$arg0).substring(0, (int)Math.min($arg1, ((String)$arg0).length()))" + } case _ => throw new IllegalArgumentException("LEFT requires 2 arguments") } } @@ -354,10 +410,12 @@ package object string { context: Option[PainlessContext] ): String = { callArgs match { - case List(arg0, arg1) => + case List(str, arg1) => if (length == 0) "" else - s"""((String)$arg0).substring(((String)$arg0).length() - (int)Math.min($arg1, ((String)$arg0).length()))""" + overReceiver(str, context) { arg0 => + s"""((String)$arg0).substring(((String)$arg0).length() - (int)Math.min($arg1, ((String)$arg0).length()))""" + } case _ => throw new IllegalArgumentException("RIGHT requires 2 arguments") } } @@ -392,8 +450,8 @@ package object string { context: Option[PainlessContext] ): String = { callArgs match { - case List(arg0, arg1, arg2) => - s"$arg0.replace($arg1, $arg2)" + case List(str, arg1, arg2) => + overReceiver(str, context)(arg0 => s"$arg0.replace($arg1, $arg2)") case _ => throw new IllegalArgumentException("REPLACE requires 3 arguments") } } @@ -435,8 +493,9 @@ package object string { context: Option[PainlessContext] ): String = { callArgs match { - case List(arg0) => s"new StringBuilder($arg0).reverse().toString()" - case _ => throw new IllegalArgumentException("REVERSE requires 1 argument") + case List(str) => + overReceiver(str, context)(arg0 => s"new StringBuilder($arg0).reverse().toString()") + case _ => throw new IllegalArgumentException("REVERSE requires 1 argument") } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 0be6cdd0..ef4ffb90 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -207,7 +207,7 @@ case class ArithmeticExpression( other: PainlessScript ): Option[ComparisonRule.NumberReading] = ComparisonRule - .readingOf(operand, ComparisonRule.declaredTypeOf(other)) + .readingOf(operand, ArithmeticExpression.reportedTypeOf(other)) .collect { case number: ComparisonRule.NumberReading => number } /** The temporal a string literal operand is read as beside the other one @@ -1128,17 +1128,25 @@ object ArithmeticExpression { private[sql] def identifierType(i: Identifier): SQLType = renderedTypeOf(i) /** The type an operand is DECLARED with -- its REPORTED type, an aggregate's being the metric it - * produces: the ONE derivation the comparison rule reads too (`ComparisonRule.declaredTypeOf`). + * produces. * * An Elasticsearch `date` field elasticsql never declared is a TIMESTAMP at the declaration seam * itself (`schema.IndexField`), so it keeps its time here as everywhere else, and every chain * over it follows: `MAX(ts)`, `COALESCE(ts, ts2)`, `CASE … THEN ts …`. + * + * 🔴 The REPORTED type in every scope, never the comparison rule's text type of a conditional + * holding a read date outside a query (`ComparisonRule.declaredTypeOf`, SCHEMA-5b R1-1): date + * arithmetic reads a temporal operand by its runtime class, text included, so `COALESCE(d, + * '2024-01-15') - d2` counts the days in an ingest processor too, where the text type would + * subtract a String. */ + private[math] def reportedTypeOf(operand: PainlessScript): SQLType = operand.reportedType + /** An operand's declared type as arithmetic reads it: an `ARRAY` column is one X element, the * value Painless reads (SCHEMA-6 D7, `SQLTypeUtils.operandType`). */ private def declaredType(operand: PainlessScript): SQLType = - SQLTypeUtils.operandType(ComparisonRule.declaredTypeOf(operand)) + SQLTypeUtils.operandType(reportedTypeOf(operand)) /** `NULL`, bare or cast: its VALUE is NULL in every venue. */ private[sql] def nullLiteral(operand: PainlessScript): Boolean = operand match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 4fd868f5..9e0f8a2e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1715,9 +1715,17 @@ package object sql { * COALESCE of DATE columns, `d + 1` are DATEs; an aggregate is the metric it produces, because * that is what an aggregate reports (`AggregateFunction.outputTypeOf`), so `COUNT(d)` is a * BIGINT. One derivation (#292): there is no aggregate arm here of its own. + * + * 🔴 Outside a query (an ingest processor, a transform) and while the computed column that + * installs such a processor is validated (A-96), a `COALESCE` or a `CASE` holding a literal + * read as a date or a timestamp is its TEXT type, the type its value has there + * (`SQLTypeUtils.textTypeOutsideQuery`, A-92, SCHEMA-5b R1-1): every comparison reads its + * operands here, so each one compares that text as 0.24.0 did. The arithmetic reads the + * reported type instead (`ArithmeticExpression`): it reads a temporal operand by its runtime + * class, text included. */ def declaredTypeOf(operand: Token): SQLType = operand match { - case p: PainlessScript => p.reportedType + case p: PainlessScript => SQLTypeUtils.textTypeOutsideQuery(p).getOrElse(p.reportedType) case other => other.out } @@ -1951,6 +1959,117 @@ package object sql { if (comparable(SQLTypeUtils.operandType(left), SQLTypeUtils.operandType(right))) None else Some((left, right)) + /** [[mismatch]] for a comparison whose operator class is known (SCHEMA-5b D6): a pair the + * implicit-cast table binds ([[implicitTarget]]) is compared, every other pair keeps + * [[mismatch]] 's answer and message. `equality`: `=`, `<>`, `!=`, IN, NULLIF, `CASE x WHEN`. + */ + def mismatch(left: SQLType, right: SQLType, equality: Boolean): Option[(SQLType, SQLType)] = + mismatch(left, right) match { + case Some(_) if implicitTarget(left, right, equality).isDefined => None + case other => other + } + + /** 🔴 The ONE implicit-cast table of a comparison (SCHEMA-5b D6): the type two operands of + * different families are compared IN, as DuckDB 1.5.5 binds them (RUN `duck_5b_ops.out`), or + * `None` -- main's rule, a refusal. Each side is read through `SQLTypeUtils.operandType` (an + * `ARRAY` column is its element X in a query, D8): + * - a text (VARCHAR, KEYWORD, TEXT, CHAR) beside a whole number: equality only, as BIGINT; + * - a text beside a DOUBLE, a REAL or a NUMERIC: equality only, as DOUBLE; + * - a text beside a BOOLEAN: equality only, as BOOLEAN; + * - a BOOLEAN beside a whole number: every comparison, BETWEEN and IN, as the whole type; + * - a BOOLEAN beside a DOUBLE or a REAL: equality only, as DOUBLE. + * + * Ordering a text against a number (and BETWEEN) is a DuckDB Binder Error, refused here too: + * one verdict for core and arrow. The text rows bind in a QUERY only (A-82): where + * `SQLTypeUtils.readsWholeValues` holds -- an ingest processor or a transform rendering, a + * computed column's validation -- a processor would swallow a text that converts to nothing + * (`ignore_failure`) where DuckDB fails the statement, so main's verdict and bytes stay. Read + * LIVE, never memoised: a node's query slot also serves the validation. A string LITERAL is + * not a side of this table: it is read by what it is compared with ([[readingOf]]). + */ + def implicitTarget(left: SQLType, right: SQLType, equality: Boolean): Option[SQLType] = { + val l = SQLTypeUtils.operandType(left) + val r = SQLTypeUtils.operandType(right) + implicitTargetOf(l, r, equality).orElse(implicitTargetOf(r, l, equality)) + } + + /** Is `t` a text type, as the implicit-cast table reads one? */ + def textType(t: SQLType): Boolean = t match { + case _: SQLVarchar | _: SQLChar => true + case _ => false + } + + /** 🔴 The literal an EQUALITY reads in place of `operand` beside `other` (SCHEMA-5b A-98 α, + * R-119), in a QUERY only: a `COALESCE` -- bare, or the one function of a nameless identifier + * -- whose first argument that is not NULL is a string literal read as a DATE whose text is + * exactly that date's `uuuu-MM-dd`, beside an operand that is neither a string literal nor + * NULL and is declared TEXT, neither side holding an aggregate. That literal is the one value + * such a `COALESCE` ever answers, and 0.24.0 compared it as text, right on every row (RUN): + * `COALESCE('2024-01-15', d) = k` holds where `k` is `'2024-01-15'`. + * + * ONE function for the type rule and the rendering of every equality reader (#292): `=`, `<>`, + * `!=` (`Expression`), `NULLIF` and its answer, `CASE x WHEN`. Ordering and BETWEEN do not + * read it (main fails there, RUN), nor does anything outside a query ([[SQLTypeUtils]] + * `.readsWholeValues`). A date-time text, a TIMESTAMP reading, a number or a boolean reading, + * a literal other side and an aggregate side keep the reading's verdict. + */ + def literalInPlaceOf(operand: Token, other: Token): Option[PainlessScript] = + (operand, other) match { + // the cheap questions first: most operands are no COALESCE, and every comparison asks + case (p: PainlessScript, o: PainlessScript) + if !SQLTypeUtils.readsWholeValues && !p.hasAggregation && !o.hasAggregation => + SQLTypeUtils.conditionalNode(p) match { + case c: app.softnetwork.elastic.sql.function.cond.Coalesce => + // the arms `readingAt` indexes (A-98 β may answer a NULLIF arm by its value) + c.answeredArms.zipWithIndex + .find { case (arm, _) => + !app.softnetwork.elastic.sql.operator.math.ArithmeticExpression.nullLiteral(arm) + } + .flatMap { case (arm, i) => + (stringLiteral(arm), c.readingAt(i)) match { + case (Some(text), Some(r: TemporalReading)) + if r.sqlType == SQLTypes.Date && r.text == text => + Some(arm) + case _ => None + } + } + .filter { _ => + stringLiteral(o).isEmpty && + !app.softnetwork.elastic.sql.operator.math.ArithmeticExpression.nullLiteral(o) && + textType(SQLTypeUtils.operandType(declaredTypeOf(o))) + } + case _ => None + } + case _ => None + } + + /** The Painless of a string literal read as text, its quoted text -- never through its `out` + * (A-98 α). + */ + def quotedText(literal: Token): Option[String] = + stringLiteral(literal).map(text => s""""${escapePainlessString(text)}"""") + + private def implicitTargetOf(a: SQLType, b: SQLType, equality: Boolean): Option[SQLType] = + if (textType(a)) { + if (!equality || SQLTypeUtils.readsWholeValues) None + else + b match { + case w if wholeNumberType(w) => Some(SQLTypes.BigInt) + case _: SQLNumeric => Some(SQLTypes.Double) + case _: SQLBool => Some(SQLTypes.Boolean) + case _ => None + } + } else + a match { + case _: SQLBool => + b match { + case w if wholeNumberType(w) => Some(w) + case _: SQLDouble | _: SQLReal if equality => Some(SQLTypes.Double) + case _ => None + } + case _ => None + } + /** The Painless a NUMBER or a BOOLEAN reading renders: the literal itself. */ def scalarPainless(reading: Reading): Option[String] = reading match { case NumberReading(painless, _) => Some(painless) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index d37c36f0..fc72f8c3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -1045,6 +1045,13 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { /** The type the left operand is DECLARED with (`ComparisonRule.declaredTypeOf`), which every * literal of this comparison is read against and the type rule judges: computed ONCE, and * memoised like [[literalReading]]. + * + * 🔴 A declared type may answer differently while a computed column is validated than in a query + * (a conditional over an `ARRAY` arm is VARCHAR there and INT in a query, SCHEMA-5b), and + * outside a query than in one (a conditional holding a literal read as a date is text there, + * A-92): this type and every reading read against it ([[literalReading]], + * [[leftLiteralReading]], an IN's and a BETWEEN's) keep one answer per scope + * (`SQLTypeUtils.PerScope`, A-95), and no scope reads another's. */ protected def operandDeclaredType: SQLType = _operandDeclaredType(ComparisonRule.declaredTypeOf(identifier)) @@ -1178,7 +1185,11 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { * by [[painlessValue]] — which is the whole defect: `ts > CAST(d AS DATE)` promoted the side * that was already a `ZonedDateTime` and left the `LocalDate` alone. */ - protected def comparisonTargetType: SQLType = { + protected def comparisonTargetType: SQLType = + implicitComparisonTarget.getOrElse(renderedTargetType) + + /** [[comparisonTargetType]] of a pair main compares: from what both sides render. */ + private def renderedTargetType: SQLType = { val operandType = operandRenderedType maybeValue match { // 🔴 Two numbers are compared as numbers, with NO widening. Painless compares boxed numerics @@ -1322,6 +1333,15 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { * call sites are all in [[painless]]. */ protected def promotedRight(ref: String, context: Option[PainlessContext]): String = + implicitComparisonTarget match { + // SCHEMA-5b D6 / D7: the right side of an implicit cast, converted to the type the pair is + // compared in; it is never NULL here (every route below tests it first) + case Some(target) => + SQLTypeUtils.implicitCast(ref, valueRenderedType, target, nullable = false, context) + case None => promotedRendered(ref, context) + } + + private def promotedRendered(ref: String, context: Option[PainlessContext]): String = if (rightNeedsTemporalPromotion(context)) SQLTypeUtils.coerce(ref, valueRenderedType, comparisonTargetType, nullable = false, context) else processorParsedRight(ref, context) @@ -1440,8 +1460,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { SQLTypeUtils .processorTemporal(chain, identifier.declaredType) .getOrElse( - SQLTypeUtils - .coerce(chain, operandType, targetedType, identifier.nullable, context) + convertedLeft(chain, operandType, targetedType, identifier.nullable, context) ) // 🔴 A temporal chain in a PROCESSOR needs no coercion at all, because story 21.8's rule 3 // has already collapsed it: every temporal value in an ingest script is a `ZonedDateTime`, @@ -1463,13 +1482,27 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { case _ => val nullable = SQLTypeUtils.mayRenderNull(identifier, context) val operand = if (nullable) chain else SQLTypeUtils.asOneValue(chain) - val converted = - SQLTypeUtils.coerce(operand, operandType, targetedType, nullable, context) + val converted = convertedLeft(operand, operandType, targetedType, nullable, context) if (converted == operand) chain else converted } (chain, coerced) } + /** The left operand converted to the type the comparison reads it as: an implicit cast's + * conversion (SCHEMA-5b D7, `SQLTypeUtils.implicitCast`), else [[SQLTypeUtils.coerce]], as + * before. + */ + private def convertedLeft( + operand: String, + from: SQLType, + to: SQLType, + nullable: Boolean, + context: Option[PainlessContext] + ): String = + if (implicitComparisonTarget.isDefined) + SQLTypeUtils.implicitCast(operand, from, to, nullable, context) + else SQLTypeUtils.coerce(operand, from, to, nullable, context) + /** The operand with its target-type coercion — [[leftOperand]] 's second element, for the callers * that do not need the chain rendering apart from it. */ @@ -1507,6 +1540,10 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { val readsMetric = isAggregation || (context.isEmpty && (referencesBucketMetric || reducesDateMetrics)) op match { + // SCHEMA-5b D6: both sides of an implicit cast are values of its target -- a number or a + // boolean -- whatever type the right-hand side was declared with + case _: ComparisonOperator if implicitComparisonTarget.isDefined => + s"$param ${op.painless(None)} $value" case _: ComparisonOperator => // 🔴 Story BIDC-8 AD-9 — dispatch on the EFFECTIVE operator, i.e. with a `NOT` folded in, // exactly as `painlessOp` does for the generic fallback below. Each arm here returns a @@ -1839,7 +1876,19 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { * `.atStartOfDay(…)` to a `ZonedDateTime`. */ override def painless(context: Option[PainlessContext]): String = - SQLTypeUtils.renderingIn(context)(comparisonPainless(context)) + literalsInPlace match { + case Some(mapped) => mapped.painless(context) + case None => SQLTypeUtils.renderingIn(context)(comparisonPainless(context)) + } + + /** 🔴 This EQUALITY (`=`, `<>`, `!=`) with each side that `ComparisonRule.literalInPlaceOf` maps + * in place of its literal (SCHEMA-5b A-98 α, R-119): `COALESCE('2024-01-15', d) = k` is + * `'2024-01-15' = k`, the text comparison 0.24.0 ran, right on every row. Read by the type rule + * ([[comparisonError]]) and the rendering ([[painless]]) alike, so what is judged is what runs; + * every reader below (the declared type, the implicit target, the target type, both operands) + * reads the mapped comparison. `None` for every other comparison, in a query or not. + */ + protected def literalsInPlace: Option[Expression] = None private def comparisonPainless(context: Option[PainlessContext]): String = { // A context-free rendering of an aggregate predicate is a bucket-pipeline rendering. @@ -2008,7 +2057,9 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { // byte-identical while sending everything else to the general path below, which binds the // operand to a local (`bindLocal`) and guards it -- the machinery `UPPER(status)` and // `ABS(amount)` already used, and the reason those were never affected. - case Some(p) if !rightNullable && p == chainRendering => + case Some(p) + if !rightNullable && p == chainRendering && + (implicitComparisonTarget.isEmpty || innerLeft == chainRendering) => if (identifier.nullable) return s"$p == null ? false : $painlessNot(${check(context, utc(p, leftUtc), utc(rightPromoted, rightUtc))})" else @@ -2124,6 +2175,12 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { * always has. A CASE condition is a script, so the rule judges it like any other comparison. */ def comparisonError(scripted: Boolean): Option[String] = + literalsInPlace match { + case Some(mapped) => mapped.comparisonError(scripted) + case None => ownComparisonError(scripted) + } + + private def ownComparisonError(scripted: Boolean): Option[String] = maybeValue.flatMap { value => if (!scripted && Expression.literalAgainstBareColumn(identifier, value)) None else @@ -2151,8 +2208,64 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { else ComparisonRule.readingOf(identifier, ComparisonRule.declaredTypeOf(value))) .map(_.sqlType) .getOrElse(operandDeclaredType) - ComparisonRule.mismatch(left, ComparisonRule.comparedType(reading, value)) - } + val right = ComparisonRule.comparedType(reading, value) + if (implicitAllowed(value, left, right)) + ComparisonRule.mismatch(left, right, equalityComparison) + else ComparisonRule.mismatch(left, right) + } + + /** Is this comparison an EQUALITY (`=`, `<>`, `!=`, IN) for the implicit-cast table + * (`ComparisonRule.implicitTarget`, SCHEMA-5b D6)? + */ + protected def equalityComparison: Boolean = effectiveOperator match { + case EQ | NE | DIFF => true + case _ => false + } + + /** May the implicit-cast table (SCHEMA-5b D6) judge the left operand against `value`, whose types + * are `left` and `right`? Neither side is a string literal (a literal is READ by what it is + * compared with, `ComparisonRule.readingOf`) or NULL; a text side reads no aggregate -- a group + * filter reads every metric as a number, converts nothing, and Elasticsearch's `max` / `min` + * read no keyword (T0-4). + */ + protected def implicitAllowed(value: Token, left: SQLType, right: SQLType): Boolean = + (effectiveOperator match { + // the comparisons DuckDB's table binds; LIKE, RLIKE, MATCH and the null tests keep main's rule + case EQ | NE | DIFF | LT | LE | GT | GE | IN | BETWEEN => true + case _ => false + }) && tableSide(identifier) && tableSide(value) && { + val text = + ComparisonRule.textType(SQLTypeUtils.operandType(left)) || + ComparisonRule.textType(SQLTypeUtils.operandType(right)) + !(text && (identifier.hasAggregation || (value match { + case p: PainlessScript => p.hasAggregation + case _ => false + }))) + } + + /** A side of the implicit-cast table: neither a string literal nor NULL. */ + protected def tableSide(token: Token): Boolean = + ComparisonRule.stringLiteral(token).isEmpty && !(token match { + case p: PainlessScript => ArithmeticExpression.nullLiteral(p) + case _ => false + }) + + /** 🔴 The type this comparison's two sides are compared IN by the implicit-cast table (SCHEMA-5b + * D6, `ComparisonRule.implicitTarget`), for a pair main's rule refuses; `None` -- every other + * comparison keeps main's rendering, byte for byte. The type rule ([[mismatchWith]]) reads the + * same table, so what is judged is what runs. Asked LIVE (A-82): the table's text rows depend on + * the scope, and a memo's query slot also serves a computed column's validation. + */ + protected def implicitComparisonTarget: Option[SQLType] = + maybeValue.flatMap { value => + val left = operandDeclaredType + val right = ComparisonRule.comparedType(literalReading, value) + if ( + ComparisonRule.stringLiteral(identifier).isDefined || + !implicitAllowed(value, left, right) || ComparisonRule.mismatch(left, right).isEmpty + ) None + else ComparisonRule.implicitTarget(left, right, equalityComparison) + } } object Expression { @@ -2189,6 +2302,26 @@ object Expression { */ private[query] def literalAgainstBareColumn(left: Token, right: Token): Boolean = (bareColumn(left) && literal(right)) || (literal(left) && bareColumn(right)) + + /** The two sides of an equality as SCHEMA-5b A-98 α reads them + * (`ComparisonRule.literalInPlaceOf`, R-119): each side a `COALESCE` whose literal it reads in + * place of it becomes that literal, a fresh one rendering its quoted text; `None` where neither + * side does. + */ + private[query] def literalsInPlace( + equality: Boolean, + left: Identifier, + right: Token + ): Option[(Identifier, Token)] = + if (!equality) None + else { + def literal(arm: PainlessScript): Option[Identifier] = + ComparisonRule.stringLiteral(arm).map(text => Identifier(StringValue(text))) + val l = ComparisonRule.literalInPlaceOf(left, right).flatMap(literal) + val r = ComparisonRule.literalInPlaceOf(right, left).flatMap(literal) + if (l.isEmpty && r.isEmpty) None + else Some((l.getOrElse(left), r.getOrElse(right))) + } } case class GenericExpression( @@ -2199,6 +2332,11 @@ case class GenericExpression( ) extends Expression { override def maybeValue: Option[Token] = Option(value) + override protected def literalsInPlace: Option[Expression] = + Expression.literalsInPlace(equalityComparison, identifier, value).map { case (l, r) => + copy(identifier = l, value = r) + } + override def negated: Option[Criteria] = Some(this.copy(maybeNot = if (maybeNot.isDefined) None else Some(NOT))) @@ -2226,6 +2364,12 @@ case class Comparison( ) extends Expression { override def maybeValue: Option[Token] = Option(value) + override protected def literalsInPlace: Option[Expression] = + Expression.literalsInPlace(equalityComparison, identifier, value).map { + case (l, r: PainlessScript) => copy(identifier = l, value = r) + case (l, _) => copy(identifier = l) + } + override def negated: Option[Criteria] = Some(this.copy(maybeNot = if (maybeNot.isDefined) None else Some(NOT))) @@ -2485,6 +2629,36 @@ case class InExpr[R, +T <: Value[R]]( s"Type mismatch: '${left.typeId}' is not compatible with '${right.typeId}' in expression: $this" } + /** `x IN (a, b)` is `x = a OR x = b`: an equality for the implicit-cast table (SCHEMA-5b D6). */ + override protected def equalityComparison: Boolean = true + + /** The implicit-cast table reads an IN whose elements are all NUMBER literals (`k IN (1, 2)`, `b + * IN (1, 0)`); a list holding a string literal keeps main's rule, its literals read by the + * operand (`readingFor`). + */ + override protected def implicitAllowed(value: Token, left: SQLType, right: SQLType): Boolean = + values.values.nonEmpty && + values.values.forall(v => ComparisonRule.stringLiteral(v).isEmpty) && + super.implicitAllowed(value, left, right) + + /** The type the operand and the elements are compared IN (SCHEMA-5b D6): each element's implicit + * target against the operand, when every element needs one -- their common supertype (`k IN (1, + * 1.5)` is compared as DOUBLE). + */ + override protected def implicitComparisonTarget: Option[SQLType] = + if (ComparisonRule.stringLiteral(identifier).isDefined) None + else { + val left = operandDeclaredType + val targets = values.values.map { v => + val right = ComparisonRule.comparedType(None, v) + if (!implicitAllowed(v, left, right) || ComparisonRule.mismatch(left, right).isEmpty) None + else ComparisonRule.implicitTarget(left, right, equality = true) + } + if (targets.nonEmpty && targets.forall(_.isDefined)) + Some(SQLTypeUtils.leastCommonSuperType(targets.flatten.toList)) + else None + } + /** What each element is READ as (`readingFor`), computed ONCE per IN and memoised (like * `literalReading`): the type rule, the type questions, both renderings and the group filter all * ask it. @@ -3029,6 +3203,28 @@ case class BetweenExpr( else if (bound eq fromTo.to) boundReadings._2 else readingFor(bound) + /** The type the operand and the bounds are compared IN (SCHEMA-5b D6): BETWEEN is no equality, so + * the table binds a BOOLEAN beside a whole number only (`b BETWEEN 0 AND 1`, `n BETWEEN b AND + * 5`). A bound main compares keeps its type; every other bound needs the table, or none applies. + */ + override protected def implicitComparisonTarget: Option[SQLType] = + if (ComparisonRule.stringLiteral(identifier).isDefined) None + else { + val left = operandDeclaredType + val perBound = Seq(fromTo.from, fromTo.to).map { b => + val right = ComparisonRule.comparedType(boundReading(b), b) + if (ComparisonRule.mismatch(left, right).isEmpty) Some(None) + else if (!implicitAllowed(b, left, right)) None + else ComparisonRule.implicitTarget(left, right, equality = false).map(Some(_)) + } + if (perBound.exists(_.isEmpty)) None + else + perBound.flatten.flatten match { + case Nil => None + case targets => Some(SQLTypeUtils.leastCommonSuperType(targets.toList)) + } + } + /** What a bare column's `range` query sends for `bound`: the whole number the one comparison rule * rounded a quoted bound to beside a whole-declared column (`queryTextOf`, R-54), else `None` -- * its own text. @@ -3096,7 +3292,19 @@ case class BetweenExpr( val rendered = readingPainless(boundReading(b), b, context) val from = boundRenderedType(b) val to = comparisonTargetType - if ( + // SCHEMA-5b D6 / D7: a BOOLEAN bound beside a whole operand is the number it is compared as + if (implicitComparisonTarget.isDefined) + SQLTypeUtils.implicitCast( + rendered, + from, + to, + b match { + case p: PainlessScript => SQLTypeUtils.mayRenderNull(p, context) + case _ => false + }, + context + ) + else if ( from != to && from.isInstanceOf[SQLTemporal] && to.isInstanceOf[SQLTemporal] && !context.exists(_.isProcessor) ) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index 6869fcc2..564eebcf 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -19,6 +19,7 @@ package app.softnetwork.elastic.sql.`type` import app.softnetwork.elastic.sql.{ painlessUtcLocalDate, painlessUtcLocalTime, + ComparisonRule, Identifier, LiteralParam, PainlessContext, @@ -352,6 +353,417 @@ object SQLTypeUtils { if (unknownBesideDate) SQLTypes.Timestamp else leastCommonSuperType(types) } + /** What the string-literal arms of a conditional are READ as (SCHEMA-5b, D1 / D2): the `target` + * -- the type the other arms report ([[armsReportedType]] over the arms that are neither a + * string literal nor NULL) -- each string literal's reading beside it, by its position among the + * arms, and whether a whole reading falls outside the target's whole range (`widened`: the read + * literals are then BIGINTs, as `GREATEST(t, '200')` over a TINYINT `t` is 200). + */ + final case class ArmReadings( + target: SQLType, + readings: Map[Int, ComparisonRule.Reading], + widened: Boolean + ) { + + /** The type every read literal takes: the target's, DuckDB's label, or BIGINT when widened. */ + def literalType: SQLType = if (widened) SQLTypes.BigInt else target + } + + /** 🔴 The ONE reading of a conditional's string-literal arms (SCHEMA-5b D1, #292), as DuckDB + * 1.5.5 types a string literal by the arguments beside it: `GREATEST(n, '1.5')` over an INT `n` + * is 2, an INTEGER; `COALESCE(d, '2024-01-15')` a DATE. Each string literal is read by the ONE + * comparison rule (`ComparisonRule.readingOf`) beside the type the other arms report: R-54's + * DuckDB cast beside a number (`'1.5'` is 2 beside a whole type), a date or a timestamp beside a + * temporal, a boolean beside a BOOLEAN. + * + * Defined only when at least one arm is typed, every typed arm's type is known, and EVERY string + * arm reads. Otherwise `None`, and the whole node keeps the verdict, the type and the bytes it + * had: `COALESCE(n, 'abc')` answers its text, `GREATEST(n, 'abc')` stays refused, a literal + * beside text stays text. All or nothing, so nothing answered becomes refused. + * + * Memoised per node and per scope (`AnswersArms.armReadings`). + */ + def armReadings(arms: List[PainlessScript]): Option[ArmReadings] = { + val literals = arms.zipWithIndex.filter(a => ComparisonRule.stringLiteral(a._1).isDefined) + if (literals.isEmpty) None + else { + val typed = arms + .filterNot(a => ComparisonRule.stringLiteral(a).isDefined) + .map(a => operandType(a.reportedType)) + .filterNot(_ == SQLTypes.Null) + if (typed.isEmpty || typed.exists(_.isUnknown)) None + else { + val target = armsReportedType(typed) + if (target.isUnknown) None + else { + val read = literals.map { case (arm, i) => i -> ComparisonRule.readingOf(arm, target) } + if (read.exists(_._2.isEmpty)) None + else { + val readings = read.collect { case (i, Some(r)) => i -> r }.toMap + Some( + ArmReadings(target, readings, readings.values.exists(outsideWholeRange(_, target))) + ) + } + } + } + } + } + + /** Is `reading` a whole number the whole type `target` cannot hold (D2)? */ + private def outsideWholeRange(reading: ComparisonRule.Reading, target: SQLType): Boolean = + reading match { + case ComparisonRule.NumberReading(painless, false) => + val v = scala.util.Try(painless.stripSuffix("L").toLong).toOption + def outside(min: Long, max: Long): Boolean = v.exists(n => n < min || n > max) + target match { + case _: SQLTinyInt => outside(scala.Byte.MinValue.toLong, scala.Byte.MaxValue.toLong) + case _: SQLSmallInt => outside(scala.Short.MinValue.toLong, scala.Short.MaxValue.toLong) + case _: SQLInt => outside(scala.Int.MinValue.toLong, scala.Int.MaxValue.toLong) + case _ => false + } + case _ => false + } + + /** The type each arm of a conditional is read as (SCHEMA-5b D2 / D5), the ONE list every reader + * of an arm's type reads (#292): a read literal is its [[ArmReadings.literalType]]; a BOOLEAN + * arm beside whole-number arms only -- no fractional, temporal, text or unknown arm, and no + * aggregate + * -- is their type, as DuckDB types `COALESCE(b, t)` a TINYINT; in a query, a read-moved arm + * whose own reading alone makes the arms mix is the VARCHAR 0.24.0 typed it + * ([[readMovedArmsAsText]], A-97); every other arm is the type it reports. + */ + def armTypesOf(arms: List[PainlessScript], readings: Option[ArmReadings]): List[SQLType] = { + val read = arms.zipWithIndex.map { case (a, i) => + readings.filter(_.readings.contains(i)).map(_.literalType).getOrElse(a.reportedType) + } + readMovedArmsAsText(arms, read).getOrElse { + val elements = read.map(operandType) + val others = elements.filterNot(t => t == SQLTypes.Null || t == SQLTypes.Boolean) + if ( + elements.contains(SQLTypes.Boolean) && others.nonEmpty && + others.forall(ComparisonRule.wholeNumberType) && !arms.exists(_.hasAggregation) + ) { + val whole = leastCommonSuperType(others) + read.zip(elements).map { case (t, e) => if (e == SQLTypes.Boolean) whole else t } + } else read + } + } + + /** 🔴 The arm types of a conditional nested beside text, in a QUERY (SCHEMA-5b A-97): where the + * arms `read` as they are read MIX ([[mixedArmTypes]]) but stop mixing once every READ-MOVED arm + * ([[readMoved]]) is put back at its 0.24.0 type, VARCHAR, those arms are VARCHAR. The node then + * keeps 0.24.0's verdict and its VARCHAR type, and converts each such arm to the text of the + * value it is read as (`convertArm`): `COALESCE(COALESCE(n, '1.5'), k)` answers `'2'` on a NULL + * `n` (0.24.0: `'1.5'`), where the nested reading refused the statement (`Cannot mix values of + * type INT and KEYWORD`) that 0.24.0 answered as text, never failing. + * + * `None` everywhere else, so the arms keep their types: outside a query ([[readsWholeValues]]: a + * processor, a transform, a computed column's validation), arms that do not mix, arms that still + * mix at 0.24.0's types (`COALESCE(COALESCE(n, '3'), d)` keeps its refusal and its message), and + * arms none of which is read-moved. Read only where the arms mix, so a node that does not mix + * scans nothing. + */ + private def readMovedArmsAsText( + arms: List[PainlessScript], + read: List[SQLType] + ): Option[List[SQLType]] = + if (readsWholeValues || mixedArmTypes(arms, read).isEmpty) None + else { + val moved = arms.map(readMoved) + if (!moved.contains(true)) None + else { + val text = read.zip(moved).map { case (t, m) => if (m) SQLTypes.Varchar else t } + if (mixedArmTypes(arms, text).isEmpty) Some(text) else None + } + } + + /** Is `arm` READ-MOVED (SCHEMA-5b A-97): a conditional ([[conditionalNode]]) whose quoted + * arguments are read beside its other arms (`AnswersArms.armReadings`) -- a type 0.24.0 reported + * as VARCHAR -- or one that answers such an arm: a nested answered arm, or a `NULLIF`'s value + * (its one answered arm)? Also read by `NullIf.answeredByFirst` (A-98 β). + */ + private[sql] def readMoved(arm: PainlessScript): Boolean = conditionalNode(arm) match { + case c: AnswersArms => c.armReadings.isDefined || c.answeredArms.exists(readMoved) + case _ => false + } + + /** 🔴 The ONE family rule of a conditional's arms (SCHEMA-5b A-97, D3 (6), #292), DuckDB 1.5.5's + * refusal of arms with NO common type: the first two of `types` -- the type each of `arms` is + * read as, position by position (`AnswersArms.armTypes`) -- whose families differ. A date (DATE, + * TIMESTAMP) or a TIME beside a number, a date beside a TIME, text beside a number or a date, a + * BOOLEAN beside a date. A BOOLEAN and a number have a common type, INTEGER. + * + * Left out, as DuckDB reads them by the other arms: a string literal (read as the type beside + * it), a NULL, an arm of a type still unknown (no schema, `date_nanos`); and every arm of a node + * holding an aggregate, whose per-group value Elasticsearch computes. Each arm is read as + * Painless reads it ([[operandType]]: an `ARRAY` arm is one X element). + * + * Read by the refusal (`mixedArmsError`) and by [[armTypesOf]] 's text rule, so the two never + * disagree on whether arms mix. + */ + def mixedArmTypes( + arms: List[PainlessScript], + types: List[SQLType] + ): Option[(SQLType, SQLType)] = { + def family(t: SQLType): Int = + if (t.isNumber) 0 + else if (t == SQLTypes.Boolean) 1 + else if ( + t == SQLTypes.Date || t == SQLTypes.DateTime || t == SQLTypes.Timestamp || + t == SQLTypes.Temporal + ) 2 + else if (t == SQLTypes.Time) 3 + else if (t.isText) 4 + else -1 + // a BOOLEAN and a number have a common type, INTEGER + def common(a: Int, b: Int): Boolean = a == b || Set(a, b) == Set(0, 1) + if (arms.exists(_.hasAggregation)) None + else { + val typed = arms + .zip(types) + .filterNot(a => ComparisonRule.stringLiteral(a._1).isDefined) + .map(a => operandType(a._2)) + .filterNot(t => t == SQLTypes.Null || t.isUnknown) + .filter(family(_) >= 0) + typed.headOption.flatMap { first => + typed.tail.find(t => !common(family(first), family(t))).map(first -> _) + } + } + } + + /** The conditional an operand IS: the operand itself, or the one function of the nameless + * identifier the parser wraps an operand in (`COALESCE(COALESCE(n, '3'), k)`'s first arm). Read + * by [[textTypeOutsideQuery]] and [[readMoved]], which see the same nested node. + */ + def conditionalNode(operand: PainlessScript): PainlessScript = operand match { + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (f: PainlessScript) :: Nil => f + case _ => operand + } + case other => other + } + + /** The Painless a READ literal arm renders (SCHEMA-5b D3 (4)): the value it is read as, a literal + * of the type `as` it takes -- `2` beside an INT, `2L` beside a BIGINT, `((byte) 2)` beside a + * TINYINT, `2.0` beside a DOUBLE -- never the text, which the arm used to answer (`COALESCE`) or + * which Painless refused to compare with a number (`GREATEST`). A date or a timestamp is the + * value it denotes (`ComparisonRule.temporalPainless`) in a query; outside one (an ingest + * processor, whose `ctx` holds JSON values, or a transform) it is its ISO text, which an + * Elasticsearch `date` field and the date parsers of 0.24.0's consumers read (A-92). + */ + def readPainless( + reading: ComparisonRule.Reading, + as: SQLType, + context: Option[PainlessContext] + ): String = + reading match { + case ComparisonRule.NumberReading(painless, fractional) => + val whole = + if (fractional) None else scala.util.Try(painless.stripSuffix("L").toLong).toOption + (as, whole) match { + case (_: SQLTinyInt, Some(v)) => s"((byte) $v)" + case (_: SQLSmallInt, Some(v)) => s"((short) $v)" + case (_: SQLInt, Some(v)) => s"$v" + case (_: SQLBigInt, Some(v)) => s"${v}L" + case (_: SQLDouble, Some(v)) => s"$v.0" + case (_: SQLDouble, None) => painless + case (_: SQLReal, Some(v)) => s"((float) $v.0)" + case (_: SQLReal, None) => s"((float) $painless)" + case _ => painless + } + // outside a query (a processor, a transform: [[rendersOutsideQuery]]; the node keeps its + // text type there and while its computed column is validated, A-92 / A-96) the ISO text, + // which the parsers there read + case t: ComparisonRule.TemporalReading => + if (rendersOutsideQuery) "\"" + t.text + "\"" + else ComparisonRule.temporalPainless(t) + case ComparisonRule.BooleanReading(value) => value.toString + } + + /** Does the BOOLEAN arm at `index` of `node` take the whole type beside it (SCHEMA-5b D5)? */ + def booleanBesideWhole(node: AnswersArms, index: Int): Boolean = + node.answeredArms.lift(index).exists(a => operandType(a.reportedType) == SQLTypes.Boolean) && + node.armTypes.lift(index).exists(ComparisonRule.wholeNumberType) + + /** A BOOLEAN read in an INGEST processor as Elasticsearch reads a boolean `_source` (SCHEMA-5b D5 + * / R-107): `true` / `"true"` are true, `false` / `"false"` / `""` false. `name` is a bound, + * non-NULL value. + */ + def processorBoolean(name: String): String = + s"($name instanceof Boolean ? (boolean) $name : 'true'.equals($name))" + + /** 🔴 The ONE runtime conversion of an implicit comparison cast (SCHEMA-5b D7), beside [[coerce]] + * and not inside it: `CAST` keeps its own strict parses (A-83, PD-1). `rendered` is the side of + * a comparison the table (`ComparisonRule.implicitTarget`) converts, read as `from`, to the type + * `to` it is compared in: + * - a TEXT to a whole number: the whole number it rounds to, half away from zero -- the + * Painless twin of `ComparisonRule.roundedWhole` (`' 1 '` is 1, `'1.5'` 2, `'-2.5'` -3, + * `'.5'` 1), the BIGINT bounds compared BEFORE any rescale (an unbounded `setScale` spent + * 8.7 s on `'1e30000000'`); + * - a TEXT to a DOUBLE: `Double.parseDouble` of the checked text, +/-Infinity past DOUBLE's + * range, DuckDB's value (A-93), where the literal rule (`numberOf`) reads nothing; + * - a TEXT to a BOOLEAN: `booleanOf`'s rule, the trimmed text in lower case equal to one of + * its words (A-81). + * + * A text is checked against `numberOf`'s grammar (`DecimalNumber`, ASCII digits) by a character + * scan -- Painless regexes are off by default on 6.8 -- and anything else THROWS, as DuckDB + * fails converting it (`'abc'`, `''`, `'1,5'`, `'NaN'`, `'0x10'`). Each text form is a prologue + * statement (`PainlessContext.bindLocalWith`), inside the operand's NULL guard: a NULL stays + * NULL. A text form needs a context; without one (a group filter, which converts nothing) the + * rendering is [[coerce]] 's. + * - a BOOLEAN to a number: [[coerce]] 's BOOLEAN -> NUMERIC arm (1 / 0), its operand read in + * an ingest processor as Elasticsearch reads a boolean `_source` ([[processorBoolean]]). + * + * Two numbers, and a side already of the target's family, are compared as they stand: Painless + * compares boxed numerics numerically. + */ + def implicitCast( + rendered: String, + from: SQLType, + to: SQLType, + nullable: Boolean, + context: Option[PainlessContext] + ): String = { + val source = operandType(from) + if (ComparisonRule.textType(source) && (to.isNumber || to == SQLTypes.Boolean)) + context match { + case Some(ctx) => textCast(rendered, to, ctx) + case None => coerce(rendered, source, to, nullable, context) + } + else if (source == SQLTypes.Boolean && to.isNumber) + context match { + case Some(ctx) => + val raw = rendered.trim + val name = + if (boundName(raw)) raw else ctx.bindLocal(asOneValue(raw), "lv") + val operand = if (ctx.isProcessor) processorBoolean(name) else name + val converted = coerce(operand, SQLTypes.Boolean, to, nullable = false, context) + if (nullable || ctx.isProcessor) s"($name == null ? null : (def)($converted))" + else converted + case None => coerce(rendered, SQLTypes.Boolean, to, nullable, context) + } + else rendered + } + + /** Is `operand` a conditional -- a `CASE`, a `COALESCE`, a `NULLIF`, a `GREATEST` / `LEAST` -- + * bare, or the one function of the nameless identifier the parser wraps an operand in? Read by + * the string functions' receiver rule (R-109). + */ + def conditional(operand: PainlessScript): Boolean = operand match { + case _: AnswersArms | _: NullIf => true + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (_: AnswersArms | _: NullIf) :: Nil => true + case _ => false + } + case _ => false + } + + /** 🔴 The TEXT type a `COALESCE` or a `CASE` holding a literal read as a DATE or a TIMESTAMP + * (`AnswersArms.readsAsTextOutsideQuery`) is DECLARED with outside a query -- in an ingest + * processor or a transform, where its value is text (`ctx` holds JSON, and the literal renders + * its ISO text, A-92): the type 0.24.0 reported for it, each such literal keeping the type it + * reports itself, and a conditional of this kind nested as an arm (or as a `NULLIF`'s value) + * keeping its own. `None` in a query, and for every other operand. + * + * Asked where a script reads whole values ([[readsWholeValues]]), so the validation of a + * computed column types the node as the processor it installs does (A-96, #292): with the node + * DATE there, NULLIF's type rule refused `NULLIF(COALESCE(d, '2024-01-15'), k)` -- which 0.24.0 + * accepts, and every ALTER re-judges -- and accepted `NULLIF(COALESCE(d, '2024-01-15'), d2)` + * over a processor comparing a String with a date. `Coalesce`, `Case` and `NullIf` answer their + * legacy `baseType` wherever it is defined. + * + * The comparison rule reads it (`ComparisonRule.declaredTypeOf`), so a comparison, an IN, a + * BETWEEN, a `NULLIF` and a `CASE x WHEN` compare that text as 0.24.0 does: reading its literal + * as a date over the text, `CASE WHEN COALESCE(d, '2024-01-15') = '2024-01-15'` rendered + * `left1.isEqual(LocalDate.parse(…))` on a String -- CREATE and ALTER were accepted, every + * document failed, and `ignore_failure` dropped the column silently (SCHEMA-5b R1-1). `GREATEST` + * / `LEAST` over dates answer an instant in every scope (`overDatesPainless`), not text, and + * keep their reported type. + */ + def textTypeOutsideQuery(operand: PainlessScript): Option[SQLType] = + if (!readsWholeValues) None + else { + conditionalNode(operand) match { + case n: NullIf => textTypeOutsideQuery(n.expr1) + case c @ (_: Coalesce | _: Case) => + val conditional = c.asInstanceOf[AnswersArms] + val arms = conditional.answeredArms + val texts = arms.indices.map { i => + if (conditional.readsAsTextOutsideQuery(i)) Some(arms(i).reportedType) + else textTypeOutsideQuery(arms(i)) + }.toList + if (texts.forall(_.isEmpty)) None + else + Some(armsReportedType(texts.zip(conditional.armTypes).map { case (text, t) => + text.getOrElse(t) + })) + case _ => None + } + } + + /** Is `raw` a NAME (a parameter or a local), which a conversion may read twice? */ + private def boundName(raw: String): Boolean = + raw.nonEmpty && (raw.head.isLetter || raw.head == '_') && + raw.forall(c => c.isLetterOrDigit || c == '_') + + /** D7's text forms: a prologue statement that reads `rendered` -- bound once -- as `to`, NULL for + * a NULL, and throws on a text that spells no value of `to`. + */ + private def textCast(rendered: String, to: SQLType, ctx: PainlessContext): String = { + val raw = rendered.trim + val x = if (boundName(raw)) raw else ctx.bindLocal(asOneValue(raw), "lv") + ctx.bindLocalWith("nv") { v => + val s = s"${v}s" + val body = + if (to == SQLTypes.Boolean) + // `booleanOf`'s exact rule: the trimmed text in lower case, equal to one of its words. + // Never `equalsIgnoreCase`, which folds U+017F 'ſ' to 's' ('yeſ' read true where the + // literal rule reads nothing, R0-2); none of the words holds an 'i', so the default + // locale's lower case reads them as Scala's does + s"String $s = String.valueOf($x).trim().toLowerCase(); " + + s"if ($s.equals('t') || $s.equals('true') || $s.equals('y') || " + + s"$s.equals('yes') || $s.equals('on') || $s.equals('1')) { $v = true; } " + + s"else if ($s.equals('f') || $s.equals('false') || $s.equals('n') || " + + s"$s.equals('no') || $s.equals('off') || $s.equals('0')) { $v = false; } " + + s"else { throw new IllegalArgumentException('Conversion Error: could not convert string [' + String.valueOf($x).trim() + '] to BOOLEAN'); }" + else { + val scan = decimalScan(x, v, to) + if (ComparisonRule.wholeNumberType(to)) + scan + + s"BigDecimal ${v}x = new BigDecimal($s); " + + s"if (${v}x.compareTo(new BigDecimal('9223372036854775807.5')) >= 0 || " + + s"${v}x.compareTo(new BigDecimal('-9223372036854775808.5')) <= 0) { " + + s"throw new ArithmeticException('Conversion Error: string [' + $s + '] is out of BIGINT range'); } " + + s"if (${v}x.abs().compareTo(BigDecimal.ONE) < 0) { " + + s"$v = ${v}x.abs().compareTo(new BigDecimal('0.5')) >= 0 ? (long) ${v}x.signum() : 0L; } " + + s"else { $v = ${v}x.setScale(0, RoundingMode.HALF_UP).longValueExact(); }" + else scan + s"$v = Double.parseDouble($s);" + } + s"def $v = null; if ($x != null) { $body } " + } + } + + /** The character scan of `numberOf`'s grammar (`[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?`, ASCII + * digits) over the trimmed text of `x`, into `s`: statements that throw unless it holds. + */ + private def decimalScan(x: String, v: String, to: SQLType): String = { + val s = s"${v}s" + val n = s"${v}n" + val i = s"${v}i" + def digit = s"($s.charAt($i) >= 48 && $s.charAt($i) <= 57)" + def sign = s"($s.charAt($i) == 43 || $s.charAt($i) == 45)" + s"String $s = String.valueOf($x).trim(); int $n = $s.length(); int $i = 0; " + + s"if ($i < $n && $sign) { $i++; } " + + s"int ${v}a = $i; while ($i < $n && $digit) { $i++; } int ${v}d = $i - ${v}a; " + + s"if ($i < $n && $s.charAt($i) == 46) { $i++; int ${v}b = $i; while ($i < $n && $digit) { $i++; } ${v}d += $i - ${v}b; } " + + s"boolean ${v}ok = ${v}d > 0; " + + s"if (${v}ok && $i < $n && ($s.charAt($i) == 101 || $s.charAt($i) == 69)) { $i++; " + + s"if ($i < $n && $sign) { $i++; } int ${v}e = $i; while ($i < $n && $digit) { $i++; } ${v}ok = $i > ${v}e; } " + + s"if (!${v}ok || $i != $n) { throw new IllegalArgumentException('Conversion Error: could not convert string [' + $s + '] to ${to.typeId}'); } " + } + def coerce(in: PainlessScript, to: SQLType, context: Option[PainlessContext]): String = coerce(in, renderedSourceType(in, context), to, context) @@ -464,16 +876,25 @@ object SQLTypeUtils { outsideQuery.withValue(true)(body) else body - /** A value a node computes ONCE PER SCOPE: one slot for a query, one for a script rendered - * outside it ([[rendersOutsideQuery]], an ingest processor or a transform). + /** A value a node computes ONCE PER SCOPE, in one of three slots picked by ONE partition of the + * scopes: + * - a script rendered outside a query ([[rendersOutsideQuery]], an ingest processor or a + * transform), whether or not a computed column is being validated; + * - else the validation of a computed column ([[validatingComputedColumns]]), which reads + * whole values ([[readsWholeValues]]: D7's element view is off there); + * - else a query. * * 🔴 Every per-node memo that reads a DECLARED type keeps its answer here (#292). A type may * depend on the scope (`DATE_TRUNC` over a date reports TIMESTAMP in a query only, a conditional - * answers its legacy type outside one), and a memo with one slot let the first scope to ask - * decide for both on a shared node: a query rendered first moved the ingest processor's bytes - * (`DATE_TRUNC(CAST(ts AS DATE), DAY) - d` in fractional days), a processor rendered first moved - * the query's type (BIGINT where a fresh query reports DOUBLE). With one slot per scope each - * scope reads what it reads on a fresh node, whatever order they render in. + * answers its legacy type outside one, and a conditional over an `ARRAY` arm reports VARCHAR + * under a computed column's validation and INT in a query), and a memo with one slot let the + * first scope to ask decide for the others on a shared node: a query rendered first moved the + * ingest processor's bytes (`DATE_TRUNC(CAST(ts AS DATE), DAY) - d` in fractional days), a + * processor rendered first moved the query's type (BIGINT where a fresh query reports DOUBLE), a + * validation first made a query render `Long.parseLong("yes")` (SCHEMA-5b). With one slot per + * scope each scope reads what it reads on a fresh node, whatever order they run in, and each + * computes it once: computed live under the validation instead, nested conditionals made a + * `CREATE TABLE` exponential (SCHEMA-5b, A-95). * * No lock, as the memos it replaces: the value is pure and immutable, so a race can only compute * it twice. `null` is "not computed yet", so `T` is never null itself (an `Option`, a type, a @@ -481,6 +902,7 @@ object SQLTypeUtils { */ final class PerScope[T <: AnyRef] { private[this] var inQuery: T = _ + private[this] var validation: T = _ private[this] var outside: T = _ def apply(compute: => T): T = @@ -491,6 +913,13 @@ object SQLTypeUtils { outside = value } value + } else if (computedColumnValidation.value) { + var value = validation + if (value eq null) { + value = compute + validation = value + } + value } else { var value = inQuery if (value eq null) { @@ -514,13 +943,17 @@ object SQLTypeUtils { * 🔴 SCHEMA-6 (lead R-66): ONE scope for D7's element view ([[operandType]]), which is off in * it. Every reader of the element view goes through `operandType`, so the processor's bytes and * the validation's verdict are 0.24.0's for an `ARRAY` column, and A-30's refusal of a - * NUMERIC or TEMPORAL input over it (`acceptableOperand`, the declared type) stays. + * NUMERIC or TEMPORAL input over it (`acceptableOperand`, the declared type) stays. The text + * type of a conditional holding a literal read as a date ([[textTypeOutsideQuery]], A-96) and + * the implicit-cast table's text rows (`ComparisonRule.implicitTarget`, A-82) read it too. */ def readsWholeValues: Boolean = outsideQuery.value || computedColumnValidation.value - /** Runs `body`, the validation of computed columns, in the scope of [[readsWholeValues]]. Only - * D7's element view reads that scope: the rest of the validation reads the types it read before - * (a conditional answers its query type, [[rendersOutsideQuery]] stays false). + /** Runs `body`, the validation of computed columns, in the scope of [[readsWholeValues]]. The + * readers of that scope ([[operandType]]'s element view, a date-reading conditional's text type, + * the table's text rows) answer there as in the processor the column installs; the rest of the + * validation reads the types it read before (a conditional answers its query type, + * [[rendersOutsideQuery]] stays false). */ def validatingComputedColumns[T](body: => T): T = if (computedColumnValidation.value) body @@ -552,9 +985,11 @@ object SQLTypeUtils { * nanosecond timestamp a TIMESTAMP, which converting it to the DATE beside it would truncate. A * NULL arm has no type and is left out. */ - def armsType(arms: List[PainlessScript], reported: SQLType): Option[SQLType] = { - // an `ARRAY` arm renders one X element (SCHEMA-6 D7, [[operandType]]) - val known = arms.map(a => operandType(a.reportedType)).filterNot(_ == SQLTypes.Null) + def armsType(node: AnswersArms, reported: SQLType): Option[SQLType] = { + val arms = node.answeredArms + // an `ARRAY` arm renders one X element (SCHEMA-6 D7, [[operandType]]); a read literal and a + // BOOLEAN beside whole numbers are the type they are read as (SCHEMA-5b, `armTypes`) + val known = node.armTypes.map(operandType).filterNot(_ == SQLTypes.Null) def all(family: SQLType => Boolean): Boolean = known.forall(family) def dateCarrying(t: SQLType): Boolean = t == SQLTypes.Date || t == SQLTypes.DateTime || t == SQLTypes.Timestamp || @@ -582,7 +1017,7 @@ object SQLTypeUtils { def armsAnswer(f: Function): Option[SQLType] = f match { case c: AnswersArms => val arms = c.answeredArms - armsType(arms, f.reportedType).filterNot(_ => + armsType(c, f.reportedType).filterNot(_ => arms.exists(a => a.reportedType != SQLTypes.Null && armTypeUnknown(a)) ) case _ => None diff --git a/sql/src/test/scala/app/softnetwork/elastic/schema/FieldCapsSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/schema/FieldCapsSpec.scala index fb322596..093e8627 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/schema/FieldCapsSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/schema/FieldCapsSpec.scala @@ -279,13 +279,17 @@ class FieldCapsSpec extends AnyFlatSpec with Matchers { val sql = place.replace("%s", expr.replace("%c", c)) withClue(s"[$origin] $sql: ") { val control = refusal(place.replace("%s", expr.replace("%c", native)), table) - control.isLeft shouldBe true + // SCHEMA-5b D6: a number beside a text, for an equality, is compared as DuckDB casts it -- + // the native column and the D1 one alike; every other pair stays refused + control.isLeft shouldBe !(expr == "%c = k" && c != "dn") refusal(sql, table) shouldBe control.left.map(_.replaceAll(s"\\b$native\\b", c)) } } - // the messages, as measured over the mapping (the review's census of the branch) - refusal("SELECT id FROM s7_t WHERE sf = k", mapping) shouldBe - Left("Type mismatch: 'DOUBLE' is not compatible with 'KEYWORD' in expression: sf = k") + // the messages, as measured over the mapping (the review's census of the branch); `sf = k` + // is answered since SCHEMA-5b (D6), refused before as 'DOUBLE' against 'KEYWORD' + refusal("SELECT id FROM s7_t WHERE sf = k", mapping) shouldBe Right(()) + refusal("SELECT id FROM s7_t WHERE sf > k", mapping) shouldBe + Left("Type mismatch: 'DOUBLE' is not compatible with 'KEYWORD' in expression: sf > k") refusal("SELECT id FROM s7_t WHERE dn > n", mapping) shouldBe Left("Type mismatch: 'TIMESTAMP' is not compatible with 'BIGINT' in expression: dn > n") // a projection of `dn * 2` / `dn / 2` takes the date-arithmetic refusal too diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/MixedArmsRefusalSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/MixedArmsRefusalSpec.scala index 6ab303eb..f5e5880d 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/MixedArmsRefusalSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/MixedArmsRefusalSpec.scala @@ -205,7 +205,13 @@ class MixedArmsRefusalSpec extends AnyFlatSpec with Matchers { val sql = statement(c) withClue(s"[$origin] $sql: ") { val control = errors(statement(native), table) - control should not be empty + // SCHEMA-5b D6: a NULLIF compares a text with a number or a boolean as DuckDB casts it, in + // a query -- the native column and the D1 one alike; every other cell stays refused + val d6 = conditional.startsWith("NULLIF") && (others match { + case `keywordArms` => other == "nb" || other == "bo" + case _ => other == "tx" && c != "dn" + }) + if (d6) control shouldBe empty else control should not be empty errors(sql, table) shouldBe control.map(_.replaceAll(s"\\b$native\\b", c)) count += 1 } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/QuotedArmReadingSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/QuotedArmReadingSpec.scala new file mode 100644 index 00000000..3c75a867 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/QuotedArmReadingSpec.scala @@ -0,0 +1,1047 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.function.cond + +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType, PainlessScript} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{AlterTable, CreateTable, SingleSearch, Update} +import app.softnetwork.elastic.sql.schema.{ + validateScriptReferences, + Column, + Schema, + ScriptProcessor, + Table => SchemaTable +} +import app.softnetwork.elastic.sql.`type`.{SQLTypeUtils, SQLTypes} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.util.Try + +/** A quoted argument of `COALESCE`, `GREATEST`, `LEAST` and a `CASE` result is read as the type of + * the arguments beside it, as DuckDB 1.5.5 reads it (SCHEMA-5b, D1 - D5): the verdict, the + * reported type and the rendering, in a query and in an ingest processor. + */ +class QuotedArmReadingSpec extends AnyFlatSpec with Matchers { + + private val table = SchemaTable( + "t", + columns = List( + Column("ti", SQLTypes.TinyInt), + Column("n", SQLTypes.Int), + Column("l", SQLTypes.BigInt), + Column("x", SQLTypes.Double), + Column("r", SQLTypes.Real), + Column("d", SQLTypes.Date), + Column("ts", SQLTypes.Timestamp), + Column("b", SQLTypes.Boolean), + Column("k", SQLTypes.Keyword), + Column("g", SQLTypes.Keyword), + Column("ab", SQLTypes("array")), + Column("an", SQLTypes("array")) + ) + ) + + private def query = Some(PainlessContext(PainlessContextType.Query)) + + private def resolved(sql: String): SingleSearch = Parser(sql) match { + case Right(s: SingleSearch) => s.update(Some(table)) + case other => fail(s"not a single search: $other") + } + + /** The verdict, the reported type and the query Painless of `SELECT AS r FROM t`. */ + private def select(expr: String): (Option[String], String, String) = { + val s = resolved(s"SELECT $expr AS r FROM t") + val field = s.select.fieldsWithComputedAliases.head + ( + s.validateResolved().left.toOption, + field.identifier.reportedType.typeId, + field.painless(query) + ) + } + + private def accepted(expr: String, reported: String): String = { + val (verdict, t, painless) = select(expr) + withClue(s"$expr: ") { + verdict shouldBe None + t shouldBe reported + } + painless + } + + private val columns = + "id KEYWORD, ti TINYINT, n INT, l BIGINT, x DOUBLE, r REAL, d DATE, ts TIMESTAMP, b BOOLEAN" + + /** The ingest processor of a computed column `c SCRIPT AS ()`. */ + private def processor(decl: String, expr: String): String = + Parser(s"CREATE TABLE t ($columns, c $decl SCRIPT AS ($expr))") match { + case Right(ct: CreateTable) => + ct.schema.defaultPipeline.processors + .collectFirst { case p: ScriptProcessor if p.column == "c" => p.source } + .getOrElse(fail(s"no processor for $expr")) + case other => fail(s"$expr: $other") + } + + /** The ingest processor of `UPDATE t SET c = `. */ + private def updateProcessor(expr: String): String = + Parser(s"UPDATE t SET c = $expr WHERE n > 0") match { + case Right(u: Update) => + u.customPipeline.processors + .collectFirst { case p: ScriptProcessor => p.source } + .getOrElse(fail(s"no processor for $expr")) + case other => fail(s"$expr: $other") + } + + /** The transform script of `SELECT AS r FROM t`. */ + private def transform(expr: String): String = + resolved(s"SELECT $expr AS r FROM t").select.fieldsWithComputedAliases.head + .painless(Some(PainlessContext(PainlessContextType.Transform))) + + behavior of "a quoted argument beside a number" + + it should "be read as the whole number it rounds to, half away from zero (R-54)" in { + // main refused it: GREATEST requires numeric arguments ... but got VARCHAR for '1.5' + val greatest = accepted("GREATEST(n, '1.5')", "INT") + greatest should not include "\"1.5\"" + greatest should include("param1 >= 2 ? param1 : 2") + accepted("LEAST('-2.5', n)", "INT") should include("-3") + accepted("GREATEST(l, '0.5')", "BIGINT") should include("1L") + } + + it should "take the operand's type, DuckDB's label (D2)" in { + accepted("GREATEST(ti, '1.5')", "TINYINT") should include("((byte) 2)") + accepted("COALESCE(x, '3')", "DOUBLE") should include("3.0") + accepted("COALESCE(n, ' 2 ')", "INT") should include("param1 : 2") + } + + it should "widen the node to BIGINT past the operand's whole range, never wrap (D2)" in { + accepted("COALESCE(ti, '200')", "BIGINT") should include("200L") + accepted("GREATEST(n, '9223372036854775807.4')", "BIGINT") should include( + "9223372036854775807L" + ) + } + + it should "answer the number in a COALESCE, ending it when the literal comes first (D3)" in { + // main answered the text "3" on every row, reported VARCHAR + accepted("COALESCE(n, '3')", "INT") shouldBe "(param1 != null ? param1 : 3)" + val first = accepted("COALESCE('3', n)", "INT") + first should not include "!= null" + first should include("3") + } + + it should "keep main's outcome for a text that reads as nothing (D11)" in { + val (verdict, t, painless) = select("COALESCE(n, 'abc')") + verdict shouldBe None + t shouldBe "VARCHAR" + painless shouldBe "(param1 != null ? param1 : \"abc\")" + select("GREATEST(n, 'abc')")._1.getOrElse("") should include("requires numeric arguments") + select("COALESCE(d, '3')")._2 shouldBe "VARCHAR" + } + + it should "read a CASE result as the other results' type (D4, A-77)" in { + val painless = accepted("CASE WHEN n > 0 THEN '1.5' ELSE n END", "INT") + painless should not include "\"1.5\"" + painless should include("2") + } + + behavior of "a quoted argument beside a date" + + it should "be read as the date it names, a date-time's date beside a DATE" in { + // 2024-01-15 is epoch day 19737, its UTC midnight 1705276800000 ms + accepted("GREATEST(d, '2024-01-15 10:30:00')", "DATE") should include("1705276800000L") + accepted("COALESCE(d, '2024-01-15')", "DATE") should include("LocalDate.ofEpochDay(19737L)") + accepted("LEAST(ts, '2024-01-15')", "TIMESTAMP") should include("1705276800000L") + } + + behavior of "a BOOLEAN beside whole numbers (D5)" + + /** The whole query script of `SELECT AS r FROM t`: its declarations, then its value. */ + private def script(expr: String, reported: String): String = { + accepted(expr, reported) + val ctx = PainlessContext(PainlessContextType.Query) + val painless = + resolved(s"SELECT $expr AS r FROM t").select.fieldsWithComputedAliases.head + .painless(Some(ctx)) + s"$ctx$painless" + } + + it should "take their type and be converted to 1 / 0 in a query" in { + // main refused GREATEST(b, 1) and answered COALESCE(b, 1)'s Boolean in a BIGINT column + script("GREATEST(b, 1)", "BIGINT") should include("? 1L : 0L") + script("LEAST(b, 1)", "BIGINT") should include("? 1L : 0L") + script("COALESCE(b, 1)", "BIGINT") should include("? 1L : 0L") + script("CASE WHEN n > 0 THEN b ELSE 1 END", "BIGINT") should include("? 1L : 0L") + } + + it should "keep main's outcome beside a fractional argument, and alone" in { + select("GREATEST(b, x)")._1.getOrElse("") should include("but got BOOLEAN for b") + select("GREATEST(b, b)")._1.getOrElse("") should include("but got BOOLEAN for b") + val (verdict, t, painless) = select("COALESCE(b, x)") + verdict shouldBe None + t shouldBe "DOUBLE" + painless shouldBe "(param1 != null ? param1 : param2)" + } + + behavior of "an ingest processor (D3 (5), R-107)" + + it should "render the value a literal is read as" in { + // main: def param1 = ctx.n; ctx.c = (param1 != null ? param1 : "3") + processor("INT", "COALESCE(n, '3')") shouldBe + "def param1 = ctx.n; ctx.c = (param1 != null ? param1 : 3)" + processor("INT", "GREATEST(n, '1.5')") should not include "\"1.5\"" + processor("DATE", "COALESCE(d, '2024-01-15 10:30:00')") should include("\"2024-01-15\"") + } + + /** Does `source` call a method on a string literal (`"2024-01-15".atStartOfDay(…)`, + * `("…").toString()`)? Painless refuses such a script when the pipeline is created. + */ + private def callsOnStringLiteral(source: String): Boolean = + "\"[^\"]*\"\\.[A-Za-z]".r.findFirstIn(source).isDefined || + "(^|[^A-Za-z0-9_])\\(\\s*\"[^\"]*\"\\s*\\)\\.[A-Za-z]".r.findFirstIn(source).isDefined + + it should "store a date read in a CASE result as its ISO text, never convert the text (A-89)" in { + // before A-89: ... : "2024-01-15".atStartOfDay(ZoneId.of('Z')) -- a method call on a string + // literal, so Painless refused a CREATE / ALTER that main and 0.24.0 accept. main: "2024-01-15" + Seq( + "CASE WHEN n > 0 THEN d ELSE '2024-01-15' END", + "CASE WHEN n > 0 THEN '2024-01-15' ELSE d END", + "CASE WHEN n > 0 THEN d ELSE '2024-01-15 10:30:00' END", + "CASE WHEN n > 0 THEN '2024-01-15 10:30:00' ELSE d END" + ).foreach { expr => + val source = processor("DATE", expr) + withClue(s"$expr -> $source: ") { + source should include("\"2024-01-15\"") + callsOnStringLiteral(source) shouldBe false + } + } + // beside a TIMESTAMP, the literal as written: main's bytes, the same instant (A-92; was the + // ISO date-time "2024-01-15T00:00:00Z", which main's `yyyy-MM-dd` parse of a consumer refuses) + val ts = processor("TIMESTAMP", "CASE WHEN n > 0 THEN ts ELSE '2024-01-15' END") + withClue(ts) { + ts should include(": \"2024-01-15\"") + ts should not include "T00:00:00Z" + callsOnStringLiteral(ts) shouldBe false + } + // toward another type (a cast over the CASE), the literal renders as on main + val cast = processor("VARCHAR", "CAST(CASE WHEN n > 0 THEN d ELSE '2024-01-15' END AS VARCHAR)") + withClue(cast)(callsOnStringLiteral(cast) shouldBe false) + } + + it should "keep a read date's text type outside a query, so a consumer parses it (A-92)" in { + // before A-92 the node was a DATE outside a query while its value was still text: + // `(param1 != null ? param1 : "2024-01-15").plus(1, ChronoUnit.DAYS)` failed every document + // (`dynamic method [java.lang.String, plus/2] not found`) and the column was silently absent + processor("DATE", "DATE_ADD(COALESCE(d, '2024-01-15'), INTERVAL 1 DAY)") shouldBe + "def param1 = ctx.d; def param2 = LocalDate.parse(((param1 != null ? param1 : \"2024-01-15\"))" + + ".replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); " + + "ctx.c = param2.plus(1, ChronoUnit.DAYS)" + val overCase = + processor("DATE", "DATE_ADD(CASE WHEN n > 0 THEN d ELSE '2024-01-15' END, INTERVAL 1 DAY)") + withClue(overCase) { + overCase should include(": \"2024-01-15\"; def param5 = LocalDate.parse((param4)") + callsOnStringLiteral(overCase) shouldBe false + } + // a TIMESTAMP reading renders the literal as written, and is tested as text: main's bytes + processor("TIMESTAMP", "COALESCE(ts, '2024-01-15 10:30:00')") shouldBe + "def param1 = ctx.ts; ctx.c = (param1 != null ? param1 : \"2024-01-15 10:30:00\")" + processor("TIMESTAMP", "COALESCE('2024-01-15 10:30:00', ts)") shouldBe + "def param1 = ctx.ts; ctx.c = (\"2024-01-15 10:30:00\" != null ? \"2024-01-15 10:30:00\" : param1)" + processor("DATE", "DATE_ADD(COALESCE(ts, '2024-01-15'), INTERVAL 1 DAY)") should include( + "(param1 != null ? param1 : \"2024-01-15\")" + ) + // under a cast, the arm renders as on main (`CAST(CASE … END AS TIMESTAMP)` keeps main's bytes) + processor( + "TIMESTAMP", + "CAST(CASE WHEN n > 0 THEN d ELSE '2024-01-15' END AS TIMESTAMP)" + ) should include(": \"2024-01-15\").replace(\" \", \"T\")") + // a transform reads text too: no java.time value is built from the literal + val transform = + resolved( + "SELECT COALESCE(d, '2024-01-15 10:30:00') AS r FROM t" + ).select.fieldsWithComputedAliases.head + .painless(Some(PainlessContext(PainlessContextType.Transform))) + withClue(transform) { + transform should include("\"2024-01-15\"") + transform should not include "LocalDate.ofEpochDay" + } + } + + it should "be compared as its text outside a query, as 0.24.0 compares it (R1-1)" in { + // before: the comparison read the node as a DATE and its literal as a date over the text a + // processor holds -- `def param2 = LocalDate.parse(("2024-01-15")…); … + // left1.isEqual(param2)` on the String left1: CREATE and ALTER were accepted, every document + // failed and the column was silently absent. The strings below are main's (and 0.24.0's). + val eq = "CASE WHEN COALESCE(d, '2024-01-15') = '2024-01-15' THEN 1 ELSE 0 END" + val eqBytes = "def param1 = ctx.d; def left1 = (param1 != null ? param1 : \"2024-01-15\"); " + + "def param2 = (left1 == null ? false : ((left1.compareTo(\"2024-01-15\") == 0))); " + + "ctx.c = param2 ? 1 : 0" + processor("INT", eq) shouldBe eqBytes + updateProcessor(eq) shouldBe eqBytes + processor( + "INT", + "CASE WHEN COALESCE(d, '2024-01-15') IN ('2024-01-15', '2024-02-01') THEN 1 ELSE 0 END" + ) shouldBe "def param1 = ctx.d; def left1 = (param1 != null ? param1 : \"2024-01-15\"); " + + "def param2 = (left1 == null ? false : (([\"2024-01-15\",\"2024-02-01\"].contains(left1)))); " + + "ctx.c = param2 ? 1 : 0" + processor("INT", "CASE COALESCE(d, '2024-01-15') WHEN '2024-01-15' THEN 1 ELSE 0 END") shouldBe + "def param1 = ctx.d; def param2 = (param1 != null ? param1 : \"2024-01-15\"); " + + "def param3 = \"2024-01-15\"; " + + "ctx.c = param2 != null && param3 != null && param2.compareTo(param3) == 0 ? 1 : 0" + processor("DATE", "NULLIF(COALESCE(d, '2024-01-15'), '2024-01-15')") shouldBe + "def param1 = ctx.d; def nif1 = (param1 != null ? param1 : \"2024-01-15\"); " + + "def param2 = nif1 == null || " + + "(\"2024-01-15\" != null && nif1.compareTo(\"2024-01-15\") == 0) ? null : nif1; ctx.c = param2" + // every operator over every such conditional, in a pipeline and in a transform: no java.time + // value is built from a compared literal, none is compared with the text + val operands = Seq( + "COALESCE(d, '2024-01-15')", + "COALESCE(d, '2024-01-15 10:30:00')", + "COALESCE('2024-01-15', d)", + "COALESCE(ts, '2024-01-15')", + "COALESCE(ts, '2024-01-15 10:30:00')", + "CASE WHEN n > 0 THEN d ELSE '2024-01-15' END", + "CASE WHEN n > 0 THEN '2024-01-15' ELSE d END", + "CASE WHEN n > 0 THEN ts ELSE '2024-01-15' END", + "CASE WHEN n > 0 THEN ts ELSE '2024-01-15 10:30:00' END", + "COALESCE(COALESCE(d, '2024-01-15'), d)" + ) + // `NULLIF(COALESCE(d, '2024-01-15'), d)` is judged as its processor types it (A-96): a text + // compared with a date, which a CREATE refuses as main does where it judges that NULLIF -- the + // operand of a `CASE … WHEN`, the value of a NULLIF; inside a WHEN condition it is accepted, as + // on main (below) + val nullIfOperand = "NULLIF(COALESCE(d, '2024-01-15'), d)" + val shapes: Seq[(String, String => String)] = Seq( + "INT" -> (o => s"CASE WHEN $o = '2024-01-15' THEN 1 ELSE 0 END"), + "INT" -> (o => s"CASE WHEN $o <> '2024-01-15' THEN 1 ELSE 0 END"), + "INT" -> (o => s"CASE WHEN $o < '2024-01-15' THEN 1 ELSE 0 END"), + "INT" -> (o => s"CASE WHEN '2024-01-15' = $o THEN 1 ELSE 0 END"), + "INT" -> (o => s"CASE WHEN $o IN ('2024-01-15', '2024-02-01') THEN 1 ELSE 0 END"), + "INT" -> (o => s"CASE WHEN $o NOT IN ('2024-01-15') THEN 1 ELSE 0 END"), + "INT" -> (o => s"CASE WHEN $o BETWEEN '2024-01-01' AND '2024-01-31' THEN 1 ELSE 0 END"), + "INT" -> (o => s"CASE $o WHEN '2024-01-15' THEN 1 ELSE 0 END"), + "DATE" -> (o => s"NULLIF($o, '2024-01-15')") + ) + val javaTimeOverText = + "isEqual\\(|isBefore\\(|isAfter\\(|(LocalDate|ZonedDateTime)\\.parse\\(\\(\"20".r + for { + o <- operands + (decl, shape) <- shapes + expr = shape(o) + (venue, source) <- Seq("CREATE" -> processor(decl, expr), "TRANSFORM" -> transform(expr)) + } withClue(s"$venue $expr -> $source: ")(javaTimeOverText.findFirstIn(source) shouldBe None) + for ((decl, shape) <- shapes) { + val expr = shape(nullIfOperand) + val transformed = transform(expr) + withClue(s"TRANSFORM $expr -> $transformed: ") { + javaTimeOverText.findFirstIn(transformed) shouldBe None + } + if (expr.startsWith("CASE WHEN")) { + val source = processor(decl, expr) + withClue(s"CREATE $expr -> $source: ")(javaTimeOverText.findFirstIn(source) shouldBe None) + } else + withClue(s"CREATE $expr: ") { + Parser(s"CREATE TABLE t ($columns, c $decl SCRIPT AS ($expr))") match { + case Left(e) => + e.msg shouldBe s"Column 'c': $nullIfOperand compares VARCHAR with TIMESTAMP: " + + "comparing a temporal value with a string is not supported yet, so cast the string " + + "to the column's type" + case other => fail(s"accepted: $other") + } + } + } + // the arithmetic keeps reading the node as a DATE: it parses a text operand itself + processor("BIGINT", "COALESCE(d, '2024-01-15') - d") should include("lv1 instanceof String") + // in a query the node is a DATE (D1): nothing moves there + accepted("COALESCE(d, '2024-01-15')", "DATE") + } + + private val ddlColumns = "id KEYWORD, n INT, d DATE, d2 DATE, ts TIMESTAMP, k KEYWORD" + + /** `CREATE TABLE t (…, c SCRIPT AS ())`: the refusal, or the column's processor. */ + private def created(decl: String, expr: String): Either[String, String] = + Parser(s"CREATE TABLE t ($ddlColumns, c $decl SCRIPT AS ($expr))") match { + case Right(ct: CreateTable) => + Right( + ct.schema.defaultPipeline.processors + .collectFirst { case p: ScriptProcessor if p.column == "c" => p.source } + .getOrElse(fail(s"no processor for $expr")) + ) + case Left(e) => Left(e.msg) + case other => fail(s"$expr: $other") + } + + /** `ALTER TABLE t ADD COLUMN c SCRIPT AS ()`, then an unrelated `ALTER TABLE t ADD + * COLUMN z INT` of the table holding it: each verdict (an ALTER judges every computed column of + * the table, R-70). + */ + private def altered(decl: String, expr: String): (Either[String, Unit], Either[String, Unit]) = { + def merged(base: Schema, alter: String): Schema = Parser(alter) match { + case Right(at: AlterTable) => base.merge(at.statements) + case other => fail(s"$alter: $other") + } + val base = Parser(s"CREATE TABLE t ($ddlColumns)") match { + case Right(ct: CreateTable) => ct.schema + case other => fail(s"base: $other") + } + val holding = merged(base, s"ALTER TABLE t ADD COLUMN c $decl SCRIPT AS ($expr)") + val next = merged(holding, "ALTER TABLE t ADD COLUMN z INT") + (validateScriptReferences(holding), validateScriptReferences(next)) + } + + it should "be judged in a computed column as its processor types it (A-96)" in { + // before: the validation typed these nodes DATE / TIMESTAMP, while the processor it installs + // types and renders them as text (A-92) -- `NULLIF(COALESCE(d, '2024-01-15'), k)` was refused + // ('compares DATE with KEYWORD') on CREATE and on every later ALTER of a table holding it, which + // main accepts, and `NULLIF(COALESCE(d, '2024-01-15'), d2)` was accepted over a processor + // whose value is a String (`YEAR` called `.get(ChronoField.YEAR)` on it), which main refuses + Seq( + "COALESCE(d, '2024-01-15')", + "COALESCE('2024-01-15', d)", + "COALESCE(ts, '2024-01-15 10:30:00')", + "COALESCE(d, ts, '2024-01-15 10:30:00')", + "CASE WHEN n > 0 THEN d ELSE '2024-01-15' END", + "CASE n WHEN 1 THEN d ELSE '2024-01-15' END", + "COALESCE(COALESCE(d, '2024-01-15'), d)", + "COALESCE(d, COALESCE(ts, '2024-01-15'))", + "CASE WHEN n > 0 THEN COALESCE(d, '2024-01-15') ELSE d END", + "NULLIF(COALESCE(d, '2024-01-15'), k)", + "NULLIF(COALESCE(d, '2024-01-15'), '2024-01-15')", + "NULLIF(CASE WHEN n > 0 THEN d ELSE '2024-01-15' END, k)" + ).foreach { expr => + val node = + resolved( + s"SELECT $expr AS r FROM t" + ).select.fieldsWithComputedAliases.head.identifier.functions.head match { + case p: PainlessScript => p + case other => fail(s"$expr: $other") + } + val validated = SQLTypeUtils.validatingComputedColumns(node.out) + withClue(s"$expr: ") { + validated shouldBe SQLTypeUtils.processorType(node) + validated.isText shouldBe true + } + } + // accepted, CREATE and ALTER, with main's bytes (0.24.0 compares the two texts) + val accepted = Seq( + "DATE" -> "NULLIF(COALESCE(d, '2024-01-15'), k)" -> + ("def param1 = ctx.k; def param2 = ctx.d; def nif1 = (param2 != null ? param2 : " + + "\"2024-01-15\"); def param3 = nif1 == null || (param1 != null && nif1.compareTo(param1) == 0)" + + " ? null : nif1; ctx.c = param3"), + "KEYWORD" -> "NULLIF(k, COALESCE(d, '2024-01-15'))" -> + ("def param1 = ctx.k; def param2 = ctx.d; def nif1 = (param2 != null ? param2 : " + + "\"2024-01-15\"); def param3 = param1 == null || (nif1 != null && param1.compareTo(nif1) == 0)" + + " ? null : param1; ctx.c = param3"), + "DATE" -> "NULLIF(CASE WHEN n > 0 THEN d ELSE '2024-01-15' END, k)" -> + ("def param1 = ctx.k; def param2 = ctx.n; def param3 = ctx.d; def param4 = param2 == null ? " + + "false : (param2 > 0); def nif1 = param4 ? (param3 instanceof String ? (param3.length() > 10 " + + "? ZonedDateTime.parse((param3).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME" + + ".withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS)" + + " : LocalDate.parse((param3).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(" + + "\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param3).atZone(" + + "ZoneId.of('Z'))) : \"2024-01-15\"; def param5 = nif1 == null || (param1 != null && " + + "nif1.compareTo(param1) == 0) ? null : nif1; ctx.c = param5") + ) + accepted.foreach { case ((decl, expr), bytes) => + withClue(s"$expr: ") { + created(decl, expr) shouldBe Right(bytes) + altered(decl, expr) shouldBe ((Right(()), Right(()))) + } + } + // refused, CREATE and ALTER, with main's message + val refusal = + "Column 'c': NULLIF(COALESCE(d, '2024-01-15'), d2) compares VARCHAR with TIMESTAMP: " + + "comparing a temporal value with a string is not supported yet, so cast the string to the " + + "column's type" + Seq( + "DATE" -> "NULLIF(COALESCE(d, '2024-01-15'), d2)", + "INT" -> "YEAR(NULLIF(COALESCE(d, '2024-01-15'), d2))", + "INT" -> ("CASE NULLIF(COALESCE(d, '2024-01-15'), d2) WHEN '2024-01-15' THEN 1 " + + "WHEN d2 THEN 2 ELSE 0 END"), + "DATE" -> "NULLIF(NULLIF(COALESCE(d, '2024-01-15'), d2), '2024-01-15')", + "BIGINT" -> "DATEDIFF(NULLIF(COALESCE(d, '2024-01-15'), d2), d2)" + ).foreach { case (decl, expr) => + withClue(s"$expr: ") { + created(decl, expr) shouldBe Left(refusal) + altered(decl, expr)._1 shouldBe Left(refusal) + } + } + // the arithmetic keeps reading the reported type, DATE, in both (A-96): unchanged + created("BIGINT", "COALESCE(d, '2024-01-15') - d2").isRight shouldBe true + } + + it should "convert a BOOLEAN beside whole numbers as Elasticsearch reads its _source" in { + // main: def param1 = ctx.b; ctx.c = (param1 == null ? 1 : Math.max(param1, 1)) + val source = processor("BIGINT", "GREATEST(b, 1)") + source should include("instanceof Boolean ? (boolean) param1 : 'true'.equals(param1)") + source should include("? 1L : 0L") + source should not include "Math.max(param1, 1)" + // COALESCE keeps main's processor bytes (D5: converted in a query only) + processor("BIGINT", "COALESCE(b, 1)") shouldBe + "def param1 = ctx.b; ctx.c = (param1 != null ? param1 : 1)" + } + + it should "keep the bytes of a conditional with no quoted argument" in { + processor("INT", "COALESCE(n, 3)") should not include "\"" + } + + behavior of "the reading memo (A-45 (2), R1)" + + it should "read each scope as a fresh node does, whatever order they render in" in { + val sql = "SELECT COALESCE(d, '2024-01-15 10:30:00') AS r FROM t" + val fresh = resolved(sql).select.fieldsWithComputedAliases.head.painless(query) + val flipped = resolved(sql).select.fieldsWithComputedAliases.head + flipped.painless(Some(PainlessContext(PainlessContextType.Processor))) should include( + "\"2024-01-15\"" + ) + flipped.painless(query) shouldBe fresh + fresh should include("LocalDate.ofEpochDay(19737L)") + } + + /** The verdict, the reported type and the query Painless of a resolved field's function, asked + * first under a computed column's validation when `validatedFirst`. + */ + private def judged(expr: String, validatedFirst: Boolean): (Option[String], String, String) = { + val field = resolved(s"SELECT $expr AS r FROM t").select.fieldsWithComputedAliases.head + val f = field.identifier.functions.head + if (validatedFirst) { + SQLTypeUtils.validatingComputedColumns(f.typeError) + SQLTypeUtils.validatingComputedColumns(field.identifier.reportedType) + } + (f.typeError, field.identifier.reportedType.typeId, field.painless(query)) + } + + it should "never carry a computed column's validation into a query over an ARRAY arm (R0-5)" in { + // the validation reads the WHOLE list (`readsWholeValues`), a query one element: both shared + // the query slot of the memo, and the validation's answer leaked into the query + Seq("GREATEST(ab, 1)", "COALESCE(an, '3')", "COALESCE(ab, 1)", "GREATEST(an, '1.5')").foreach { + expr => + withClue(s"$expr: ") { + judged(expr, validatedFirst = true) shouldBe judged(expr, validatedFirst = false) + } + } + judged("GREATEST(ab, 1)", validatedFirst = true)._1 shouldBe None + judged("COALESCE(an, '3')", validatedFirst = true)._2 shouldBe "INT" + } + + it should "never carry a comparison's validation into a query, or the reverse (R0-4)" in { + // the comparison memoised the type its operand is declared with, and the readings of its + // literals: a conditional over an ARRAY arm is VARCHAR under the validation, INT in a query + def validation(s: SingleSearch): Option[String] = + SQLTypeUtils.validatingComputedColumns(s.validateResolved().left.toOption) + def queried(s: SingleSearch): (Option[String], String) = { + val ctx = PainlessContext(PainlessContextType.Query) + val painless = s.select.fieldsWithComputedAliases.head.painless(Some(ctx)) + (s.validateResolved().left.toOption, s"$ctx$painless") + } + Seq( + "CASE WHEN COALESCE(an, '3') = k THEN 1 ELSE 0 END", + "CASE WHEN COALESCE(an, '3') = '3' THEN 1 ELSE 0 END", + "CASE WHEN COALESCE(an, '3') IN ('3', '4') THEN 1 ELSE 0 END", + "CASE WHEN COALESCE(an, '3') BETWEEN '1' AND '5' THEN 1 ELSE 0 END", + "CASE WHEN '3' = COALESCE(an, '3') THEN 1 ELSE 0 END", + "NULLIF(COALESCE(an, '3'), '3')" + ).foreach { expr => + val sql = s"SELECT $expr AS r FROM t" + val (freshValidation, freshQuery) = (validation(resolved(sql)), queried(resolved(sql))) + withClue(s"$expr, validated first: ") { + val node = resolved(sql) + validation(node) shouldBe freshValidation + queried(node) shouldBe freshQuery + } + withClue(s"$expr, queried first: ") { + val node = resolved(sql) + queried(node) shouldBe freshQuery + validation(node) shouldBe freshValidation + } + } + } + + /** What `s` answers in one scope: `v` a computed column's validation (verdict, reported type), + * `q` a query (verdict, reported type, Painless and its parameters), `p` an ingest processor. + */ + private def askedIn(s: SingleSearch, scope: Char): String = + Try { + val field = s.select.fieldsWithComputedAliases.head + scope match { + case 'v' => + SQLTypeUtils.validatingComputedColumns( + s"${s.validateResolved().left.toOption}|${field.identifier.reportedType.typeId}" + ) + case 'q' => + val ctx = PainlessContext(PainlessContextType.Query) + val painless = field.painless(Some(ctx)) + val reported = field.identifier.reportedType.typeId + s"${s.validateResolved().left.toOption}|$reported|$ctx$painless" + case _ => + val ctx = PainlessContext(PainlessContextType.Processor) + val painless = field.painless(Some(ctx)) + s"$ctx$painless" + } + }.recover { case e => s"ERR:${e.getClass.getSimpleName}:${e.getMessage}" }.get + + it should "answer each scope as a fresh node does, in all six orders (A-95)" in { + // one memo slot per scope -- a query, a computed column's validation, a script outside a query: + // with the validation in the query slot, a validation first made the query render + // `Long.parseLong("yes")` (CASE COALESCE(ab, 'true') WHEN 'yes') or refuse an arithmetic a + // fresh query answers, and a query first refused `CASE COALESCE(an, '3') WHEN k` in the + // validation (45 of these 180 orders leaked); computed live there instead, nested conditionals + // made a CREATE TABLE exponential + val shapes = Seq( + "CASE WHEN COALESCE(an, '3') = k THEN 1 ELSE 0 END", + "CASE WHEN COALESCE(an, '3') = '3' THEN 1 ELSE 0 END", + "CASE WHEN COALESCE(an, '3') = n THEN 1 ELSE 0 END", + "CASE WHEN COALESCE(an, '3') IN ('3', '4') THEN 1 ELSE 0 END", + "CASE WHEN COALESCE(an, '3') BETWEEN k AND k THEN 1 ELSE 0 END", + "CASE WHEN COALESCE(an, '3') BETWEEN '1' AND '5' THEN 1 ELSE 0 END", + "CASE WHEN COALESCE(an, '3') > '2' THEN 1 ELSE 0 END", + "CASE WHEN COALESCE(an, '3') <> k THEN 1 ELSE 0 END", + "CASE WHEN k = COALESCE(an, '3') THEN 1 ELSE 0 END", + "CASE WHEN '3' = COALESCE(an, '3') THEN 1 ELSE 0 END", + "CASE COALESCE(an, '3') WHEN '3' THEN 1 ELSE 0 END", + "CASE COALESCE(an, '3') WHEN k THEN 1 ELSE 0 END", + "CASE COALESCE(an, '3') WHEN 3 THEN 1 ELSE 0 END", + "CASE CASE WHEN n > 0 THEN an ELSE '3' END WHEN '3' THEN 1 ELSE 0 END", + "CASE GREATEST(an, '1.5') WHEN '2' THEN 1 ELSE 0 END", + "COALESCE(an, '3') + '1'", + "'1' + COALESCE(an, '3')", + "COALESCE(an, '3') * '2'", + "CASE WHEN n > 0 THEN an ELSE '3' END + '1'", + "COALESCE(an, '3') + 1", + "NULLIF(COALESCE(an, '3'), '3')", + "NULLIF(COALESCE(an, '3'), k)", + "NULLIF(k, COALESCE(an, '3'))", + "GREATEST(COALESCE(an, '3'), '2')", + "COALESCE(COALESCE(an, '3'), '4')", + "CASE WHEN n > 0 THEN COALESCE(an, '3') ELSE '4' END", + "UPPER(COALESCE(an, '3'))", + "CASE WHEN ab = k THEN 1 ELSE 0 END", + "CASE WHEN COALESCE(ab, 'true') = k THEN 1 ELSE 0 END", + "CASE COALESCE(ab, 'true') WHEN 'yes' THEN 1 ELSE 0 END", + // a read date: text outside a query only (R1-1) + "CASE WHEN COALESCE(d, '2024-01-15') = '2024-01-15' THEN 1 ELSE 0 END", + "CASE WHEN CASE WHEN n > 0 THEN ts ELSE '2024-01-15' END IN ('2024-01-15') THEN 1 ELSE 0 END", + "CASE COALESCE(d, '2024-01-15 10:30:00') WHEN '2024-01-15' THEN 1 ELSE 0 END", + "NULLIF(COALESCE(d, '2024-01-15'), '2024-01-15')", + "COALESCE(COALESCE(d, '2024-01-15'), d) + 1" + ) + shapes.foreach { expr => + val sql = s"SELECT $expr AS r FROM t" + val fresh = "vqp".map(scope => scope -> askedIn(resolved(sql), scope)).toMap + "vqp".permutations.foreach { order => + val node = resolved(sql) + order.foreach { scope => + withClue(s"$expr, order $order, scope $scope: ") { + askedIn(node, scope) shouldBe fresh(scope) + } + } + } + } + } + + behavior of "a string function over a read conditional (R-109)" + + it should "read a non-text receiver as its text, and a conditional receiver as one value" in { + // main: param2 ? param1 : "3".toUpperCase() -- the call on the last arm alone + val upper = accepted("UPPER(CASE WHEN n > 0 THEN n ELSE '3' END)", "VARCHAR") + upper should include("String.valueOf(lv1).toUpperCase()") + upper should not include "3.toUpperCase()" + // main failed on every row: dynamic method [java.lang.Long, toUpperCase/0] not found + accepted("UPPER(n)", "VARCHAR") should include("String.valueOf(param1).toUpperCase()") + accepted("LENGTH(n)", "BIGINT") should include("String.valueOf(param1).length()") + // typed `def` beside its NULL guard: `(param1 == null) ? null : ` failed to compile on + // Elasticsearch 8.18.3 ("Cannot cast from [int] to [java.lang.Object]") + accepted("LENGTH(n)", "BIGINT") should include("((def)(String.valueOf(param1).length()))") + accepted("UPPER(COALESCE(n, '3'))", "VARCHAR") should include( + "String.valueOf((param1 != null ? param1 : 3)).toUpperCase()" + ) + accepted("SUBSTRING(n, 1, 1)", "VARCHAR") should include("((String)String.valueOf(param1))") + // a text conditional: one value -- main lowered the ELSE arm only + accepted("LOWER(CASE WHEN n > 0 THEN 'A' ELSE 'B' END)", "VARCHAR") should include( + "(param2 ? \"A\" : \"B\").toLowerCase()" + ) + // a conditional that can be NULL is bound and guarded: never the text "null" + accepted("UPPER(CASE WHEN n > 0 THEN k END)", "VARCHAR") should include( + "(lv1 == null ? null : (def)(lv1.toUpperCase()))" + ) + } + + it should "keep the bytes of a text receiver" in { + accepted("UPPER(k)", "VARCHAR") shouldBe "(param1 == null) ? null : param1.toUpperCase()" + accepted("UPPER(TRIM(k))", "VARCHAR") shouldBe + "(param1 == null) ? null : param1.trim().toUpperCase()" + accepted("LOWER(COALESCE(k, 'x'))", "VARCHAR") shouldBe + "(param1 != null ? param1 : \"x\").toLowerCase()" + } + + behavior of "a BOOLEAN cast to a temporal (R-110)" + + it should "be refused in every scope, as DuckDB refuses it" in { + // main answered the Boolean in a DATE column, or failed on every row once its text was parsed + Seq( + "CAST(b AS DATE)" -> "DATE", + "CAST(b AS TIMESTAMP)" -> "TIMESTAMP", + "CAST(b AS TIME)" -> "TIME", + "TRY_CAST(b AS DATE)" -> "DATE", + "CAST(COALESCE(b, 'true') AS DATE)" -> "DATE" + ).foreach { case (expr, target) => + select(expr)._1.getOrElse("") should include(s"cannot cast BOOLEAN to $target") + } + Parser(s"CREATE TABLE t ($columns, c DATE SCRIPT AS (CAST(b AS DATE)))") match { + case Left(e) => e.msg should include("cannot cast BOOLEAN to DATE") + case other => fail(s"accepted: $other") + } + select("CAST(b AS BIGINT)")._1 shouldBe None + } + + behavior of "a read conditional nested beside text in a query (A-97)" + + /** The statements a nested reading refused in a query (`Cannot mix values of type INT and + * KEYWORD`), which 0.24.0 answered as text: each answers 0.24.0's VARCHAR again. + */ + private val nestedBesideText = Seq( + "COALESCE(COALESCE(n, '3'), k)", + "COALESCE(k, COALESCE(n, '3'))", + "COALESCE(COALESCE(n, '3'), g, k)", + "CASE WHEN n > 0 THEN COALESCE(n, '3') ELSE k END", + "CASE WHEN n > 0 THEN k ELSE COALESCE(n, '3') END", + "COALESCE(CASE WHEN n > 0 THEN n ELSE '3' END, k)", + "COALESCE(COALESCE(x, '1.5'), k)", + "COALESCE(COALESCE(b, 'true'), k)", + "COALESCE(COALESCE(d, '2024-01-15'), k)", + "CASE WHEN n > 0 THEN COALESCE(d, '2024-01-15') ELSE k END", + "COALESCE(COALESCE(n, '3'), CAST(n AS VARCHAR))", + "COALESCE(COALESCE(n, '1.5'), k)", + // read x read, a NULLIF's value, GREATEST over dates and numbers, a deeper nesting + "COALESCE(COALESCE(n, '3'), COALESCE(d, '2024-01-15'))", + "COALESCE(COALESCE(d, '2024-01-15'), COALESCE(n, '3'))", + "COALESCE(NULLIF(COALESCE(n, '3'), k), g)", + "COALESCE(GREATEST(d, '2024-01-15'), k)", + "COALESCE(GREATEST(n, '3'), k)", + "COALESCE(COALESCE(COALESCE(n, '3'), g), k)" + ) + + it should "keep 0.24.0's verdict and VARCHAR type, answering each read arm's value as text" in { + nestedBesideText.foreach { expr => + val painless = accepted(expr, "VARCHAR") + withClue(s"$expr: ") { + // each read arm is converted to the text of its value: no Integer under a VARCHAR label + painless should include("String.valueOf") + } + } + // the inner node keeps its one reading: '1.5' is 2 beside the INT n, answered as '2' + val n15 = accepted("COALESCE(COALESCE(n, '1.5'), k)", "VARCHAR") + n15 should not include "1.5" + n15 should include(": 2)") + accepted("COALESCE(n, '1.5')", "INT") + } + + it should "leave every other node as it was: no mix, a mix at 0.24.0's types, no read arm" in { + // no mix at the read types: the read type stays (a BOOLEAN beside a number is an INT) + accepted("COALESCE(COALESCE(n, '3'), x)", "DOUBLE") + accepted("CASE WHEN n > 0 THEN COALESCE(n, '3') ELSE COALESCE(b, 'true') END", "INT") + accepted("COALESCE(COALESCE(n, '3'), 'abc')", "VARCHAR") + // still mixing with the read arm at VARCHAR: r4's refusal and message + select("COALESCE(COALESCE(n, '3'), d)")._1 shouldBe Some( + "Cannot mix values of type INT and DATE in COALESCE operator - an explicit cast is required" + ) + select("COALESCE(COALESCE(n, '3'), x, k)")._1 shouldBe Some( + "Cannot mix values of type INT and KEYWORD in COALESCE operator - an explicit cast is required" + ) + // no read-moved arm: main's refusal + select("COALESCE(COALESCE(n, 3), k)")._1 shouldBe Some( + "Cannot mix values of type BIGINT and KEYWORD in COALESCE operator - an explicit cast is " + + "required" + ) + select("COALESCE(n, k)")._1 shouldBe Some( + "Cannot mix values of type INT and KEYWORD in COALESCE operator - an explicit cast is required" + ) + // a GREATEST over text stays refused, as on main + select("GREATEST(COALESCE(n, '3'), k)")._1.getOrElse("") should include( + "requires numeric arguments" + ) + } + + it should "reach every consumer of the outer node as its VARCHAR" in { + accepted("UPPER(COALESCE(COALESCE(n, '3'), k))", "VARCHAR") + select("CASE WHEN COALESCE(COALESCE(n, '3'), k) = '1' THEN 1 ELSE 0 END")._1 shouldBe None + resolved("SELECT g FROM t ORDER BY COALESCE(COALESCE(n, '3'), k)") + .validateResolved() shouldBe Right(()) + resolved( + "SELECT COALESCE(COALESCE(n, '3'), k) AS r FROM t GROUP BY COALESCE(COALESCE(n, '3'), k)" + ) + .validateResolved() shouldBe Right(()) + } + + it should "keep the read types outside a query: a processor, a transform, a computed column" in { + // `id`, a KEYWORD of the computed column's table + val expr = "COALESCE(COALESCE(n, '3'), id)" + val source = processor("KEYWORD", expr) + source should not include "String.valueOf" + source should not include "\"3\"" + transform("COALESCE(COALESCE(n, '3'), k)") should not include "String.valueOf" + created("KEYWORD", expr) shouldBe a[Right[_, _]] + } + + behavior of "COALESCE over a CASE arm (R-113)" + + it should "test the whole CASE for NULL, in a query and in a processor" in { + // main tested the ELSE alone (`c ? a : b != null ? … : k`): NULL where c held and a was NULL + val q = accepted("COALESCE(CASE WHEN n IS NULL THEN l ELSE n END, 7)", "BIGINT") + q should startWith("((") + q should include(") != null ?") + val p = processor("BIGINT", "COALESCE(CASE WHEN n IS NULL THEN l ELSE n END, 7)") + p should include(") != null ?") + // a NULL ELSE: `null != null` is never tested alone + accepted("COALESCE(CASE WHEN k = '3' THEN n ELSE NULL END, 7)", "BIGINT") should not include + "null != null" + // a CASE as a middle arm, and a read CASE beside a typed arm (r4 tested `3 != null`) + accepted("COALESCE(ti, CASE WHEN n IS NULL THEN l ELSE n END, 7)", "BIGINT") should include( + ") != null ?" + ) + val p01 = accepted("COALESCE(CASE WHEN n > 0 THEN n ELSE '3' END, n)", "INT") + p01 should include(": 3) != null ?") + p01 should not include ": 3 != null" + // the testkit's RUN rows (text arms) + Seq( + "COALESCE(CASE WHEN k = '3' THEN CAST(n AS VARCHAR) ELSE g END, g)", + "COALESCE(CASE WHEN k = '3' THEN CAST(n AS VARCHAR) ELSE NULL END, g)", + "COALESCE(CAST(ti AS VARCHAR), CASE WHEN k = '3' THEN CAST(n AS VARCHAR) ELSE g END, g)" + ).foreach(e => accepted(e, "VARCHAR") should include(") != null ?")) + } + + it should "keep the bytes of an arm that is no CASE" in { + accepted("COALESCE(n, l)", "BIGINT") should startWith("(param") + accepted("COALESCE(COALESCE(n, l), 7)", "BIGINT") should startWith("((param") + } + + // ---- run 6 (A-98, A-99): pins P1 - P4 ---- + + private val cdr = "COALESCE('2024-01-15', d)" + private val caseD = "CASE WHEN n > 0 THEN d ELSE '2024-01-15' END" + private val caseTs = "CASE WHEN n > 0 THEN ts ELSE '2024-01-15 10:30:00' END" + private val simpleD = "CASE n WHEN 3 THEN d ELSE '2024-01-15' END" + + /** The verdict, the reported type and the query Painless -- the WHERE criteria if there is one, + * else the first field -- with the declarations it registered. + */ + private def rendered(sql: String): (Option[String], String, String) = { + val s = resolved(sql) + val c = PainlessContext(PainlessContextType.Query) + val p = s.where.flatMap(_.criteria) match { + case Some(criteria) => criteria.painless(Some(c)) + case None => s.select.fieldsWithComputedAliases.head.painless(Some(c)) + } + ( + s.validateResolved().left.toOption, + s.select.fieldsWithComputedAliases.head.identifier.reportedType.typeId, + s"$c$p" + ) + } + + private def refusal(expr: String): String = + select(expr)._1.getOrElse(fail(s"$expr: accepted")) + + behavior of "a literal-first COALESCE beside text in an equality (A-98 α, R-119)" + + it should "compare its literal's text, with main's verdict and type (P1)" in { + Seq( + s"NULLIF($cdr, k)" -> "VARCHAR", + s"NULLIF(k, $cdr)" -> "KEYWORD", + s"CASE WHEN NULLIF($cdr, k) = '2024-01-15' THEN 1 ELSE 0 END" -> "BIGINT", + s"COALESCE(NULLIF($cdr, k), k)" -> "VARCHAR", + s"CASE $cdr WHEN k THEN 1 ELSE 0 END" -> "BIGINT", + s"CASE WHEN $cdr = k THEN 1 ELSE 0 END" -> "BIGINT", + s"CASE k WHEN $cdr THEN 1 ELSE 0 END" -> "BIGINT", + s"CASE WHEN k = $cdr THEN 1 ELSE 0 END" -> "BIGINT", + s"CASE WHEN $cdr <> k THEN 1 ELSE 0 END" -> "BIGINT", + s"CASE WHEN $cdr != k THEN 1 ELSE 0 END" -> "BIGINT", + s"CASE WHEN NOT $cdr = k THEN 1 ELSE 0 END" -> "BIGINT", + s"CASE WHEN $cdr = UPPER(k) THEN 1 ELSE 0 END" -> "BIGINT", + "CASE WHEN COALESCE(NULL, '2024-01-15', d) = k THEN 1 ELSE 0 END" -> "BIGINT", + "NULLIF(COALESCE(NULL, '2024-01-15', d), k)" -> "VARCHAR", + "CASE WHEN COALESCE('2024-01-15', '2024-01-16', d) = k THEN 1 ELSE 0 END" -> "BIGINT" + ).foreach { case (e, t) => + val (verdict, reported, painless) = rendered(s"SELECT $e AS r FROM t") + withClue(s"$e: ") { + verdict shouldBe None + reported shouldBe t + painless should include("\"2024-01-15\"") + painless should not include "LocalDate.ofEpochDay" + } + } + val (verdict, _, where) = rendered(s"SELECT k FROM t WHERE $cdr = k") + verdict shouldBe None + where should include("(\"2024-01-15\" == param") + // the NULLIF beside its own operand: alpha answers it, A-97 converts the read arm to its text + select(s"COALESCE(NULLIF(k, $cdr), $cdr)")._1 shouldBe None + } + + it should "keep the reading's verdict everywhere else (P1 controls)" in { + // ordering keeps r5's refusal and message (main fails at runtime, RUN M6, R-119) + Seq(s"$cdr < k", s"$cdr >= k").foreach { c => + refusal(s"CASE WHEN $c THEN 1 ELSE 0 END") should startWith( + "Type mismatch: 'DATE' is not compatible with 'KEYWORD'" + ) + } + select(cdr)._2 shouldBe "DATE" + refusal(s"CASE WHEN $cdr = n THEN 1 ELSE 0 END") should startWith( + "Type mismatch: 'DATE' is not compatible with 'INT'" + ) + refusal("CASE WHEN COALESCE('3', n) > k THEN 1 ELSE 0 END") should startWith( + "Type mismatch: 'INT' is not compatible with 'KEYWORD'" + ) + refusal("CASE WHEN COALESCE(d, '2024-01-15') = k THEN 1 ELSE 0 END") should startWith( + "Type mismatch: 'DATE' is not compatible with 'KEYWORD'" + ) + // one rendered operand: a CASE whose values do not all read the literal keeps the reading + refusal(s"CASE $cdr WHEN k THEN 1 WHEN d THEN 2 ELSE 0 END") shouldBe + "CASE WHEN conditions must be of the same type as the expression" + Seq( + s"CASE WHEN $cdr = '2024-01-15 00:00:00' THEN 1 ELSE 0 END", + "CASE WHEN COALESCE('3', n) = k THEN 1 ELSE 0 END", + "CASE WHEN COALESCE('true', b) = k THEN 1 ELSE 0 END" + ).foreach(e => rendered(s"SELECT $e AS r FROM t")._1 shouldBe None) + } + + it should "refuse the classes main answers wrongly (P3)" in { + Seq( + "COALESCE('2024-01-15 10:30:00', d)", + "COALESCE('2024-01-15', ts)", + "COALESCE('2024-01-15 10:30:00', ts)" + ).foreach { f => + refusal(s"CASE WHEN $f = k THEN 1 ELSE 0 END") should startWith("Type mismatch: ") + refusal(s"NULLIF($f, k)") should include( + "comparing a temporal value with a string is not supported yet" + ) + } + Seq(caseD, caseTs, simpleD).foreach { f => + refusal(s"NULLIF($f, k)") should include(" compares ") + refusal(s"NULLIF(k, $f)") should include(" compares ") + refusal(s"CASE WHEN NULLIF($f, k) = '2024-01-15' THEN 1 ELSE 0 END") should include( + " compares " + ) + refusal(s"CASE $f WHEN k THEN 1 ELSE 0 END") shouldBe + "CASE WHEN conditions must be of the same type as the expression" + refusal(s"CASE WHEN $f = k THEN 1 ELSE 0 END") should startWith("Type mismatch: ") + } + } + + behavior of "COALESCE(NULLIF(a, b), b) over a read-moved conditional (A-98 β)" + + /** A rendering with its parameter names erased. */ + private def shape(painless: String): String = + painless.replaceAll("\\b(param|nif|lv|left|right)\\d+\\b", "P") + + it should "answer COALESCE(a, b), VARCHAR, rendered as COALESCE(a, b) is (P2)" in { + Seq( + s"COALESCE(NULLIF($caseD, k), k)" -> s"COALESCE($caseD, k)", + s"COALESCE(NULLIF($caseTs, k), k)" -> s"COALESCE($caseTs, k)", + s"COALESCE(NULLIF($simpleD, k), k)" -> s"COALESCE($simpleD, k)", + "COALESCE(NULLIF(COALESCE(d, '2024-01-15'), k), k)" -> "COALESCE(COALESCE(d, '2024-01-15'), k)", + s"COALESCE(NULLIF($caseD, k), k, g)" -> s"COALESCE($caseD, k, g)", + s"COALESCE(g, NULLIF($caseD, k), k)" -> s"COALESCE(g, $caseD, k)", + s"COALESCE(NULLIF(k, $caseD), $caseD)" -> s"COALESCE(k, $caseD)" + ).foreach { case (beta, reference) => + val (verdict, reported, painless) = rendered(s"SELECT $beta AS r FROM t") + withClue(s"$beta: ") { + verdict shouldBe None + reported shouldBe "VARCHAR" + shape(painless) shouldBe shape(rendered(s"SELECT $reference AS r FROM t")._3) + } + } + accepted(s"UPPER(COALESCE(NULLIF($caseD, k), k))", "VARCHAR") + } + + it should "keep r5's verdict and message for every other NULLIF (P2 controls)" in { + refusal(s"COALESCE(NULLIF($caseD, k), g)") should include(" compares DATE with KEYWORD") + refusal("COALESCE(NULLIF(d, k), k)") shouldBe + "Cannot mix values of type DATE and KEYWORD in COALESCE operator - an explicit cast is required" + refusal("COALESCE(NULLIF(d, 'yesterday'), 'yesterday')") should startWith( + "Cannot parse 'yesterday'" + ) + accepted("COALESCE(NULLIF(n, 0), 0)", "BIGINT") + // one pass: the inner NULLIF of a replaced arm is judged, and refused + refusal("COALESCE(NULLIF(NULLIF(COALESCE(d, '2024-01-15'), k), k), k)") should include( + " compares DATE with KEYWORD" + ) + } + + behavior of "COALESCE over a GREATEST / LEAST holding a literal (A-99, R-116)" + + /** How many groups a primitive cast opens directly tested for NULL (`((long) …) != null`). */ + private def primitivesTested(painless: String): Int = + "\\) != null".r.findAllMatchIn(painless).count { m => + var depth = 0 + var i = m.start + var open = -1 + while (i >= 0 && open < 0) { + painless.charAt(i) match { + case ')' => depth += 1 + case '(' => + depth -= 1 + if (depth == 0) open = i + case _ => + } + i -= 1 + } + open >= 0 && painless + .substring(open) + .matches("(?s)^\\(\\(\\s*(int|long|double|float|byte|short)\\).*") + } + + private val keywordColumns = s"$columns, k KEYWORD, g KEYWORD" + + private def processorWithK(decl: String, expr: String): String = + Parser(s"CREATE TABLE t ($keywordColumns, c $decl SCRIPT AS ($expr))") match { + case Right(ct: CreateTable) => + ct.schema.defaultPipeline.processors + .collectFirst { case p: ScriptProcessor if p.column == "c" => p.source } + .getOrElse(fail(s"no processor for $expr")) + case other => fail(s"$expr: $other") + } + + it should "end at that arm in every scope: no primitive is tested for NULL (P4)" in { + val rows = Seq( + "COALESCE(GREATEST(n, 3), l)" -> "BIGINT", + "COALESCE(LEAST(x, 1.5), x)" -> "DOUBLE", + "COALESCE(GREATEST(n, 3), n, 7)" -> "BIGINT", + "COALESCE(GREATEST(n, '2'), k)" -> "VARCHAR", + "COALESCE(GREATEST(n, '3'), n)" -> "INT", + "COALESCE(LEAST(x, '1.5'), x)" -> "DOUBLE", + "COALESCE(GREATEST(n, '3'), k)" -> "VARCHAR" + ) ++ (for { + f <- Seq("GREATEST", "LEAST") + t <- Seq("ti", "n", "x") + x <- Seq("'3'", "'1.5'", "'200'") + } yield s"COALESCE($f($t, $x), $t)" -> "DOUBLE") + rows.foreach { case (e, decl) => + withClue(s"$e: ") { + val (verdict, _, q) = rendered(s"SELECT $e AS r FROM t") + verdict shouldBe None + primitivesTested(q) shouldBe 0 + primitivesTested(processorWithK(decl, e)) shouldBe 0 + primitivesTested(updateProcessor(e)) shouldBe 0 + primitivesTested(transform(e)) shouldBe 0 + } + } + // the cut follows `NumericReducer.neverNull`: a processor ends at the reducer, a reducer with + // no literal (or a NULL one) is still tested (#292) + processorWithK("BIGINT", "COALESCE(GREATEST(n, 3), l)") should not include "!= null" + rendered("SELECT COALESCE(GREATEST(n, l), x) AS r FROM t")._3 should include("!= null ?") + rendered("SELECT COALESCE(GREATEST(n, NULL), l) AS r FROM t")._3 should include("!= null ?") + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/ImplicitComparisonCastSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/ImplicitComparisonCastSpec.scala new file mode 100644 index 00000000..0e53e628 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/ImplicitComparisonCastSpec.scala @@ -0,0 +1,396 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.{ + ComparisonRule, + PainlessContext, + PainlessContextType, + StringValue +} +import app.softnetwork.elastic.sql.function.cond.NullIf +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.schema.{ + validateScriptReferences, + Column, + Schema, + ScriptProcessor, + Table => SchemaTable +} +import app.softnetwork.elastic.sql.`type`.{SQLTypeUtils, SQLTypes} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** A comparison between a text and a number or a boolean, and between a boolean and a number, + * answers for the operators DuckDB 1.5.5 binds, as DuckDB casts it (SCHEMA-5b D6 / D7): the + * binding table both ways, the runtime conversions in every emission context, and the scope rule + * that keeps a text side outside a query as 0.24.0 judged it (A-82). + */ +class ImplicitComparisonCastSpec extends AnyFlatSpec with Matchers { + + private val table = SchemaTable( + "t", + columns = List( + Column("g", SQLTypes.Keyword), + Column("k", SQLTypes.Keyword), + Column("tx", SQLTypes.Text), + Column("ak", SQLTypes("array")), + Column("n", SQLTypes.Int), + Column("l", SQLTypes.BigInt), + Column("x", SQLTypes.Double), + Column("b", SQLTypes.Boolean), + Column("ab", SQLTypes("array")), + Column("d", SQLTypes.Date) + ) + ) + + private def resolved(sql: String): SingleSearch = Parser(sql) match { + case Right(s: SingleSearch) => s.update(Some(table)) + case other => fail(s"$sql: not a single search: $other") + } + + private def verdict(sql: String): Option[String] = resolved(sql).validateResolved().left.toOption + + /** The whole query script of the first SELECT field: its declarations, then its value. */ + private def selectScript(sql: String): String = { + val ctx = PainlessContext(PainlessContextType.Query) + val painless = resolved(sql).select.fieldsWithComputedAliases.head.painless(Some(ctx)) + s"$ctx$painless" + } + + /** The whole WHERE script. */ + private def whereScript(sql: String): String = { + val ctx = PainlessContext(PainlessContextType.Query) + val painless = resolved(sql).where.flatMap(_.criteria).get.painless(Some(ctx)) + s"$ctx$painless" + } + + // ---- P-B2: DuckDB 1.5.5's binding table (`duck_5b_ops.out`), compiled ---- + + private sealed trait Family + private case object Whole extends Family + private case object Fractional extends Family + private case object Bool extends Family + + /** Does DuckDB bind a text (`text = true`) or a boolean against `right`, for an equality or an + * ordering? D6's rows: text x number / boolean, equality only; boolean x whole, every + * comparison; boolean x fractional, equality only. + */ + private def binds(text: Boolean, right: Family, equality: Boolean): Boolean = + if (text) equality + else right == Whole || equality + + private val textRights: Seq[(String, Family)] = + Seq( + "n" -> Whole, + "l" -> Whole, + "x" -> Fractional, + "b" -> Bool, + "1" -> Whole, + "1.5" -> Fractional + ) + + private val boolRights: Seq[(String, Family)] = + Seq( + "n" -> Whole, + "l" -> Whole, + "x" -> Fractional, + "1" -> Whole, + "0" -> Whole, + "1.5" -> Fractional + ) + + private def literal(r: String): Boolean = r.headOption.exists(_.isDigit) + + /** Every P-B2 statement, with whether DuckDB answers it. */ + private val pb2: Seq[(String, Boolean)] = { + val lefts = Seq("k" -> true, "ak" -> true, "b" -> false, "ab" -> false) + for { + (l, text) <- lefts + (r, right) <- if (text) textRights else boolRights + (sql, equality) <- Seq( + s"SELECT CASE WHEN $l = $r THEN 1 ELSE 0 END AS v FROM t" -> true, + s"SELECT CASE WHEN $l <> $r THEN 1 ELSE 0 END AS v FROM t" -> true, + s"SELECT CASE WHEN $l < $r THEN 1 ELSE 0 END AS v FROM t" -> false, + s"SELECT CASE WHEN $l >= $r THEN 1 ELSE 0 END AS v FROM t" -> false, + s"SELECT CASE WHEN $l BETWEEN $r AND $r THEN 1 ELSE 0 END AS v FROM t" -> false, + s"SELECT NULLIF($l, $r) AS v FROM t" -> true, + s"SELECT CASE $l WHEN $r THEN 1 ELSE 0 END AS v FROM t" -> true + ) ++ (if (literal(r)) + Seq(s"SELECT CASE WHEN $l IN ($r) THEN 1 ELSE 0 END AS v FROM t" -> true) + else + Seq( + s"SELECT g FROM t WHERE $l = $r" -> true, + s"SELECT g FROM t WHERE $r = $l" -> true, + s"SELECT g FROM t WHERE $r < $l" -> false, + s"SELECT CASE WHEN $r <> $l THEN 1 ELSE 0 END AS v FROM t" -> true, + s"SELECT NULLIF($r, $l) AS v FROM t" -> true + )) + } yield sql -> binds(text, right, equality) + } + + /** main's refusal messages, unchanged where DuckDB refuses. */ + private def mainRefusal(message: String): Boolean = + message.contains("Type mismatch") || message.contains("comparable types") || + message.contains("CASE WHEN conditions must be of the same type") + + behavior of "the implicit-cast table (D6), both ways" + + it should "answer every pair x operator DuckDB binds" in { + val answered = pb2.filter(_._2).map(_._1) + answered should not be empty + answered.foreach(sql => withClue(s"$sql: ")(verdict(sql) shouldBe None)) + } + + /** A BETWEEN whose bounds are COLUMNS: `BetweenExpr.update` resolves the operand only, so the + * bounds keep an unknown type and main accepts it whatever the operand (pre-existing, recorded) + * -- the verdict stays main's. + */ + private val columnBounds = "BETWEEN [a-z]+ AND".r + + it should "keep main's refusal and message for every pair x operator DuckDB refuses" in { + val refused = + pb2.filterNot(_._2).map(_._1).filter(sql => columnBounds.findFirstIn(sql).isEmpty) + refused should not be empty + refused.foreach { sql => + withClue(s"$sql: ") { + val v = verdict(sql) + v shouldBe defined + mainRefusal(v.get) shouldBe true + } + } + } + + it should "keep main's verdict for a BETWEEN over column bounds (pre-existing)" in { + Seq("k BETWEEN n AND n", "k BETWEEN x AND x", "b BETWEEN x AND x", "b BETWEEN n AND n") + .foreach { cond => + verdict(s"SELECT CASE WHEN $cond THEN 1 ELSE 0 END AS v FROM t") shouldBe None + } + // `BetweenExpr.update` resolves its operand only: column bounds keep an unknown type, so the + // table never applies and both renderings stay main's + selectScript("SELECT CASE WHEN b BETWEEN n AND n THEN 1 ELSE 0 END AS v FROM t") should include( + "((param1 >= param2 && param1 <= param2))" + ) + selectScript( + "SELECT CASE WHEN k BETWEEN n AND n THEN 1 ELSE 0 END AS v FROM t" + ) should not include + "nv1" + } + + it should "leave a string literal to the comparison rule's reading" in { + // 'abc' spells no number: refused as text against a number, as on main + verdict("SELECT CASE WHEN n = 'abc' THEN 1 ELSE 0 END AS v FROM t") shouldBe defined + verdict("SELECT NULLIF(k, 'abc') AS v FROM t") shouldBe None + } + + it should "read a group filter's BOOLEAN metric with no conversion (R10)" in { + Seq(">", "=", ">=", "<").foreach { op => + verdict(s"SELECT g, MAX(b) AS m FROM t GROUP BY g HAVING MAX(b) $op 1") shouldBe None + } + verdict("SELECT g, MAX(b) AS m FROM t GROUP BY g HAVING MAX(b) = 1.5") shouldBe None + verdict("SELECT g, MAX(b) AS m FROM t GROUP BY g HAVING MAX(b) < 1.5") shouldBe defined + val selector = resolved("SELECT g, MAX(b) AS m FROM t GROUP BY g HAVING MAX(b) > 1").having + .flatMap(_.criteria) + .map(MetricSelectorScript.metricSelector) + .get + selector should include("> 1") + selector should not include "? 1" + // a text metric converts nothing per group, and Elasticsearch's max reads no keyword (T0-4) + verdict("SELECT g, MAX(k) AS m FROM t GROUP BY g HAVING MAX(k) = 2") shouldBe defined + } + + it should "keep the refusals R-102 excludes (A-78)" in { + verdict( + "SELECT g, MAX(k) AS m FROM t GROUP BY g HAVING MAX(k) > '2024-01-01'" + ).getOrElse("") should include("compares a metric as text") + } + + behavior of "the runtime conversion (D7)" + + it should "read a text as the whole number it rounds to, inside the operand's NULL guard" in { + val script = selectScript("SELECT CASE WHEN k = 1 THEN 1 ELSE 0 END AS v FROM t") + script should include("def nv1 = null; if (param1 != null) {") + script should include("String.valueOf(param1).trim()") + script should include("new BigDecimal('9223372036854775807.5')") + script should include("setScale(0, RoundingMode.HALF_UP).longValueExact()") + script should include("throw new IllegalArgumentException") + script should not include "Long.parseLong" + script should include("(left2 == null ? false : ((left2 == 1)))") + } + + it should "read a text as a DOUBLE beside a DOUBLE, a BOOLEAN beside a BOOLEAN" in { + whereScript("SELECT g FROM t WHERE k = x") should include("Double.parseDouble(nv1s)") + val boolean = whereScript("SELECT g FROM t WHERE k = b") + boolean should include("String nv1s = String.valueOf(param1).trim().toLowerCase();") + boolean should include("nv1s.equals('on')") + boolean should not include "equalsIgnoreCase" + boolean should not include "Boolean.parseBoolean" + } + + it should "convert a BOOLEAN beside a number to 1 / 0, and leave the number as it is" in { + whereScript("SELECT g FROM t WHERE b > n") should include( + "def left1 = (param1 == null ? null : (def)((param1 ? 1 : 0)))" + ) + whereScript("SELECT g FROM t WHERE x = b") should include("((left1 == (param2 ? 1.0 : 0.0)))") + } + + it should "read an ARRAY column as its element in a query (D8)" in { + whereScript("SELECT g FROM t WHERE ak = n") should include("String.valueOf(param1).trim()") + selectScript("SELECT CASE WHEN ab = 1 THEN 1 ELSE 0 END AS v FROM t") should include( + "(param1 ? 1L : 0L)" + ) + } + + it should "compare a NULLIF's arguments by the table and answer its first argument" in { + val keyword = selectScript("SELECT NULLIF(k, 0) AS v FROM t") + keyword should include("param1 == null || nv1 == 0 ? null : param1") + keyword should not include "0 != null" + selectScript("SELECT NULLIF(b, 0) AS v FROM t") should include( + "param1 == null || (param1 ? 1L : 0L) == 0 ? null : param1" + ) + } + + it should "compare a simple CASE's expression and values by the table" in { + val keyword = selectScript("SELECT CASE k WHEN 1 THEN 'one' ELSE 'other' END AS v FROM t") + keyword should include("def param2 = nv1;") + keyword should include("param2 != null && param3 != null && param2 == param3 ? \"one\"") + selectScript("SELECT CASE b WHEN 1 THEN 'one' ELSE 'other' END AS v FROM t") should include( + "(param1 == null ? null : (def)((param1 ? 1L : 0L)))" + ) + } + + it should "keep a transform's text comparison as main renders it (A-82)" in { + val field = + resolved( + "SELECT CASE WHEN k = 1 THEN 1 ELSE 0 END AS v FROM t" + ).select.fieldsWithComputedAliases.head + val transform = field.painless(Some(PainlessContext(PainlessContextType.Transform))) + transform should not include "nv1" + } + + it should "spell exactly the literal rule's booleans (A-81 twin pin)" in { + val script = whereScript("SELECT g FROM t WHERE k = b") + val spelled = "equals(?:IgnoreCase)?\\('([^']*)'\\)".r + .findAllMatchIn(script) + .map(_.group(1)) + .toSet + // the rendered rule, as the JVM runs it: the methods the script calls, not a Scala restatement + // (R0-2: `equalsIgnoreCase` folds U+017F 'ſ' to 's', so 'yeſ' read true where `booleanOf` reads + // nothing; its lower case is still 'yeſ') + val ignoringCase = script.contains(".equalsIgnoreCase(") + val lowered = script.contains(".trim().toLowerCase()") + def rendered(text: String): Boolean = { + val s = if (lowered) text.trim.toLowerCase else text.trim + spelled.exists(word => if (ignoringCase) s.equalsIgnoreCase(word) else s == word) + } + Seq( + "t", + "TRUE", + " true ", + "Yes", + "y", + "on", + "OFF", + "1", + "0", + "f", + "no", + "n", + "false", + "2", + "", + "maybe", + "nan", + "ye\u017F", + "YE\u017F", + "fal\u017Fe" + ).foreach { text => + withClue(s"[$text]: ") { + ComparisonRule.readingOf(StringValue(text), SQLTypes.Boolean).isDefined shouldBe + rendered(text) + } + } + } + + behavior of "a text side outside a query (A-82)" + + private val columns = + "id KEYWORD, n INT, x DOUBLE, b BOOLEAN, k KEYWORD, ak ARRAY" + + private def createVerdict(decl: String, expr: String): Either[String, String] = + Parser(s"CREATE TABLE t ($columns, c $decl SCRIPT AS ($expr))") match { + case Left(e) => Left(e.msg) + case Right(ct: CreateTable) => + Right( + ct.schema.defaultPipeline.processors + .collectFirst { case p: ScriptProcessor if p.column == "c" => p.source } + .getOrElse("-") + ) + case other => fail(s"$expr: $other") + } + + private def alterVerdict(decl: String, expr: String): Either[String, String] = { + val base: Schema = Parser(s"CREATE TABLE t ($columns)") match { + case Right(ct: CreateTable) => ct.schema + case other => fail(s"base: $other") + } + Parser(s"ALTER TABLE t ADD COLUMN c $decl SCRIPT AS ($expr)") match { + case Left(e) => Left(e.msg) + case Right(at: AlterTable) => + val merged = base.merge(at.statements) + validateScriptReferences(merged).map(_ => + merged.defaultPipeline.processors + .collectFirst { case p: ScriptProcessor if p.column == "c" => p.source } + .getOrElse("-") + ) + case other => fail(s"$expr: $other") + } + } + + it should "refuse NULLIF(k, 0) in a computed column, on CREATE and ALTER (#382)" in { + createVerdict("KEYWORD", "NULLIF(k, 0)").left.getOrElse("") should include("comparable types") + alterVerdict("KEYWORD", "NULLIF(k, 0)").left.getOrElse("") should include("comparable types") + } + + it should "keep main's processor bytes for a text comparison" in { + createVerdict("INT", "CASE WHEN k = 1 THEN 1 ELSE 0 END") shouldBe + Right( + "def param1 = ctx.k; def param2 = param1 == null ? false : (param1 == 1); ctx.c = param2 ? 1 : 0" + ) + } + + it should "convert a BOOLEAN beside a number in a processor as Elasticsearch reads its _source" in { + val source = createVerdict("INT", "CASE WHEN b = 1 THEN 1 ELSE 0 END").getOrElse("") + source should include("param1 instanceof Boolean ? (boolean) param1 : 'true'.equals(param1)") + source should include("(param1 == null ? null : (def)(") + } + + it should "judge one NULLIF node as each scope does, in either order (R1, R11)" in { + def node(): NullIf = + resolved( + "SELECT NULLIF(k, 0) AS v FROM t" + ).select.fieldsWithComputedAliases.head.identifier.functions.collectFirst { case n: NullIf => + n + }.get + val first = node() + SQLTypeUtils.validatingComputedColumns(first.typeError) shouldBe defined + first.typeError shouldBe None + val second = node() + second.typeError shouldBe None + SQLTypeUtils.validatingComputedColumns(second.typeError) shouldBe defined + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/NamelessKeyFilterSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/NamelessKeyFilterSpec.scala index e9a11d20..b4b57a41 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/NamelessKeyFilterSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/NamelessKeyFilterSpec.scala @@ -138,8 +138,7 @@ class NamelessKeyFilterSpec extends AnyFlatSpec with Matchers { it should "be closed with the mapping for a key over a date" in { Seq( - "CONCAT(d, '')" -> "'2024-01-03'" -> "it reads the date column d", - "COALESCE(d, '2024-01-01')" -> "'2024-01-03'" -> "it reads the date column d" + "CONCAT(d, '')" -> "'2024-01-03'" -> "it reads the date column d" ).foreach { case ((key, value), cause) => val sql = s"SELECT $key AS k, COUNT(*) AS c FROM t GROUP BY $key HAVING $key = $value" withClue(sql) { @@ -150,6 +149,18 @@ class NamelessKeyFilterSpec extends AnyFlatSpec with Matchers { } } + it should "close a key whose quoted argument is read as a date as a DATE grouping level" in { + // Re-pinned (SCHEMA-5b, R-47): `COALESCE(d, '2024-01-01')` was a VARCHAR key (the literal kept + // its text) and was refused as a key reading the date column d. Its literal is now read as the + // date it names, as DuckDB 1.5.5 reads it beside a DATE, so the key IS a DATE and is refused by + // the DATE grouping level's own reason -- still refused, with the reason that now applies. + val key = "COALESCE(d, '2024-01-01')" + val sql = s"SELECT $key AS k, COUNT(*) AS c FROM t GROUP BY $key HAVING $key = '2024-01-03'" + val reason = resolved(sql).left.getOrElse("") + reason should startWith(s"HAVING cannot filter a DATE grouping level: $key = '2024-01-03'") + reason should include(s"Restrict the rows in WHERE instead, with the same condition on $key") + } + it should "be open with the mapping for a decimal key and for a whole number written 1.0" in { // These were refused ("writes such a number as 1 or as 1.0", "a whole number is written 1 // there, never 1.0"): the filter sent the literal's own spelling, which the key's script value diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala index 2bb25275..4191c441 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala @@ -375,7 +375,12 @@ class NullIfOperandSpec extends AnyFlatSpec with Matchers with TableDrivenProper it should "emit compareTo only when BOTH operands render text, over the whole corpus" in { val offenders = corpus.flatMap { case (expr, dataType) => everyVenueEmission(expr, dataType).collect { - case (venue, emitted) if emitted.contains(".compareTo(") && mixesTextWithNonText(expr) => + // (SCHEMA-5b D7: a text compared with a number in a query is read as one, and that + // conversion bounds the text's BigDecimal with `compareTo` -- not a String comparison) + case (venue, emitted) + if emitted + .replaceAll("nv\\d+x(\\.abs\\(\\))?\\.compareTo\\(", "") + .contains(".compareTo(") && mixesTextWithNonText(expr) => s"[$venue] [$expr] -> [$emitted]" } } @@ -627,34 +632,40 @@ class NullIfOperandSpec extends AnyFlatSpec with Matchers with TableDrivenProper ("NULLIF(1.5, CAST(n AS KEYWORD))", "KEYWORD", "DOUBLE", "KEYWORD") ) ) { (expr, dataType, left, right) => - everyVenueVerdict(expr, dataType).foreach { case (venue, verdict) => - withClue(s"[$venue] [$expr] -> $verdict ") { - val reason = verdict.left.getOrElse(fail(s"[$venue] [$expr] was ACCEPTED")) - // 🔴 Story 21.4's lesson: a rejection test is UNFALSIFIABLE while any `Left` passes it. - // The message must NAME BOTH rendered types and must not be an internal-error label -- - // restoring the defect yields a `Left` too. - // - // 🔴 Read from the TYPE clause, not with `include` (issue #382, found by review). The - // message echoes the whole expression AND, in the DDL venue, up to 200 characters of the - // caller's SQL (#262) -- which declares `s KEYWORD, n BIGINT`. So `include("KEYWORD")` - // and `include("BIGINT")` were free, and a message naming only ONE type, or the WRONG - // pair, passed. - val named = NamedTypes - .findFirstMatchIn(reason) - .orElse(NamedTypesAtParse.findFirstMatchIn(reason)) - .getOrElse(fail(s"[$venue] [$expr] message names no type pair: $reason")) - (named.group(1), named.group(2)) shouldBe ((left, right)) - reason should not include "Internal" - reason should not include "Operation failed" - // ...and the DDL venue, which is the one the ruling repairs, must NAME THE EXPRESSION - // and the column. (A `CAST( AS )` pair is already refused at PARSE in the - // query venues by the pre-existing `NullIf.validate()`, whose message predates #382 and - // names only the two types -- see the release note.) - if (venue == "computed column") { - reason should include(expr.substring(expr.indexOf("NULLIF"))) - reason should include("Column 'c'") + everyVenueVerdict(expr, dataType).foreach { + // SCHEMA-5b D6: in a QUERY a text is compared with a number as DuckDB casts it (`NULLIF(s, + // 0)` reads `s` as a BIGINT); a computed column keeps this refusal (A-82), and so does a + // string literal, read by the comparison rule ('x' spells no number) + case (venue, verdict) if venue != "computed column" && !expr.contains("'") => + withClue(s"[$venue] [$expr] ")(verdict shouldBe Right(())) + case (venue, verdict) => + withClue(s"[$venue] [$expr] -> $verdict ") { + val reason = verdict.left.getOrElse(fail(s"[$venue] [$expr] was ACCEPTED")) + // 🔴 Story 21.4's lesson: a rejection test is UNFALSIFIABLE while any `Left` passes it. + // The message must NAME BOTH rendered types and must not be an internal-error label -- + // restoring the defect yields a `Left` too. + // + // 🔴 Read from the TYPE clause, not with `include` (issue #382, found by review). The + // message echoes the whole expression AND, in the DDL venue, up to 200 characters of the + // caller's SQL (#262) -- which declares `s KEYWORD, n BIGINT`. So `include("KEYWORD")` + // and `include("BIGINT")` were free, and a message naming only ONE type, or the WRONG + // pair, passed. + val named = NamedTypes + .findFirstMatchIn(reason) + .orElse(NamedTypesAtParse.findFirstMatchIn(reason)) + .getOrElse(fail(s"[$venue] [$expr] message names no type pair: $reason")) + (named.group(1), named.group(2)) shouldBe ((left, right)) + reason should not include "Internal" + reason should not include "Operation failed" + // ...and the DDL venue, which is the one the ruling repairs, must NAME THE EXPRESSION + // and the column. (A `CAST( AS )` pair is already refused at PARSE in the + // query venues by the pre-existing `NullIf.validate()`, whose message predates #382 and + // names only the two types -- see the release note.) + if (venue == "computed column") { + reason should include(expr.substring(expr.indexOf("NULLIF"))) + reason should include("Column 'c'") + } } - } } } } @@ -741,7 +752,8 @@ class NullIfOperandSpec extends AnyFlatSpec with Matchers with TableDrivenProper ("expr", "dataType", "left", "right"), ("NULLIF(d, 0)", "DATE", "TIMESTAMP", "BIGINT"), ("NULLIF(0, d)", "DATE", "BIGINT", "TIMESTAMP"), - ("NULLIF(b, 0)", "BOOLEAN", "BOOLEAN", "BIGINT"), + // (`NULLIF(b, 0)` is compared since SCHEMA-5b, DuckDB's implicit cast in every scope (D6): + // pinned below, with the pairs an arm expresses) ("NULLIF(b, 'x')", "BOOLEAN", "BOOLEAN", "VARCHAR"), ("NULLIF(s, d)", "KEYWORD", "KEYWORD", "TIMESTAMP"), ("NULLIF(d, b)", "DATE", "TIMESTAMP", "BOOLEAN") @@ -770,7 +782,9 @@ class NullIfOperandSpec extends AnyFlatSpec with Matchers with TableDrivenProper ("NULLIF(n, 0)", "BIGINT"), // both numeric ("NULLIF(b, false)", "BOOLEAN"), // both boolean ("NULLIF(d, d)", "DATE"), // both temporal - ("NULLIF(CAST(d AS DATE), CAST('2025-01-01' AS DATE))", "DATE") + ("NULLIF(CAST(d AS DATE), CAST('2025-01-01' AS DATE))", "DATE"), + // a boolean against a whole number, as 1 / 0 (SCHEMA-5b D6; refused before) + ("NULLIF(b, 0)", "BOOLEAN") ) ) { (expr, dataType) => everyVenueVerdict(expr, dataType).foreach { case (venue, verdict) => @@ -793,11 +807,16 @@ class NullIfOperandSpec extends AnyFlatSpec with Matchers with TableDrivenProper ("NULLIF(NULLIF(s, status), 0)", "KEYWORD") ) ) { (expr, dataType) => - everyVenueVerdict(expr, dataType).foreach { case (venue, verdict) => - withClue(s"[$venue] [$expr] -> $verdict ") { - verdict.left.getOrElse(fail(s"[$venue] [$expr] was ACCEPTED")) should - include("requires two arguments of comparable types") - } + everyVenueVerdict(expr, dataType).foreach { + // SCHEMA-5b D6: a query compares a text with a number as DuckDB casts it; a computed + // column keeps the refusal (A-82) + case (venue, verdict) if venue != "computed column" => + withClue(s"[$venue] [$expr] ")(verdict shouldBe Right(())) + case (venue, verdict) => + withClue(s"[$venue] [$expr] -> $verdict ") { + verdict.left.getOrElse(fail(s"[$venue] [$expr] was ACCEPTED")) should + include("requires two arguments of comparable types") + } } } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala index 435f1382..45bf5eb9 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala @@ -75,7 +75,9 @@ class TypeCheckResolutionSpec extends AnyFlatSpec with Matchers with TableDriven "SELECT id FROM t WHERE n + 1 > 'abc'", "'BIGINT' is not compatible with 'VARCHAR'" ), - ("comparison", "SELECT id FROM t WHERE s = n", "'KEYWORD' is not compatible with 'INT'"), + // (`s = n` is answered since SCHEMA-5b, DuckDB's implicit cast (D6); ordering a text + // against a number is a DuckDB Binder Error, still refused) + ("comparison", "SELECT id FROM t WHERE s < n", "'KEYWORD' is not compatible with 'INT'"), ("comparison (TIME)", "SELECT id FROM t WHERE CAST(ts AS TIME) > ts", "a TIME value"), // IN: the operand against the list's element type ( @@ -100,7 +102,9 @@ class TypeCheckResolutionSpec extends AnyFlatSpec with Matchers with TableDriven "output 'INT' is not compatible with input 'VARCHAR'" ), // NULLIF: #382's rule, and the matching rule over the types the statement fixes - ("NULLIF (#382)", "SELECT NULLIF(s, 1) AS v FROM t", "compares KEYWORD with BIGINT"), + // (`NULLIF(s, 1)` is answered in a query since SCHEMA-5b (D6); a computed column keeps + // #382's refusal, A-82 -- `ImplicitComparisonCastSpec`) + ("NULLIF (#382)", "SELECT NULLIF(s, d) AS v FROM t", "compares KEYWORD with TIMESTAMP"), // (a DATE against a TIMESTAMP is comparable, the ONE comparison rule: TIME is not) ( "NULLIF (match)", @@ -180,7 +184,10 @@ class TypeCheckResolutionSpec extends AnyFlatSpec with Matchers with TableDriven "SELECT LEAST(d, ts, CURRENT_DATE) AS v FROM t", "SELECT id FROM t WHERE GREATEST(d, d2) > CAST('2024-02-01' AS DATE)", // over aggregates, in a HAVING (in a SELECT calculation it is refused, as it can be NULL) - "SELECT g FROM t GROUP BY g HAVING GREATEST(MAX(d), MAX(d2)) > CAST('2024-02-01' AS DATE)" + "SELECT g FROM t GROUP BY g HAVING GREATEST(MAX(d), MAX(d2)) > CAST('2024-02-01' AS DATE)", + // a text against a number, for an equality DuckDB binds (SCHEMA-5b D6; refused before) + "SELECT id FROM t WHERE s = n", + "SELECT NULLIF(s, 1) AS v FROM t" ) ) { sql => withClue(s"[$sql] ")(resolved(sql) shouldBe Right(())) } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala index 88799b40..1b196ef9 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala @@ -3379,7 +3379,9 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with // the ONE comparison rule accepts: it RUNS, in "the ONE comparison rule" above) s"SELECT id, NULLIF(n, 'a') AS x FROM $table" -> Seq("compares INT with VARCHAR"), // one true mismatch per rule that waits for the column types - s"SELECT id FROM $table WHERE id = n" -> Seq("'KEYWORD' is not compatible with 'INT'"), + // (`id = n` is bound since SCHEMA-5b, DuckDB's implicit cast (D6): read per row below; + // ordering a text against a number is a DuckDB Binder Error, still refused) + s"SELECT id FROM $table WHERE id < n" -> Seq("'KEYWORD' is not compatible with 'INT'"), s"SELECT id FROM $table WHERE CAST(ts AS TIME) > ts" -> Seq( "a TIME value cannot be compared" ), @@ -3395,7 +3397,9 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with s"SELECT id FROM $table WHERE CAST(id AS INT) BETWEEN 'a' AND 'b'" -> Seq( "output 'INT' is not compatible with input 'VARCHAR'" ), - s"SELECT id, NULLIF(id, 1) AS x FROM $table" -> Seq("compares KEYWORD with BIGINT"), + // (`NULLIF(id, 1)` is bound in a query since SCHEMA-5b (D6): read per row below; a computed + // column keeps #382's refusal, A-82) + s"SELECT id, NULLIF(id, d) AS x FROM $table" -> Seq("compares KEYWORD with TIMESTAMP"), s"SELECT id, CASE WHEN n THEN 1 ELSE 0 END AS x FROM $table" -> Seq( "CASE WHEN conditions must be of type BOOLEAN" ), @@ -3418,8 +3422,28 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with case Left(t) => Left(t.toString) }) } + // SCHEMA-5b D6 / D7: a text against a number, for an equality DuckDB binds, is read as a + // number on each row; 'r1' spells none, so the statement fails as it runs, as DuckDB's + // "Could not convert string 'r1'" does -- no longer refused before anything runs + val readPerRow = Seq( + s"SELECT id FROM $table WHERE id = n", + s"SELECT id, NULLIF(id, 1) AS x FROM $table" + ).map { sql => + sql -> (Try(Await.result(client.run(sql), 60.seconds)).toEither match { + case Right(ElasticFailure(error)) => Right(Seq(s"refused as [${error.message}]")) + case Right(ElasticSuccess(QueryStream(stream, _, _))) => + Try(Await.result(stream.runWith(Sink.seq), 60.seconds)).toEither match { + case Left(t) => + info(s"[$sql] failed as it ran: ${String.valueOf(t.getMessage).take(300)}") + Right(Nil) + case Right(rs) => Right(Seq(s"answered ${rs.size} rows instead of failing as it runs")) + } + case Right(other) => Right(Seq(s"answered $other instead of failing as it runs")) + case Left(t) => Left(t.toString) + }) + } Try(Await.result(client.run("DROP TABLE IF EXISTS date_arithmetic_refused"), 60.seconds)) - verdict("refusals", outcomes) + verdict("refusals", outcomes ++ readPerRow) } "a CREATE OR REPLACE TABLE ... AS SELECT the rules refuse" should diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/QuotedArgumentConditionalSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/QuotedArgumentConditionalSpec.scala new file mode 100644 index 00000000..5e987ff8 --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/QuotedArgumentConditionalSpec.scala @@ -0,0 +1,821 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ + ElasticFailure, + ElasticResult, + ElasticSuccess, + QueryResult +} +import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.elastic.sql.{ComparisonRule, StringValue => SqlStringValue} +import app.softnetwork.elastic.sql.`type`.SQLTypes + +import java.time.{LocalDate, ZonedDateTime} +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.language.implicitConversions +import scala.util.Try + +/** A quoted argument of `COALESCE`, `GREATEST`, `LEAST` and a `CASE` result is read as the type + * beside it, and a comparison between a text and a number or a boolean -- or a boolean and a + * number -- answers as DuckDB 1.5.5 casts it (SCHEMA-5b): each row is the DuckDB twin's value and + * Java class, on a real cluster. A text that converts to nothing fails the query, as DuckDB fails + * it; the runtime conversions answer what the literal rule (`ComparisonRule`) reads (the twin + * pin). Fixtures `s5b_*`. + */ +trait QuotedArgumentConditionalSpec extends GatewayIntegrationTestKit { + self: ElasticTestKit => + + private lazy val api: ElasticClientApi = client.asInstanceOf[ElasticClientApi] + + private def exec(sql: String): ElasticResult[QueryResult] = + Await.result(client.run(sql), 60.seconds) + + /** A statement over the fixtures, loaded on first use: a runner mixes `ElasticDockerTestKit` in + * last, and its `beforeAll` does not call this trait's. + */ + private def run(sql: String): ElasticResult[QueryResult] = { + fixtures + exec(sql) + } + + private def ddl(sql: String): Unit = exec(sql) match { + case ElasticSuccess(_) => + case ElasticFailure(error) => fail(s"[$sql] failed: ${error.message}") + } + + private def rows(sql: String): Seq[Map[String, Any]] = run(sql) match { + case ElasticFailure(error) => fail(s"[$sql] failed: ${error.message}") + case ok => collectRows(System.nanoTime(), ok) + } + + /** The failure of `sql` -- refused, or failed by Elasticsearch -- if it fails. */ + private def failure(sql: String): Option[String] = + Try(run(sql)).toEither match { + case Left(e) => Some(String.valueOf(e.getMessage)) + case Right(ElasticFailure(error)) => Some(error.message) + case Right(ok) => + Try(collectRows(System.nanoTime(), ok)).failed.toOption.map(e => + String.valueOf(e.getMessage) + ) + } + + private def bulk(index: String, docs: List[String]): Unit = { + implicit val bulkOptions: BulkOptions = BulkOptions(defaultIndex = index, logEvery = 100000) + implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = + Source.fromIterator(() => list.iterator) + api.bulk[String](docs, identity, idKey = Some(Set("id"))) match { + case ElasticSuccess(_) => + case ElasticFailure(error) => fail(s"bulk into $index failed: ${error.message}") + } + api.refresh(index) + } + + private def json(s: String): String = + "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"") + "\"" + + /** `id -> column` over the rows of `sql`. */ + private def byId(sql: String, column: String = "r"): Map[String, Any] = + rows(sql).map(r => String.valueOf(r("id")) -> r.getOrElse(column, null)).toMap + + private def ids(sql: String): Set[String] = rows(sql).map(r => String.valueOf(r("id"))).toSet + + /** Does `actual` hold the value `expected` names -- a number by value, a date by its day, a + * timestamp by its instant (`2024-01-15T00:00:00Z`), anything else by its text? + */ + private def same(actual: Any, expected: Any): Boolean = (actual, expected) match { + case (null, null) => true + case (null, _) | (_, null) => false + case (a: Number, e: Number) => BigDecimal(a.toString) == BigDecimal(e.toString) + case (a: ZonedDateTime, e) => a.toInstant.toString == String.valueOf(e) + case (a: LocalDate, e) => a.toString == String.valueOf(e) + case (a, e) => String.valueOf(a) == String.valueOf(e) + } + + /** Each `id -> value` of `SELECT id, AS r FROM `, and the Java class of every + * non-NULL value, the class the DuckDB twin's type reads as. + */ + private def answers( + index: String, + expr: String, + cls: Class[_], + expected: (String, Any)* + ): Unit = { + val sql = s"SELECT id, $expr AS r FROM $index" + val got = byId(sql) + withClue(s"[$sql] -> $got: ") { + expected.foreach { case (id, value) => + withClue(s"$id: ") { + same(got.getOrElse(id, null), value) shouldBe true + Option(got.getOrElse(id, null)).foreach(v => v.getClass shouldBe cls) + } + } + } + } + + private val IntC = classOf[java.lang.Integer] + private val LongC = classOf[java.lang.Long] + private val DoubleC = classOf[java.lang.Double] + private val BoolC = classOf[java.lang.Boolean] + private val TextC = classOf[String] + private val DateC = classOf[LocalDate] + private val TimestampC = classOf[ZonedDateTime] + + private lazy val fixtures: Unit = { + ddl( + "CREATE TABLE s5b_t (id KEYWORD, g KEYWORD, ti TINYINT, n INT, l BIGINT, x DOUBLE, d DATE, " + + "ts TIMESTAMP, b BOOLEAN, k KEYWORD, PRIMARY KEY (id))" + ) + bulk( + "s5b_t", + List( + """{"id":"t1","g":"g1","ti":1,"n":1,"l":1,"x":1.5,"d":"2024-01-10","ts":"2024-01-10T08:00:00Z","b":true,"k":"1"}""", + """{"id":"t2","g":"g2","k":"3"}""", + """{"id":"t3","g":"g3","ti":5,"n":3,"l":5,"x":0.5,"d":"2024-02-01","ts":"2024-02-01T00:00:00Z","b":false,"k":"3"}""" + ) + ) + ddl( + "CREATE TABLE s5b_c (id KEYWORD, g KEYWORD, k KEYWORD, n INT, l BIGINT, x DOUBLE, b BOOLEAN, " + + "tx TEXT, ak ARRAY, ab ARRAY, PRIMARY KEY (id))" + ) + bulk( + "s5b_c", + List( + """{"id":"c1","g":"g1","k":"1","n":1,"l":1,"x":1.5,"b":true,"tx":"1","ak":["1"],"ab":[true]}""", + """{"id":"c2","g":"g2","k":"0","n":0,"l":0,"x":0.5,"b":false,"tx":"0","ak":["0"],"ab":[false]}""" + ) + ) + ddl( + "CREATE TABLE s5b_k (id KEYWORD, k KEYWORD, l BIGINT, x DOUBLE, b BOOLEAN, PRIMARY KEY (id))" + ) + bulk( + "s5b_k", + wholeTexts.zipWithIndex.map { case (t, i) => + s"""{"id":"w$i","k":${json(t)},"l":${ComparisonRule.roundedWhole(t).getOrElse(0L)}}""" + }.toList ++ + booleanTexts.zipWithIndex.map { case (t, i) => + s"""{"id":"b$i","k":${json(t)},"b":${booleanOf(t).getOrElse(true)}}""" + } ++ + doubleTexts.zipWithIndex.map { case (t, i) => + s"""{"id":"x$i","k":${json(t)},"x":${doubleOf(t).getOrElse(0.0)}}""" + } + ) + ddl( + "CREATE TABLE s5b_p (id KEYWORD, n INT, b BOOLEAN, d DATE, ts TIMESTAMP, " + + "c1 INT SCRIPT AS (COALESCE(n, '3')), c2 INT SCRIPT AS (GREATEST(n, '1.5')), " + + "c3 BIGINT SCRIPT AS (GREATEST(b, 1)), c4 BIGINT SCRIPT AS (LEAST(b, 1)), " + + "c5 DATE SCRIPT AS (COALESCE(d, '2024-01-15')), " + + "c6 INT SCRIPT AS (CASE WHEN b = 1 THEN 1 ELSE 0 END), " + + "c7 BOOLEAN SCRIPT AS (NULLIF(b, 0)), " + + "c8 DATE SCRIPT AS (DATE_ADD(COALESCE(d, '2024-01-15'), INTERVAL 1 DAY)), " + + "c9 DATE SCRIPT AS (DATE_ADD(COALESCE(ts, '2024-01-15'), INTERVAL 1 DAY)), " + + "c10 INT SCRIPT AS (CASE WHEN b BETWEEN 1 AND 1 THEN 1 ELSE 0 END), PRIMARY KEY (id))" + ) + bulk( + "s5b_p", + List( + """{"id":"p1","n":1,"b":true,"d":"2024-01-10"}""", + """{"id":"p2","n":"5","b":"true"}""", + """{"id":"p3","b":""}""", + """{"id":"p4","b":false}""", + """{"id":"p5","b":"false"}""", + """{"id":"p6"}""" + ) + ) + } + + // ---- the texts of the twin pins (D7): what the literal rule reads each as ---- + + private val wholeTexts: Seq[String] = Seq( + "3", + "1.5", + "-2.5", + "0.5", + "1e3", + " 2 ", + "200", + "9223372036854775807.4", + "01", + ".5", + "-0.5", + "0.4999", + "5.", + "1e+2", + "1e-30000000", + "1e30000000", + "9223372036854775807.5", + "-9223372036854775808.5", + "abc", + "", + "1,5", + "1d", + "0x10", + "1_000", + "NaN", + "Infinity", + "inf", + "+.", + "1e", + "2024-01-15", + "true", + "yes" + ) + + // U+017F ('ſ') upper-cases to 'S': a case-insensitive comparison read 'yeſ' as true (R0-2), + // where the literal rule -- the text in lower case -- reads nothing + private val booleanTexts: Seq[String] = Seq( + "t", + "TRUE", + " true ", + "on", + "OFF", + "yes", + "Y", + "1", + "0", + "f", + "no", + "n", + "false", + "2", + "", + "maybe", + "nan", + "ye\u017F", + "YE\u017F", + "fal\u017Fe" + ) + + private val doubleTexts: Seq[String] = + Seq("1.5", "1e3", " 2 ", ".5", "abc", "1d", "0x1p4", "NaN", "1e400", "-1e400") + + private def booleanOf(text: String): Option[Boolean] = + ComparisonRule.readingOf(SqlStringValue(text), SQLTypes.Boolean).collect { + case ComparisonRule.BooleanReading(value) => value + } + + private def doubleOf(text: String): Option[Double] = + ComparisonRule.numberOf(text).map(_.painless.stripSuffix("L").toDouble) + + /** What a stored text beside a DOUBLE is compared as (A-93): the number the literal rule reads, + * or +/-Infinity for a number past DOUBLE's range -- `Double.parseDouble`'s overflow, DuckDB + * 1.5.5's value -- where the literal rule reads nothing (D16); `None`, a failure, otherwise. + */ + private def doubleTwinOf(text: String): Option[Double] = + doubleOf(text).orElse { + val t = text.trim + if (t.matches("[+-]?(\\d+\\.?\\d*|\\.\\d+)([eE][+-]?\\d+)?")) + Some(t.toDouble).filter(_.isInfinite) + else None + } + + // ---- AC1: a quoted argument beside a typed one ---- + + "a quoted argument beside a number" should "answer the number DuckDB reads, in its type" in { + answers("s5b_t", "COALESCE(n, '3')", IntC, "t1" -> 1, "t2" -> 3, "t3" -> 3) + answers("s5b_t", "COALESCE('3', n)", IntC, "t1" -> 3, "t2" -> 3, "t3" -> 3) + answers("s5b_t", "GREATEST(n, '1.5')", IntC, "t1" -> 2, "t2" -> 2, "t3" -> 3) + answers("s5b_t", "LEAST('-2.5', n)", IntC, "t1" -> -3, "t2" -> -3, "t3" -> -3) + answers("s5b_t", "GREATEST(l, '1e3')", LongC, "t1" -> 1000, "t2" -> 1000, "t3" -> 1000) + answers("s5b_t", "COALESCE(x, '0.5')", DoubleC, "t1" -> 1.5, "t2" -> 0.5, "t3" -> 0.5) + answers("s5b_t", "COALESCE(n, ' 2 ')", IntC, "t1" -> 1, "t2" -> 2, "t3" -> 3) + answers("s5b_t", "COALESCE(n, l, '3')", LongC, "t1" -> 1, "t2" -> 3, "t3" -> 3) + answers("s5b_t", "GREATEST(n, '1.5', NULL)", IntC, "t1" -> 2, "t2" -> 2, "t3" -> 3) + answers( + "s5b_t", + "CASE WHEN n > 0 THEN '1.5' ELSE n END", + IntC, + "t1" -> 2, + "t2" -> null, + "t3" -> 2 + ) + } + + it should "widen past the operand's whole range to BIGINT, never wrap (D2)" in { + answers("s5b_t", "GREATEST(ti, '200')", LongC, "t1" -> 200, "t2" -> 200, "t3" -> 200) + } + + it should "keep a text that reads as nothing as main answered it (D11)" in { + answers("s5b_t", "COALESCE(n, 'abc')", TextC, "t1" -> "1", "t2" -> "abc", "t3" -> "3") + } + + it should "answer a constant-only conditional (6.8 refuses some constant scripts)" in { + answers("s5b_t", "GREATEST(3, '1.5')", LongC, "t1" -> 3, "t2" -> 3, "t3" -> 3) + answers("s5b_t", "GREATEST(1.5, '2.5')", DoubleC, "t1" -> 2.5, "t2" -> 2.5, "t3" -> 2.5) + } + + "a quoted argument beside a date, a timestamp, a boolean" should "answer its value" in { + answers( + "s5b_t", + "COALESCE(d, '2024-01-15')", + DateC, + "t1" -> "2024-01-10", + "t2" -> "2024-01-15", + "t3" -> "2024-02-01" + ) + answers( + "s5b_t", + "GREATEST(d, '2024-01-15 10:30:00')", + DateC, + "t1" -> "2024-01-15", + "t2" -> "2024-01-15", + "t3" -> "2024-02-01" + ) + answers( + "s5b_t", + "LEAST(ts, '2024-01-15')", + TimestampC, + "t1" -> "2024-01-10T08:00:00Z", + "t2" -> "2024-01-15T00:00:00Z", + "t3" -> "2024-01-15T00:00:00Z" + ) + answers("s5b_t", "COALESCE(b, 'true')", BoolC, "t1" -> true, "t2" -> true, "t3" -> false) + } + + "a BOOLEAN beside whole numbers (D5, R-107)" should "answer 1 / 0 in their type" in { + answers("s5b_t", "GREATEST(b, 1)", LongC, "t1" -> 1, "t2" -> 1, "t3" -> 1) + answers("s5b_t", "LEAST(b, 1)", LongC, "t1" -> 1, "t2" -> 1, "t3" -> 0) + answers("s5b_t", "COALESCE(b, 1)", LongC, "t1" -> 1, "t2" -> 1, "t3" -> 0) + } + + "a read conditional" should "be filtered, grouped and filtered per group by its value" in { + ids("SELECT id FROM s5b_t WHERE COALESCE(n, '3') = 3") shouldBe Set("t2", "t3") + ids("SELECT id FROM s5b_t WHERE GREATEST(x, '1') > 1") shouldBe Set("t1") + // main failed on every row: Cannot cast java.lang.String to java.time.chrono.ChronoZonedDateTime + ids("SELECT id FROM s5b_t WHERE COALESCE(d, '2024-01-15') = '2024-01-15'") shouldBe Set("t2") + ids("SELECT id FROM s5b_t WHERE COALESCE(b, 'false') = 'false'") shouldBe Set("t2", "t3") + rows("SELECT g, MAX(n) AS m FROM s5b_t GROUP BY g HAVING COALESCE(MAX(n), '0') > 1") + .map(r => String.valueOf(r("g"))) shouldBe Seq("g3") + val groups = rows( + "SELECT COALESCE(n, '3') AS r, COUNT(*) AS c FROM s5b_t GROUP BY COALESCE(n, '3')" + ).map(r => r("r") -> r("c")) + // main grouped the TEXTS "1" and "3" + groups.foreach { case (key, _) => key shouldBe a[Number] } + groups.map { case (key, c) => + BigDecimal(key.toString) -> BigDecimal(c.toString) + }.toMap shouldBe + Map(BigDecimal(1) -> BigDecimal(1), BigDecimal(3) -> BigDecimal(2)) + } + + it should "compare with a text column as DuckDB casts it (A-84, D6)" in { + // main answered t2 only: an Integer is never == a String + ids("SELECT id FROM s5b_t WHERE COALESCE(n, '3') = k") shouldBe Set("t1", "t2", "t3") + ids("SELECT id FROM s5b_t WHERE CASE WHEN n > 0 THEN n ELSE '3' END = k") shouldBe + Set("t1", "t2", "t3") + ids("SELECT id FROM s5b_t WHERE COALESCE(n, '3') IN ('3', '1')") shouldBe Set("t1", "t2", "t3") + // a literal that reads as nothing: refused, as its unquoted twin is + failure("SELECT id FROM s5b_t WHERE COALESCE(n, '3') IN ('3', 'abc')") shouldBe defined + } + + it should "be read as its text by a string function (R-109)" in { + answers( + "s5b_t", + "UPPER(CASE WHEN n > 0 THEN n ELSE '3' END)", + TextC, + "t1" -> "1", + "t2" -> "3", + "t3" -> "3" + ) + answers("s5b_t", "UPPER(n)", TextC, "t1" -> "1", "t2" -> null, "t3" -> "3") + answers("s5b_t", "LENGTH(n)", LongC, "t1" -> 1, "t2" -> null, "t3" -> 1) + answers("s5b_t", "UPPER(COALESCE(n, '3'))", TextC, "t1" -> "1", "t2" -> "3", "t3" -> "3") + answers( + "s5b_t", + "LOWER(CASE WHEN n > 0 THEN 'A' ELSE 'B' END)", + TextC, + "t1" -> "a", + "t2" -> "b", + "t3" -> "a" + ) + } + + "a BOOLEAN cast to a temporal (R-110)" should "be refused" in { + failure("SELECT id, CAST(b AS DATE) AS r FROM s5b_t").getOrElse("") should include( + "cannot cast BOOLEAN to DATE" + ) + } + + "a read conditional nested beside text (A-97)" should "answer its value's text, 0.24.0's type" in { + // nested beside a text argument, the read conditional answers the text of the value it is read + // as -- the same row's CAST( AS VARCHAR) -- where the nested reading refused the + // statement ('Cannot mix values of type INT and KEYWORD') that 0.24.0 answered as text + def text(expr: String, inner: String, expected: (String, Any)*): Unit = { + answers("s5b_t", expr, TextC, expected: _*) + answers("s5b_t", s"CAST($inner AS VARCHAR)", TextC, expected: _*) + } + // 0.24.0 answered '1.5' on the NULL row; the inner COALESCE reads it as 2 beside the INT n + text( + "COALESCE(COALESCE(n, '1.5'), k)", + "COALESCE(n, '1.5')", + "t1" -> "1", + "t2" -> "2", + "t3" -> "3" + ) + text( + "COALESCE(CASE WHEN n > 0 THEN n ELSE '3' END, k)", + "CASE WHEN n > 0 THEN n ELSE '3' END", + "t1" -> "1", + "t2" -> "3", + "t3" -> "3" + ) + text( + "COALESCE(COALESCE(n, '3'), COALESCE(d, '2024-01-15'))", + "COALESCE(n, '3')", + "t1" -> "1", + "t2" -> "3", + "t3" -> "3" + ) + // main refused the inner GREATEST (a text beside a number): refused -> answered + text( + "COALESCE(GREATEST(n, '2'), k)", + "GREATEST(n, '2')", + "t1" -> "2", + "t2" -> "2", + "t3" -> "3" + ) + // the date arm answers on the rows n > 0 (its ISO date); the NULL row takes k + answers( + "s5b_t", + "CASE WHEN n > 0 THEN COALESCE(d, '2024-01-15') ELSE k END", + TextC, + "t1" -> "2024-01-10", + "t2" -> "3", + "t3" -> "2024-02-01" + ) + answers( + "s5b_t", + "CAST(COALESCE(d, '2024-01-15') AS VARCHAR)", + TextC, + "t1" -> "2024-01-10", + "t3" -> "2024-02-01" + ) + // beside a typed arm the read CASE keeps its INT, its NULL test enclosed (r4: `3 != null`) + answers( + "s5b_t", + "COALESCE(CASE WHEN n > 0 THEN n ELSE '3' END, n)", + IntC, + "t1" -> 1, + "t2" -> 3, + "t3" -> 3 + ) + } + + "a literal-first COALESCE beside text (A-98)" should "compare its literal's text, as main does" in { + // main answers both right on every row (RUN); DuckDB refuses them at bind time + answers( + "s5b_t", + "CASE WHEN COALESCE('2024-01-15', d) = k THEN 1 ELSE 0 END", + LongC, + "t1" -> 0, + "t2" -> 0, + "t3" -> 0 + ) + answers( + "s5b_t", + "NULLIF(COALESCE('2024-01-15', d), k)", + TextC, + "t1" -> "2024-01-15", + "t2" -> "2024-01-15", + "t3" -> "2024-01-15" + ) + } + + it should "answer COALESCE(NULLIF(F, k), k) as COALESCE(F, k) (β)" in { + Seq( + "CASE WHEN n > 0 THEN d ELSE '2024-01-15' END", + "CASE WHEN n > 0 THEN ts ELSE '2024-01-15 10:30:00' END" + ).foreach { f => + // the oracle: COALESCE(F, k) on the same tree, its date or instant's text (A-97) + val oracle = byId(s"SELECT id, COALESCE($f, k) AS r FROM s5b_t") + answers("s5b_t", s"COALESCE(NULLIF($f, k), k)", TextC, oracle.toSeq: _*) + } + answers( + "s5b_t", + "COALESCE(NULLIF(CASE WHEN n > 0 THEN d ELSE '2024-01-15' END, k), k)", + TextC, + "t1" -> "2024-01-10", + "t2" -> "2024-01-15", + "t3" -> "2024-02-01" + ) + } + + "a COALESCE over a GREATEST / LEAST holding a literal (A-99)" should "answer, never NULL" in { + // main and r4 tested the reducer's primitive for NULL: Painless refused to compile it + answers("s5b_t", "COALESCE(GREATEST(n, 3), l)", LongC, "t1" -> 3, "t2" -> 3, "t3" -> 3) + answers("s5b_t", "COALESCE(GREATEST(n, '3'), n)", IntC, "t1" -> 3, "t2" -> 3, "t3" -> 3) + } + + it should "store its value in a computed column" in { + // one column per table: one failing pipeline hides nothing + ddl( + "CREATE TABLE s5b_u1 (id KEYWORD, n INT, l BIGINT, k KEYWORD, " + + "c BIGINT SCRIPT AS (COALESCE(GREATEST(n, 3), l)), PRIMARY KEY (id))" + ) + ddl( + "CREATE TABLE s5b_u2 (id KEYWORD, n INT, l BIGINT, k KEYWORD, " + + "c VARCHAR SCRIPT AS (COALESCE(GREATEST(n, '3'), k)), PRIMARY KEY (id))" + ) + val docs = List( + """{"id":"q1","n":1,"l":5,"k":"a"}""", + """{"id":"q2","k":"b"}""", + """{"id":"q3","n":7,"l":2,"k":"c"}""" + ) + bulk("s5b_u1", docs) + bulk("s5b_u2", docs) + Seq( + "s5b_u1" -> Seq("q1" -> 3, "q2" -> 3, "q3" -> 7), + "s5b_u2" -> Seq("q1" -> "3", "q2" -> "3", "q3" -> "7") + ) + .foreach { case (index, expected) => + val got = byId(s"SELECT id, c FROM $index", "c") + expected.foreach { case (id, value) => + withClue(s"$index $id $got: ")(same(got.getOrElse(id, null), value) shouldBe true) + } + } + } + + "COALESCE over a CASE (R-113)" should "test the whole CASE for NULL" in { + // main tested the ELSE alone (`c ? a : b != null ? … : k`): where the condition held and the + // THEN was NULL it answered NULL, not the next argument (t2: k = '3', n missing) + answers( + "s5b_t", + "COALESCE(CASE WHEN k = '3' THEN CAST(n AS VARCHAR) ELSE g END, g)", + TextC, + "t1" -> "g1", + "t2" -> "g2", + "t3" -> "3" + ) + // a NULL ELSE + answers( + "s5b_t", + "COALESCE(CASE WHEN k = '3' THEN CAST(n AS VARCHAR) ELSE NULL END, g)", + TextC, + "t1" -> "g1", + "t2" -> "g2", + "t3" -> "3" + ) + // a CASE nested in the fold, as a middle argument + answers( + "s5b_t", + "COALESCE(CAST(ti AS VARCHAR), CASE WHEN k = '3' THEN CAST(n AS VARCHAR) ELSE g END, g)", + TextC, + "t1" -> "1", + "t2" -> "g2", + "t3" -> "5" + ) + } + + it should "test the whole CASE for NULL in a computed column too" in { + ddl( + "CREATE TABLE s5b_n (id KEYWORD, k KEYWORD, n INT, g KEYWORD, " + + "c KEYWORD SCRIPT AS (COALESCE(CASE WHEN k = '3' THEN CAST(n AS VARCHAR) ELSE g END, g)), " + + "PRIMARY KEY (id))" + ) + bulk( + "s5b_n", + List( + """{"id":"n1","k":"3","g":"g1"}""", + """{"id":"n2","k":"3","n":5,"g":"g2"}""", + """{"id":"n3","k":"1","g":"g3"}""" + ) + ) + byId("SELECT id, c FROM s5b_n", "c") shouldBe Map("n1" -> "g1", "n2" -> "5", "n3" -> "g3") + } + + // ---- an ingest processor (D3 (5), R-107, D6 / D7): string and native _source ---- + + "a computed column" should "store the value its quoted argument or its BOOLEAN is read as" in { + val got = rows("SELECT id, c1, c2, c3, c4, c5, c6, c7 FROM s5b_p") + .map(r => String.valueOf(r("id")) -> r) + .toMap + def column(c: String, expected: (String, Any)*): Unit = + expected.foreach { case (id, value) => + withClue(s"$c $id ${got.get(id)}: ")(same(got(id).getOrElse(c, null), value) shouldBe true) + } + column("c1", "p1" -> 1, "p2" -> 5, "p3" -> 3, "p4" -> 3, "p5" -> 3, "p6" -> 3) + column("c2", "p1" -> 2, "p3" -> 2, "p4" -> 2, "p5" -> 2, "p6" -> 2) + // main stored nothing for a document with a boolean: Math.max over a Boolean threw + column("c3", "p1" -> 1, "p2" -> 1, "p3" -> 1, "p4" -> 1, "p5" -> 1, "p6" -> 1) + column("c4", "p1" -> 1, "p2" -> 1, "p3" -> 0, "p4" -> 0, "p5" -> 0, "p6" -> 1) + column( + "c5", + "p1" -> "2024-01-10", + "p2" -> "2024-01-15", + "p3" -> "2024-01-15", + "p6" -> "2024-01-15" + ) + // main compared a Boolean with 1: never equal, 0 on every document + column("c6", "p1" -> 1, "p2" -> 1, "p3" -> 0, "p4" -> 0, "p5" -> 0, "p6" -> 0) + column("c7", "p1" -> true, "p2" -> true, "p3" -> null, "p4" -> null, "p6" -> null) + } + + it should "keep a quoted date's text type outside a query, so DATE_ADD parses it (A-92)" in { + // before A-92 the conditional was a DATE in the processor while its value was text, and + // DATE_ADD called `.plus` on a String: every document failed and the column was absent + val got = rows("SELECT id, c8, c9 FROM s5b_p").map(r => String.valueOf(r("id")) -> r).toMap + def stored(id: String, c: String): Any = got.get(id).flatMap(_.get(c)).orNull + withClue(s"c8 p1 $got: ")(same(stored("p1", "c8"), "2024-01-11") shouldBe true) + Seq("p2", "p3", "p4", "p5", "p6").foreach { id => + withClue(s"c8 $id $got: ")(same(stored(id, "c8"), "2024-01-16") shouldBe true) + // a timestamp read in a processor is the literal as written, which DATE_ADD's date parse reads + withClue(s"c9 $id $got: ")(same(stored(id, "c9"), "2024-01-16") shouldBe true) + } + } + + it should "compare a BOOLEAN BETWEEN two numbers as 1 / 0 (A-91)" in { + // main compared the Boolean itself with the bounds: the processor failed on every document + val got = byId("SELECT id, c10 FROM s5b_p", "c10") + Seq("p1" -> 1, "p2" -> 1, "p3" -> 0, "p4" -> 0, "p5" -> 0, "p6" -> 0).foreach { case (id, v) => + withClue(s"c10 $id $got: ")(same(got.getOrElse(id, null), v) shouldBe true) + } + } + + it should "store a date read in a CASE result as its ISO text (A-89)" in { + // before A-89 the processor called a method on the string literal ("2024-01-15".atStartOfDay): + // Elasticsearch refused the pipeline, so this CREATE failed where main and 0.24.0 accept it + ddl( + "CREATE TABLE s5b_q (id KEYWORD, n INT, d DATE, " + + "c DATE SCRIPT AS (CASE WHEN n > 0 THEN d ELSE '2024-01-15' END), " + + "c2 DATE SCRIPT AS (CASE WHEN n > 0 THEN '2024-01-15' ELSE d END), PRIMARY KEY (id))" + ) + bulk( + "s5b_q", + List( + """{"id":"q1","n":1,"d":"2024-01-10"}""", + """{"id":"q2","n":0,"d":"2024-01-10"}""", + """{"id":"q3","n":-1,"d":"2024-01-20"}""", + """{"id":"q4","d":"2024-01-10"}""" + ) + ) + val got = rows("SELECT id, c, c2 FROM s5b_q").map(r => String.valueOf(r("id")) -> r).toMap + def stored(id: String, c: String): Any = got.get(id).flatMap(_.get(c)).orNull + Seq("q2", "q3", "q4").foreach { id => + withClue(s"c $id $got: ")(same(stored(id, "c"), "2024-01-15") shouldBe true) + } + withClue(s"c2 q1 $got: ")(same(stored("q1", "c2"), "2024-01-15") shouldBe true) + // the date column's arm renders as on main (A-89 moves only the literal): recorded, not pinned + info(s"c q1 = ${stored("q1", "c")}; c2 q2 = ${stored("q2", "c2")}") + } + + it should "compare a conditional holding a quoted date as its text, as 0.24.0 does (R1-1)" in { + // before R1-1 the processor read the conditional as a DATE over the text it holds and called + // `isEqual` / `isBefore` on a String: every document failed and the column was absent + ddl( + "CREATE TABLE s5b_r (id KEYWORD, n INT, d DATE, ts TIMESTAMP, " + + "c1 INT SCRIPT AS (CASE WHEN COALESCE(d, '2024-01-15') = '2024-01-15' THEN 1 ELSE 0 END), " + + "c2 INT SCRIPT AS (CASE WHEN COALESCE(d, '2024-01-15') BETWEEN '2024-01-01' AND " + + "'2024-01-31' THEN 1 ELSE 0 END), " + + "c3 INT SCRIPT AS (CASE WHEN CASE WHEN n > 0 THEN d ELSE '2024-01-15' END " + + "IN ('2024-01-15', '2024-03-01') THEN 1 ELSE 0 END), " + + "c4 INT SCRIPT AS (CASE WHEN COALESCE(ts, '2024-01-15 10:30:00') < '2024-01-12' " + + "THEN 1 ELSE 0 END), " + + "c5 DATE SCRIPT AS (NULLIF(COALESCE(d, '2024-01-15'), '2024-01-15')), PRIMARY KEY (id))" + ) + bulk( + "s5b_r", + List( + """{"id":"r1","n":1,"d":"2024-01-15","ts":"2024-01-10T08:00:00Z"}""", + """{"id":"r2","n":1,"d":"2024-02-01","ts":"2024-01-20T08:00:00Z"}""", + """{"id":"r3","n":0,"d":"2024-01-10"}""", + """{"id":"r4"}""" + ) + ) + val got = rows("SELECT id, c1, c2, c3, c4, c5 FROM s5b_r") + .map(r => String.valueOf(r("id")) -> r) + .toMap + def column(c: String, expected: (String, Any)*): Unit = + expected.foreach { case (id, value) => + withClue(s"$c $id ${got.get(id)}: ") { + same(got.get(id).flatMap(_.get(c)).orNull, value) shouldBe true + } + } + // DuckDB's values on these documents (a missing d is the literal) + column("c1", "r1" -> 1, "r2" -> 0, "r3" -> 0, "r4" -> 1) + column("c2", "r1" -> 1, "r2" -> 0, "r3" -> 1, "r4" -> 1) + // the literal's rows; the date column's arm renders as on main (recorded, not pinned) + column("c3", "r3" -> 1, "r4" -> 1) + column("c4", "r1" -> 1, "r2" -> 0, "r3" -> 0, "r4" -> 0) + column("c5", "r1" -> null, "r2" -> "2024-02-01", "r3" -> "2024-01-10", "r4" -> null) + def stored(id: String): Any = got.get(id).flatMap(_.get("c3")).orNull + info(s"c3 r1 = ${stored("r1")}; c3 r2 = ${stored("r2")}") + } + + // ---- AC3: R-60's cells (R-102: 17 delivered), the twin fixture and its boolean discriminator ---- + + "R-60's comparison cells" should "answer DuckDB's rows" in { + Seq( + "n = b" -> Set("c1", "c2"), + "l = b" -> Set("c1", "c2"), + "x = b" -> Set.empty[String], + "k = n" -> Set("c1", "c2"), + "k = x" -> Set.empty[String], + "k = b" -> Set("c1", "c2"), + "b = n" -> Set("c1", "c2"), + "b = x" -> Set.empty[String], + // over an ARRAY column, the element (D8) + "ak = n" -> Set("c1", "c2"), + "ab = n" -> Set("c1", "c2") + ).foreach { case (cond, expected) => + withClue(cond)(ids(s"SELECT id FROM s5b_c WHERE $cond") shouldBe expected) + } + answers("s5b_c", "NULLIF(k, 0)", TextC, "c1" -> "1", "c2" -> null) + answers("s5b_c", "CASE WHEN k = 1 THEN 1 ELSE 0 END", LongC, "c1" -> 1, "c2" -> 0) + answers( + "s5b_c", + "CASE k WHEN 1 THEN 'one' ELSE 'other' END", + TextC, + "c1" -> "one", + "c2" -> "other" + ) + answers("s5b_c", "NULLIF(b, 0)", BoolC, "c1" -> true, "c2" -> null) + answers("s5b_c", "CASE WHEN b = 1 THEN 1 ELSE 0 END", LongC, "c1" -> 1, "c2" -> 0) + answers( + "s5b_c", + "CASE b WHEN 1 THEN 'one' ELSE 'other' END", + TextC, + "c1" -> "one", + "c2" -> "other" + ) + answers("s5b_c", "GREATEST(b, 1)", LongC, "c1" -> 1, "c2" -> 1) + answers("s5b_c", "LEAST(b, 1)", LongC, "c1" -> 1, "c2" -> 0) + answers("s5b_c", "CASE WHEN ak = 1 THEN 1 ELSE 0 END", LongC, "c1" -> 1, "c2" -> 0) + answers("s5b_c", "LEAST(ab, 1)", LongC, "c1" -> 1, "c2" -> 0) + } + + it should "filter groups on a BOOLEAN metric as 1 / 0, with no conversion (R10)" in { + rows("SELECT g, MAX(b) AS m FROM s5b_c GROUP BY g HAVING MAX(b) > 1") shouldBe empty + rows("SELECT g, MAX(b) AS m FROM s5b_c GROUP BY g HAVING MAX(b) > 0") + .map(r => String.valueOf(r("g"))) shouldBe Seq("g1") + } + + it should "leave a text column to Elasticsearch, and keep R-102's exclusions (A-78, A-79)" in { + // a script over a `text` field fails in Elasticsearch (fielddata), never in core's type rule + val text = failure("SELECT id FROM s5b_c WHERE tx = n") + text shouldBe defined + text.get should not include "Type mismatch" + failure("SELECT g, MAX(k) AS m FROM s5b_c GROUP BY g HAVING MAX(k) > '2024-01-01'") + .getOrElse("") should include("compares a metric as text") + } + + // ---- D7's twin pins: the runtime conversion answers what the literal rule reads ---- + + "a stored text beside a number" should "be read as roundedWhole reads it, or fail the query" in { + wholeTexts.zipWithIndex.foreach { case (t, i) => + val sql = + s"SELECT id, CASE WHEN k = l THEN 'eq' ELSE 'ne' END AS r FROM s5b_k WHERE id = 'w$i'" + withClue(s"[$t] $sql: ") { + ComparisonRule.roundedWhole(t) match { + case Some(_) => byId(sql) shouldBe Map(s"w$i" -> "eq") + case None => failure(sql) shouldBe defined + } + } + } + } + + "a stored text beside a boolean" should "be read as the literal rule reads it, or fail (A-81)" in { + booleanTexts.zipWithIndex.foreach { case (t, i) => + val sql = + s"SELECT id, CASE WHEN k = b THEN 'eq' ELSE 'ne' END AS r FROM s5b_k WHERE id = 'b$i'" + withClue(s"[$t] $sql: ") { + booleanOf(t) match { + case Some(_) => byId(sql) shouldBe Map(s"b$i" -> "eq") + case None => failure(sql) shouldBe defined + } + } + } + } + + "a stored text beside a DOUBLE" should "be read as the number it spells, or fail" in { + doubleTexts.zipWithIndex.foreach { case (t, i) => + val sql = + s"SELECT id, CASE WHEN k = x THEN 'eq' ELSE 'ne' END AS r FROM s5b_k WHERE id = 'x$i'" + withClue(s"[$t] $sql: ") { + doubleTwinOf(t) match { + // past DOUBLE's range: +/-Infinity, which equals no stored (finite) x (A-93) + case Some(v) if v.isInfinite => byId(sql) shouldBe Map(s"x$i" -> "ne") + case Some(_) => byId(sql) shouldBe Map(s"x$i" -> "eq") + case None => failure(sql) shouldBe defined + } + } + } + } +} From 0b368d3e2f784ec17a2ba0fb1823a86b6331d0b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 9 Nov 2026 07:39:24 +0100 Subject: [PATCH 2/4] fix(sql): literal-first COALESCE equality stays query-only at every script entry (SCHEMA-5b) Story SCHEMA-5b. Reviewed: r5:clean. Co-Authored-By: Claude Opus 5.5 --- .../softnetwork/elastic/sql/query/Where.scala | 12 +++-- .../function/cond/QuotedArmReadingSpec.scala | 48 +++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index fc72f8c3..e292e788 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -1874,11 +1874,17 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { * operands render there. Read outside that scope, a transform's `COALESCE(d2, d) > ts` took the * query's DATE for a conditional whose arms render their doc values there, and appended * `.atStartOfDay(…)` to a `ZonedDateTime`. + * + * The scope is entered BEFORE [[literalsInPlace]] is asked, which reads it (A-98 α applies in a + * query only): asked first, a processor or a transform entering `COALESCE('2024-01-15', d) = k` + * read the query's literal in place of the `COALESCE`, where its bytes are the comparison's. */ override def painless(context: Option[PainlessContext]): String = - literalsInPlace match { - case Some(mapped) => mapped.painless(context) - case None => SQLTypeUtils.renderingIn(context)(comparisonPainless(context)) + SQLTypeUtils.renderingIn(context) { + literalsInPlace match { + case Some(mapped) => mapped.painless(context) + case None => comparisonPainless(context) + } } /** 🔴 This EQUALITY (`=`, `<>`, `!=`) with each side that `ComparisonRule.literalInPlaceOf` maps diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/QuotedArmReadingSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/QuotedArmReadingSpec.scala index 3c75a867..c93bd439 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/QuotedArmReadingSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/function/cond/QuotedArmReadingSpec.scala @@ -940,6 +940,54 @@ class QuotedArmReadingSpec extends AnyFlatSpec with Matchers { } } + private val scopeTable = table.copy(columns = table.columns :+ Column("d2", SQLTypes.Date)) + + /** `t`'s declarations and the WHERE criteria of `sql` rendered in `t`, on a fresh node: `scoped` + * renders it inside the scope of `t` (`SQLTypeUtils.renderingIn`), as an ingest processor's + * script does; else the criteria itself is the rendering's entry, as a bridge script query is. + */ + private def criteriaIn(sql: String, t: PainlessContextType, scoped: Boolean): String = { + val c = PainlessContext(t) + val criteria = Parser(sql) match { + case Right(s: SingleSearch) => + s.update(Some(scopeTable)).where.flatMap(_.criteria).getOrElse(fail(s"no WHERE: $sql")) + case other => fail(s"not a single search: $other") + } + val p = + if (scoped) SQLTypeUtils.renderingIn(Some(c))(criteria.painless(Some(c))) + else criteria.painless(Some(c)) + s"$c$p" + } + + private val outsideQuery = + Seq(PainlessContextType.Transform -> "doc['d']", PainlessContextType.Processor -> "ctx.d") + + it should "be read in a query only, also where the comparison enters the script (RV6-1)" in { + val sql = s"SELECT k FROM t WHERE $cdr = k" + outsideQuery.foreach { case (t, d) => + withClue(s"$t: ") { + val entry = criteriaIn(sql, t, scoped = false) + // the bytes of the comparison rendered in its scope: the COALESCE is read whole + entry shouldBe criteriaIn(sql, t, scoped = true) + entry should include(d) + } + } + } + + it should "keep the query's bytes, and those of a comparison it does not read (RV6-1)" in { + val alpha = s"SELECT k FROM t WHERE $cdr = k" + // the literal compared with k as text, d never read (A-98 α) + val query = criteriaIn(alpha, PainlessContextType.Query, scoped = false) + query should include("(\"2024-01-15\" == param1)") + query should not include "doc['d']" + val control = "SELECT k FROM t WHERE COALESCE(d, '2024-01-15') = d2" + outsideQuery.foreach { case (t, _) => + withClue(s"$t: ") { + criteriaIn(control, t, scoped = false) shouldBe criteriaIn(control, t, scoped = true) + } + } + } + behavior of "COALESCE(NULLIF(a, b), b) over a read-moved conditional (A-98 β)" /** A rendering with its parameter names erased. */ From 96b5b85877056cc4a365c619c1dcd26e953fa34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 9 Nov 2026 14:06:42 +0100 Subject: [PATCH 3/4] fix(sql): ISNULL / ISNOTNULL over an operand with no column run as a script (SCHEMA-5b) A conditional operand (NULLIF, COALESCE, CASE) has no column for `exists` to test, so Elasticsearch rejected the query. Such an operand now takes the script path, in both bridges, and is tested as one value. The #382 integration test now asserts what a query does with a text against a number: accepted, and failing when a text that spells no number is read. The computed-column refusal is unchanged. Story SCHEMA-5b. Reviewed: r6:clean, r7:clean. Co-Authored-By: Claude Opus 5.5 --- .../elastic/sql/bridge/package.scala | 16 ++++++ .../elastic/sql/bridge/package.scala | 16 ++++++ .../softnetwork/elastic/sql/query/Where.scala | 6 ++- .../client/GatewayApiIntegrationSpec.scala | 49 ++++++++++++++----- .../QuotedArgumentConditionalSpec.scala | 12 ++++- 5 files changed, 84 insertions(+), 15 deletions(-) diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 6e8ba206..b82da1d9 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -1085,16 +1085,32 @@ package object bridge { existsQuery(identifier.name) } + /** Does this `ISNULL` / `ISNOTNULL` operand have no column to test? `exists` needs a field name + * and a conditional (`NULLIF(a, b)`, `COALESCE(a, b)`) has none: Elasticsearch rejects an + * `exists` on an empty field, so the script path, which renders the operand, answers it. + */ + private[bridge] def hasNoField(identifier: Identifier): Boolean = identifier.name.trim.isEmpty + implicit def isNullCriteriaToQuery( isNull: IsNullCriteria + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { + if (hasNoField(isNull.identifier)) return scriptQueryOf(isNull) import isNull._ not(existsQuery(identifier.name)) } implicit def isNotNullCriteriaToQuery( isNotNull: IsNotNullCriteria + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { + if (hasNoField(isNotNull.identifier)) return scriptQueryOf(isNotNull) import isNotNull._ existsQuery(identifier.name) } diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 643c97c6..20162fda 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -1072,16 +1072,32 @@ package object bridge { existsQuery(identifier.name) } + /** Does this `ISNULL` / `ISNOTNULL` operand have no column to test? `exists` needs a field name + * and a conditional (`NULLIF(a, b)`, `COALESCE(a, b)`) has none: Elasticsearch rejects an + * `exists` on an empty field, so the script path, which renders the operand, answers it. + */ + private[bridge] def hasNoField(identifier: Identifier): Boolean = identifier.name.trim.isEmpty + implicit def isNullCriteriaToQuery( isNull: IsNullCriteria + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { + if (hasNoField(isNull.identifier)) return scriptQueryOf(isNull) import isNull._ not(existsQuery(identifier.name)) } implicit def isNotNullCriteriaToQuery( isNotNull: IsNotNullCriteria + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { + if (hasNoField(isNotNull.identifier)) return scriptQueryOf(isNotNull) import isNotNull._ existsQuery(identifier.name) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index e292e788..3f0f05fd 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -2485,7 +2485,8 @@ case class IsNullCriteria(identifier: Identifier) extends CriteriaWithConditiona case None => s"def left = $rendered; left == null" } } - s"${left(context)} == null" + // ONE value: a nameless operand can render `c ? a : b`, which the test would split + s"${SQLTypeUtils.asOneValue(left(context))} == null" } } @@ -2529,7 +2530,8 @@ case class IsNotNullCriteria(identifier: Identifier) case None => s"def left = $rendered; left != null" } } - s"${left(context)} != null" + // ONE value: a nameless operand can render `c ? a : b`, which the test would split + s"${SQLTypeUtils.asOneValue(left(context))} != null" } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala index c7989624..e06d0be1 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -1497,8 +1497,10 @@ trait GatewayApiIntegrationSpec extends GatewayIntegrationTestKit { } } - /** 🔴 Issue #382, the LEAD RULING of 2026-09-23 — a `NULLIF` that compares TEXT with a NUMBER is - * a TYPE ERROR, and the engine refuses it instead of building a script around it. + /** 🔴 Issue #382, the LEAD RULING of 2026-09-23 — in a computed column, a `NULLIF` that compares + * TEXT with a NUMBER is a TYPE ERROR, and the engine refuses it instead of building a script + * around it. A query compares the two as DuckDB does, and a text that spells no number fails + * that query when its rows are read: the refusal is for the venue where the failure is silent. * * Executed, because the thing being asserted is that NOTHING is deployed and NOTHING is stored. * Before the ruling, the DDL form of this statement answered `200`, created the table, and then @@ -1510,7 +1512,7 @@ trait GatewayApiIntegrationSpec extends GatewayIntegrationTestKit { * column and BOTH rendered types, or a future regression that merely produces some other `Left` * would satisfy the row (story 21.4's lesson). */ - it should "refuse a NULLIF that compares text with a number, in every venue (#382)" in { + it should "refuse a NULLIF of text and a number in a computed column, and fail the query on a text that spells no number (#382)" in { val create = """CREATE TABLE IF NOT EXISTS nullif_mismatch ( | id INT, @@ -1530,18 +1532,43 @@ trait GatewayApiIntegrationSpec extends GatewayIntegrationTestKit { // ...and the table it would have created does not exist, so nothing was deployed half-way client.indexExists("nullif_mismatch", pattern = false) shouldBe ElasticSuccess(false) - // the QUERY venue of the same rule: a projection and a predicate over a table that DOES exist + // The QUERY venue compares them as DuckDB does: the text is read as a BIGINT, so a projection + // and a predicate are accepted, and this fixture's 'x' and 'yy', which spell no number, fail + // them when the rows are read. CONTROL: the same NULLIF over the same column, with nothing to + // convert, answers, so the number literal is the only difference between the two. + val control = collectRows( + System.nanoTime(), + client.run("SELECT id, NULLIF(s, 'x') AS c FROM nullif_fn").futureValue + ).map(r => String.valueOf(scalarOf(r, "id")).toDouble.toLong -> r).toMap + withClue(s"$control ") { + control.keySet shouldBe Set(1L, 2L) + control(1L).get("c").filterNot(v => v == null || v == Nil) shouldBe None + String.valueOf(scalarOf(control(2L), "c")) shouldBe "yy" + } + // ...and as a predicate: a NULLIF has no column for `exists` to test, the script answers it + Seq("ISNOTNULL" -> Set(2L), "ISNULL" -> Set(1L)).foreach { case (function, expected) => + val sql = s"SELECT id FROM nullif_fn WHERE $function(NULLIF(s, 'x'))" + withClue(s"[$sql] ") { + collectRows(System.nanoTime(), client.run(sql).futureValue) + .map(r => String.valueOf(scalarOf(r, "id")).toDouble.toLong) + .toSet shouldBe expected + } + } + Seq( "SELECT id, NULLIF(s, 0) AS c FROM nullif_fn", "SELECT id FROM nullif_fn WHERE ISNOTNULL(NULLIF(s, 0))" ).foreach { sql => - val res = client.run(sql).futureValue - withClue(s"[$sql] $res ") { - res.isFailure shouldBe true - val message = String.valueOf(res) - message should include("NULLIF(s, 0)") - message should include("KEYWORD") - message should include("BIGINT") + val thrown = scala.util + .Try(collectRows(System.nanoTime(), client.run(sql).futureValue)) + .failed + .getOrElse(fail(s"[$sql] over a text that spells no number was expected to fail")) + val reported = throwableText(thrown) + // the comparison's own conversion failing in Elasticsearch, not the engine refusing the + // statement: a client reports the thrown message or the response body quoting the script + withClue(s"[$sql] reported failure text was [$reported]: ") { + reported should include("could not convert string [") + reported should include("] to BIGINT") } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/QuotedArgumentConditionalSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/QuotedArgumentConditionalSpec.scala index 5e987ff8..67a6097b 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/QuotedArgumentConditionalSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/QuotedArgumentConditionalSpec.scala @@ -737,8 +737,16 @@ trait QuotedArgumentConditionalSpec extends GatewayIntegrationTestKit { ).foreach { case (cond, expected) => withClue(cond)(ids(s"SELECT id FROM s5b_c WHERE $cond") shouldBe expected) } - answers("s5b_c", "NULLIF(k, 0)", TextC, "c1" -> "1", "c2" -> null) - answers("s5b_c", "CASE WHEN k = 1 THEN 1 ELSE 0 END", LongC, "c1" -> 1, "c2" -> 0) + answers("s5b_c", "NULLIF(k, 0)", TextC, "c1" -> "1", "c2" -> null) + // a predicate over a conditional has no column for `exists` to test: the script answers it + ids("SELECT id FROM s5b_c WHERE ISNOTNULL(NULLIF(k, 0))") shouldBe Set("c1") + ids("SELECT id FROM s5b_c WHERE ISNULL(NULLIF(k, 0))") shouldBe Set("c2") + ids("SELECT id FROM s5b_c WHERE ISNOTNULL(COALESCE(k, 'z'))") shouldBe Set("c1", "c2") + // the operand is ONE value: unenclosed, `c ? b : y == null` would answer the arm `b` + ids("SELECT id FROM s5b_c WHERE ISNULL(CASE WHEN n > 0 THEN b ELSE b END)") shouldBe empty + ids("SELECT id FROM s5b_c WHERE ISNOTNULL(CASE WHEN n = 0 THEN b ELSE b END)") shouldBe + Set("c1", "c2") + answers("s5b_c", "CASE WHEN k = 1 THEN 1 ELSE 0 END", LongC, "c1" -> 1, "c2" -> 0) answers( "s5b_c", "CASE k WHEN 1 THEN 'one' ELSE 'other' END", From 68a74c3921e7cfe2fc0567498ebb8caca8bd3c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 10 Nov 2026 10:31:29 +0100 Subject: [PATCH 4/4] perf(sql): comparisons and NULLIF compute their implicit-cast reading once per scope (SCHEMA-5b) The implicit-cast table made every comparison re-derive its target type at each of its renderers, and NULLIF re-ask, on every read of an argument's type, whether a literal-first COALESCE stands in for that argument. - Expression.implicitComparisonTarget is memoised per scope (PerScope), as literalReading is; InExpr and BetweenExpr override the computation. - ComparisonRule.literalInPlaceOf asks whether the operand is a COALESCE before it walks both chains for aggregates. - NullIf.first / second are memoised per scope. No verdict and no generated script changes. Two specs pin that one node answers each scope (query, transform, processor, computed-column validation) as a fresh node does, in any order. Co-Authored-By: Claude Opus 5.5 --- .../elastic/sql/function/cond/package.scala | 11 ++- .../app/softnetwork/elastic/sql/package.scala | 16 ++-- .../softnetwork/elastic/sql/query/Where.scala | 18 +++-- .../query/ImplicitComparisonCastSpec.scala | 81 +++++++++++++++++++ 4 files changed, 111 insertions(+), 15 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 322263d9..8b59a98e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -626,14 +626,19 @@ package object cond { * is read in place of beside a text second argument (SCHEMA-5b A-98 α, * `ComparisonRule.literalInPlaceOf`), else `expr1`. `NULLIF(COALESCE('2024-01-15', d), k)` * compares and answers `'2024-01-15'`, VARCHAR, as 0.24.0 did (right on every row, RUN). Every - * reader of an argument's type, reading or rendering reads [[first]] and [[second]]. + * reader of an argument's type, reading or rendering reads [[first]] and [[second]], so each + * is computed ONCE PER SCOPE (`literalInPlaceOf` answers in a query only). */ private[cond] def first: PainlessScript = - ComparisonRule.literalInPlaceOf(expr1, expr2).getOrElse(expr1) + _first(ComparisonRule.literalInPlaceOf(expr1, expr2).getOrElse(expr1)) + + private[this] val _first = new SQLTypeUtils.PerScope[PainlessScript] /** The second argument as NULLIF compares it ([[first]]). */ private[cond] def second: PainlessScript = - ComparisonRule.literalInPlaceOf(expr2, expr1).getOrElse(expr2) + _second(ComparisonRule.literalInPlaceOf(expr2, expr1).getOrElse(expr2)) + + private[this] val _second = new SQLTypeUtils.PerScope[PainlessScript] override def answeredArms: List[PainlessScript] = List(first) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 9e0f8a2e..76ab17e7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1983,9 +1983,10 @@ package object sql { * one verdict for core and arrow. The text rows bind in a QUERY only (A-82): where * `SQLTypeUtils.readsWholeValues` holds -- an ingest processor or a transform rendering, a * computed column's validation -- a processor would swallow a text that converts to nothing - * (`ignore_failure`) where DuckDB fails the statement, so main's verdict and bytes stay. Read - * LIVE, never memoised: a node's query slot also serves the validation. A string LITERAL is - * not a side of this table: it is read by what it is compared with ([[readingOf]]). + * (`ignore_failure`) where DuckDB fails the statement, so main's verdict and bytes stay. A + * node that keeps the answer keeps it per scope (`SQLTypeUtils.PerScope`, A-95), whose slots + * split the scopes as `readsWholeValues` does. A string LITERAL is not a side of this table: + * it is read by what it is compared with ([[readingOf]]). */ def implicitTarget(left: SQLType, right: SQLType, equality: Boolean): Option[SQLType] = { val l = SQLTypeUtils.operandType(left) @@ -2015,11 +2016,12 @@ package object sql { */ def literalInPlaceOf(operand: Token, other: Token): Option[PainlessScript] = (operand, other) match { - // the cheap questions first: most operands are no COALESCE, and every comparison asks - case (p: PainlessScript, o: PainlessScript) - if !SQLTypeUtils.readsWholeValues && !p.hasAggregation && !o.hasAggregation => + // the cheap questions first: most operands are no COALESCE, and every comparison asks; + // the aggregate questions walk both chains, so they come after it + case (p: PainlessScript, o: PainlessScript) => SQLTypeUtils.conditionalNode(p) match { - case c: app.softnetwork.elastic.sql.function.cond.Coalesce => + case c: app.softnetwork.elastic.sql.function.cond.Coalesce + if !SQLTypeUtils.readsWholeValues && !p.hasAggregation && !o.hasAggregation => // the arms `readingAt` indexes (A-98 β may answer a NULLIF arm by its value) c.answeredArms.zipWithIndex .find { case (arm, _) => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 3f0f05fd..a2f1406a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -2259,10 +2259,18 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { /** 🔴 The type this comparison's two sides are compared IN by the implicit-cast table (SCHEMA-5b * D6, `ComparisonRule.implicitTarget`), for a pair main's rule refuses; `None` -- every other * comparison keeps main's rendering, byte for byte. The type rule ([[mismatchWith]]) reads the - * same table, so what is judged is what runs. Asked LIVE (A-82): the table's text rows depend on - * the scope, and a memo's query slot also serves a computed column's validation. + * same table, so what is judged is what runs. The table's text rows depend on the scope (A-82), + * so it is computed ONCE PER SCOPE and memoised like [[literalReading]]: every renderer of the + * comparison asks it, several times each. */ - protected def implicitComparisonTarget: Option[SQLType] = + protected final def implicitComparisonTarget: Option[SQLType] = + _implicitComparisonTarget(computeImplicitComparisonTarget) + + private[this] val _implicitComparisonTarget = new SQLTypeUtils.PerScope[Option[SQLType]] + + /** What [[implicitComparisonTarget]] computes; `InExpr` and `BetweenExpr` read their own sides. + */ + protected def computeImplicitComparisonTarget: Option[SQLType] = maybeValue.flatMap { value => val left = operandDeclaredType val right = ComparisonRule.comparedType(literalReading, value) @@ -2653,7 +2661,7 @@ case class InExpr[R, +T <: Value[R]]( * target against the operand, when every element needs one -- their common supertype (`k IN (1, * 1.5)` is compared as DOUBLE). */ - override protected def implicitComparisonTarget: Option[SQLType] = + override protected def computeImplicitComparisonTarget: Option[SQLType] = if (ComparisonRule.stringLiteral(identifier).isDefined) None else { val left = operandDeclaredType @@ -3215,7 +3223,7 @@ case class BetweenExpr( * the table binds a BOOLEAN beside a whole number only (`b BETWEEN 0 AND 1`, `n BETWEEN b AND * 5`). A bound main compares keeps its type; every other bound needs the table, or none applies. */ - override protected def implicitComparisonTarget: Option[SQLType] = + override protected def computeImplicitComparisonTarget: Option[SQLType] = if (ComparisonRule.stringLiteral(identifier).isDefined) None else { val left = operandDeclaredType diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/ImplicitComparisonCastSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/ImplicitComparisonCastSpec.scala index 0e53e628..4a4bb2b1 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/ImplicitComparisonCastSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/ImplicitComparisonCastSpec.scala @@ -393,4 +393,85 @@ class ImplicitComparisonCastSpec extends AnyFlatSpec with Matchers { second.typeError shouldBe None SQLTypeUtils.validatingComputedColumns(second.typeError) shouldBe defined } + + /** A scope's answer, or the failure it raises. */ + private def answer[N](n: N, ask: N => Any): Any = + scala.util.Try(ask(n)).fold(e => s"throws ${e.getMessage}", identity) + + /** Each scope's answer on ONE node, asked in every order, is what that scope answers on a fresh + * node (A-95): a memo that lets the first scope to ask decide for the others fails an order. + */ + private def answersEachScopeAsFresh[N](node: () => N, scopes: List[(String, N => Any)]): Unit = { + val fresh = scopes.map { case (scope, ask) => scope -> answer(node(), ask) }.toMap + scopes.permutations.foreach { order => + val one = node() + order.foreach { case (scope, ask) => + withClue(s"${order.map(_._1).mkString(" > ")}, $scope: ") { + answer(one, ask) shouldBe fresh(scope) + } + } + } + } + + /** The whole script `painless` renders from a node in a fresh context of type `t`. */ + private def renderedIn[N]( + t: PainlessContextType + )(painless: (N, PainlessContext) => String): N => String = { n => + val ctx = PainlessContext(t) + val p = painless(n, ctx) + s"$ctx$p" + } + + it should "answer one comparison node as each scope does, in any order (A-95)" in { + val render = (c: Criteria, ctx: PainlessContext) => c.painless(Some(ctx)) + val scopes: List[(String, Criteria => Any)] = List( + "query" -> renderedIn(PainlessContextType.Query)(render), + "transform" -> renderedIn(PainlessContextType.Transform)(render), + "processor" -> renderedIn(PainlessContextType.Processor)(render), + "type rule" -> ((c: Criteria) => c.typeErrors), + "validation" -> ((c: Criteria) => SQLTypeUtils.validatingComputedColumns(c.typeErrors)), + "validation render" -> ((c: Criteria) => + SQLTypeUtils.validatingComputedColumns(renderedIn(PainlessContextType.Query)(render)(c)) + ) + ) + Seq("k = 1", "k IN (1, 2)", "b BETWEEN 0 AND n").foreach { p => + withClue(s"$p: ") { + val node = () => resolved(s"SELECT g FROM t WHERE $p").where.flatMap(_.criteria).get + if (p.startsWith("k")) { + // the text row converts in a query only (A-82): the scopes differ, so the pin bites + answer(node(), scopes.head._2).toString should include("nv1") + answer(node(), scopes(1)._2).toString should not include "nv1" + answer(node(), scopes(2)._2).toString should not include "nv1" + answer(node(), scopes(5)._2).toString should not include "nv1" + } + answersEachScopeAsFresh(node, scopes) + } + } + } + + it should "answer one literal-first NULLIF node as each scope does, in any order (A-95)" in { + val node = () => + resolved( + "SELECT NULLIF(COALESCE('2024-01-15', d), k) AS v FROM t" + ).select.fieldsWithComputedAliases.head + def nullIf(f: Field): NullIf = f.identifier.functions.collectFirst { case n: NullIf => n }.get + val render = (f: Field, ctx: PainlessContext) => f.painless(Some(ctx)) + val scopes: List[(String, Field => Any)] = List( + "query" -> renderedIn(PainlessContextType.Query)(render), + "processor" -> renderedIn(PainlessContextType.Processor)(render), + "type rule" -> ((f: Field) => nullIf(f).typeError), + "validation" -> ((f: Field) => SQLTypeUtils.validatingComputedColumns(nullIf(f).typeError)), + "validation render" -> ((f: Field) => + SQLTypeUtils.validatingComputedColumns(renderedIn(PainlessContextType.Query)(render)(f)) + ) + ) + // the literal is read in a query only (A-98 α): a query compares its text, a processor reads + // the COALESCE whole -- the scopes differ + val literalCompared = "\"2024-01-15\".compareTo(param" + answer(node(), scopes.head._2).toString should include(literalCompared) + val processor = answer(node(), scopes(1)._2).toString + processor should include("ctx.d") + processor should not include literalCompared + answersEachScopeAsFresh(node, scopes) + } }