diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 3dcdd718..329dc833 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -2025,6 +2025,13 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override def describe(statement: query.Statement): ElasticResult[Option[ResultSchema]] = delegate.describe(statement) + /** SCHEMA-15 (D10) -- the delegate's: its schema cache, its miss and reload memos, counted once. + */ + override private[elastic] def describePerTable( + statement: query.Statement + ): ElasticResult[Option[ResultSchema]] = + delegate.describePerTable(statement) + // ==================== Extensions (delegate) ==================== /** #163 — the delegator inherits ExtensionApi's lazy vals, so `this.extensionRegistry` would be a diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index a562ffeb..136a6f95 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -2281,7 +2281,8 @@ trait GatewayApi extends IndicesApi with ElasticClientHelpers { * than core's handles answers `None` (its layer types it); a statement `run` refuses before * executing anything is refused with `run`'s status and message, through the SAME resolve body * (`resolveWithSchema(_, executeSubqueries = false)`). A refusal that needs a subquery's values - * or Elasticsearch surfaces at `run` only. `None` where no schema is known. + * or Elasticsearch surfaces at `run` only. `None` where no schema is known. A JOIN body's types + * per table, which `run` refuses here: [[describePerTable]]. */ def describe(sql: String): ElasticResult[Option[ResultSchema]] = GatewayApi.splitStatements(sql) match { @@ -2316,6 +2317,40 @@ trait GatewayApi extends IndicesApi with ElasticClientHelpers { ) } + /** SCHEMA-15 (D1, D3, D4, D7) -- the types a statement's columns take from their OWN tables' + * mappings, running nothing (a mapping may be read, as [[describe]] reads one). A JOIN body -- + * one FROM table with standard JOIN legs, and no derived table, CTE, set operation, correlated + * or WHERE subquery: the bodies a materialized view accepts -- is typed per table by describe's + * builder, whatever extension would run it; `None` where a leg's schema is unknown, a column's + * key names no leg or an item is a star; a type error is a 400 (`operation = "describe"`). Any + * other statement is [[describe]] 's answer, verbatim. + * + * The typing API of the materialized views, which store these types: a view is materialized by + * Elasticsearch, not by the engine that would run the body as a query, so a JOIN body `run` + * refuses without the relational engine is typed here, never run. + */ + private[elastic] def describePerTable( + statement: Statement + ): ElasticResult[Option[ResultSchema]] = { + // D3 (c) -- `shapeOf` names every closure shape, and each closure story adds its own there + def joinBody(single: SingleSearch): Boolean = + single.from.tables.size == 1 && single.from.enrichmentRequired && + RelationalClosureGuard.shapeOf(single) == RelationalClosureGuard.JoinShape && + !app.softnetwork.elastic.sql.query.whereSubqueriesPresent(single) + val body: Option[SingleSearch] = statement match { + case select: SelectStatement => + select.statement.collect { + case single: SingleSearch if joinBody(single) => single.copy(score = select.score) + } + case single: SingleSearch if joinBody(single) => Some(single) + case _ => None + } + body match { + case Some(single) => resolvePerTable(single).map(_.flatMap(statementSchema)) + case None => describe(statement) + } + } + /** [[describe]] over a query core's own handler runs. */ private def describeQuery(statement: Statement): ElasticResult[Option[ResultSchema]] = statement match { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/RelationalClosureGuard.scala b/core/src/main/scala/app/softnetwork/elastic/client/RelationalClosureGuard.scala index 9db8463c..15f6b664 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/RelationalClosureGuard.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/RelationalClosureGuard.scala @@ -55,7 +55,13 @@ object RelationalClosureGuard { else if (ctesPresent(statement)) CteShape else if (closureSearches(statement).exists(_.hasCorrelatedSubqueries)) CorrelatedShape else if (derivedTablesPresent(statement)) "A derived table (subquery in FROM/JOIN)" - else "A cross-index JOIN" + else JoinShape + + /** SCHEMA-15 (D3) -- the shape of a cross-index JOIN, and of a statement no other arm names. + * `GatewayApi.describePerTable` types a JOIN body only when [[shapeOf]] answers this, so a + * closure construct a later story names above leaves that branch with its own message. + */ + private[client] val JoinShape: String = "A cross-index JOIN" /** Story 22.5. 🔴 The words `WITH clause` sit in the first 25 characters on purpose: * `GatewayApi.excerpt` caps a rejection at 200 characters and elides the MIDDLE (head 120 + tail diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 27b8acc3..7e6d717e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -29,6 +29,7 @@ import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.function.aggregate.{PercentileAgg, RankingWindow} import app.softnetwork.elastic.sql.macros.SQLQueryMacros import app.softnetwork.elastic.sql.query.{ + Field, GroupedRowOrder, Limit, MultiSearch, @@ -341,27 +342,35 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC } /** The paths `statement` selects explicitly (a bare column, not `*`, not a function, not an - * UNNEST element, not a metadata field). + * UNNEST element, not a metadata field), among the items `of` keeps (SCHEMA-15: one JOIN leg's). */ - private def explicitPaths(statement: SingleSearch): Seq[String] = + private def explicitPaths( + statement: SingleSearch, + of: Field => Boolean = _ => true + ): Seq[String] = statement.select.fields.collect { case f if f.identifier.functions.isEmpty && !f.identifier.nested && f.identifier.name.nonEmpty && f.identifier.name != "*" && - !f.identifier.name.startsWith("_") => + !f.identifier.name.startsWith("_") && of(f) => f.identifier.name }.distinct - /** `schema`, or -- when `statement` selects a path it lacks on ONE concrete source, not reloaded - * for that path within the source's TTL -- the source's mapping read again + /** [[reloadedFor]] over the paths `statement` selects explicitly. */ + private def reloadedFor(statement: SingleSearch, sources: Seq[String], schema: Schema): Schema = + reloadedFor(explicitPaths(statement), sources, schema) + + /** `schema`, or -- when `paths` holds a path it lacks on ONE concrete source, not reloaded for + * that path within the source's TTL -- the source's mapping read again * (`IndicesApi.invalidateSchema`, then [[lookupSchema]]); `schema` when the reload fails. + * `paths` is by name: read only for a concrete source. */ - private def reloadedFor(statement: SingleSearch, sources: Seq[String], schema: Schema): Schema = + private def reloadedFor(paths: => Seq[String], sources: Seq[String], schema: Schema): Schema = this match { case indices: IndicesApi if sources.size == 1 && !sources.head.contains("*") && !sources.head.contains(",") => val source = sources.head - val missing = explicitPaths(statement).filter(p => schema.find(p).isEmpty) + val missing = paths.filter(p => schema.find(p).isEmpty) if (missing.isEmpty) schema else { val now = System.currentTimeMillis() @@ -437,6 +446,76 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC case _ => None } + /** SCHEMA-15 (D5, D6, D7) -- `single`, a cross-index JOIN body (one FROM table, standard JOIN + * legs over indices), with every leg typed from its OWN index, by the typing single-table + * `describe` attaches: [[attachDerived]] 's `DerivedTable.attach`, then `validateResolved`. It + * executes nothing and guards nothing (the caller chose the body). + * + * `ElasticSuccess(None)`, before any read, when a key no leg registers would resolve against the + * main table (one index name under two qualifiers, an UNNEST alias), or an item is a star (the + * main table's columns only would be a short schema); after the reads, when any leg's schema is + * unknown -- every leg is read first, then all or none, so a joined column never takes the main + * table's same-named column. A type error is a 400 with `validateResolved`'s reason. + * + * Cost: one [[lookupSchema]] per leg (the 5-minute cache `run` shares), one attach, one + * validation; a leg's explicitly selected path its cached mapping lacks reloads it once per TTL + * ([[reloadedFor]], the memo `run` shares). + */ + private[client] def resolvePerTable(single: SingleSearch): ElasticResult[Option[SingleSearch]] = { + val from = single.from + val main = from.mainTable.name + val legs = (main +: from.joinedTables).distinct + if ( + from.unnests.nonEmpty || !from.aliasesToTable.values.forall(legs.contains) || + single.select.fields.exists(HitReading.isStar) + ) ElasticResult.success(None) + else { + // every leg read (a 404 remembered) before any is used + val read = legs.map { leg => + leg -> legSchema(leg, explicitPaths(single, _.identifier.table.getOrElse(main) == leg)) + } + if (read.exists(_._2.isEmpty)) ElasticResult.success(None) + else { + val schemas = read.collect { case (leg, Some(schema)) => leg -> schema }.toMap + def refused(reason: String): ElasticResult[Option[SingleSearch]] = { + logger.error(s"❌ $reason") + ElasticResult.failure( + ElasticError( + message = reason, + statusCode = Some(400), + index = Some(legs.mkString(",")), + operation = Some("describe") + ) + ) + } + app.softnetwork.elastic.sql.query.DerivedTable + .attach(single, sources => schemas.get(sources.mkString(","))) match { + case Left(refusal) => refused(refusal) + case Right(attached) => + attached.validateResolved() match { + case Left(reason) => refused(reason) + case Right(_) => ElasticResult.success(Some(attached)) + } + } + } + } + } + + /** One leg of [[resolvePerTable]]: [[lookupSchema]] 's schema (a 404 remembered as the attach of + * [[resolveWithSchema]] remembers it), reloaded for the `paths` it lacks ([[reloadedFor]]); + * `None` when unknown. + */ + private def legSchema(leg: String, paths: => Seq[String]): Option[Schema] = + lookupSchema(Seq(leg)) match { + case Some(Success(ElasticSuccess(cached))) => + schemaMisses.remove(leg) + Some(reloadedFor(paths, Seq(leg), cached)) + case Some(Success(ElasticFailure(error))) => + if (error.statusCode.contains(404)) rememberSchemaMiss(leg) + None + case _ => None + } + /** The mapped type of a column PROJECTED BY AN INNER STATEMENT, when it can be known for free * (story 22.2). * diff --git a/core/src/test/scala/app/softnetwork/elastic/client/DescribeSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/DescribeSpec.scala index 85799c64..1bed6728 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/DescribeSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/DescribeSpec.scala @@ -19,8 +19,9 @@ package app.softnetwork.elastic.client import akka.actor.ActorSystem import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.schema.Index -import app.softnetwork.elastic.sql.`type`.SQLTypes -import app.softnetwork.elastic.sql.query.Statement +import app.softnetwork.elastic.sql.`type`.{SQLTypeUtils, SQLTypes} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{DerivedTable, MultiSearch, SingleSearch, Statement} import app.softnetwork.elastic.sql.schema.Schema import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import com.typesafe.config.ConfigFactory @@ -82,6 +83,14 @@ class DescribeSpec extends AnyFlatSpec with Matchers with ScalaFutures with Befo val searches = new AtomicInteger(0) + /** Every schema lookup a statement makes, a cache hit included (SCHEMA-15 D5). */ + val reads = new AtomicInteger(0) + + override private[client] def loadQuerySchema(source: String): ElasticResult[Schema] = { + reads.incrementAndGet() + super.loadQuerySchema(source) + } + override private[client] def executeVersion(): ElasticResult[String] = ElasticResult.success("8.18.3") @@ -274,4 +283,476 @@ class DescribeSpec extends AnyFlatSpec with Matchers with ScalaFutures with Befo c.describe("SELECT id FROM nowhere_t") shouldBe ElasticSuccess(None) c.searches.get() shouldBe 0 } + + // --------------------------------------------------------------------------------------------- + // SCHEMA-15 -- `describePerTable`: a JOIN body typed per table, by describe's builder, running + // nothing; every other statement answered as `describe` answers it + // --------------------------------------------------------------------------------------------- + + private def mappingOf(columns: Seq[(String, String)]): String = + columns.map { case (c, m) => s""""$c": $m""" }.mkString("{", ", ", "}") + + private val ptAColumns: Seq[(String, String)] = Seq( + "aid" -> """{"type": "keyword"}""", + "bid" -> """{"type": "keyword"}""", + "k" -> """{"type": "keyword"}""", + "t" -> """{"type": "text"}""", + "n" -> """{"type": "integer"}""", + "big_a" -> """{"type": "long"}""", + "x" -> """{"type": "double"}""", + "r" -> """{"type": "float"}""", + "ts" -> """{"type": "date"}""", + "dc" -> """{"type": "date", "format": "yyyy/MM/dd"}""", + "de" -> """{"type": "date", "format": "epoch_second"}""", + "b" -> """{"type": "boolean"}""", + "g" -> """{"type": "geo_point"}""", + "obj" -> """{"properties": {"a": {"type": "integer"}}}""", + "items" -> """{"type": "nested", "properties": {"p": {"type": "integer"}}}""", + "tags" -> """{"type": "integer"}""", + "pz" -> """{"type": "date", "format": "yyyy-MM-dd HH:mm z"}""", + "dn" -> """{"type": "date_nanos"}""" + ) + + /** `k` and `n` collide with `pt_a`'s, `n` with another type: a wrong attribution is visible. */ + private val ptBColumns: Seq[(String, String)] = Seq( + "bid" -> """{"type": "keyword"}""", + "label" -> """{"type": "keyword"}""", + "name" -> """{"type": "text"}""", + "m" -> """{"type": "integer"}""", + "big" -> """{"type": "long"}""", + "d2" -> """{"type": "date"}""", + "addr" -> """{"properties": {"zip": {"type": "keyword"}}}""", + "k" -> """{"type": "keyword"}""", + "n" -> """{"type": "long"}""" + ) + + private val ptMeta = """{"de": {"data_type": "DATE"}, "tags": {"data_type": "ARRAY"}}""" + + private val ptA: Schema = schemaOf("pt_a", mappingOf(ptAColumns), ptMeta) + private val ptB: Schema = schemaOf("pt_b", mappingOf(ptBColumns)) + + /** The twin: every column of both tables (`pt_b`'s `bid`, `k`, `n` are `pt_a`'s). */ + private val ptAB: Schema = schemaOf( + "pt_ab", + mappingOf(ptAColumns ++ ptBColumns.filterNot(c => Set("bid", "k", "n").contains(c._1))), + ptMeta + ) + + private def ptClient(dialect: String = "8.18.3"): CountingClient = { + val c = new CountingClient { + override private[client] def executeVersion(): ElasticResult[String] = + ElasticResult.success(dialect) + } + c.updateSchema("pt_a", ptA) + c.updateSchema("pt_b", ptB) + c.updateSchema("pt_ab", ptAB) + c + } + + private def parsed(sql: String): Statement = + Parser(sql) match { + case Right(statement) => statement + case Left(error) => fail(s"[$sql] does not parse: ${error.msg}") + } + + private def single(sql: String): SingleSearch = + parsed(sql) match { + case s: SingleSearch => s + case other => fail(s"[$sql] is not a single search: $other") + } + + /** The item over its own table: the qualifier dropped. */ + private def unqualified(sql: String): String = + sql.replaceAll("\\b(?:a2|b2|a|b|pt_a|pt_b)\\.", "") + + private val onBid = "ON a.bid = b.bid" + + private val joinKinds: Seq[String] = + Seq("JOIN", "INNER JOIN", "LEFT JOIN", "RIGHT JOIN", "FULL JOIN") + + private def oneTable(q: String, c: String): Seq[String] = + Seq(s"$q.$c", s"$q.$c AS v", s"CAST($q.$c AS VARCHAR) AS s") + + /** G1 -- (item, the table its oracle reads). */ + private val g1: Seq[(String, String)] = + ptAColumns.map(_._1).flatMap(c => oneTable("a", c).map(_ -> "pt_a")) ++ + ptBColumns.map(_._1).flatMap(c => oneTable("b", c).map(_ -> "pt_b")) ++ + Seq("a.obj.a" -> "pt_a", "b.addr.zip AS zip" -> "pt_b") + + /** G2 -- items over both tables, typed as over the twin. */ + private val g2: Seq[(String, String)] = Seq( + "COALESCE(a.n, b.m) AS v", + "a.n + b.big AS v", + "GREATEST(a.x, b.m) AS v", + "CASE WHEN a.b THEN a.ts ELSE b.d2 END AS v", + "NULLIF(a.k, b.label) AS v", + "CONCAT(a.k, b.label) AS v", + "UPPER(b.name) AS v", + "a.ts - b.d2 AS v", + "DATEDIFF(a.ts, b.d2) AS v" + ).map(_ -> "pt_ab") + + /** G3 / B8 -- (select list, group by). */ + private val g3: Seq[(String, String)] = Seq( + "b.label, COUNT(*) AS c, COUNT(DISTINCT a.k) AS dk, SUM(a.n) AS s, AVG(a.x) AS av, " + + "MIN(a.ts) AS mi, MAX(a.dc) AS ma" -> "b.label", + "b.label, a.k, COUNT(*) AS c, SUM(a.n) AS s" -> "b.label, a.k" + ) + + private val b2 = + "a.aid, b.label, b.addr.zip AS customer_zip, UPPER(b.name) AS uname" + + /** B9 -- every B1 type of `pt_a` (keyword, text, int, long, double, date, timestamp, boolean). */ + private val b9 = "a.aid, a.t, a.n, a.big_a, a.x, a.de, a.ts, a.b" + + /** A cell: the JOIN body, and the single-table statement whose answer is its oracle. */ + private case class Cell(body: String, oracle: String) + + private def joinCells(from: String, items: Seq[(String, String)]): Seq[Cell] = + items.map { case (item, table) => + Cell(s"SELECT $item FROM $from", s"SELECT ${unqualified(item)} FROM $table") + } + + /** Every G1 / G2 cell under every join form, the unaliased, self-join and three-leg forms, G3. */ + private lazy val parityCells: Seq[Cell] = { + val kinds = joinKinds.flatMap(kind => joinCells(s"pt_a a $kind pt_b b $onBid", g1 ++ g2)) + val unaliased = (g1 ++ g2).map { case (item, table) => + val written = item.replaceAll("\\ba\\.", "pt_a.").replaceAll("\\bb\\.", "pt_b.") + Cell( + s"SELECT $written FROM pt_a JOIN pt_b ON pt_a.bid = pt_b.bid", + s"SELECT ${unqualified(item)} FROM $table" + ) + } + val ptAItems = g1.filter(_._2 == "pt_a") + val selfJoin = joinCells( + "pt_a a JOIN pt_a a2 ON a.bid = a2.bid", + ptAItems ++ ptAItems.map { case (item, t) => item.replaceAll("\\ba\\.", "a2.") -> t } + ) + val ptBItems = g1.filter(_._2 == "pt_b") + val threeLegs = joinCells( + s"pt_a a JOIN pt_b b $onBid JOIN pt_b b2 ON a.bid = b2.bid", + g1 ++ g2 ++ ptBItems.map { case (item, t) => item.replaceAll("\\bb\\.", "b2.") -> t } + ) + val aggregates = g3.flatMap { case (items, keys) => + Seq("JOIN", "LEFT JOIN").map { kind => + Cell( + s"SELECT $items FROM pt_a a $kind pt_b b $onBid GROUP BY $keys", + s"SELECT ${unqualified(items)} FROM pt_ab GROUP BY ${unqualified(keys)}" + ) + } + } + val consumer = Seq( + Cell(s"SELECT $b2 FROM pt_a a JOIN pt_b b $onBid", s"SELECT ${unqualified(b2)} FROM pt_ab"), + Cell(s"SELECT $b9 FROM pt_a a JOIN pt_b b $onBid", s"SELECT ${unqualified(b9)} FROM pt_a") + ) + kinds ++ unaliased ++ selfJoin ++ threeLegs ++ aggregates ++ consumer + } + + /** Both answers equal: the same schema, or the same status and message. */ + private def sameAnswer( + perTable: ElasticResult[Option[ResultSchema]], + oracle: ElasticResult[Option[ResultSchema]] + ): Unit = + (perTable, oracle) match { + case (ElasticSuccess(p), ElasticSuccess(o)) => p shouldBe o + case (ElasticFailure(p), ElasticFailure(o)) => + p.statusCode shouldBe o.statusCode + p.message shouldBe o.message + case other => fail(s"describePerTable and its oracle disagree: $other") + } + + private def columnsOf(answer: ElasticResult[Option[ResultSchema]]): Seq[(String, String)] = + answer match { + case ElasticSuccess(Some(schema)) => schema.columns.map(c => c.name -> c.sqlType.typeId) + case other => fail(s"no schema: $other") + } + + "T0-2 (SCHEMA-15 premises)" should "attach both legs, and fall back to the main table for a leg the attach misses" in { + val byName = Map("pt_a" -> ptA, "pt_b" -> ptB) + // (i) both legs registered under their bare names; a joined column typed from its own leg + val s = single(s"SELECT a.aid, b.big FROM pt_a a JOIN pt_b b $onBid") + DerivedTable.attach(s, sources => byName.get(sources.mkString(","))) match { + case Right(attached) => + attached.schemas.keySet shouldBe Set("pt_a", "pt_b") + MultiSearch.fieldType(attached.select.fields(1)).sqlType shouldBe SQLTypes.BigInt + case Left(refusal) => fail(s"(i) attach refused: $refusal") + } + // (ii) an index pattern as a JOIN leg is a leg, keyed by the pattern (MappingTypeReadingSpec + // types both sides) + val pattern = single("SELECT t.k, p.k FROM s7_t t JOIN s7_p* p ON t.k = p.k") + pattern.from.aliasesToTable.toSeq shouldBe Seq("t" -> "s7_t", "p" -> "s7_p*") + pattern.from.joinedTables shouldBe Seq("s7_p*") + // (iii) a qualified star parses as a star: `HitReading.isStar` is D7's whole test + def starForm(sql: String): Seq[(String, Option[String], Option[String], Boolean)] = + single(sql).select.fields.map { f => + ( + f.identifier.identifierName, + f.identifier.tableAlias, + f.identifier.table, + HitReading.isStar(f) + ) + } + starForm(s"SELECT a.* FROM pt_a a JOIN pt_b b $onBid") shouldBe + Seq(("*", Some("a"), Some("pt_a"), true)) + starForm(s"SELECT a.aid, b.* FROM pt_a a JOIN pt_b b $onBid") shouldBe + Seq(("aid", Some("a"), Some("pt_a"), false), ("*", Some("b"), Some("pt_b"), true)) + // (iv) G3 / B8 (and B2, B9) pass the resolved-type rules once attached + val bodies = g3.map { case (items, keys) => + s"SELECT $items FROM pt_a a JOIN pt_b b $onBid GROUP BY $keys" + } ++ Seq( + s"SELECT $b2 FROM pt_a a JOIN pt_b b $onBid", + s"SELECT $b9 FROM pt_a a JOIN pt_b b $onBid" + ) + bodies.foreach { sql => + withClue(s"(iv) [$sql] ") { + DerivedTable.attach(single(sql), sources => byName.get(sources.mkString(","))) match { + case Right(attached) => attached.validateResolved() shouldBe Right(()) + case Left(refusal) => fail(s"attach refused: $refusal") + } + } + } + // (v) the main table's schema answers for a leg the attach did not register (D5's reason) + val missing = single(s"SELECT b.n FROM pt_a a JOIN pt_b b $onBid") + DerivedTable.attach(missing, sources => Map("pt_a" -> ptA).get(sources.mkString(","))) match { + case Right(attached) => + attached.schemas.keySet shouldBe Set("pt_a") + MultiSearch.fieldType(attached.select.fields.head).sqlType shouldBe SQLTypes.Int + case Left(refusal) => fail(s"(v) attach refused: $refusal") + } + } + + "describePerTable (SCHEMA-15)" should "type every JOIN body per table as describe types its single-table oracle, searching nothing (AC1)" in { + val c = ptClient() + val oracles = parityCells.map(cell => cell -> c.describe(cell.oracle)) + // a refused oracle is compared by status and message; a schema-less one would prove nothing + oracles + .collect { case (cell, o) if !o.toOption.exists(_.isDefined) => cell -> o } + .distinct + .foreach { case (cell, o) => println(s"[SCHEMA-15 oracle not typed] ${cell.oracle}: $o") } + oracles.collect { case (cell, ElasticSuccess(None)) => cell.oracle } shouldBe empty + val statements = oracles.map { case (cell, o) => (cell, parsed(cell.body), o) } + statements.foreach { case (cell, statement, oracle) => + withClue(s"[${cell.body}] vs [${cell.oracle}] ") { + sameAnswer(c.describePerTable(statement), oracle) + } + } + c.searches.get() shouldBe 0 + } + + it should "attribute a bare name to the main table and keep repeated names, as the view does (A-110)" in { + val oracles = ptClient() + val mainN = columnsOf(oracles.describe("SELECT n FROM pt_a")) + val joinedN = columnsOf(oracles.describe("SELECT n FROM pt_b")) + mainN should not be joinedN + // a fresh client per cell: a path no mapping holds reloads (and, in this test double, drops) + // the cached mapping of its leg (D10), as single-table describe does + Seq( + "aid" -> columnsOf(oracles.describe("SELECT aid FROM pt_a")), + "k" -> columnsOf(oracles.describe("SELECT k FROM pt_a")), + "n" -> mainN, + "label" -> Seq("label" -> "ANY"), + "b.aid" -> Seq("aid" -> "ANY"), + "a.n, b.n" -> (mainN ++ joinedN), + "a.nope" -> Seq("nope" -> "ANY") + ).foreach { case (items, expected) => + withClue(s"[$items] ") { + val c = ptClient() + columnsOf( + c.describePerTable(parsed(s"SELECT $items FROM pt_a a JOIN pt_b b $onBid")) + ) shouldBe + expected + c.searches.get() shouldBe 0 + } + } + } + + it should "answer None for an unknown leg, an unregistered key or a star, never the main table's column (AC3)" in { + // an unknown leg, joined or main, and a same-named column of the main table + Seq( + s"SELECT a.aid, z.q FROM pt_a a JOIN pt_zz z ON a.bid = z.bid", + s"SELECT z.n FROM pt_a a JOIN pt_zz z ON a.bid = z.bid", + s"SELECT z.q, b.label FROM pt_zz z JOIN pt_b b ON z.bid = b.bid" + ).foreach { sql => + withClue(s"[$sql] ") { + val c = ptClient() + c.describePerTable(parsed(sql)) shouldBe ElasticSuccess(None) + // remembered: answered the same way again + c.describePerTable(parsed(sql)) shouldBe ElasticSuccess(None) + c.searches.get() shouldBe 0 + } + } + // a failed read (not a 404) + val failing = new CountingClient { + override private[client] def loadQuerySchema(source: String): ElasticResult[Schema] = + if (source == "pt_b") + ElasticResult.failure(ElasticError(message = "boom", statusCode = Some(500))) + else super.loadQuerySchema(source) + } + failing.updateSchema("pt_a", ptA) + failing.describePerTable(parsed(s"SELECT a.n, b.m FROM pt_a a JOIN pt_b b $onBid")) shouldBe + ElasticSuccess(None) + // a key no leg registers: one index under two qualifiers (a fallback would type `p.n` BIGINT + // from `pt_b`), an UNNEST alias -- no mapping read + Seq( + """SELECT p.n FROM pt_b x JOIN "c1".pt_a p ON x.bid = p.bid JOIN "c2".pt_a q ON x.bid = q.bid""", + s"SELECT a.aid, i.p FROM pt_a a JOIN UNNEST(a.items) AS i JOIN pt_b b $onBid" + ).foreach { sql => + withClue(s"[$sql] ") { + val c = ptClient() + val statement = parsed(sql) + c.reads.set(0) + c.describePerTable(statement) shouldBe ElasticSuccess(None) + c.reads.get() shouldBe 0 + c.searches.get() shouldBe 0 + } + } + // a star: the main table's columns only would be a short schema + Seq( + s"SELECT * FROM pt_a a JOIN pt_b b $onBid", + s"SELECT a.* FROM pt_a a JOIN pt_b b $onBid", + s"SELECT a.aid, b.* FROM pt_a a JOIN pt_b b $onBid" + ).foreach { sql => + withClue(s"[$sql] ") { + ptClient().describePerTable(parsed(sql)) shouldBe ElasticSuccess(None) + } + } + } + + it should "answer describe's answer for every other shape, and refuse a type error as over the twin (AC2, AC3)" in { + val shapes = Seq( + s"SELECT a.aid, d.label FROM pt_a a JOIN (SELECT bid, label FROM pt_b) d ON a.bid = d.bid", + "WITH d AS (SELECT bid, label FROM pt_b) SELECT a.aid, d.label FROM pt_a a JOIN d ON a.bid = d.bid", + s"SELECT a.aid, b.label FROM pt_a a JOIN pt_b b $onBid WHERE a.k IN (SELECT k FROM pt_b)", + s"SELECT a.aid FROM pt_a a JOIN pt_b b $onBid WHERE a.n IN (SELECT c.m FROM pt_ab c WHERE c.aid = a.aid)", + "SELECT a.aid FROM pt_a a, pt_b b JOIN pt_ab c ON b.bid = c.bid", + s"SELECT a.aid FROM pt_a a JOIN pt_b b $onBid UNION SELECT a.aid FROM pt_a a JOIN pt_b b $onBid" + ) + val statements = shapes.map(sql => sql -> parsed(sql)) + statements.foreach { case (sql, statement) => + withClue(s"[$sql] ") { + val c = ptClient() + sameAnswer(c.describePerTable(statement), c.describe(statement)) + c.searches.get() shouldBe 0 + } + } + // P1: a non-JOIN statement is describe's, verbatim + p1.foreach { sql => + withClue(s"[$sql] ") { + val c = client() + val statement = parsed(sql) + c.describePerTable(statement) shouldBe c.describe(statement) + } + } + // a type error: validateResolved's reason, as single-table describe gives it over the twin; + // the reason quotes the expression as written, so the body's qualifiers are dropped to compare + val twin = ptClient().describe("SELECT aid FROM pt_ab WHERE CAST(ts AS TIME) = d2") + val typeError = + parsed(s"SELECT a.aid FROM pt_a a JOIN pt_b b $onBid WHERE CAST(a.ts AS TIME) = b.d2") + (ptClient().describePerTable(typeError), twin) match { + case (ElasticFailure(p), ElasticFailure(o)) => + p.statusCode shouldBe Some(400) + p.operation shouldBe Some("describe") + unqualified(p.message) shouldBe o.message + case other => fail(s"a type error: $other") + } + } + + it should "read dates as single-table describe does under 6.8.23 and 8.18.3 (AC4)" in { + Seq("6.8.23", "8.18.3").foreach { dialect => + withClue(s"[$dialect] ") { + val c = ptClient(dialect) + val oracle = c.describe("SELECT dc, de, pz, ts, dn FROM pt_a") + columnsOf(oracle) should not be empty + sameAnswer( + c.describePerTable( + parsed(s"SELECT a.dc, a.de, a.pz, a.ts, a.dn FROM pt_a a JOIN pt_b b $onBid") + ), + oracle + ) + sameAnswer( + c.describePerTable(parsed(s"SELECT b.d2 FROM pt_a a JOIN pt_b b $onBid")), + c.describe("SELECT d2 FROM pt_b") + ) + } + } + } + + it should "ignore a foreign extension for a JOIN body only (D4)" in { + val foreign = new ExtensionSpi { + override def extensionId: String = "describe-per-table-foreign-test" + override def extensionName: String = "Foreign handler (test double)" + override def version: String = "test" + override def priority: Int = 1 + override def initialize( + config: com.typesafe.config.Config, + licenseRefreshStrategy: app.softnetwork.elastic.licensing.LicenseRefreshStrategy + ): Either[String, Unit] = Right(()) + override def canHandle(statement: Statement): Boolean = true + override def execute(statement: Statement, client: ElasticClientApi)(implicit + system: ActorSystem + ): Future[ElasticResult[QueryResult]] = Future.successful(ElasticSuccess(QueryRows(Nil))) + override def supportedSyntax: Seq[String] = Seq.empty + } + val withForeign = new CountingClient { + override lazy val extensionRegistry: ExtensionRegistry = + new ExtensionRegistry(ConfigFactory.load(), licenseRefreshStrategy) { + override lazy val extensions: Seq[ExtensionSpi] = Seq(foreign) + } + } + Seq("pt_a" -> ptA, "pt_b" -> ptB).foreach { case (i, s) => withForeign.updateSchema(i, s) } + val join = parsed(s"SELECT $b9 FROM pt_a a JOIN pt_b b $onBid") + withForeign.describe(join) shouldBe ElasticSuccess(None) + val typed = withForeign.describePerTable(join) + typed shouldBe ptClient().describePerTable(join) + typed.toOption.flatten should not be empty + // a single-table statement: describe's answer under that extension + withForeign.describePerTable(parsed("SELECT aid FROM pt_a")) shouldBe ElasticSuccess(None) + withForeign.searches.get() shouldBe 0 + } + + it should "answer each scope as a fresh parse: after a validation pass on the same AST (AC6)" in { + val schemas = Map("pt_a" -> ptA, "pt_b" -> ptB) + Seq( + s"SELECT $b9, a.tags FROM pt_a a JOIN pt_b b $onBid", + s"SELECT a.tags, COALESCE(a.dc, b.d2) AS v, CASE WHEN a.b THEN a.ts ELSE b.d2 END AS w " + + s"FROM pt_a a JOIN pt_b b $onBid", + s"SELECT ${g3.head._1} FROM pt_a a JOIN pt_b b $onBid GROUP BY ${g3.head._2}" + ).foreach { sql => + withClue(s"[$sql] ") { + val c = ptClient() + val fresh = c.describePerTable(single(sql)) + val shared = single(sql) + val verdict = + SQLTypeUtils.validatingComputedColumns(shared.updateAll(schemas).validateResolved()) + c.describePerTable(shared) shouldBe fresh + // and the validation pass, after describePerTable, answers as on a fresh parse + SQLTypeUtils.validatingComputedColumns( + shared.updateAll(schemas).validateResolved() + ) shouldBe verdict + SQLTypeUtils.validatingComputedColumns( + single(sql).updateAll(schemas).validateResolved() + ) shouldBe verdict + } + } + } + + it should "forward through a delegator: the delegate's answer, its member run once (D10)" in { + val calls = new AtomicInteger(0) + val recording = new CountingClient { + override private[elastic] def describePerTable( + statement: Statement + ): ElasticResult[Option[ResultSchema]] = { + calls.incrementAndGet() + super.describePerTable(statement) + } + } + Seq("pt_a" -> ptA, "pt_b" -> ptB).foreach { case (i, s) => recording.updateSchema(i, s) } + val wrapper = new ElasticClientDelegator { override val delegate: ElasticClientApi = recording } + val join = parsed(s"SELECT $b9 FROM pt_a a JOIN pt_b b $onBid") + val answer = wrapper.describePerTable(join) + calls.get() shouldBe 1 + answer shouldBe ptClient().describePerTable(join) + answer.toOption.flatten should not be empty + recording.searches.get() shouldBe 0 + } } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/perf/PerTableDescribeCostProbe.scala b/core/src/test/scala/app/softnetwork/elastic/client/perf/PerTableDescribeCostProbe.scala new file mode 100644 index 00000000..b2ad503b --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/perf/PerTableDescribeCostProbe.scala @@ -0,0 +1,173 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.perf + +import app.softnetwork.elastic.client.{NopeClientApi, ResultSchema} +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.Statement +import app.softnetwork.elastic.sql.schema.Schema +import org.slf4j.{Logger, LoggerFactory} + +import scala.util.Try + +/** SCHEMA-15 (AC7) -- what `describePerTable` costs on a JOIN body, against what `describe` costs + * on its two single-table halves, and what single-table `describe` costs (a path the story leaves + * unchanged: equal within noise between a control tree and this one). + * + * Each sample parses the statement OUTSIDE the timed region (a fresh AST, as a request has: the + * per-node type memos of a re-used AST would answer from the previous sample), then times the one + * call. The schemas are seeded in the client's cache: the timed work is the typing, never a + * mapping read. Medians in microseconds, after a warm-up. + * + * `describePerTable` is reached reflectively (a `private[elastic]` member is public in bytecode), + * so the SAME file compiles in a control tree, where that row reads "absent". + * + * ==No assertion== + * A `main`, never a test with a ceiling (#269/#270). Compare two trees by running it in each, + * INTERLEAVED, in one session. + * + * {{{ + * sbt "core/Test/runMain app.softnetwork.elastic.client.perf.PerTableDescribeCostProbe" + * sbt "core/Test/runMain app.softnetwork.elastic.client.perf.PerTableDescribeCostProbe 300 1000" + * }}} + */ +object PerTableDescribeCostProbe { + + private val DefaultWarmup = 300 + private val DefaultRuns = 1000 + + private val testLogger: Logger = LoggerFactory.getLogger(getClass) + + private def schemaOf(index: String, properties: String, meta: String = "{}"): Schema = + Index( + index, + s"""{"$index": {"aliases": {}, "mappings": {"_meta": {"columns": $meta}, + | "properties": $properties}, "settings": {"index": {"number_of_shards": "1"}}}}""".stripMargin + ).schema + + private val ptA: Schema = schemaOf( + "pt_a", + """{"aid": {"type": "keyword"}, "bid": {"type": "keyword"}, "k": {"type": "keyword"}, + | "t": {"type": "text"}, "n": {"type": "integer"}, "big_a": {"type": "long"}, + | "x": {"type": "double"}, "r": {"type": "float"}, "ts": {"type": "date"}, + | "dc": {"type": "date", "format": "yyyy/MM/dd"}, + | "de": {"type": "date", "format": "epoch_second"}, "b": {"type": "boolean"}, + | "obj": {"properties": {"a": {"type": "integer"}}}, "tags": {"type": "integer"}}""".stripMargin, + """{"de": {"data_type": "DATE"}, "tags": {"data_type": "ARRAY"}}""" + ) + + private val ptB: Schema = schemaOf( + "pt_b", + """{"bid": {"type": "keyword"}, "label": {"type": "keyword"}, "name": {"type": "text"}, + | "m": {"type": "integer"}, "big": {"type": "long"}, "d2": {"type": "date"}, + | "addr": {"properties": {"zip": {"type": "keyword"}}}, "k": {"type": "keyword"}, + | "n": {"type": "long"}}""".stripMargin + ) + + private class ProbeClient extends NopeClientApi { + override protected def logger: Logger = testLogger + override private[client] def executeVersion(): ElasticResult[String] = + ElasticResult.success("8.18.3") + } + + /** B9 (every B1 type of `pt_a`) widened by two columns of the joined table, and its halves. */ + private val joinBody = + "SELECT a.aid, a.t, a.n, a.big_a, a.x, a.de, a.ts, a.b, b.label, b.m " + + "FROM pt_a a JOIN pt_b b ON a.bid = b.bid" + private val mainHalf = "SELECT aid, t, n, big_a, x, de, ts, b FROM pt_a" + private val joinedHalf = "SELECT label, m FROM pt_b" + + /** Single-table shapes: the path the story leaves unchanged. */ + private val singleTable: List[(String, String)] = List( + "B9 half (pt_a)" -> mainHalf, + "joined half" -> joinedHalf, + "computed items" -> "SELECT UPPER(k) AS u, n + 1 AS v, CAST(ts AS VARCHAR) AS s FROM pt_a", + "group by" -> "SELECT k, COUNT(*) AS c, SUM(n) AS s, MAX(dc) AS ma FROM pt_a GROUP BY k", + "star" -> "SELECT * FROM pt_a", + "union all" -> "SELECT n AS v FROM pt_a UNION ALL SELECT m AS v FROM pt_b", + "join (refused, 400)" -> joinBody + ) + + private def parse(sql: String): Statement = + Parser(sql) match { + case Right(statement) => statement + case Left(e) => throw new IllegalArgumentException(s"[$sql] ${e.msg}") + } + + private def median(xs: Array[Long]): Double = { + val sorted = xs.sorted + val n = sorted.length + if (n == 0) 0d + else if (n % 2 == 1) sorted(n / 2).toDouble + else (sorted(n / 2 - 1) + sorted(n / 2)) / 2.0 + } + + /** The median of `body` over a fresh `setup` per sample, `setup` untimed. */ + private def timeMedian[S](warmup: Int, runs: Int)(setup: => S)(body: S => Any): Double = { + var i = 0 + while (i < warmup) { body(setup); i += 1 } + val samples = new Array[Long](runs) + i = 0 + while (i < runs) { + val s = setup + val t0 = System.nanoTime() + body(s) + samples(i) = System.nanoTime() - t0 + i += 1 + } + median(samples) / 1000.0 + } + + def main(args: Array[String]): Unit = { + val warmup = Try(args(0).toInt).getOrElse(DefaultWarmup) + val runs = Try(args(1).toInt).getOrElse(DefaultRuns) + val client = new ProbeClient + client.updateSchema("pt_a", ptA) + client.updateSchema("pt_b", ptB) + val perTable: Option[java.lang.reflect.Method] = + Try(client.getClass.getMethod("describePerTable", classOf[Statement])).toOption + def describePerTable(s: Statement): ElasticResult[Option[ResultSchema]] = + perTable.get.invoke(client, s).asInstanceOf[ElasticResult[Option[ResultSchema]]] + + // global warm-up over every shape + (singleTable.map(_._2) :+ joinBody).foreach { sql => + var i = 0 + while (i < warmup) { client.describe(parse(sql)); i += 1 } + } + + println(s"PerTableDescribeCostProbe warmup=$warmup runs=$runs (medians, microseconds)") + println("shape\tdescribe") + val medians = singleTable.map { case (name, sql) => + val m = timeMedian(warmup, runs)(parse(sql))((s: Statement) => client.describe(s)) + println(f"$name\t$m%.1f") + name -> m + }.toMap + + perTable match { + case None => println("describePerTable\tabsent (control tree)") + case Some(_) => + println(s"[answer] ${describePerTable(parse(joinBody))}") + val joined = + timeMedian(warmup, runs)(parse(joinBody))((s: Statement) => describePerTable(s)) + val halves = medians("B9 half (pt_a)") + medians("joined half") + println("join body\tdescribePerTable\tsum of describe over its halves\tratio") + println(f"B9 + 2 joined columns\t$joined%.1f\t$halves%.1f\t${joined / halves}%.2f") + } + } +} diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientPerTableDescribeSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientPerTableDescribeSpec.scala new file mode 100644 index 00000000..c4cce83a --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientPerTableDescribeSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JestClientPerTableDescribeSpec extends PerTableDescribeSpec diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPerTableDescribeSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPerTableDescribeSpec.scala new file mode 100644 index 00000000..c6fe949f --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPerTableDescribeSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientPerTableDescribeSpec extends PerTableDescribeSpec diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPerTableDescribeSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPerTableDescribeSpec.scala new file mode 100644 index 00000000..c6fe949f --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPerTableDescribeSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientPerTableDescribeSpec extends PerTableDescribeSpec diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientReadOnlyUserSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientReadOnlyUserSpec.scala index ecbec4c4..b67be4be 100644 --- a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientReadOnlyUserSpec.scala +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientReadOnlyUserSpec.scala @@ -327,6 +327,25 @@ class JavaClientReadOnlyUserSpec extends AnyFlatSpecLike with ElasticDockerTestK } } + "A read-only user's JOIN body" should "be typed per table from _field_caps (SCHEMA-15)" in { + val main = Seq("id", "k", "d", "sf", "hf", "ip", "u", "dn", "ck") + val joined = Seq("n", "x") + val body = app.softnetwork.elastic.sql.parser.Parser( + s"SELECT ${main.map(c => s"a.$c").mkString(", ")}, ${joined.map(c => s"b.$c").mkString(", ")} " + + s"FROM $table a JOIN $table b ON a.id = b.id" + ) match { + case Right(statement) => statement + case Left(error) => fail(s"the JOIN body does not parse: ${error.msg}") + } + Seq("read-only" -> roClient, "privileged" -> client).foreach { case (who, on) => + withClue(s"[$who] ") { + val single = on.describe(s"SELECT ${(main ++ joined).mkString(", ")} FROM $table") + single.toOption.flatten should not be empty + on.describePerTable(body) shouldBe single + } + } + } + // ------------------------------------------------------------------------------------------- // A-63 (R-85) -- un-LIMITed reads page by the build major // ------------------------------------------------------------------------------------------- diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/extensions/JavaClientPerTableDescribeSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/extensions/JavaClientPerTableDescribeSpec.scala new file mode 100644 index 00000000..89eedc6b --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/extensions/JavaClientPerTableDescribeSpec.scala @@ -0,0 +1,57 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.extensions + +import app.softnetwork.elastic.client.{ElasticClientApi, PerTableDescribeSpec} +import app.softnetwork.elastic.sql.query.CreateMaterializedView + +/** SCHEMA-15 (AC5) -- the ES 8 runner of [[PerTableDescribeSpec]], in the materialized-view + * consumer's package on purpose: `describePerTable` is reached from that package exactly as the + * view extension calls it, `client.describePerTable(create.search)`, over a client of the + * consumer's static type (`ElasticClientApi`, from `ElasticClientFactory`, i.e. a delegator) and a + * parsed `CREATE MATERIALIZED VIEW … AS `. + */ +class JavaClientPerTableDescribeSpec extends PerTableDescribeSpec { + + "A view body" should "be typed from describePerTable(create.search) (AC5)" in { + val consumer: ElasticClientApi = client + val (b8Items, b8Keys) = g3.head + val bodies = Seq( + // B2: a joined object field and a function over a joined column + s"SELECT a.aid, b.label, b.addr.zip AS customer_zip, UPPER(b.name) AS uname " + + s"FROM $ptA a JOIN $ptB b $onBid" -> + s"SELECT aid, label, addr.zip AS customer_zip, UPPER(name) AS uname FROM $ptAB", + // B8: the aggregates + s"SELECT $b8Items FROM $ptA a JOIN $ptB b $onBid GROUP BY $b8Keys" -> + s"SELECT ${unqualified(b8Items)} FROM $ptAB GROUP BY ${unqualified(b8Keys)}", + // B9: every B1 type of the main table + s"SELECT a.aid, a.t, a.n, a.big_a, a.x, a.de, a.ts, a.b FROM $ptA a JOIN $ptB b $onBid" -> + s"SELECT aid, t, n, big_a, x, de, ts, b FROM $ptA" + ) + bodies.foreach { case (body, oracle) => + withClue(s"[$body] ") { + val create = parsed(s"CREATE MATERIALIZED VIEW v AS $body") match { + case view: CreateMaterializedView => view + case other => fail(s"not a view: $other") + } + val typed = consumer.describePerTable(create.search) + typed.toOption.flatten should not be empty + sameAnswer(typed, consumer.describe(oracle)) + } + } + } +} diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPerTableDescribeSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPerTableDescribeSpec.scala new file mode 100644 index 00000000..10283a57 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPerTableDescribeSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientPerTableDescribeSpec extends PerTableDescribeSpec diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/MappingTypeReadingSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/MappingTypeReadingSpec.scala index 1897b776..c289aad9 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/MappingTypeReadingSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/MappingTypeReadingSpec.scala @@ -20,6 +20,7 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.Sink import app.softnetwork.elastic.client.result.{ ElasticFailure, + ElasticResult, ElasticSuccess, QueryResult, QueryRows, @@ -329,6 +330,36 @@ trait MappingTypeReadingSpec extends AnyFlatSpecLike with ElasticDockerTestKit w values.foreach(_ shouldBe a[String]) } + "A JOIN body over a pattern and the SCHEMA-7 types" should "be typed as single-table describe types each side (SCHEMA-15)" in { + val columns = "id" +: typedColumns + def body(sql: String) = Parser(sql) match { + case Right(statement) => statement + case Left(error) => fail(s"the JOIN body does not parse: ${error.msg}") + } + val pattern = client.describe("SELECT k FROM s7_p*") + val typed = client.describe(s"SELECT ${columns.mkString(", ")} FROM $table") + def joined( + first: ElasticResult[Option[ResultSchema]], + second: ElasticResult[Option[ResultSchema]] + ) = + for { + f <- first + s <- second + } yield for { + a <- f + b <- s + } yield ResultSchema(a.columns ++ b.columns) + joined(pattern, typed).toOption.flatten should not be empty + val tItems = columns.map(c => s"t.$c").mkString(", ") + // the pattern on the FROM side, then on the JOIN side + client.describePerTable( + body(s"SELECT p.k, $tItems FROM s7_p* p JOIN $table t ON p.k = t.k") + ) shouldBe joined(pattern, typed) + client.describePerTable( + body(s"SELECT $tItems, p.k FROM $table t JOIN s7_p* p ON t.k = p.k") + ) shouldBe joined(typed, pattern) + } + // ------------------------------------------------------------------------------------------- // P1 -- the consumers // ------------------------------------------------------------------------------------------- diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/PerTableDescribeSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/PerTableDescribeSpec.scala new file mode 100644 index 00000000..3dbe0ddd --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/PerTableDescribeSpec.scala @@ -0,0 +1,298 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} +import app.softnetwork.elastic.client.spi.ElasticClientFactory +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.Statement +import app.softnetwork.persistence.generateUUID +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.http.util.EntityUtils +import org.elasticsearch.client.Request +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.util.Try + +/** SCHEMA-15 (AC1, RUN) -- `describePerTable` types each output column of a JOIN body from its own + * table's mapping, as single-table `describe` types the same item over that table (one-table + * items) or over the twin index mapping both tables' columns (two-table items, aggregates), and + * searches nothing: every index's `_stats/search` `query_total` is unchanged, while one `run` (the + * positive control) moves it. + * + * Fixtures: `pt_a`, `pt_b` (its `k` and `n` collide with `pt_a`'s, `n` with another type, so a + * wrong attribution is visible) and their twin `pt_ab`; `pz` and `dn` from Elasticsearch 7 on. + */ +trait PerTableDescribeSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Matchers { + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + lazy val client: ElasticClientApi = ElasticClientFactory.create(elasticConfig) + + protected lazy val major: Int = Try(elasticVersion.takeWhile(_ != '.').toInt).getOrElse(8) + + /** `date_nanos` (7.0) and a zone-name format (6.8's Joda-Time cannot parse a zone name). */ + protected lazy val modern: Boolean = major >= 7 + + private val mapper = new ObjectMapper() + + protected val ptA = "pt_a" + protected val ptB = "pt_b" + protected val ptAB = "pt_ab" + + private def indices: Seq[String] = Seq(ptA, ptB, ptAB) + + protected def ptAColumns: Seq[(String, String)] = Seq( + "aid" -> """{"type": "keyword"}""", + "bid" -> """{"type": "keyword"}""", + "k" -> """{"type": "keyword"}""", + "t" -> """{"type": "text"}""", + "n" -> """{"type": "integer"}""", + "big_a" -> """{"type": "long"}""", + "x" -> """{"type": "double"}""", + "r" -> """{"type": "float"}""", + "ts" -> """{"type": "date"}""", + "dc" -> """{"type": "date", "format": "yyyy/MM/dd"}""", + "de" -> """{"type": "date", "format": "epoch_second"}""", + "b" -> """{"type": "boolean"}""", + "g" -> """{"type": "geo_point"}""", + "obj" -> """{"properties": {"a": {"type": "integer"}}}""", + "items" -> """{"type": "nested", "properties": {"p": {"type": "integer"}}}""", + "tags" -> """{"type": "integer"}""" + ) ++ (if (modern) + Seq( + "pz" -> """{"type": "date", "format": "yyyy-MM-dd HH:mm z"}""", + "dn" -> """{"type": "date_nanos"}""" + ) + else Nil) + + protected val ptBColumns: Seq[(String, String)] = Seq( + "bid" -> """{"type": "keyword"}""", + "label" -> """{"type": "keyword"}""", + "name" -> """{"type": "text"}""", + "m" -> """{"type": "integer"}""", + "big" -> """{"type": "long"}""", + "d2" -> """{"type": "date"}""", + "addr" -> """{"properties": {"zip": {"type": "keyword"}}}""", + "k" -> """{"type": "keyword"}""", + "n" -> """{"type": "long"}""" + ) + + private val meta = """{"de": {"data_type": "DATE"}, "tags": {"data_type": "ARRAY"}}""" + + private def mappingOf(columns: Seq[(String, String)], withMeta: Boolean): String = { + val properties = columns.map { case (c, m) => s""""$c": $m""" }.mkString("{", ", ", "}") + if (withMeta) s"""{"_meta": {"columns": $meta}, "properties": $properties}""" + else s"""{"properties": $properties}""" + } + + private def drop(index: String): Unit = { + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $index"), 60.seconds)) + Try(client.deleteIndex(index)) + } + + private def load(index: String, mappingJson: String, documents: Seq[(String, String)]): Unit = { + client + .createIndex(index, settings = """{"number_of_shards": 2, "number_of_replicas": 0}""") + .get shouldBe true + client.setMapping(index, mappingJson).get shouldBe true + documents.foreach { case (id, source) => + client.index(index, id, source) match { + case ElasticSuccess(true) => + case other => fail(s"indexing $id into $index: $other") + } + } + client.refresh(index) + () + } + + override def beforeAll(): Unit = { + super.beforeAll() + indices.foreach(drop) + load( + ptA, + mappingOf(ptAColumns, withMeta = true), + Seq( + "a1" -> """{"aid":"a1","bid":"b1","k":"x","n":1,"big_a":10,"x":1.5,"ts":"2024-01-31T10:30:00Z","b":true,"tags":[1,2]}""", + "a2" -> """{"aid":"a2","bid":"b2","k":"y","n":2}""" + ) + ) + load( + ptB, + mappingOf(ptBColumns, withMeta = false), + Seq( + "b1" -> """{"bid":"b1","label":"L1","name":"one","m":3,"big":30,"d2":"2024-01-30T00:00:00Z","k":"x","n":5}""", + "b2" -> """{"bid":"b2","label":"L2","m":4}""" + ) + ) + load( + ptAB, + mappingOf( + ptAColumns ++ ptBColumns.filterNot(c => Set("bid", "k", "n").contains(c._1)), + withMeta = true + ), + Seq("ab1" -> """{"aid":"a1","bid":"b1","label":"L1"}""") + ) + } + + override def afterAll(): Unit = { + indices.foreach(drop) + super.afterAll() + } + + protected def parsed(sql: String): Statement = + Parser(sql) match { + case Right(statement) => statement + case Left(error) => fail(s"[$sql] does not parse: ${error.msg}") + } + + /** The item over its own table: the qualifier dropped. */ + protected def unqualified(sql: String): String = + sql.replaceAll("\\b(?:a2|b2|a|b|pt_a|pt_b)\\.", "") + + protected val onBid = "ON a.bid = b.bid" + + /** `query_total` summed over the three fixtures. */ + protected def queryTotal(): Long = { + val response = + restClient.performRequest(new Request("GET", s"/${indices.mkString(",")}/_stats/search")) + mapper + .readTree(EntityUtils.toString(response.getEntity)) + .path("_all") + .path("total") + .path("search") + .path("query_total") + .asLong(-1L) + } + + /** Both answers equal: the same schema, or the same status and message. */ + protected def sameAnswer( + perTable: ElasticResult[Option[ResultSchema]], + oracle: ElasticResult[Option[ResultSchema]] + ): Unit = + (perTable, oracle) match { + case (ElasticSuccess(p), ElasticSuccess(o)) => p shouldBe o + case (ElasticFailure(p), ElasticFailure(o)) => + p.statusCode shouldBe o.statusCode + p.message shouldBe o.message + case other => fail(s"describePerTable and its oracle disagree: $other") + } + + protected def columnsOf(answer: ElasticResult[Option[ResultSchema]]): Seq[(String, String)] = + answer match { + case ElasticSuccess(Some(schema)) => schema.columns.map(c => c.name -> c.sqlType.typeId) + case other => fail(s"no schema: $other") + } + + private def oneTable(q: String, c: String): Seq[String] = + Seq(s"$q.$c", s"$q.$c AS v", s"CAST($q.$c AS VARCHAR) AS s") + + /** G1 -- (item, the table its oracle reads). */ + protected def g1: Seq[(String, String)] = + ptAColumns.map(_._1).flatMap(c => oneTable("a", c).map(_ -> ptA)) ++ + ptBColumns.map(_._1).flatMap(c => oneTable("b", c).map(_ -> ptB)) ++ + Seq("a.obj.a" -> ptA, "b.addr.zip AS zip" -> ptB) + + /** G2 -- items over both tables, typed as over the twin. */ + protected val g2: Seq[(String, String)] = Seq( + "COALESCE(a.n, b.m) AS v", + "a.n + b.big AS v", + "GREATEST(a.x, b.m) AS v", + "CASE WHEN a.b THEN a.ts ELSE b.d2 END AS v", + "NULLIF(a.k, b.label) AS v", + "CONCAT(a.k, b.label) AS v", + "UPPER(b.name) AS v", + "a.ts - b.d2 AS v", + "DATEDIFF(a.ts, b.d2) AS v" + ).map(_ -> "pt_ab") + + /** G3 / B8 -- (select list, group by). */ + protected val g3: Seq[(String, String)] = Seq( + "b.label, COUNT(*) AS c, COUNT(DISTINCT a.k) AS dk, SUM(a.n) AS s, AVG(a.x) AS av, " + + "MIN(a.ts) AS mi, MAX(a.dc) AS ma" -> "b.label", + "b.label, a.k, COUNT(*) AS c, SUM(a.n) AS s" -> "b.label, a.k" + ) + + /** (JOIN body, the single-table statement whose answer is its oracle). */ + protected def cells: Seq[(String, String)] = { + def over(from: String, items: Seq[(String, String)]): Seq[(String, String)] = + items.map { case (item, table) => + s"SELECT $item FROM $from" -> s"SELECT ${unqualified(item)} FROM $table" + } + val aliased = Seq("INNER JOIN", "LEFT JOIN").flatMap { kind => + over(s"$ptA a $kind $ptB b $onBid", g1 ++ g2) + } + val unaliased = (g1 ++ g2).map { case (item, table) => + val written = item.replaceAll("\\ba\\.", "pt_a.").replaceAll("\\bb\\.", "pt_b.") + s"SELECT $written FROM pt_a JOIN pt_b ON pt_a.bid = pt_b.bid" -> + s"SELECT ${unqualified(item)} FROM $table" + } + val ptAItems = g1.filter(_._2 == ptA) + val selfJoin = over( + s"$ptA a JOIN $ptA a2 ON a.bid = a2.bid", + ptAItems ++ ptAItems.map { case (item, t) => item.replaceAll("\\ba\\.", "a2.") -> t } + ) + val aggregates = g3.map { case (items, keys) => + s"SELECT $items FROM $ptA a JOIN $ptB b $onBid GROUP BY $keys" -> + s"SELECT ${unqualified(items)} FROM $ptAB GROUP BY ${unqualified(keys)}" + } + aliased ++ unaliased ++ selfJoin ++ aggregates + } + + "describePerTable" should "type a JOIN body per table, searching nothing (AC1)" in { + val oracles = cells.map { case (body, oracle) => (body, oracle, client.describe(oracle)) } + oracles.collect { case (_, oracle, ElasticSuccess(None)) => oracle } shouldBe empty + val statements = oracles.map { case (body, oracle, answer) => + (body, oracle, parsed(body), answer) + } + val bare = parsed(s"SELECT n FROM $ptA a JOIN $ptB b $onBid") + val joinedOnly = parsed(s"SELECT b.aid FROM $ptA a JOIN $ptB b $onBid") + val unknown = parsed(s"SELECT a.aid, z.n FROM $ptA a JOIN pt_zz z ON a.bid = z.bid") + val mainN = columnsOf(client.describe(s"SELECT n FROM $ptA")) + mainN should not be columnsOf(client.describe(s"SELECT n FROM $ptB")) + + val before = queryTotal() + before should be >= 0L + statements.foreach { case (body, oracle, statement, answer) => + withClue(s"[$body] vs [$oracle] ") { + sameAnswer(client.describePerTable(statement), answer) + } + } + // G4 (A-110): a bare name is the main table's; a column only the main table maps is ANY under + // the joined qualifier + columnsOf(client.describePerTable(bare)) shouldBe mainN + columnsOf(client.describePerTable(joinedOnly)) shouldBe Seq("aid" -> "ANY") + // G5: an unknown leg answers None, never the main table's same-named column + client.describePerTable(unknown) shouldBe ElasticSuccess(None) + queryTotal() shouldBe before + + // the positive control: one run moves the counter + Await.result(client.run(s"SELECT aid FROM $ptA LIMIT 10"), 60.seconds) match { + case ElasticSuccess(_) => + case ElasticFailure(error) => fail(s"positive control: ${error.message}") + } + queryTotal() should be > before + } +}