Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2025,6 +2025,13 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes {
override def describe(statement: query.Statement): ElasticResult[Option[ResultSchema]] =
delegate.describe(statement)

/** SCHEMA-15 (D10) -- the delegate's: its schema cache, its miss and reload memos, counted once.
*/
override private[elastic] def describePerTable(
statement: query.Statement
): ElasticResult[Option[ResultSchema]] =
delegate.describePerTable(statement)

// ==================== Extensions (delegate) ====================

/** #163 — the delegator inherits ExtensionApi's lazy vals, so `this.extensionRegistry` would be a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2281,7 +2281,8 @@ trait GatewayApi extends IndicesApi with ElasticClientHelpers {
* than core's handles answers `None` (its layer types it); a statement `run` refuses before
* executing anything is refused with `run`'s status and message, through the SAME resolve body
* (`resolveWithSchema(_, executeSubqueries = false)`). A refusal that needs a subquery's values
* or Elasticsearch surfaces at `run` only. `None` where no schema is known.
* or Elasticsearch surfaces at `run` only. `None` where no schema is known. A JOIN body's types
* per table, which `run` refuses here: [[describePerTable]].
*/
def describe(sql: String): ElasticResult[Option[ResultSchema]] =
GatewayApi.splitStatements(sql) match {
Expand Down Expand Up @@ -2316,6 +2317,40 @@ trait GatewayApi extends IndicesApi with ElasticClientHelpers {
)
}

/** SCHEMA-15 (D1, D3, D4, D7) -- the types a statement's columns take from their OWN tables'
* mappings, running nothing (a mapping may be read, as [[describe]] reads one). A JOIN body --
* one FROM table with standard JOIN legs, and no derived table, CTE, set operation, correlated
* or WHERE subquery: the bodies a materialized view accepts -- is typed per table by describe's
* builder, whatever extension would run it; `None` where a leg's schema is unknown, a column's
* key names no leg or an item is a star; a type error is a 400 (`operation = "describe"`). Any
* other statement is [[describe]] 's answer, verbatim.
*
* The typing API of the materialized views, which store these types: a view is materialized by
* Elasticsearch, not by the engine that would run the body as a query, so a JOIN body `run`
* refuses without the relational engine is typed here, never run.
*/
private[elastic] def describePerTable(
statement: Statement
): ElasticResult[Option[ResultSchema]] = {
// D3 (c) -- `shapeOf` names every closure shape, and each closure story adds its own there
def joinBody(single: SingleSearch): Boolean =
single.from.tables.size == 1 && single.from.enrichmentRequired &&
RelationalClosureGuard.shapeOf(single) == RelationalClosureGuard.JoinShape &&
!app.softnetwork.elastic.sql.query.whereSubqueriesPresent(single)
val body: Option[SingleSearch] = statement match {
case select: SelectStatement =>
select.statement.collect {
case single: SingleSearch if joinBody(single) => single.copy(score = select.score)
}
case single: SingleSearch if joinBody(single) => Some(single)
case _ => None
}
body match {
case Some(single) => resolvePerTable(single).map(_.flatMap(statementSchema))
case None => describe(statement)
}
}

/** [[describe]] over a query core's own handler runs. */
private def describeQuery(statement: Statement): ElasticResult[Option[ResultSchema]] =
statement match {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,13 @@ object RelationalClosureGuard {
else if (ctesPresent(statement)) CteShape
else if (closureSearches(statement).exists(_.hasCorrelatedSubqueries)) CorrelatedShape
else if (derivedTablesPresent(statement)) "A derived table (subquery in FROM/JOIN)"
else "A cross-index JOIN"
else JoinShape

/** SCHEMA-15 (D3) -- the shape of a cross-index JOIN, and of a statement no other arm names.
* `GatewayApi.describePerTable` types a JOIN body only when [[shapeOf]] answers this, so a
* closure construct a later story names above leaves that branch with its own message.
*/
private[client] val JoinShape: String = "A cross-index JOIN"

/** Story 22.5. 🔴 The words `WITH clause` sit in the first 25 characters on purpose:
* `GatewayApi.excerpt` caps a rejection at 200 characters and elides the MIDDLE (head 120 + tail
Expand Down
93 changes: 86 additions & 7 deletions core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import app.softnetwork.elastic.sql.PainlessContextType
import app.softnetwork.elastic.sql.function.aggregate.{PercentileAgg, RankingWindow}
import app.softnetwork.elastic.sql.macros.SQLQueryMacros
import app.softnetwork.elastic.sql.query.{
Field,
GroupedRowOrder,
Limit,
MultiSearch,
Expand Down Expand Up @@ -341,27 +342,35 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC
}

/** The paths `statement` selects explicitly (a bare column, not `*`, not a function, not an
* UNNEST element, not a metadata field).
* UNNEST element, not a metadata field), among the items `of` keeps (SCHEMA-15: one JOIN leg's).
*/
private def explicitPaths(statement: SingleSearch): Seq[String] =
private def explicitPaths(
statement: SingleSearch,
of: Field => Boolean = _ => true
): Seq[String] =
statement.select.fields.collect {
case f
if f.identifier.functions.isEmpty && !f.identifier.nested &&
f.identifier.name.nonEmpty && f.identifier.name != "*" &&
!f.identifier.name.startsWith("_") =>
!f.identifier.name.startsWith("_") && of(f) =>
f.identifier.name
}.distinct

/** `schema`, or -- when `statement` selects a path it lacks on ONE concrete source, not reloaded
* for that path within the source's TTL -- the source's mapping read again
/** [[reloadedFor]] over the paths `statement` selects explicitly. */
private def reloadedFor(statement: SingleSearch, sources: Seq[String], schema: Schema): Schema =
reloadedFor(explicitPaths(statement), sources, schema)

/** `schema`, or -- when `paths` holds a path it lacks on ONE concrete source, not reloaded for
* that path within the source's TTL -- the source's mapping read again
* (`IndicesApi.invalidateSchema`, then [[lookupSchema]]); `schema` when the reload fails.
* `paths` is by name: read only for a concrete source.
*/
private def reloadedFor(statement: SingleSearch, sources: Seq[String], schema: Schema): Schema =
private def reloadedFor(paths: => Seq[String], sources: Seq[String], schema: Schema): Schema =
this match {
case indices: IndicesApi
if sources.size == 1 && !sources.head.contains("*") && !sources.head.contains(",") =>
val source = sources.head
val missing = explicitPaths(statement).filter(p => schema.find(p).isEmpty)
val missing = paths.filter(p => schema.find(p).isEmpty)
if (missing.isEmpty) schema
else {
val now = System.currentTimeMillis()
Expand Down Expand Up @@ -437,6 +446,76 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC
case _ => None
}

/** SCHEMA-15 (D5, D6, D7) -- `single`, a cross-index JOIN body (one FROM table, standard JOIN
* legs over indices), with every leg typed from its OWN index, by the typing single-table
* `describe` attaches: [[attachDerived]] 's `DerivedTable.attach`, then `validateResolved`. It
* executes nothing and guards nothing (the caller chose the body).
*
* `ElasticSuccess(None)`, before any read, when a key no leg registers would resolve against the
* main table (one index name under two qualifiers, an UNNEST alias), or an item is a star (the
* main table's columns only would be a short schema); after the reads, when any leg's schema is
* unknown -- every leg is read first, then all or none, so a joined column never takes the main
* table's same-named column. A type error is a 400 with `validateResolved`'s reason.
*
* Cost: one [[lookupSchema]] per leg (the 5-minute cache `run` shares), one attach, one
* validation; a leg's explicitly selected path its cached mapping lacks reloads it once per TTL
* ([[reloadedFor]], the memo `run` shares).
*/
private[client] def resolvePerTable(single: SingleSearch): ElasticResult[Option[SingleSearch]] = {
val from = single.from
val main = from.mainTable.name
val legs = (main +: from.joinedTables).distinct
if (
from.unnests.nonEmpty || !from.aliasesToTable.values.forall(legs.contains) ||
single.select.fields.exists(HitReading.isStar)
) ElasticResult.success(None)
else {
// every leg read (a 404 remembered) before any is used
val read = legs.map { leg =>
leg -> legSchema(leg, explicitPaths(single, _.identifier.table.getOrElse(main) == leg))
}
if (read.exists(_._2.isEmpty)) ElasticResult.success(None)
else {
val schemas = read.collect { case (leg, Some(schema)) => leg -> schema }.toMap
def refused(reason: String): ElasticResult[Option[SingleSearch]] = {
logger.error(s"❌ $reason")
ElasticResult.failure(
ElasticError(
message = reason,
statusCode = Some(400),
index = Some(legs.mkString(",")),
operation = Some("describe")
)
)
}
app.softnetwork.elastic.sql.query.DerivedTable
.attach(single, sources => schemas.get(sources.mkString(","))) match {
case Left(refusal) => refused(refusal)
case Right(attached) =>
attached.validateResolved() match {
case Left(reason) => refused(reason)
case Right(_) => ElasticResult.success(Some(attached))
}
}
}
}
}

/** One leg of [[resolvePerTable]]: [[lookupSchema]] 's schema (a 404 remembered as the attach of
* [[resolveWithSchema]] remembers it), reloaded for the `paths` it lacks ([[reloadedFor]]);
* `None` when unknown.
*/
private def legSchema(leg: String, paths: => Seq[String]): Option[Schema] =
lookupSchema(Seq(leg)) match {
case Some(Success(ElasticSuccess(cached))) =>
schemaMisses.remove(leg)
Some(reloadedFor(paths, Seq(leg), cached))
case Some(Success(ElasticFailure(error))) =>
if (error.statusCode.contains(404)) rememberSchemaMiss(leg)
None
case _ => None
}

/** The mapped type of a column PROJECTED BY AN INNER STATEMENT, when it can be known for free
* (story 22.2).
*
Expand Down
Loading
Loading