diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6cd9fcc..8f8cd15 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -222,6 +222,21 @@ jobs: echo "::error::integration tests failed on Java $VERSION" failed="$failed $VERSION" fi + + # The optional Arrow module uses Arrow Java's C Data JNI on the same staged + # release pair. All A1/A2 registration and A5 insert tests must run on rc.3; + # no local-engine or skipEngineVersionCheck override is allowed in PR CI. + if mvn $MAVEN_ARGS -pl chdb-arrow -am test \ + -Djvm="$HOME_DIR/bin/java" \ + -Dchdb.arrow.it=true \ + -Dchdb.library.path="$LIBS" \ + -DargLine=--add-opens=java.base/java.nio=ALL-UNNAMED \ + -DfailIfNoTests=true; then + echo "Arrow Java $VERSION passed (see report for the separate core conversion regression)" + else + echo "::error::Arrow integration tests failed on Java $VERSION" + failed="$failed $VERSION-arrow" + fi echo "::endgroup::" done @@ -282,7 +297,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: debug-symbols-${{ matrix.platform }}-java${{ matrix.java }} + name: debug-symbols-${{ matrix.platform }} path: target/debug-symbols/ if-no-files-found: ignore retention-days: 14 @@ -291,9 +306,14 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: test-reports-${{ matrix.platform }}-java${{ matrix.java }} + name: test-reports-${{ matrix.platform }} path: | chdb-jdbc/target/surefire-reports/ + chdb-arrow/target/surefire-reports/ + chdb-arrow/target/hs_err_pid*.log + chdb-arrow/target/replay_pid*.log + chdb-arrow/*.dump + chdb-arrow/*.dumpstream chdb-integration-tests/target/failsafe-reports/ chdb-integration-tests/target/hs_err_pid*.log chdb-integration-tests/target/replay_pid*.log @@ -354,6 +374,7 @@ jobs: *) ARCH=x86_64 ;; esac LIBS="$PWD/chdb-native-${{ matrix.platform }}/target/native/META-INF/chdb/native/linux/$ARCH" + ENGINE_VERSION=$(sed -n 's/^engine.version=//p' scripts/engine.properties | head -1) # The workspace and the local Maven repository are mounted at the paths they already # have, so the classpath file needs no rewriting. @@ -370,7 +391,7 @@ jobs: ldd '"$LIBS"'/libchdb_java_jni.so echo - java -Xmx512m -Dchdb.library.path='"$LIBS"' -Dchdb.it.engine.version=26.11.0-rc.2 -cp "'"$PWD"'/chdb-jdbc/target/classes:'"$PWD"'/chdb-integration-tests/target/test-classes:'"$(cat target/it-cp.txt)"'" org.junit.platform.console.ConsoleLauncher execute --select-package=org.chdb.it --include-classname=".*IT" --details=summary --disable-banner --fail-if-no-tests + java -Xmx512m -Dchdb.library.path='"$LIBS"' -Dchdb.it.engine.version='"$ENGINE_VERSION"' -cp "'"$PWD"'/chdb-jdbc/target/classes:'"$PWD"'/chdb-integration-tests/target/test-classes:'"$(cat target/it-cp.txt)"'" org.junit.platform.console.ConsoleLauncher execute --select-package=org.chdb.it --include-classname=".*IT" --details=summary --disable-banner --fail-if-no-tests ' - name: A query over the memory limit returns an exception, not an OOM kill diff --git a/README.md b/README.md index 4836619..d47f6ab 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ streaming, forward-only result sets over ClickHouse SQL. > **Status: release candidate.** The current Maven release candidate is `v1.0.0-rc.2`, built with > chDB Core `26.11.0-rc.2`. RC artifacts are available from the public chDB Maven repository; the -> public API is not frozen. See +> public API is not frozen. This development branch targets core `26.11.0-rc.3` and adds an +> optional `chdb-arrow` module; those Java artifacts have not been published. See > [What works today](#what-works-today). ```java @@ -23,8 +24,9 @@ try (Connection connection = DriverManager.getConnection("jdbc:chdb::memory:"); ## Support matrix -Every one of the four platforms builds, passes the full test suite on Java 11, 17, 21 and 25, -and is loaded from a real packaged JAR, in CI: +The existing JDBC binding has built and passed its full suite on these four platforms and +Java 11, 17, 21 and 25 in CI. The new Arrow module is added to the PR matrix for rc.3, +but is not yet a published-platform claim: | | Linux glibc | macOS | |---|---|---| @@ -40,7 +42,8 @@ and is loaded from a real packaged JAR, in CI: engine does. Each package records the figure it was built against in its `manifest.properties`, and CI runs the whole test suite on AlmaLinux 8 to demonstrate the floor rather than infer it from symbol versions. -- **Engine:** chDB Core **26.11.0-rc.2**, pinned. The C ABI is version-locked, so the driver +- **Engine in this source branch:** chDB Core **26.11.0-rc.3**, pinned. Published Java + `1.0.0-rc.2` still uses core `26.11.0-rc.2`. The C ABI is version-locked, so the driver refuses to run against a different engine build rather than risking a struct-layout mismatch. That release is a chdb-core *pre-release*, pinned to validate the ClickHouse 26.11 baseline; a binding on an engine RC is itself an RC and is not the V1 GA @@ -114,7 +117,8 @@ all-platforms package, on purpose: it would be the sum of those four. SemVer, on the binding alone: `1.0.0` is the first release and a major bump means a breaking change to the Java API. The engine version is not part of it — it is in each package's `manifest.properties`, pinned in [`scripts/engine.properties`](scripts/engine.properties) and -named in the release notes (`26.11.0-rc.2` today), and the driver refuses to load any other build. +named in the release notes (`26.11.0-rc.2` for the published Java RC, `26.11.0-rc.3` +in this development branch), and the driver refuses to load any other build. The first test version is the RC `1.0.0-rc.1`; later candidates increment the final number, and the first stable Maven release is `1.0.0`. RCs use the permanent `com.clickhouse.chdb` groupId, diff --git a/chdb-arrow/README.md b/chdb-arrow/README.md new file mode 100644 index 0000000..541f0e3 --- /dev/null +++ b/chdb-arrow/README.md @@ -0,0 +1,57 @@ +# chDB Arrow (in development) + +This optional artifact currently contains experimental A1 batch/A2 reader registration, +A3 materialized output, A4 incremental output, and A5 synchronous insert. It is not an implementation of the complete +A1–A6 native Arrow contract. +It does not use Arrow IPC, and it does not promise zero copies inside ClickHouse. + +`ArrowSession.openInMemory()` opens a dedicated JDBC-managed connection to the process-wide +in-memory database; `open(jdbcUrl)` also supports the JDBC driver's path rules. +`queryArrowMaterialized(sql, allocator)` executes the complete query before it +returns an Apache Arrow Java `ArrowReader`. The caller closes the reader and then its +allocator. The session does not share another JDBC connection. `registerOwnedReader(reader, allocator)` +transfers reader ownership and returns a closable scope with `fromSql()`. Use that SQL +fragment only on the owning session, for one scan. Keep the allocator alive until the scope +is closed. A close requested while a materialized query runs waits for it to complete, +then unregisters before releasing the C stream. `queryArrow(sql, allocator)` returns an +`ArrowQuery` reader that pulls one core batch at a time. Close it after partial consumption, +or call `cancel()` to stop future fetches; cancellation waits for a fetch already in progress. +Only one incremental query may be active on a session. +`insertBatch(table, root, allocator)` and `insertOwnedReader(table, reader, allocator)` +use core's synchronous C Stream insert. Only simple ASCII table names and distinctly named +source columns are accepted; this excludes core's unnamed-column positional fallback. +There is no transaction guarantee and a failed insert can leave partial writes. +`ArrowOptions.DEFAULT` rejects unsupported output types, materializes LowCardinality and +exports String as UTF-8. Both query forms accept an explicit `ArrowOptions` for binary +String output, dictionary LowCardinality and lossy unsupported-type output. DateTime +defaults to Arrow unsigned 32-bit seconds without timezone; use DateTime64 in SQL for +timestamp semantics. +`registerBatch(root, allocator)` adapts one current `VectorSchemaRoot` batch; the caller +retains ownership of the root and must not mutate or close it before unregistering. + +The test `ArrowSessionTest` is opt-in (`-Dchdb.arrow.it=true`) and requires a matching +engine and JNI shim in the directory given by `-Dchdb.library.path`. The previously +published Java `1.0.0-rc.2` platform JAR has JNI ABI 2 and cannot run this work-in-progress +ABI 3 module. Arrow Java 19.0.0 on classpath also requires +`--add-opens=java.base/java.nio=ALL-UNNAMED` on supported JDKs. + +The current source pins chDB core `26.11.0-rc.3`. For a macOS arm64 release-asset +smoke test, compile the Java side (`mvn -pl chdb-jdbc -am compile`), then run +`scripts/build-native.sh macos-aarch64`. That script downloads the exact rc.3 asset, +verifies its SHA-256 and stages the engine plus JNI shim in +`chdb-native-macos-aarch64/target/native/`. Run the opt-in test with the staged pair: + +```sh +mvn -pl chdb-arrow -am test -Dchdb.arrow.it=true \ + -Dchdb.library.path="$PWD/chdb-native-macos-aarch64/target/native/META-INF/chdb/native/macos/aarch64" \ + -DargLine=--add-opens=java.base/java.nio=ALL-UNNAMED +``` + +The earlier core `26.11.0-rc.2` omitted a `get_schema` release during registration: +a value-correct Java scan left 148 bytes in the Arrow allocator. The rc.3 release +includes that fix, so **A1/A2/A5 tests are not skipped** on this baseline. One +separate regression for an unparseable String inserted into Int64 remains disabled +pending core investigation; it must not be reported as passing. CI exercises the +same suite on all four native platforms and Java 11/17/21/25 without a version +override. No production deployment should use a local `--local-engine` package or +`chdb.skipEngineVersionCheck`. diff --git a/chdb-arrow/pom.xml b/chdb-arrow/pom.xml new file mode 100644 index 0000000..40d5da5 --- /dev/null +++ b/chdb-arrow/pom.xml @@ -0,0 +1,88 @@ + + + 4.0.0 + + com.clickhouse.chdb + chdb-java-parent + 1.0.0-rc.3-SNAPSHOT + + chdb-arrow + chDB Arrow + Optional Apache Arrow Java integration for chDB. + + + com.clickhouse.chdb + chdb-jdbc + ${project.version} + + + org.apache.arrow + arrow-c-data + ${arrow.java.version} + + + org.apache.arrow + arrow-vector + ${arrow.java.version} + + + org.apache.arrow + arrow-memory-netty + ${arrow.java.version} + + + org.junit.jupiter + junit-jupiter + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.chdb.arrow + ${project.version} + ${chdb.engine.version} + ${chdb.jni.abi.version} + ${arrow.java.version} + + + + + + + + + release + + + + org.apache.maven.plugins + maven-source-plugin + + attach-sourcesjar-no-fork + + + + org.apache.maven.plugins + maven-javadoc-plugin + + attach-javadocsjar + + + all,-accessibility + true + true + + + + + + + diff --git a/chdb-arrow/src/main/java/org/chdb/arrow/ArrowOptions.java b/chdb-arrow/src/main/java/org/chdb/arrow/ArrowOptions.java new file mode 100644 index 0000000..63744f0 --- /dev/null +++ b/chdb-arrow/src/main/java/org/chdb/arrow/ArrowOptions.java @@ -0,0 +1,27 @@ +package org.chdb.arrow; + +/** + * Explicit core Arrow output type choices. Defaults reject unsupported types, materialize + * LowCardinality, and expose String as UTF-8. Alternative mappings can lose type fidelity; + * callers opt into them rather than silently changing JDBC or Arrow defaults. + */ +public final class ArrowOptions { + public static final ArrowOptions DEFAULT = new ArrowOptions(false, false, true); + + private final boolean unsupportedAsBinary; + private final boolean lowCardinalityAsDictionary; + private final boolean stringAsString; + + public ArrowOptions(boolean unsupportedAsBinary, boolean lowCardinalityAsDictionary, + boolean stringAsString) { + this.unsupportedAsBinary = unsupportedAsBinary; + this.lowCardinalityAsDictionary = lowCardinalityAsDictionary; + this.stringAsString = stringAsString; + } + + int nativeFlags() { + return (unsupportedAsBinary ? 1 : 0) + | (lowCardinalityAsDictionary ? 2 : 0) + | (stringAsString ? 4 : 0); + } +} diff --git a/chdb-arrow/src/main/java/org/chdb/arrow/ArrowQuery.java b/chdb-arrow/src/main/java/org/chdb/arrow/ArrowQuery.java new file mode 100644 index 0000000..4a40e14 --- /dev/null +++ b/chdb-arrow/src/main/java/org/chdb/arrow/ArrowQuery.java @@ -0,0 +1,181 @@ +package org.chdb.arrow; + +import java.io.IOException; +import java.sql.SQLException; +import org.apache.arrow.c.ArrowArrayStream; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.VectorUnloader; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.pojo.Schema; +import org.chdb.jdbc.ChdbConnection; + +/** + * A typed incremental query. Each load pulls one core batch without IPC serialization. + * The current root is valid until the next load or close. Cancel is serialized after any + * in-flight fetch; it does not interrupt an engine call already running on this connection. + */ +public final class ArrowQuery extends ArrowReader { + private final ChdbConnection connection; + private final BufferAllocator allocator; + private long handle; + private ArrowReader current; + private boolean exhausted; + + ArrowQuery(ChdbConnection connection, BufferAllocator allocator, long handle) { + super(allocator); + this.connection = connection; + this.allocator = allocator; + this.handle = handle; + } + + synchronized boolean isFinished() { + return handle == 0; + } + + private ArrowReader fetchOne() throws IOException { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + try { + connection.arrowStreamFetchNative(handle, stream.memoryAddress()); + } catch (SQLException e) { + throw new IOException("Failed to fetch an Arrow batch", e); + } + // The imported reader owns the moved callback. The original struct is empty. + return Data.importArrayStream(allocator, stream); + } + } + + @Override + protected synchronized Schema readSchema() throws IOException { + try { + if (current == null) { + current = fetchOne(); + } + return current.getVectorSchemaRoot().getSchema(); + } catch (IOException | RuntimeException e) { + try { + closeReadSource(); + } catch (IOException cleanup) { + e.addSuppressed(cleanup); + } + throw e; + } + } + + @Override + public synchronized boolean loadNextBatch() throws IOException { + if (exhausted) { + return false; + } + try { + prepareLoadNextBatch(); + if (current == null) { + current = fetchOne(); + } + if (!current.loadNextBatch()) { + current.close(); + current = null; + exhausted = true; + releaseNative(false); + return false; + } + VectorSchemaRoot batch = current.getVectorSchemaRoot(); + try (ArrowRecordBatch record = new VectorUnloader(batch).getRecordBatch()) { + loadRecordBatch(record); + } + current.close(); + current = null; + return true; + } catch (IOException | RuntimeException e) { + try { + closeReadSource(); + } catch (IOException cleanup) { + e.addSuppressed(cleanup); + } + throw e; + } + } + + @Override + public long bytesRead() { + return 0; // C Data exchanges buffers, not a byte stream. + } + + @Override + public synchronized void close(boolean closeReadSource) throws IOException { + // ArrowReader.close(boolean) closes its VectorSchemaRoot before calling our + // closeReadSource(). Synchronize the whole operation against loadNextBatch(), not + // just the latter callback, or a concurrent close can free the root mid-load. + // The native result must never outlive this reader, including when a caller uses + // ArrowReader's public close(false) overload. + try { + super.close(true); + } finally { + if (handle != 0) { + closeReadSource(); + } + } + } + + /** Stops future batches and closes the reader. Waits for a concurrent fetch to finish. */ + public synchronized void cancel() throws IOException { + close(); + } + + private void releaseNative(boolean cancel) throws IOException { + if (handle == 0) { + return; + } + long closing = handle; + handle = 0; + IOException failure = null; + try { + if (cancel) { + connection.arrowStreamCancelNative(closing); + } + } catch (SQLException e) { + failure = new IOException("Failed to cancel Arrow query", e); + } finally { + try { + connection.arrowStreamCloseNative(closing, this); + } catch (SQLException e) { + if (failure == null) { + failure = new IOException("Failed to close Arrow query", e); + } else { + failure.addSuppressed(e); + } + } + } + if (failure != null) { + throw failure; + } + } + + @Override + protected synchronized void closeReadSource() throws IOException { + IOException failure = null; + try { + if (current != null) { + current.close(); + current = null; + } + } catch (IOException e) { + failure = e; + } + exhausted = true; + try { + releaseNative(true); + } catch (IOException e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + if (failure != null) { + throw failure; + } + } +} diff --git a/chdb-arrow/src/main/java/org/chdb/arrow/ArrowSession.java b/chdb-arrow/src/main/java/org/chdb/arrow/ArrowSession.java new file mode 100644 index 0000000..69245ff --- /dev/null +++ b/chdb-arrow/src/main/java/org/chdb/arrow/ArrowSession.java @@ -0,0 +1,304 @@ +package org.chdb.arrow; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import org.apache.arrow.c.ArrowArrayStream; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.VectorUnloader; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.pojo.Schema; +import org.chdb.jdbc.ChdbConnection; + +/** + * Experimental Arrow session. It owns a dedicated JDBC-managed connection, not the caller's + * allocator or the returned reader. It does not share another JDBC connection's session state. + */ +public final class ArrowSession implements AutoCloseable { + private ChdbConnection connection; + private final Set sources = new HashSet<>(); + private final Set queries = new HashSet<>(); + private final AutoCloseable connectionCloseHook = this::closeBeforeConnection; + + private ArrowSession(ChdbConnection connection) { + this.connection = connection; + connection.registerArrowCloseHook(connectionCloseHook); + } + + /** Opens a dedicated connection under the JDBC driver's storage and shutdown rules. */ + public static ArrowSession open(String jdbcUrl) throws SQLException { + Connection connection = DriverManager.getConnection(Objects.requireNonNull(jdbcUrl)); + try { + return new ArrowSession(connection.unwrap(ChdbConnection.class)); + } catch (SQLException | RuntimeException e) { + connection.close(); + throw e; + } + } + + /** Opens a dedicated connection to chDB's process-wide in-memory database. */ + public static ArrowSession openInMemory() throws SQLException { + return open("jdbc:chdb::memory:"); + } + + /** + * Registers the current batch of a caller-owned root. Do not close or modify the root + * until the returned source is closed. Its reader adapter is owned by the registration; + * the root itself remains the caller's responsibility. + */ + public RegisteredArrowSource registerBatch(VectorSchemaRoot root, BufferAllocator allocator) + throws IOException, SQLException { + Objects.requireNonNull(root, "root"); + return registerOwnedReader(new SingleBatchReader(allocator, root), allocator); + } + + private static final class SingleBatchReader extends ArrowReader { + private final VectorSchemaRoot source; + private boolean consumed; + + SingleBatchReader(BufferAllocator allocator, VectorSchemaRoot source) { + super(Objects.requireNonNull(allocator, "allocator")); + this.source = source; + } + + @Override protected Schema readSchema() { return source.getSchema(); } + @Override public long bytesRead() { return 0; } + @Override protected void closeReadSource() { /* Caller owns the root. */ } + + @Override + public boolean loadNextBatch() throws IOException { + if (consumed) { + return false; + } + prepareLoadNextBatch(); + try (ArrowRecordBatch batch = new VectorUnloader(source).getRecordBatch()) { + loadRecordBatch(batch); + } + consumed = true; + return true; + } + } + + /** + * Registers an in-memory, one-shot Arrow reader for SQL scans. This method transfers + * ownership of {@code reader} once an open session begins registration, including if + * native registration fails. If the session is already closed, ownership stays with the caller. + * The caller keeps the allocator alive through {@link RegisteredArrowSource#close()}. + * A returned source is only for queries on this session; repeated scans are unsupported. + */ + public synchronized RegisteredArrowSource registerOwnedReader( + ArrowReader reader, BufferAllocator allocator) throws IOException, SQLException { + Objects.requireNonNull(reader, "reader"); + Objects.requireNonNull(allocator, "allocator"); + if (connection == null) { + throw new IllegalStateException("Arrow session is closed"); + } + requireNoActiveQuery(); + ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator); + try { + Data.exportArrayStream(allocator, reader, stream); + // UUID uses only ASCII characters safe inside the SQL string literal below. + String name = "chdb_java_" + UUID.randomUUID().toString().replace("-", ""); + RegisteredArrowSource source = new RegisteredArrowSource(this, reader, stream, name); + sources.add(source); + try { + connection.registerArrowStreamNative(name, stream.memoryAddress()); + } catch (SQLException | RuntimeException e) { + sources.remove(source); + throw e; + } + return source; + } catch (SQLException | RuntimeException e) { + try { + stream.close(); + } catch (RuntimeException closeError) { + e.addSuppressed(closeError); + } + try { + reader.close(); + } catch (IOException closeError) { + e.addSuppressed(closeError); + } + throw e; + } + } + + synchronized void unregister(RegisteredArrowSource source) throws IOException, SQLException { + if (!sources.contains(source)) { + return; + } + // An arbitrary SQL query may refer to any registered name. Conservatively finish + // the only active query before unregistering a source; no SQL parsing is involved. + closeQueries(); + // A failed unregister leaves the borrowed C pointer live in core. Keep the stream, + // reader and session alive so the caller can retry rather than free under the registry. + connection.unregisterArrowStreamNative(source.name()); + sources.remove(source); + source.releaseOwnedResources(); + } + + /** + * Returns the materialized SQL result through Arrow C Stream, without IPC serialization. + * The native query runs to completion before this method returns. Close the reader before + * closing its allocator; retaining a batch beyond the next load requires a copy/transfer. + * + * @throws IOException if Arrow Java cannot import the returned C stream + */ + public synchronized ArrowReader queryArrowMaterialized(String sql, BufferAllocator allocator) + throws IOException, SQLException { + return queryArrowMaterialized(sql, allocator, ArrowOptions.DEFAULT); + } + + /** Materialized output with explicit core Arrow type mapping choices. */ + public synchronized ArrowReader queryArrowMaterialized( + String sql, BufferAllocator allocator, ArrowOptions options) + throws IOException, SQLException { + Objects.requireNonNull(sql, "sql"); + Objects.requireNonNull(allocator, "allocator"); + Objects.requireNonNull(options, "options"); + if (connection == null) { + throw new IllegalStateException("Arrow session is closed"); + } + requireNoActiveQuery(); + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + connection.queryArrowMaterializedNative(sql, stream.memoryAddress(), options.nativeFlags()); + // importArrayStream moves the release callback into the returned reader. Closing + // the now-empty struct is safe; on failure it releases any unconsumed callback. + return Data.importArrayStream(allocator, stream); + } + } + + /** + * Starts a lazy typed Arrow query. Its reader holds this connection's statement slot + * through EOF, cancel or close. Close it before issuing another query on this session. + */ + public synchronized ArrowQuery queryArrow(String sql, BufferAllocator allocator) + throws SQLException { + return queryArrow(sql, allocator, ArrowOptions.DEFAULT); + } + + /** Incremental output with explicit core Arrow type mapping choices. */ + public synchronized ArrowQuery queryArrow( + String sql, BufferAllocator allocator, ArrowOptions options) throws SQLException { + Objects.requireNonNull(sql, "sql"); + Objects.requireNonNull(allocator, "allocator"); + Objects.requireNonNull(options, "options"); + if (connection == null) { + throw new IllegalStateException("Arrow session is closed"); + } + requireNoActiveQuery(); + ArrowQuery query = connection.arrowStreamOpenNative( + sql, options.nativeFlags(), handle -> new ArrowQuery(connection, allocator, handle)); + queries.add(query); + return query; + } + + /** + * Inserts a one-shot reader into an existing table without Arrow IPC. Only simple ASCII + * table or database.table names are accepted; core receives no arbitrary settings text. + * All source columns must be named uniquely, so core cannot silently fall back to + * positional insertion. The reader is owned and closed after schema validation, including + * on query failure. Like ordinary chDB inserts, this is not transactional. + * + * @return the core-reported rows written + */ + public synchronized long insertOwnedReader( + String tableName, ArrowReader reader, BufferAllocator allocator) + throws IOException, SQLException { + Objects.requireNonNull(tableName, "tableName"); + Objects.requireNonNull(reader, "reader"); + Objects.requireNonNull(allocator, "allocator"); + if (connection == null) { + throw new IllegalStateException("Arrow session is closed"); + } + requireNoActiveQuery(); + if (!tableName.matches("[A-Za-z_][A-Za-z0-9_]*(\\.[A-Za-z_][A-Za-z0-9_]*)?")) { + throw new IllegalArgumentException("Expected a simple table or database.table name"); + } + Set columns = new HashSet<>(); + for (org.apache.arrow.vector.types.pojo.Field field : reader.getVectorSchemaRoot() + .getSchema().getFields()) { + if (field.getName() == null || field.getName().isEmpty() + || !columns.add(field.getName())) { + throw new IllegalArgumentException("Arrow insert requires distinct, named columns"); + } + } + if (columns.isEmpty()) { + throw new IllegalArgumentException("Arrow insert needs at least one column"); + } + try (ArrowReader owned = reader; + ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + Data.exportArrayStream(allocator, owned, stream); + return connection.insertArrowStreamNative(tableName, stream.memoryAddress()); + } + } + + /** Inserts one caller-owned batch; the root remains valid after the call returns. */ + public long insertBatch(String tableName, VectorSchemaRoot root, BufferAllocator allocator) + throws IOException, SQLException { + Objects.requireNonNull(root, "root"); + return insertOwnedReader(tableName, new SingleBatchReader(allocator, root), allocator); + } + + private void requireNoActiveQuery() { + queries.removeIf(ArrowQuery::isFinished); + if (!queries.isEmpty()) { + throw new IllegalStateException("Close the active Arrow query before using this session again"); + } + } + + private void closeQueries() throws IOException { + for (ArrowQuery query : new ArrayList<>(queries)) { + query.close(); + } + queries.clear(); + } + + private synchronized void closeBeforeConnection() throws IOException { + closeQueries(); + IOException failure = null; + for (RegisteredArrowSource source : new ArrayList<>(sources)) { + try { + // The JDBC close gate is already shut. Bypass the ordinary execution-start + // check while the native connection still exists, then release the borrow. + connection.unregisterArrowStreamOnClose(source.name()); + sources.remove(source); + source.releaseOwnedResources(); + } catch (IOException | RuntimeException e) { + IOException wrapped = new IOException("Failed to unregister Arrow source on close", e); + if (failure == null) { + failure = wrapped; + } else { + failure.addSuppressed(wrapped); + } + } + } + connection = null; + if (failure != null) { + throw failure; + } + } + + @Override + public synchronized void close() throws SQLException, IOException { + if (connection != null) { + closeQueries(); + for (RegisteredArrowSource source : new ArrayList<>(sources)) { + source.close(); + } + ChdbConnection current = connection; + connection = null; + current.unregisterArrowCloseHook(connectionCloseHook); + current.close(); + } + } +} diff --git a/chdb-arrow/src/main/java/org/chdb/arrow/RegisteredArrowSource.java b/chdb-arrow/src/main/java/org/chdb/arrow/RegisteredArrowSource.java new file mode 100644 index 0000000..1fa2c1e --- /dev/null +++ b/chdb-arrow/src/main/java/org/chdb/arrow/RegisteredArrowSource.java @@ -0,0 +1,54 @@ +package org.chdb.arrow; + +import java.io.IOException; +import java.sql.SQLException; +import org.apache.arrow.c.ArrowArrayStream; +import org.apache.arrow.vector.ipc.ArrowReader; + +/** + * A borrowed native registration of a one-shot Arrow reader. Close it before its allocator. + * A close during a materialized query waits for the session's query to finish; it cannot + * unregister and release the native stream while that query is scanning it. + */ +public final class RegisteredArrowSource implements AutoCloseable { + private final ArrowSession session; + private final ArrowReader reader; + private final ArrowArrayStream stream; + private final String name; + private boolean closed; + + RegisteredArrowSource(ArrowSession session, ArrowReader reader, ArrowArrayStream stream, String name) { + this.session = session; + this.reader = reader; + this.stream = stream; + this.name = name; + } + + String name() { + return name; + } + + /** SQL fragment for a scan, e.g. {@code FROM arrowstream('...')}. */ + public String fromSql() { + synchronized (session) { + if (closed) { + throw new IllegalStateException("Arrow source is closed"); + } + return "arrowstream('" + name + "')"; + } + } + + void releaseOwnedResources() throws IOException { + closed = true; + try { + stream.close(); + } finally { + reader.close(); + } + } + + @Override + public void close() throws IOException, SQLException { + session.unregister(this); + } +} diff --git a/chdb-arrow/src/test/java/org/chdb/arrow/ArrowSessionTest.java b/chdb-arrow/src/test/java/org/chdb/arrow/ArrowSessionTest.java new file mode 100644 index 0000000..c8f8563 --- /dev/null +++ b/chdb-arrow/src/test/java/org/chdb/arrow/ArrowSessionTest.java @@ -0,0 +1,661 @@ +package org.chdb.arrow; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.sql.SQLException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.VectorUnloader; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.chdb.jdbc.ChdbConnection; +import org.chdb.internal.ChdbNative; + +/** Run with a matching locally built libchdb + JNI shim via -Dchdb.library.path. */ +@EnabledIfSystemProperty(named = "chdb.arrow.it", matches = "true") +class ArrowSessionTest { + @AfterEach + void allNativeArrowQueriesAreReleased() { + assertEquals(0L, ChdbNative.openHandleCount(ChdbNative.KIND_ARROW_STREAM)); + } + + private static class OneBatchReader extends ArrowReader { + private final VectorSchemaRoot source; + private boolean consumed; + + OneBatchReader(RootAllocator allocator, VectorSchemaRoot source) { + super(allocator); + this.source = source; + } + + @Override protected Schema readSchema() { return source.getSchema(); } + @Override public long bytesRead() { return 0; } + @Override protected void closeReadSource() { /* The source root belongs to the test. */ } + + @Override + public boolean loadNextBatch() throws IOException { + if (consumed) { + return false; + } + prepareLoadNextBatch(); + try (ArrowRecordBatch batch = new VectorUnloader(source).getRecordBatch()) { + loadRecordBatch(batch); + } + consumed = true; + return true; + } + } + + private static final class MultipleBatchReader extends ArrowReader { + private final List batches; + private int next; + + MultipleBatchReader(RootAllocator allocator, VectorSchemaRoot... batches) { + super(allocator); + this.batches = Arrays.asList(batches); + } + + @Override protected Schema readSchema() { return batches.get(0).getSchema(); } + @Override public long bytesRead() { return 0; } + @Override protected void closeReadSource() { /* Caller owns the roots. */ } + + @Override + public boolean loadNextBatch() throws IOException { + if (next == batches.size()) { + return false; + } + prepareLoadNextBatch(); + try (ArrowRecordBatch batch = new VectorUnloader(batches.get(next++)).getRecordBatch()) { + loadRecordBatch(batch); + } + return true; + } + + int batchesProduced() { return next; } + } + + private static VectorSchemaRoot idBatch(RootAllocator allocator, long... ids) { + BigIntVector vector = new BigIntVector("id", allocator); + vector.allocateNew(ids.length); + for (int i = 0; i < ids.length; i++) { + vector.setSafe(i, ids[i]); + } + vector.setValueCount(ids.length); + VectorSchemaRoot root = VectorSchemaRoot.of(vector); + root.setRowCount(ids.length); + return root; + } + + @Test + void readerInputConsumesSeveralBatchesOnlyWhenQueried() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory(); + VectorSchemaRoot first = idBatch(allocator, 1, 2); + VectorSchemaRoot second = idBatch(allocator, 3, 4, 5)) { + MultipleBatchReader reader = new MultipleBatchReader(allocator, first, second); + try (RegisteredArrowSource source = session.registerOwnedReader(reader, allocator)) { + assertEquals(0, reader.batchesProduced(), "registration must not read all input"); + try (ArrowReader result = session.queryArrowMaterialized( + "SELECT sum(id) AS total, count() AS n FROM " + source.fromSql(), allocator)) { + assertTrue(result.loadNextBatch()); + assertEquals(15L, + ((BigIntVector) result.getVectorSchemaRoot().getVector("total")).get(0)); + assertEquals(5L, + ((Number) result.getVectorSchemaRoot().getVector("n").getObject(0)) + .longValue()); + } + assertEquals(2, reader.batchesProduced()); + } + assertEquals(5L, ((BigIntVector) second.getVector("id")).get(2)); + } + } + + @Test + void inputCallbackFailureReleasesRegistrationAndConnectionSlot() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory(); + VectorSchemaRoot root = idBatch(allocator, 7)) { + OneBatchReader failing = new OneBatchReader(allocator, root) { + @Override public boolean loadNextBatch() throws IOException { + throw new IOException("deliberate input callback failure"); + } + }; + try (RegisteredArrowSource source = session.registerOwnedReader(failing, allocator)) { + assertThrows(SQLException.class, () -> session.queryArrowMaterialized( + "SELECT id FROM " + source.fromSql(), allocator)); + } + try (ArrowReader next = session.queryArrowMaterialized("SELECT toInt64(17) AS id", allocator)) { + assertTrue(next.loadNextBatch()); + assertEquals(17L, ((BigIntVector) next.getVectorSchemaRoot().getVector("id")).get(0)); + } + } + } + + @Test + void closeFalseStillReleasesLazyNativeHandle() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + ArrowQuery query = session.queryArrow("SELECT number FROM numbers(200000)", allocator); + assertTrue(query.loadNextBatch()); + query.close(false); // ArrowReader exposes this overload to callers. + assertEquals(0L, ChdbNative.openHandleCount(ChdbNative.KIND_ARROW_STREAM)); + try (ArrowReader next = session.queryArrowMaterialized("SELECT toInt64(19) AS id", allocator)) { + assertTrue(next.loadNextBatch()); + } + } + } + + @Test + void registeredReaderCanBeScannedWithoutIpc() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + BigIntVector values = new BigIntVector("id", allocator); + values.allocateNew(2); + values.setSafe(0, 7); + values.setSafe(1, 9); + values.setValueCount(2); + try (VectorSchemaRoot root = VectorSchemaRoot.of(values)) { + root.setRowCount(2); + try (RegisteredArrowSource source = session.registerOwnedReader( + new OneBatchReader(allocator, root), allocator)) { + try (ArrowReader result = session.queryArrowMaterialized( + "SELECT sum(id) AS total FROM " + source.fromSql(), allocator)) { + assertTrue(result.loadNextBatch()); + assertEquals(16L, + ((BigIntVector) result.getVectorSchemaRoot().getVector("total")).get(0)); + assertFalse(result.loadNextBatch()); + } + } + } + } + } + + @Test + void batchAdapterLeavesCallerRootOpen() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + BigIntVector values = new BigIntVector("id", allocator); + values.allocateNew(1); + values.setSafe(0, 23); + values.setValueCount(1); + try (VectorSchemaRoot root = VectorSchemaRoot.of(values)) { + root.setRowCount(1); + try (RegisteredArrowSource source = session.registerBatch(root, allocator); + ArrowReader result = session.queryArrowMaterialized( + "SELECT id FROM " + source.fromSql(), allocator)) { + assertTrue(result.loadNextBatch()); + assertEquals(23L, + ((BigIntVector) result.getVectorSchemaRoot().getVector("id")).get(0)); + } + assertEquals(23L, values.get(0)); + } + } + } + + @Test + void closeDuringScanWaitsForBorrowedReader() throws Exception { + CountDownLatch reading = new CountDownLatch(1); + CountDownLatch continueReading = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + BigIntVector values = new BigIntVector("id", allocator); + values.allocateNew(1); + values.setSafe(0, 31); + values.setValueCount(1); + try (VectorSchemaRoot root = VectorSchemaRoot.of(values)) { + root.setRowCount(1); + OneBatchReader reader = new OneBatchReader(allocator, root) { + @Override + public boolean loadNextBatch() throws IOException { + reading.countDown(); + try { + if (!continueReading.await(10, TimeUnit.SECONDS)) { + throw new IOException("test reader timed out"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("test reader interrupted", e); + } + return super.loadNextBatch(); + } + }; + try (RegisteredArrowSource source = session.registerOwnedReader(reader, allocator)) { + Future query = executor.submit(() -> { + try (ArrowReader result = session.queryArrowMaterialized( + "SELECT id FROM " + source.fromSql(), allocator)) { + assertTrue(result.loadNextBatch()); + return ((BigIntVector) result.getVectorSchemaRoot().getVector("id")).get(0); + } + }); + assertTrue(reading.await(10, TimeUnit.SECONDS)); + Future close = executor.submit(() -> { + source.close(); + return null; + }); + try { + assertThrows(TimeoutException.class, () -> close.get(150, TimeUnit.MILLISECONDS)); + } finally { + continueReading.countDown(); + } + assertEquals(31L, query.get(10, TimeUnit.SECONDS)); + close.get(10, TimeUnit.SECONDS); + } finally { + continueReading.countDown(); + } + } + } finally { + continueReading.countDown(); + executor.shutdownNow(); + } + } + + @Test + void closeDuringLazyFetchWaitsBeforeUnregistering() throws Exception { + CountDownLatch reading = new CountDownLatch(1); + CountDownLatch continueReading = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + BigIntVector values = new BigIntVector("id", allocator); + values.allocateNew(1); + values.setSafe(0, 37); + values.setValueCount(1); + try (VectorSchemaRoot root = VectorSchemaRoot.of(values)) { + root.setRowCount(1); + OneBatchReader reader = new OneBatchReader(allocator, root) { + @Override public boolean loadNextBatch() throws IOException { + reading.countDown(); + try { + if (!continueReading.await(10, TimeUnit.SECONDS)) { + throw new IOException("lazy reader timed out"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("lazy reader interrupted", e); + } + return super.loadNextBatch(); + } + }; + try (RegisteredArrowSource source = session.registerOwnedReader(reader, allocator)) { + ArrowQuery query = session.queryArrow("SELECT id FROM " + source.fromSql(), allocator); + Future fetch = executor.submit(() -> { + assertTrue(query.loadNextBatch()); + return ((BigIntVector) query.getVectorSchemaRoot().getVector("id")).get(0); + }); + assertTrue(reading.await(10, TimeUnit.SECONDS)); + Future close = executor.submit(() -> { + source.close(); + return null; + }); + try { + assertThrows(TimeoutException.class, () -> close.get(150, TimeUnit.MILLISECONDS)); + } finally { + continueReading.countDown(); + } + assertEquals(37L, fetch.get(10, TimeUnit.SECONDS)); + close.get(10, TimeUnit.SECONDS); + } finally { + continueReading.countDown(); + } + } + } finally { + continueReading.countDown(); + executor.shutdownNow(); + } + } + + @Test + void materializedQueryPreservesValuesNullAndUtf8() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory(); + ArrowReader reader = session.queryArrowMaterialized( + "SELECT toInt64(number + 10) AS id, " + + "if(number = 0, CAST(NULL, 'Nullable(String)'), " + + "CAST('你好', 'Nullable(String)')) AS name FROM numbers(2)", + allocator)) { + assertTrue(reader.loadNextBatch()); + assertEquals(2, reader.getVectorSchemaRoot().getRowCount()); + BigIntVector ids = (BigIntVector) reader.getVectorSchemaRoot().getVector("id"); + VarCharVector names = (VarCharVector) reader.getVectorSchemaRoot().getVector("name"); + assertEquals(10L, ids.get(0)); + assertEquals(11L, ids.get(1)); + assertTrue(names.isNull(0)); + assertEquals("你好", names.getObject(1).toString()); + assertFalse(reader.loadNextBatch()); + } + } + + @Test + void emptyResultHasSchemaAndErrorsDoNotPoisonConnection() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + assertThrows(SQLException.class, + () -> session.queryArrowMaterialized("SELECT * FROM missing_arrow_table", allocator)); + try (ArrowReader reader = session.queryArrowMaterialized( + "SELECT toInt64(1) AS id WHERE 0", allocator)) { + assertEquals("id", reader.getVectorSchemaRoot().getSchema().getFields().get(0).getName()); + assertFalse(reader.loadNextBatch()); + } + } + } + + @Test + void materializedStreamOutlivesTheSession() throws Exception { + try (RootAllocator allocator = new RootAllocator()) { + ArrowReader reader; + try (ArrowSession session = ArrowSession.openInMemory()) { + reader = session.queryArrowMaterialized("SELECT toInt64(42) AS id", allocator); + } + try (ArrowReader owned = reader) { + assertTrue(owned.loadNextBatch()); + assertEquals(42L, + ((BigIntVector) owned.getVectorSchemaRoot().getVector("id")).get(0)); + assertFalse(owned.loadNextBatch()); + } + } + } + + @Test + void streamedQueryReadsMultipleBatchesWithoutMaterializingFirst() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory(); + ArrowQuery query = session.queryArrow( + "SELECT number AS id FROM numbers(200000)", allocator)) { + assertEquals("id", query.getVectorSchemaRoot().getSchema().getFields().get(0).getName()); + int batches = 0; + int rows = 0; + while (query.loadNextBatch()) { + rows += query.getVectorSchemaRoot().getRowCount(); + batches++; + } + assertEquals(200000, rows); + assertTrue(batches > 1, "expected more than one core fetch"); + assertFalse(query.loadNextBatch()); + } + } + + @Test + void streamedEmptyResultRetainsSchemaAndPartialCloseFreesConnection() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + try (ArrowQuery empty = session.queryArrow( + "SELECT toInt64(1) AS id WHERE 0", allocator)) { + assertEquals("id", empty.getVectorSchemaRoot().getSchema().getFields().get(0).getName()); + assertFalse(empty.loadNextBatch()); + } + try (ArrowQuery partial = session.queryArrow( + "SELECT number FROM numbers(200000)", allocator)) { + assertTrue(partial.loadNextBatch()); + } + try (ArrowReader next = session.queryArrowMaterialized("SELECT toInt64(8) AS id", allocator)) { + assertTrue(next.loadNextBatch()); + assertEquals(8L, ((BigIntVector) next.getVectorSchemaRoot().getVector("id")).get(0)); + } + } + } + + @Test + void closingSourceAlsoFinishesLazyQueryBeforeUnregister() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + BigIntVector values = new BigIntVector("id", allocator); + values.allocateNew(1); + values.setSafe(0, 41); + values.setValueCount(1); + try (VectorSchemaRoot root = VectorSchemaRoot.of(values)) { + root.setRowCount(1); + try (RegisteredArrowSource source = session.registerBatch(root, allocator)) { + ArrowQuery query = session.queryArrow("SELECT id FROM " + source.fromSql(), allocator); + assertTrue(query.loadNextBatch()); + source.close(); // The query is still open, so this must end it first. + try (ArrowReader next = session.queryArrowMaterialized( + "SELECT toInt64(9) AS id", allocator)) { + assertTrue(next.loadNextBatch()); + } + } + } + } + } + + @Test + void cancelAndFailedOpenReleaseTheStatementSlot() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + try (ArrowQuery failed = session.queryArrow( + "SELECT * FROM missing_arrow_stream_table", allocator)) { + // Streaming init can defer SQL execution/error until the first fetch. + assertThrows(IOException.class, failed::loadNextBatch); + } + try (ArrowQuery query = session.queryArrow( + "SELECT number FROM numbers(200000)", allocator)) { + assertTrue(query.loadNextBatch()); + query.cancel(); + } + try (ArrowReader next = session.queryArrowMaterialized("SELECT toInt64(12) AS id", allocator)) { + assertTrue(next.loadNextBatch()); + assertEquals(12L, ((BigIntVector) next.getVectorSchemaRoot().getVector("id")).get(0)); + } + } + } + + @Test + void insertBatchMapsNamedColumnsAndPreservesCallerRoot() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + Connection ddl = DriverManager.getConnection("jdbc:chdb::memory:"); + ArrowSession session = ArrowSession.openInMemory()) { + try (Statement statement = ddl.createStatement()) { + statement.execute("CREATE TABLE arrow_java_insert_test (id Int64, name Nullable(String)) " + + "ENGINE=MergeTree ORDER BY id"); + } + VarCharVector names = new VarCharVector("name", allocator); + BigIntVector ids = new BigIntVector("id", allocator); + names.allocateNew(); + ids.allocateNew(2); + names.setNull(0); + names.setSafe(1, "你好".getBytes(StandardCharsets.UTF_8)); + ids.setSafe(0, 1); + ids.setSafe(1, 2); + names.setValueCount(2); + ids.setValueCount(2); + try (VectorSchemaRoot root = VectorSchemaRoot.of(names, ids)) { + root.setRowCount(2); + assertEquals(2L, session.insertBatch("arrow_java_insert_test", root, allocator)); + assertEquals(2L, ids.get(1)); + assertThrows(IllegalArgumentException.class, + () -> session.insertBatch("arrow_java_insert_test;DROP TABLE x", root, allocator)); + } + try (ArrowReader result = session.queryArrowMaterialized( + "SELECT id, name FROM arrow_java_insert_test ORDER BY id", allocator)) { + assertTrue(result.loadNextBatch()); + assertEquals(2, result.getVectorSchemaRoot().getRowCount()); + VarCharVector out = (VarCharVector) result.getVectorSchemaRoot().getVector("name"); + assertTrue(out.isNull(0)); + assertEquals("你好", out.getObject(1).toString()); + } + } + } + + @Test + void failedInsertIntoMissingTableDoesNotConsumeCallerBatch() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory(); + VectorSchemaRoot root = idBatch(allocator, 61, 62)) { + for (int attempt = 0; attempt < 3; attempt++) { + assertThrows(SQLException.class, + () -> session.insertBatch("missing_arrow_insert_destination", root, allocator)); + assertEquals(62L, ((BigIntVector) root.getVector("id")).get(1)); + } + try (ArrowReader next = session.queryArrowMaterialized("SELECT toInt64(63) AS id", allocator)) { + assertTrue(next.loadNextBatch()); + } + } + } + + @Test + @Disabled("core 26.11.0-rc.3 ArrowStream inserts invalid String into Int64 as 0; " + + "ordinary INSERT SELECT rejects it with CANNOT_PARSE_TEXT") + void invalidInsertValueReportsErrorAndKeepsSessionUsable() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + Connection ddl = DriverManager.getConnection("jdbc:chdb::memory:"); + ArrowSession session = ArrowSession.openInMemory()) { + try (Statement statement = ddl.createStatement()) { + statement.execute("CREATE TABLE arrow_java_bad_insert_test (id Int64) " + + "ENGINE=MergeTree ORDER BY id"); + } + VarCharVector ids = new VarCharVector("id", allocator); + ids.allocateNew(); + ids.setSafe(0, "not-an-integer".getBytes(StandardCharsets.UTF_8)); + ids.setValueCount(1); + try (VectorSchemaRoot root = VectorSchemaRoot.of(ids)) { + root.setRowCount(1); + assertThrows(SQLException.class, + () -> session.insertBatch("arrow_java_bad_insert_test", root, allocator)); + assertEquals("not-an-integer", ids.getObject(0).toString()); + } + try (ArrowReader next = session.queryArrowMaterialized("SELECT toInt64(71) AS id", allocator)) { + assertTrue(next.loadNextBatch()); + } + } + } + + @Test + void duplicateSourceColumnNamesAreRejectedBeforeNativeInsert() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + BigIntVector first = new BigIntVector("id", allocator); + BigIntVector second = new BigIntVector("id", allocator); + first.allocateNew(1); + second.allocateNew(1); + first.setSafe(0, 1); + second.setSafe(0, 2); + first.setValueCount(1); + second.setValueCount(1); + try (VectorSchemaRoot root = VectorSchemaRoot.of(first, second)) { + root.setRowCount(1); + assertThrows(IllegalArgumentException.class, + () -> session.insertBatch("arrow_java_bad_insert_test", root, allocator)); + assertEquals(1L, first.get(0)); + } + } + } + + @Test + void outputTypeOptionsAreExplicitForMaterializedAndStreamedQueries() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + String sql = "SELECT toString('abc') AS text, " + + "toDateTime('2020-01-02 03:04:05') AS dt, " + + "toDecimal64(12.34, 2) AS amount"; + try (ArrowReader defaults = session.queryArrowMaterialized(sql, allocator)) { + Schema schema = defaults.getVectorSchemaRoot().getSchema(); + assertTrue(schema.getFields().get(0).getType() instanceof ArrowType.Utf8); + assertTrue(schema.getFields().get(1).getType() instanceof ArrowType.Int); + assertTrue(schema.getFields().get(2).getType() instanceof ArrowType.Decimal); + assertTrue(defaults.loadNextBatch()); + } + ArrowOptions binaryString = new ArrowOptions(false, false, false); + try (ArrowReader binary = session.queryArrowMaterialized(sql, allocator, binaryString)) { + assertTrue(binary.getVectorSchemaRoot().getSchema().getFields().get(0).getType() + instanceof ArrowType.Binary); + } + try (ArrowQuery streamed = session.queryArrow(sql, allocator, binaryString)) { + assertTrue(streamed.getVectorSchemaRoot().getSchema().getFields().get(0).getType() + instanceof ArrowType.Binary); + assertTrue(streamed.loadNextBatch()); + } + } + } + + @Test + void dictionaryTimestampAndNestedTypesHaveObservableSchemas() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + String sql = "SELECT CAST('x', 'LowCardinality(String)') AS lc, " + + "toDateTime64('2020-01-02 03:04:05', 3, 'UTC') AS ts, [1,2] AS arr"; + try (ArrowReader defaults = session.queryArrowMaterialized(sql, allocator)) { + Schema schema = defaults.getVectorSchemaRoot().getSchema(); + assertTrue(schema.getFields().get(0).getType() instanceof ArrowType.Utf8); + assertTrue(schema.getFields().get(1).getType() instanceof ArrowType.Timestamp); + assertTrue(schema.getFields().get(2).getType() instanceof ArrowType.List); + } + try (ArrowQuery dictionaries = session.queryArrow( + sql, allocator, new ArrowOptions(false, true, true))) { + assertTrue(dictionaries.getVectorSchemaRoot().getSchema().getFields().get(0) + .getDictionary() != null); + assertTrue(dictionaries.loadNextBatch()); + } + } + } + + @Test + void dictionarySchemaIsStableAcrossStreamedBatches() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory(); + ArrowQuery query = session.queryArrow( + "SELECT toLowCardinality(if(number % 2 = 0, 'even', 'odd')) AS lc " + + "FROM numbers(200000)", + allocator, new ArrowOptions(false, true, true))) { + Schema initial = query.getVectorSchemaRoot().getSchema(); + assertTrue(initial.getFields().get(0).getDictionary() != null); + int batches = 0; + int rows = 0; + while (query.loadNextBatch()) { + assertEquals(initial, query.getVectorSchemaRoot().getSchema()); + rows += query.getVectorSchemaRoot().getRowCount(); + batches++; + } + assertEquals(200000, rows); + assertTrue(batches > 1); + } + } + + @Test + void jdbcConnectionCloseCascadesToArrowQueryAndBorrowedSource() throws Exception { + try (RootAllocator allocator = new RootAllocator(); + ArrowSession session = ArrowSession.openInMemory()) { + BigIntVector values = new BigIntVector("id", allocator); + values.allocateNew(1); + values.setSafe(0, 53); + values.setValueCount(1); + try (VectorSchemaRoot root = VectorSchemaRoot.of(values)) { + root.setRowCount(1); + RegisteredArrowSource source = session.registerBatch(root, allocator); + session.queryArrow("SELECT id FROM " + source.fromSql(), allocator); + // Exercise JDBC's independent close/shutdown route, which cannot be invoked + // through ArrowSession's public interface. The reflection is test-only. + Field field = ArrowSession.class.getDeclaredField("connection"); + field.setAccessible(true); + ((ChdbConnection) field.get(session)).close(); + assertThrows(IllegalStateException.class, source::fromSql); + assertThrows(IllegalStateException.class, + () -> session.queryArrow("SELECT 1", allocator)); + } + } + } +} diff --git a/chdb-bom/pom.xml b/chdb-bom/pom.xml index d75d84a..24f9dae 100644 --- a/chdb-bom/pom.xml +++ b/chdb-bom/pom.xml @@ -7,7 +7,7 @@ com.clickhouse.chdb chdb-java-parent - 1.0.0-SNAPSHOT + 1.0.0-rc.3-SNAPSHOT chdb-bom @@ -27,6 +27,11 @@ chdb-jdbc ${project.version} + + com.clickhouse.chdb + chdb-arrow + ${project.version} + com.clickhouse.chdb chdb-native-macos-aarch64 diff --git a/chdb-examples/pom.xml b/chdb-examples/pom.xml index 104cfe5..e6c2bd0 100644 --- a/chdb-examples/pom.xml +++ b/chdb-examples/pom.xml @@ -7,7 +7,7 @@ com.clickhouse.chdb chdb-java-parent - 1.0.0-SNAPSHOT + 1.0.0-rc.3-SNAPSHOT chdb-examples diff --git a/chdb-integration-tests/pom.xml b/chdb-integration-tests/pom.xml index 28e8089..99180e9 100644 --- a/chdb-integration-tests/pom.xml +++ b/chdb-integration-tests/pom.xml @@ -7,7 +7,7 @@ com.clickhouse.chdb chdb-java-parent - 1.0.0-SNAPSHOT + 1.0.0-rc.3-SNAPSHOT chdb-integration-tests diff --git a/chdb-jdbc/pom.xml b/chdb-jdbc/pom.xml index 91a3169..9240602 100644 --- a/chdb-jdbc/pom.xml +++ b/chdb-jdbc/pom.xml @@ -7,7 +7,7 @@ com.clickhouse.chdb chdb-java-parent - 1.0.0-SNAPSHOT + 1.0.0-rc.3-SNAPSHOT chdb-jdbc diff --git a/chdb-jdbc/src/main/java/org/chdb/internal/ChdbNative.java b/chdb-jdbc/src/main/java/org/chdb/internal/ChdbNative.java index db66e60..6605297 100644 --- a/chdb-jdbc/src/main/java/org/chdb/internal/ChdbNative.java +++ b/chdb-jdbc/src/main/java/org/chdb/internal/ChdbNative.java @@ -28,7 +28,7 @@ public final class ChdbNative { * Must equal {@code chdb_jni::kJniAbiVersion} in the shim and {@code * chdb.jni.abi.version} in the poms. Verified at load time before any other call. */ - public static final int JNI_ABI_VERSION = 2; + public static final int JNI_ABI_VERSION = 3; /** Handle kinds, for {@link #openHandleCount(int)}. */ public static final int KIND_CONNECTION = 1; @@ -38,6 +38,7 @@ public final class ChdbNative { * make an old shim and a new driver agree on a number while disagreeing on its meaning. */ public static final int KIND_ROW_BINARY = 4; + public static final int KIND_ARROW_STREAM = 5; static { // Loads libchdb and then the shim, in that order, and verifies the ABI. @@ -57,7 +58,7 @@ private ChdbNative() { /** ABI version the loaded shim was built with. */ public static native int jniAbiVersion(); - /** {@code chdb_version()} of the loaded libchdb, e.g. {@code "26.11.0-rc.2"}. */ + /** {@code chdb_version()} of the loaded libchdb, e.g. {@code "26.11.0-rc.3"}. */ public static native String engineVersion(); /** Build provenance of the shim: git commit, compiler, engine headers it compiled against. */ @@ -123,6 +124,35 @@ private ChdbNative() { public static native long query( long connection, byte[] sql, byte[] format, byte[][] paramNames, byte[][] paramValues); + /** + * Materializes a query and transfers an Arrow C stream into the caller-allocated struct. + * The caller must release that stream, including when this method throws. The address is + * obtained from Arrow Java's ArrowArrayStream, never from a Java heap object. + */ + public static native void queryArrowMaterialized( + long connection, byte[] sql, long streamAddress, int optionFlags); + + /** Registers a borrowed Arrow C Stream under a process-unique internal name. */ + public static native void registerArrowStream(long connection, byte[] name, long streamAddress); + + /** Unregisters before the caller releases the borrowed Arrow C Stream. */ + public static native void unregisterArrowStream(long connection, byte[] name); + + /** Starts a lazy Arrow query and returns a registry ID, not a native pointer. */ + public static native long arrowStreamOpen(long connection, byte[] sql, int optionFlags); + + /** Fills a caller-allocated, initially empty C Stream with one fetched batch or EOF. */ + public static native void arrowStreamFetch(long connection, long stream, long outAddress); + + /** Cooperative cancellation; serialized against fetch on this connection. */ + public static native void arrowStreamCancel(long connection, long stream); + + /** Destroys the query handle after all fetched C streams have been released. */ + public static native void arrowStreamClose(long stream); + + /** Synchronous insert from a borrowed Arrow C Stream; no SQL text or settings are accepted. */ + public static native long insertArrowStream(long connection, byte[] tableName, long streamAddress); + /** Result payload bytes, or an empty array for a statement that produced none. */ public static native byte[] resultBytes(long result); @@ -142,7 +172,7 @@ public static native long query( * {@code chdb_classify_query_n}. Decides whether a statement has a result set to stream * without executing it, using the engine's own parser. * - *

Optional symbol: it landed in chdb-core v26.7.2-rc.2, so the pinned v26.11.0-rc.2 + *

Optional symbol: it landed in chdb-core v26.7.2-rc.2, so the pinned v26.11.0-rc.3 * baseline has it, so it is resolved by {@code dlsym} rather than linked. It is present on * the baseline; the null path is what an older engine gets, and means the caller must fall * back to {@code StatementShape}'s own analysis. @@ -204,7 +234,7 @@ public static native long rowBinaryOpen( * every result destroyed first, and the engine cannot be used again in this process. * *

Optional symbol, like {@link #classifyQuery(long, byte[])}: it landed in chdb-core - * v26.7.2-rc.2, so the pinned v26.11.0-rc.2 baseline has it, and it is resolved by {@code + * v26.7.2-rc.2, so the pinned v26.11.0-rc.3 baseline has it, and it is resolved by {@code * dlsym} rather than linked. Not having it is not a failure -- the threads it would join are * reaped by process exit either way. * diff --git a/chdb-jdbc/src/main/java/org/chdb/internal/NativeLibraryLoader.java b/chdb-jdbc/src/main/java/org/chdb/internal/NativeLibraryLoader.java index 587bfcf..981d53c 100644 --- a/chdb-jdbc/src/main/java/org/chdb/internal/NativeLibraryLoader.java +++ b/chdb-jdbc/src/main/java/org/chdb/internal/NativeLibraryLoader.java @@ -41,8 +41,8 @@ * a Maven coordinate in the message, not something to download at runtime (section 2.3). * *

Why the order matters

- * The shim records a dependency on {@code @rpath/libchdb.so} resolved through {@code - * @loader_path}/{@code $ORIGIN}. Loading {@code libchdb} first by absolute path puts it in + * The shim records a dependency on {@code @rpath/libchdb.so} resolved through + * {@code @loader_path}/{@code $ORIGIN}. Loading {@code libchdb} first by absolute path puts it in * the process's global symbol namespace, so the shim's dependency is already satisfied when * it is opened -- and {@code chdb_classify_query_n}, resolved by {@code dlsym(RTLD_DEFAULT)}, * is findable. diff --git a/chdb-jdbc/src/main/java/org/chdb/jdbc/ChdbConnection.java b/chdb-jdbc/src/main/java/org/chdb/jdbc/ChdbConnection.java index e9eae31..ee9ec09 100644 --- a/chdb-jdbc/src/main/java/org/chdb/jdbc/ChdbConnection.java +++ b/chdb-jdbc/src/main/java/org/chdb/jdbc/ChdbConnection.java @@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.LongFunction; import org.chdb.internal.ChdbNative; import org.chdb.internal.ChdbNativeException; import org.chdb.internal.NativeLibraryLoader; @@ -84,6 +85,14 @@ public final class ChdbConnection implements Connection { private final Set openStatements = Collections.newSetFromMap(new ConcurrentHashMap()); + /** Optional Arrow readers must be closed before their connection, like JDBC result sets. */ + private final Set openArrowQueries = + Collections.newSetFromMap(new ConcurrentHashMap()); + + /** An optional Arrow session can have borrowed sources that outlive a query. */ + private final Set arrowCloseHooks = + Collections.newSetFromMap(new ConcurrentHashMap()); + /** * Arbitrates between a thread starting a statement here and the shutdown hook closing this * connection, which must never overlap: closing a connection whose statement is executing @@ -181,6 +190,170 @@ StatementSlot statementSlot() { return statementSlot; } + /** + * Low-level bridge for the optional chdb-arrow artifact. The address belongs to a + * caller-allocated Arrow C Stream struct; this method never owns that struct. Keeping the + * native call here shares the JDBC connection's execution gate and statement slot. + */ + public void queryArrowMaterializedNative(String sql, long streamAddress, int optionFlags) + throws SQLException { + statementSlot.acquire(sql); + boolean entered = false; + try { + entered = executionStarted(); + if (!entered || isClosed()) { + throw ChdbExceptions.closed("Connection"); + } + ChdbNative.queryArrowMaterialized(handle, Utf8.encode(sql), streamAddress, optionFlags); + } catch (ChdbNativeException e) { + throw ChdbExceptions.wrap("Arrow query failed", e); + } finally { + if (entered) { + executionFinished(); + } + statementSlot.release(); + } + } + + /** Internal bridge for a borrowed Arrow C Stream; the caller retains the stream. */ + public void registerArrowStreamNative(String name, long streamAddress) throws SQLException { + arrowRegistrationNative(name, streamAddress, true); + } + + /** Internal bridge; the caller may release its C Stream only after this returns. */ + public void unregisterArrowStreamNative(String name) throws SQLException { + arrowRegistrationNative(name, 0, false); + } + + /** Registers cleanup to run after queries but before the native connection is destroyed. */ + public void registerArrowCloseHook(AutoCloseable hook) { + synchronized (closeLock) { + if (closed.get()) { + throw new IllegalStateException("Connection is closed"); + } + arrowCloseHooks.add(hook); + } + } + + public void unregisterArrowCloseHook(AutoCloseable hook) { + arrowCloseHooks.remove(hook); + } + + /** Used only by a close hook after the execution gate has stopped new statements. */ + public void unregisterArrowStreamOnClose(String name) { + ChdbNative.unregisterArrowStream(handle, Utf8.encode(name)); + } + + private void arrowRegistrationNative(String name, long address, boolean register) throws SQLException { + statementSlot.acquire(register ? "register Arrow source" : "unregister Arrow source"); + boolean entered = false; + try { + entered = executionStarted(); + if (!entered || isClosed()) { + throw ChdbExceptions.closed("Connection"); + } + if (register) { + ChdbNative.registerArrowStream(handle, Utf8.encode(name), address); + } else { + ChdbNative.unregisterArrowStream(handle, Utf8.encode(name)); + } + } catch (ChdbNativeException e) { + throw ChdbExceptions.wrap("Arrow registration failed", e); + } finally { + if (entered) { + executionFinished(); + } + statementSlot.release(); + } + } + + /** Opens one streaming query and holds the statement slot until arrowStreamCloseNative. */ + public T arrowStreamOpenNative( + String sql, int optionFlags, LongFunction ownerFactory) throws SQLException { + statementSlot.acquire(sql); + boolean entered = false; + boolean handedOff = false; + long stream = 0; + try { + entered = executionStarted(); + if (!entered || isClosed()) { + throw ChdbExceptions.closed("Connection"); + } + stream = ChdbNative.arrowStreamOpen(handle, Utf8.encode(sql), optionFlags); + T owner = ownerFactory.apply(stream); + // Published before the execution gate is released, so close() cannot miss a + // stream in the interval between native open and Java owner construction. + openArrowQueries.add(owner); + handedOff = true; + return owner; + } catch (ChdbNativeException e) { + throw ChdbExceptions.wrap("Arrow stream open failed", e); + } finally { + if (entered) { + executionFinished(); + } + if (!handedOff) { + try { + if (stream != 0) { + ChdbNative.arrowStreamClose(stream); + } + } finally { + statementSlot.release(); + } + } + } + } + + /** Fills one Arrow C Stream with a single batch; the caller releases it. */ + public void arrowStreamFetchNative(long stream, long outAddress) throws SQLException { + try { + ChdbNative.arrowStreamFetch(handle, stream, outAddress); + } catch (ChdbNativeException e) { + throw ChdbExceptions.wrap("Arrow stream fetch failed", e); + } + } + + /** Cooperative cancellation after any in-flight fetch; not simultaneous engine calls. */ + public void arrowStreamCancelNative(long stream) throws SQLException { + try { + ChdbNative.arrowStreamCancel(handle, stream); + } catch (ChdbNativeException e) { + throw ChdbExceptions.wrap("Arrow stream cancel failed", e); + } + } + + /** Frees the native result and the connection's statement slot, even on close failure. */ + public void arrowStreamCloseNative(long stream, AutoCloseable owner) throws SQLException { + try { + ChdbNative.arrowStreamClose(stream); + } catch (ChdbNativeException e) { + throw ChdbExceptions.wrap("Arrow stream close failed", e); + } finally { + openArrowQueries.remove(owner); + statementSlot.release(); + } + } + + /** Synchronous native Arrow insert, using the same single-statement execution rules. */ + public long insertArrowStreamNative(String tableName, long streamAddress) throws SQLException { + statementSlot.acquire("insert Arrow stream"); + boolean entered = false; + try { + entered = executionStarted(); + if (!entered || isClosed()) { + throw ChdbExceptions.closed("Connection"); + } + return ChdbNative.insertArrowStream(handle, Utf8.encode(tableName), streamAddress); + } catch (ChdbNativeException e) { + throw ChdbExceptions.wrap("Arrow insert failed", e); + } finally { + if (entered) { + executionFinished(); + } + statementSlot.release(); + } + } + /** * Announces that this thread is about to start a statement in the engine. * @@ -415,6 +588,34 @@ private void doClose() throws SQLException { // ExecutionGate.closeToNewEntrantsWaiting() for why waiting here costs nothing. executionGate.closeToNewEntrantsWaiting(); + for (AutoCloseable query : openArrowQueries.toArray(new AutoCloseable[0])) { + try { + query.close(); + } catch (Exception e) { + SQLException wrapped = new SQLException("Failed to close an Arrow query", e); + if (firstFailure == null) { + firstFailure = wrapped; + } else { + firstFailure.addSuppressed(wrapped); + } + } + } + openArrowQueries.clear(); + + for (AutoCloseable hook : arrowCloseHooks.toArray(new AutoCloseable[0])) { + try { + hook.close(); + } catch (Exception e) { + SQLException wrapped = new SQLException("Failed to close an Arrow source", e); + if (firstFailure == null) { + firstFailure = wrapped; + } else { + firstFailure.addSuppressed(wrapped); + } + } + } + arrowCloseHooks.clear(); + // Statements first, so their streams and chunks are released before the connection // they were opened on. The shim keeps a strong reference from stream to connection, // so the other order would leak rather than crash -- but leaking is still wrong. diff --git a/chdb-jdbc/src/main/java/org/chdb/jdbc/ChdbStatement.java b/chdb-jdbc/src/main/java/org/chdb/jdbc/ChdbStatement.java index a009140..843023c 100644 --- a/chdb-jdbc/src/main/java/org/chdb/jdbc/ChdbStatement.java +++ b/chdb-jdbc/src/main/java/org/chdb/jdbc/ChdbStatement.java @@ -365,7 +365,7 @@ private void checkDeadlineSurvivedTheOpen(String sql) throws SQLException { private StatementShape.Route route(String sql) throws SQLException { int[] analysis; try { - // Null when the engine predates chdb_classify_query_n, which the v26.11.0-rc.2 + // Null when the engine predates chdb_classify_query_n, which the v26.11.0-rc.3 // baseline does not -- but a loader pointed at another libchdb still can, in which // case StatementShape falls back to its keyword scan. analysis = ChdbNative.classifyQuery(connection.handle(), Utf8.encode(sql)); diff --git a/chdb-jdbc/src/main/java/org/chdb/jdbc/StatementShape.java b/chdb-jdbc/src/main/java/org/chdb/jdbc/StatementShape.java index a655d2f..b09b32c 100644 --- a/chdb-jdbc/src/main/java/org/chdb/jdbc/StatementShape.java +++ b/chdb-jdbc/src/main/java/org/chdb/jdbc/StatementShape.java @@ -29,7 +29,7 @@ *
    *
  1. {@code chdb_classify_query_n}, which parses the SQL with the engine's own parser and * settings and executes nothing. Available from engine v26.7.2-rc.2, so present on the - * pinned v26.11.0-rc.2 baseline and asked in practice. Authoritative about + * pinned v26.11.0-rc.3 baseline and asked in practice. Authoritative about * whether there is a result set, but silent on which door delivers it: * {@code CHDB_QUERY_READ_ONLY} covers a {@code SELECT} and a {@code SHOW} alike, so it * cannot tell the two entry points apart. See {@link #route(int[], String)}. diff --git a/chdb-jdbc/src/test/java/org/chdb/jdbc/StatementShapeTest.java b/chdb-jdbc/src/test/java/org/chdb/jdbc/StatementShapeTest.java index 9dafbf7..57923c4 100644 --- a/chdb-jdbc/src/test/java/org/chdb/jdbc/StatementShapeTest.java +++ b/chdb-jdbc/src/test/java/org/chdb/jdbc/StatementShapeTest.java @@ -13,7 +13,7 @@ * an entry point that refuses it -- which is what issue #12 was -- so all three are covered in * both directions. * - *

    The pinned v26.11.0-rc.2 baseline does export chdb_classify_query_n, so the scan is no + *

    The pinned v26.11.0-rc.3 baseline does export chdb_classify_query_n, so the scan is no * longer the only thing deciding. It is still what decides whether a result set is streamed or * materialized, because the classifier does not distinguish those, as well as * deciding both halves whenever the classifier is absent or refuses the text -- hence the cases diff --git a/chdb-jni/include/chdb.h b/chdb-jni/include/chdb.h index 1bef187..2eba9e9 100644 --- a/chdb-jni/include/chdb.h +++ b/chdb-jni/include/chdb.h @@ -12,7 +12,7 @@ extern "C" { #define CHDB_EXPORT __attribute__((visibility("default"))) -#define CHDB_VERSION "26.11.0-rc.2" +#define CHDB_VERSION "26.11.0-rc.3" /** * Returns the version of the linked chDB library. @@ -784,6 +784,31 @@ CHDB_EXPORT uint64_t chdb_result_bytes_written(chdb_result * result); */ CHDB_EXPORT const char * chdb_result_error(chdb_result * result); +/** + * Gets the number of columns in the result set, the same for every output + * format. With several statements, counts the last result set; for a stream, + * each fetched result carries it. + * @param result The query result handle + * @return Column count; 0 for DDL, INSERT, a failed query or NULL + */ +CHDB_EXPORT int chdb_result_column_count(chdb_result * result); + +/** + * Gets the name of a result column + * @param result The query result handle + * @param col 0-based column index + * @return Column name owned by the result, or NULL if col is out of range + */ +CHDB_EXPORT const char * chdb_result_column_name(chdb_result * result, int col); + +/** + * Gets the ClickHouse type of a result column, e.g. "Nullable(String)" + * @param result The query result handle + * @param col 0-based column index + * @return Type name owned by the result, or NULL if col is out of range + */ +CHDB_EXPORT const char * chdb_result_column_type(chdb_result * result, int col); + //===--------------------------------------------------------------------===// // Arrow Integration //===--------------------------------------------------------------------===// @@ -1108,6 +1133,46 @@ CHDB_EXPORT chdb_state chdb_classify_query_n( size_t target_database_len, chdb_query_analysis_v1 * out_analysis); +//===--------------------------------------------------------------------===// +// Connection Settings +//===--------------------------------------------------------------------===// + +/** + * Does what `SET key = value` would on this connection only, without SQL. + * + * Prefer one connection per thread; calls sharing a connection run one at a + * time, and per-call parameters never leak between them. + * + * A setting key goes through SET's constraint checks; NULL resets it. + * A param_ key stores query parameter for every later query to + * use as {name:Type}; NULL removes it. chdb_query_with_params() overrides it + * for one call. Credentials passed this way never appear in the SQL text: + * + * chdb_set(conn, "param_s3_key", "AKIA..."); + * chdb_set(conn, "param_s3_secret", "wJal..."); + * chdb_query(conn, "SELECT * FROM s3('https://b.s3.amazonaws.com/f.parquet'," + * " {s3_key:String}, {s3_secret:String})", "Arrow"); + * + * @param conn Connection to change + * @param key Setting name, or param_ + * @param value Value, or NULL to reset or remove + * @return CHDBError on a NULL argument, a closed connection, an unknown + * setting, a rejected value, or an active streaming insert; the + * connection is then unchanged + */ +CHDB_EXPORT chdb_state chdb_set(chdb_connection conn, const char * key, const char * value); + +/** + * Gets a setting's current value as getSetting() returns it, or the value of + * param_. + * @param conn Connection to read + * @param key Setting name, or param_ + * @return Value valid until the next chdb_get() on this thread; NULL on a NULL + * argument, a closed connection, an unknown setting or an unset + * parameter + */ +CHDB_EXPORT const char * chdb_get(chdb_connection conn, const char * key); + //===--------------------------------------------------------------------===// // Signal Handler Control //===--------------------------------------------------------------------===// diff --git a/chdb-jni/src/chdb_jni.cpp b/chdb-jni/src/chdb_jni.cpp index e052165..f358734 100644 --- a/chdb-jni/src/chdb_jni.cpp +++ b/chdb-jni/src/chdb_jni.cpp @@ -246,6 +246,17 @@ std::string resultError(chdb_result * result) return error == nullptr ? std::string() : std::string(error); } +bool arrowOptions(JNIEnv * env, jint flags, chdb_arrow_options & out) +{ + if ((flags & ~7) != 0) + { + throwNative(env, "invalid Arrow output option flags"); + return false; + } + out = chdb_arrow_options{(flags & 1) != 0, (flags & 2) != 0, (flags & 4) != 0}; + return true; +} + } // namespace // ====================================================================== identity @@ -434,6 +445,411 @@ Java_org_chdb_internal_ChdbNative_closeConnection(JNIEnv * env, jclass, jlong id // ====================================================================== non-streaming query +extern "C" JNIEXPORT void JNICALL +Java_org_chdb_internal_ChdbNative_queryArrowMaterialized( + JNIEnv * env, jclass, jlong connection_id, jbyteArray sql_bytes, jlong stream_address, + jint option_flags) +{ + try + { + if (stream_address == 0) + { + throwNative(env, "Arrow stream address is null"); + return; + } + auto connection = requireConnection(env, connection_id); + if (!connection) + return; + std::string sql; + if (sql_bytes == nullptr || !toBytes(env, sql_bytes, sql)) + { + if (!env->ExceptionCheck()) + throwNative(env, "SQL must not be null"); + return; + } + chdb_arrow_options options; + if (!arrowOptions(env, option_flags, options)) + return; + chdb_result * raw = nullptr; + { + std::lock_guard lock(connection->mutex); + if (connection->conn == nullptr) + { + throwNative(env, "connection was closed before the Arrow query started"); + return; + } + raw = chdb_query_arrow_n( + *connection->conn, sql.data(), sql.size(), + reinterpret_cast(static_cast(stream_address)), &options); + } + std::unique_ptr result( + raw, &chdb_destroy_query_result); + const std::string error = resultError(result.get()); + if (!error.empty()) + throwNative(env, error); + // The ArrowArrayStream release callback belongs to Java after this point. The + // metrics result is a separate object and is destroyed by the unique_ptr above. + } + catch (const std::bad_alloc &) + { + throwOutOfMemory(env, "out of memory executing an Arrow query"); + } + catch (const std::exception & e) + { + throwNative(env, std::string("Arrow query failed: ") + e.what()); + } + catch (...) + { + throwNative(env, "Arrow query failed with an unknown native exception"); + } +} + +extern "C" JNIEXPORT void JNICALL +Java_org_chdb_internal_ChdbNative_registerArrowStream( + JNIEnv * env, jclass, jlong connection_id, jbyteArray name_bytes, jlong stream_address) +{ + try + { + if (stream_address == 0) + { + throwNative(env, "Arrow stream address is null"); + return; + } + auto connection = requireConnection(env, connection_id); + if (!connection) + return; + std::string name; + if (name_bytes == nullptr || !toBytes(env, name_bytes, name)) + return; + if (name.empty() || name.find('\0') != std::string::npos) + { + throwNative(env, "Arrow registration name is empty or contains a NUL byte"); + return; + } + std::lock_guard lock(connection->mutex); + if (connection->conn == nullptr) + { + throwNative(env, "connection is closed"); + return; + } + if (chdb_arrow_scan(*connection->conn, name.c_str(), + reinterpret_cast(static_cast(stream_address))) != CHDBSuccess) + throwNative(env, "chDB rejected the Arrow stream registration"); + } + catch (const std::bad_alloc &) + { + throwOutOfMemory(env, "out of memory registering an Arrow stream"); + } + catch (const std::exception & e) + { + throwNative(env, std::string("Arrow registration failed: ") + e.what()); + } + catch (...) + { + throwNative(env, "Arrow registration failed with an unknown native exception"); + } +} + +extern "C" JNIEXPORT void JNICALL +Java_org_chdb_internal_ChdbNative_unregisterArrowStream( + JNIEnv * env, jclass, jlong connection_id, jbyteArray name_bytes) +{ + try + { + auto connection = requireConnection(env, connection_id); + if (!connection) + return; + std::string name; + if (name_bytes == nullptr || !toBytes(env, name_bytes, name)) + return; + if (name.empty() || name.find('\0') != std::string::npos) + { + throwNative(env, "Arrow registration name is empty or contains a NUL byte"); + return; + } + std::lock_guard lock(connection->mutex); + if (connection->conn == nullptr) + { + throwNative(env, "connection is closed"); + return; + } + if (chdb_arrow_unregister_table(*connection->conn, name.c_str()) != CHDBSuccess) + throwNative(env, "chDB rejected the Arrow stream unregister request"); + } + catch (const std::exception & e) + { + throwNative(env, std::string("Arrow unregister failed: ") + e.what()); + } + catch (...) + { + throwNative(env, "Arrow unregister failed with an unknown native exception"); + } +} + +namespace +{ +struct ArrowStreamHandle : HandleBase +{ + ArrowStreamHandle() : HandleBase(kKindArrowStream) { } + ~ArrowStreamHandle() override { closeNow(); } + void closeNow() + { + if (result != nullptr) + { + chdb_destroy_query_result(result); + result = nullptr; + } + } + chdb_result * result = nullptr; + std::shared_ptr owner; +}; + +std::shared_ptr requireArrowStream(JNIEnv * env, jlong id) +{ + auto base = HandleRegistry::instance().get(id, kKindArrowStream); + if (!base) + { + throwNative(env, "Arrow query handle " + std::to_string(id) + " is not open"); + return nullptr; + } + return std::static_pointer_cast(base); +} +} // namespace + +extern "C" JNIEXPORT jlong JNICALL +Java_org_chdb_internal_ChdbNative_arrowStreamOpen( + JNIEnv * env, jclass, jlong connection_id, jbyteArray sql_bytes, jint option_flags) +{ + try + { + auto connection = requireConnection(env, connection_id); + if (!connection) + return 0; + std::string sql; + if (sql_bytes == nullptr || !toBytes(env, sql_bytes, sql)) + return 0; + chdb_arrow_options options; + if (!arrowOptions(env, option_flags, options)) + return 0; + chdb_result * raw = nullptr; + { + std::lock_guard lock(connection->mutex); + if (connection->conn == nullptr) + { + throwNative(env, "connection was closed before Arrow stream open"); + return 0; + } + raw = chdb_stream_query_arrow_n(*connection->conn, sql.data(), sql.size(), &options); + } + std::unique_ptr result( + raw, &chdb_destroy_query_result); + const std::string error = resultError(result.get()); + if (!error.empty()) + { + throwNative(env, error); + return 0; + } + auto handle = std::make_shared(); + handle->owner = connection; + handle->result = result.release(); + return static_cast(HandleRegistry::instance().insert(handle)); + } + catch (const std::bad_alloc &) + { + throwOutOfMemory(env, "out of memory starting an Arrow query"); + return 0; + } + catch (const std::exception & e) + { + throwNative(env, std::string("Arrow stream open failed: ") + e.what()); + return 0; + } +} + +extern "C" JNIEXPORT void JNICALL +Java_org_chdb_internal_ChdbNative_arrowStreamFetch( + JNIEnv * env, jclass, jlong connection_id, jlong stream_id, jlong out_address) +{ + try + { + if (out_address == 0) + { + throwNative(env, "Arrow output stream address is null"); + return; + } + auto stream = requireArrowStream(env, stream_id); + if (!stream) + return; + auto connection = requireConnection(env, connection_id); + if (!connection) + return; + if (stream->owner != connection) + { + throwNative(env, "Arrow query belongs to another connection"); + return; + } + std::lock_guard stream_lock(stream->mutex); + std::lock_guard conn_lock(connection->mutex); + if (stream->result == nullptr || connection->conn == nullptr) + { + throwNative(env, "Arrow query or connection is closed"); + return; + } + const auto state = chdb_stream_fetch_arrow( + *connection->conn, stream->result, + reinterpret_cast(static_cast(out_address))); + if (state != CHDBSuccess) + { + const std::string error = resultError(stream->result); + throwNative(env, error.empty() ? "Arrow stream fetch failed" : error); + } + } + catch (const std::exception & e) + { + throwNative(env, std::string("Arrow stream fetch failed: ") + e.what()); + } +} + +extern "C" JNIEXPORT void JNICALL +Java_org_chdb_internal_ChdbNative_arrowStreamCancel( + JNIEnv * env, jclass, jlong connection_id, jlong stream_id) +{ + try + { + auto stream = requireArrowStream(env, stream_id); + if (!stream) + return; + auto connection = requireConnection(env, connection_id); + if (!connection) + return; + if (stream->owner != connection) + { + throwNative(env, "Arrow query belongs to another connection"); + return; + } + std::lock_guard stream_lock(stream->mutex); + std::lock_guard conn_lock(connection->mutex); + if (stream->result != nullptr && connection->conn != nullptr) + chdb_stream_cancel_query(*connection->conn, stream->result); + } + catch (const std::exception & e) + { + throwNative(env, std::string("Arrow stream cancel failed: ") + e.what()); + } +} + +extern "C" JNIEXPORT void JNICALL +Java_org_chdb_internal_ChdbNative_arrowStreamClose(JNIEnv * env, jclass, jlong stream_id) +{ + try + { + auto base = HandleRegistry::instance().remove(stream_id, kKindArrowStream); + if (!base) + return; + auto stream = std::static_pointer_cast(base); + std::lock_guard lock(stream->mutex); + stream->closeNow(); + } + catch (const std::exception & e) + { + throwNative(env, std::string("Arrow stream close failed: ") + e.what()); + } +} + +extern "C" JNIEXPORT jlong JNICALL +Java_org_chdb_internal_ChdbNative_insertArrowStream( + JNIEnv * env, jclass, jlong connection_id, jbyteArray table_bytes, jlong stream_address) +{ + try + { + if (stream_address == 0) + { + throwNative(env, "Arrow input stream address is null"); + return 0; + } + auto connection = requireConnection(env, connection_id); + if (!connection) + return 0; + std::string table; + if (table_bytes == nullptr || !toBytes(env, table_bytes, table)) + return 0; + // core appends dest_table to SQL. Accept only simple, optionally qualified ASCII + // identifiers and quote each segment here; no user SQL or SETTINGS cross this seam. + std::string quoted; + size_t start = 0; + unsigned segments = 0; + while (start < table.size()) + { + if (++segments > 2) + break; + size_t end = table.find('.', start); + if (end == std::string::npos) + end = table.size(); + if (end == start || !(table[start] == '_' || + (table[start] >= 'A' && table[start] <= 'Z') || + (table[start] >= 'a' && table[start] <= 'z'))) + break; + bool valid = true; + for (size_t i = start + 1; i < end; ++i) + { + const char c = table[i]; + if (!(c == '_' || (c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'))) + valid = false; + } + if (!valid) + break; + if (!quoted.empty()) + quoted += '.'; + quoted += '`'; + quoted.append(table, start, end - start); + quoted += '`'; + if (end == table.size()) + { + start = end; + break; + } + start = end + 1; + } + if (table.empty() || start != table.size() || table.back() == '.' || segments > 2) + { + throwNative(env, "Arrow destination must be a simple table or database.table identifier"); + return 0; + } + chdb_result * raw = nullptr; + { + std::lock_guard lock(connection->mutex); + if (connection->conn == nullptr) + { + throwNative(env, "connection closed before Arrow insert"); + return 0; + } + raw = chdb_insert_arrow_stream( + *connection->conn, quoted.c_str(), + reinterpret_cast(static_cast(stream_address)), nullptr); + } + std::unique_ptr result( + raw, &chdb_destroy_query_result); + const std::string error = resultError(result.get()); + if (!error.empty()) + { + throwNative(env, error); + return 0; + } + return static_cast(chdb_result_rows_written(result.get())); + } + catch (const std::bad_alloc &) + { + throwOutOfMemory(env, "out of memory inserting an Arrow stream"); + return 0; + } + catch (const std::exception & e) + { + throwNative(env, std::string("Arrow insert failed: ") + e.what()); + return 0; + } +} + extern "C" JNIEXPORT jlong JNICALL Java_org_chdb_internal_ChdbNative_query( JNIEnv * env, @@ -622,7 +1038,7 @@ Java_org_chdb_internal_ChdbNative_classifyQuery(JNIEnv * env, jclass, jlong conn { try { - // Optional symbol: present from v26.7.2-rc.2, so on the pinned v26.11.0-rc.2 baseline, + // Optional symbol: present from v26.7.2-rc.2, so on the pinned v26.11.0-rc.3 baseline, // absent on anything older. Null tells the Java side to use its own statement-shape // heuristic rather than failing the query. const auto classify = optionalApi().classify_query_n; diff --git a/chdb-jni/src/chdb_jni_handles.h b/chdb-jni/src/chdb_jni_handles.h index c6ccc4e..bca7998 100644 --- a/chdb-jni/src/chdb_jni_handles.h +++ b/chdb-jni/src/chdb_jni_handles.h @@ -24,13 +24,14 @@ namespace chdb_jni // Bumped only on an incompatible change to the Java <-> shim contract. Must match // ChdbNative.JNI_ABI_VERSION on the Java side and chdb.jni.abi.version in the poms. -constexpr int32_t kJniAbiVersion = 2; +constexpr int32_t kJniAbiVersion = 3; enum HandleKind : int32_t { kKindConnection = 1, kKindResult = 2, kKindRowBinary = 4, + kKindArrowStream = 5, }; // Lock order, for every mutex in the shim. Acquire left to right, never right to left. @@ -144,6 +145,7 @@ class HandleRegistry counts_.emplace(kKindConnection, 0); counts_.emplace(kKindResult, 0); counts_.emplace(kKindRowBinary, 0); + counts_.emplace(kKindArrowStream, 0); } mutable std::mutex mutex_; diff --git a/chdb-jni/src/chdb_optional_api.h b/chdb-jni/src/chdb_optional_api.h index 13c9e4f..fed0e1d 100644 --- a/chdb-jni/src/chdb_optional_api.h +++ b/chdb-jni/src/chdb_optional_api.h @@ -1,6 +1,6 @@ // chDB C API entry points the shim resolves at runtime rather than linking. // -// The baseline is chdb-core v26.11.0-rc.2 (work plan section 2.1). Both of these arrived in +// The baseline is chdb-core v26.11.0-rc.3 (work plan section 2.1). Both of these arrived in // v26.7.2-rc.2 -- neither is in v26.7.0 or v26.7.1-rc.1, whose export lists and headers have // neither name -- and both are exported by the current baseline, checked in its release // library rather than in an export list: @@ -44,7 +44,7 @@ struct QueryAnalysisV1 uint32_t query_class; }; -// chdb_query_class values, as of v26.11.0-rc.2. Kept in sync with ChdbNative.classifyQuery's +// chdb_query_class values, as of v26.11.0-rc.3. Kept in sync with ChdbNative.classifyQuery's // documented return contract. enum QueryClass : uint32_t { diff --git a/chdb-native-linux-aarch64-gnu/pom.xml b/chdb-native-linux-aarch64-gnu/pom.xml index ddf224d..64331af 100644 --- a/chdb-native-linux-aarch64-gnu/pom.xml +++ b/chdb-native-linux-aarch64-gnu/pom.xml @@ -7,7 +7,7 @@ com.clickhouse.chdb chdb-java-parent - 1.0.0-SNAPSHOT + 1.0.0-rc.3-SNAPSHOT chdb-native-linux-aarch64-gnu diff --git a/chdb-native-linux-x86_64-gnu/pom.xml b/chdb-native-linux-x86_64-gnu/pom.xml index aacb9ff..5dc19c2 100644 --- a/chdb-native-linux-x86_64-gnu/pom.xml +++ b/chdb-native-linux-x86_64-gnu/pom.xml @@ -7,7 +7,7 @@ com.clickhouse.chdb chdb-java-parent - 1.0.0-SNAPSHOT + 1.0.0-rc.3-SNAPSHOT chdb-native-linux-x86_64-gnu diff --git a/chdb-native-macos-aarch64/pom.xml b/chdb-native-macos-aarch64/pom.xml index 492df9d..7f0f76b 100644 --- a/chdb-native-macos-aarch64/pom.xml +++ b/chdb-native-macos-aarch64/pom.xml @@ -7,7 +7,7 @@ com.clickhouse.chdb chdb-java-parent - 1.0.0-SNAPSHOT + 1.0.0-rc.3-SNAPSHOT chdb-native-macos-aarch64 diff --git a/chdb-native-macos-x86_64/pom.xml b/chdb-native-macos-x86_64/pom.xml index 586d521..8992452 100644 --- a/chdb-native-macos-x86_64/pom.xml +++ b/chdb-native-macos-x86_64/pom.xml @@ -7,7 +7,7 @@ com.clickhouse.chdb chdb-java-parent - 1.0.0-SNAPSHOT + 1.0.0-rc.3-SNAPSHOT chdb-native-macos-x86_64 diff --git a/docs/memory.md b/docs/memory.md index 08beaa6..09d393b 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -192,7 +192,7 @@ Assert this at the end of a test suite. Every non-zero count is a `ResultSet`, ` `chdb_shutdown()` joins every engine thread, for a host that runs its own teardown — global destructors, a finalizing runtime, a sanitizer exit handler — that would otherwise race them. -On the pinned v26.11.0-rc.2 that means all of them, jemalloc's background threads included, +On the previously measured v26.11.0-rc.2 that means all of them, jemalloc's background threads included, and it succeeds after `MergeTree` use as well. Through v26.11.0-rc.1 it returned 1 once a `MergeTree` table had been used, and left jemalloc's threads running; see [upstream findings §9](upstream-findings.md). @@ -206,7 +206,7 @@ int result = org.chdb.internal.ChdbNative.shutdown(); A process that simply exits does not need to call it while the driver's shutdown hook is installed, which is the default. A host that turns the hook off should close its connections and call it before exiting, rather than leave the engine to tear itself down from its own exit -handlers with connections open. The pinned 26.11.0-rc.2 engine exports it, so 2 is what an older engine returns; see +handlers with connections open. The pinned 26.11.0-rc.3 engine exports it, so 2 is what an older engine returns; see [upstream findings §6](upstream-findings.md). The driver's shutdown hook already calls it after closing the connections it knows about, so a host that leaves the hook installed does not need to. diff --git a/docs/publishing.md b/docs/publishing.md index 30a4b4c..60d06b8 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -252,8 +252,8 @@ Apache-2.0, `LICENSE` in the repository, declared in the POM. Nothing to do. ### The redistributed engine Each platform package ships a ~326 MB `libchdb.so` verbatim from a chdb-core release. It -statically links ClickHouse and **1082 third-party components** (as of v26.11.0-rc.2, the same -as v26.11.0-rc.1; 968 through v26.7.3). +statically links ClickHouse and **1081 third-party components** on v26.11.0-rc.3 +(1082 on v26.11.0-rc.1/rc.2; 968 through v26.7.3). That number, and the licence of each one, comes from the engine itself: @@ -294,6 +294,10 @@ The fourth, for v26.11.0-rc.2 — chdb-core #312 on the same ClickHouse baseline byte-identical to v26.11.0-rc.1's again: the same 1082 entries and the same SHA-256, regenerated from the new engine rather than copied. +The fifth, for v26.11.0-rc.3, has 1081 entries: `gdeflate` (Apache) is absent. +No copyleft entry changed. Generated from the published macOS arm64 binary and checked +against that binary by `LicenseInventoryIT`; the other three platform binaries are checked in CI. + ### The one open question Reading the inventory rather than guessing at it: most components are permissive, and several diff --git a/docs/release-readiness.md b/docs/release-readiness.md index 26575aa..9a1fce6 100644 --- a/docs/release-readiness.md +++ b/docs/release-readiness.md @@ -340,7 +340,8 @@ statically links the ClickHouse tree. contrib submodules by scanning their licence files and found seven LGPL and eight GPL. That method has since been replaced by querying the binary we actually ship — `SELECT DISTINCT library_name, license_type FROM system.licenses`, which ClickHouse generates -at build time — and the answer is 1082 components on v26.11.0-rc.1 and v26.11.0-rc.2 (968 through v26.7.3), of +at build time — and the answer is 1081 components on v26.11.0-rc.3, 1082 on +v26.11.0-rc.1/rc.2 (968 through v26.7.3), of which eleven carry a copyleft licence with no permissive alternative, in two families rather than one. A filename scan calls every dual-licensed component GPL and describes a source checkout rather than an artifact, so the diff --git a/docs/upstream-findings.md b/docs/upstream-findings.md index 66a1a39..7768b19 100644 --- a/docs/upstream-findings.md +++ b/docs/upstream-findings.md @@ -233,7 +233,7 @@ tag v26.7.0, stale constant included, rather than to the header inside the relea **Consequence for this binding:** `chdb-jni/include/chdb.h` is now taken from the release tarball, which is the artifact this binding is built beside and ships against, and -`shim.built.against.engine.header` reads `26.11.0-rc.2` accordingly. The rule is recorded in +`shim.built.against.engine.header` reads `26.11.0-rc.3` in this development branch. The rule is recorded in `chdb-jni/CMakeLists.txt` where the vendored header is included: copy it from the tarball, not from the source tree, and do not edit it — its value is being byte-identical to what upstream shipped, which is what makes the next comparison able to detect drift. This finding was made @@ -366,8 +366,8 @@ recognize. Validating values without validating names catches the rarer mistake. **Severity: expected; still handled as optional symbols.** Neither is in v26.7.0 or v26.7.1-rc.1. Both are in v26.7.2-rc.2, in v26.7.3, in v26.11.0-rc.1 -and in v26.11.0-rc.2, which is now the baseline — exported by all four of its platform -libraries. +and in v26.11.0-rc.2 — exported by all four of its platform libraries. The current +rc.3 baseline must be checked by the new CI matrix. Checked in the release library rather than in the source's export list, because the two are not the same claim: diff --git a/docs/v1-progress.md b/docs/v1-progress.md index 087c10b..a35b2be 100644 --- a/docs/v1-progress.md +++ b/docs/v1-progress.md @@ -13,11 +13,10 @@ macOS arm64 and x86_64 — each running the full suite on **Java 11, 17, 21 and sanitizer runs on one Linux and one macOS toolchain. Sixteen platform-and-JDK combinations, all green. -The baseline has since moved four times, to **engine 26.7.2-rc.2** (issue #8), then to -**engine 26.7.3**, to **engine 26.11.0-rc.1**, and now to **engine 26.11.0-rc.2**. The first two -moves were verified on macOS arm64 only; the two 26.11 moves were verified locally on Linux -x86_64 only. So the sixteen-way result above describes 26.7.0, and the other platforms are -unverified on the current engine until CI says otherwise. +The baseline subsequently moved to **engine 26.7.2-rc.2** (issue #8), **26.7.3**, +**26.11.0-rc.1** and **26.11.0-rc.2**. This development branch now targets +**26.11.0-rc.3** for Arrow registration fixes. The sixteen-way result above describes +26.7.0; rc.3 still needs the four-platform CI matrix before making that claim. 26.7.3 is a **stable** chdb-core release, which cleared the [work plan §4.3 ](../CHDB_JAVA_V1_WORK_PLAN.md) bar that 26.7.2-rc.2 could not: a binding built on an engine @@ -37,6 +36,10 @@ versions can be cut while it is the baseline, and the stable release has to be p has to be pinned before `1.0.0`. The signal-handler fix holds on it too: `measure` reports zero and `stress` survives 6 of 6 on Linux x86_64. +26.11.0-rc.3 includes chdb-core #323's Arrow registration-schema and failed array-wrapper +cleanup fixes. The optional `chdb-arrow` module is under development against this published +core RC; local macOS arm64 tests do not substitute for its four-platform CI run. + --- ## Phase 0 — Freeze V1 decisions and upstream dependencies @@ -48,7 +51,7 @@ zero and `stress` survives 6 of 6 on Linux x86_64. | ✅ | Four glibc/macOS platforms | `Platform`, with musl detected and refused | | ✅ | Direct JNI is the mainline, ADBC the experiment | no Arrow Java dependency anywhere | | ✅ | Engine release, dynamic-library checksum and symbol set pinned | `scripts/engine.properties`, SHA-256 per platform from the GitHub release digests | -| ✅ | Consistency check across `chdb_version()`, tag and header version | all three agree on v26.11.0-rc.2. The `26.5.1-rc.3` constant blamed on upstream turned out to be ours: the vendored header had been copied from chdb-core's source tree instead of its release tarball — [findings §2](upstream-findings.md). The v26.7.3 bump re-confirmed that rule the hard way: `programs/local/chdb.h` at the git tag still says `26.7.2`, while the header in the release tarball says `26.7.3`; at the v26.11.0-rc.1 tag it says `26.11.0-dev`. Still enforced against `engine.properties` rather than the header | +| 🟡 | Consistency check across `chdb_version()`, tag and header version | all three agreed on v26.11.0-rc.2. The rc.3 macOS arm64 release tarball header is now vendored and reports `26.11.0-rc.3`; runtime and the other platforms need CI confirmation. The `26.5.1-rc.3` constant blamed on upstream turned out to be ours: the vendored header had been copied from chdb-core's source tree instead of its release tarball — [findings §2](upstream-findings.md). Still enforced against `engine.properties` rather than the header | | ✅ | C API split into required and optional symbols | 18 required, linker- and load-checked; `chdb_classify_query_n` and `chdb_shutdown` optional via `dlsym`, and present on the current baseline | | ⬜ | C ABI version and compatibility promise confirmed with Core | needs upstream agreement | | ✅ | Signal-handler requirement raised upstream | reproduced, measured and written up in [findings §1](upstream-findings.md), filed as chdb-core #221, and fixed by chdb-core #224 in engine v26.7.3 — the shim's guard now reports no handler to repair, and `run-signal-window-test.sh measure` reports zero | diff --git a/licenses/engine-third-party-26.11.0-rc.3.tsv b/licenses/engine-third-party-26.11.0-rc.3.tsv new file mode 100644 index 0000000..9dcad01 --- /dev/null +++ b/licenses/engine-third-party-26.11.0-rc.3.tsv @@ -0,0 +1,1081 @@ +abseil-cpp Apache +addr2line Apache-2.0 OR MIT +adler2 0BSD OR MIT OR Apache-2.0 +aead MIT OR Apache-2.0 +aes MIT OR Apache-2.0 +aes-gcm Apache-2.0 OR MIT +ahash MIT OR Apache-2.0 +aho-corasick Unlicense OR MIT +ai-sdk-cpp Apache +alloc-no-stdlib BSD-3-Clause +alloc-stdlib BSD-3-Clause +alloca MIT +allocator-api2 MIT OR Apache-2.0 +alp Apache-2.0 +ambient-authority Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +AMQP-CPP Apache +android_system_properties MIT OR Apache-2.0 +android_system_properties MIT/Apache-2.0 +anes MIT OR Apache-2.0 +anstream MIT OR Apache-2.0 +anstyle MIT OR Apache-2.0 +anstyle-parse MIT OR Apache-2.0 +anstyle-query MIT OR Apache-2.0 +anstyle-wincon MIT OR Apache-2.0 +antlr4-cpp-runtime BSD 2-clause +anyhow MIT OR Apache-2.0 +approx Apache-2.0 +ar_archive_writer Apache-2.0 WITH LLVM-exception +arbitrary MIT OR Apache-2.0 +arboard MIT OR Apache-2.0 +arbtest MIT OR Apache-2.0 +arc-swap MIT OR Apache-2.0 +arcref MIT +ariadne MIT +arrayref BSD-2-Clause +arrayvec MIT OR Apache-2.0 +arrow Apache +arrow Apache-2.0 +arrow-arith Apache-2.0 +arrow-array Apache-2.0 +arrow-array Apache-2.0 AND MIT +arrow-buffer Apache-2.0 +arrow-cast Apache-2.0 +arrow-csv Apache-2.0 +arrow-data Apache-2.0 +arrow-ipc Apache-2.0 +arrow-json Apache-2.0 +arrow-ord Apache-2.0 +arrow-row Apache-2.0 +arrow-schema Apache-2.0 +arrow-select Apache-2.0 +arrow-string Apache-2.0 +ascii Apache-2.0 OR MIT +async-channel Apache-2.0 OR MIT +async-executor Apache-2.0 OR MIT +async-fs Apache-2.0 OR MIT +async-io Apache-2.0 OR MIT +async-lock Apache-2.0 OR MIT +async-net Apache-2.0 OR MIT +async-process Apache-2.0 OR MIT +async-signal Apache-2.0 OR MIT +async-stream MIT +async-stream-impl MIT +async-task Apache-2.0 OR MIT +async-trait MIT OR Apache-2.0 +atoi MIT +atomic-waker Apache-2.0 OR MIT +autocfg Apache-2.0 OR MIT +avro Apache +aws Apache +aws-c-auth Apache +aws-c-cal Apache +aws-c-common Apache +aws-c-compression Apache +aws-c-event-stream Apache +aws-c-http Apache +aws-c-io Apache +aws-c-mqtt Apache +aws-c-s3 Apache +aws-c-sdkutils Apache +aws-checksums Apache +aws-crt-cpp Apache +aws-lc-rs ISC AND (Apache-2.0 OR ISC) +aws-lc-sys ISC AND (Apache-2.0 OR ISC) AND Apache-2.0 AND MIT AND BSD-3-Clause AND (Apache-2.0 OR ISC OR MIT) AND (Apache-2.0 OR ISC OR MIT-0) +aws-lc-sys ISC AND (Apache-2.0 OR ISC) AND OpenSSL +axum MIT +axum-core MIT +azure MIT +backtrace MIT OR Apache-2.0 +base64 MIT OR Apache-2.0 +base64ct Apache-2.0 OR MIT +bech32 MIT +beef MIT OR Apache-2.0 +better_io Apache-2.0 +bincode MIT +bindgen BSD-3-Clause +bit-set Apache-2.0 OR MIT +bit-vec Apache-2.0 OR MIT +bitflags MIT OR Apache-2.0 +bitflags MIT/Apache-2.0 +bitvec MIT +blake3 CC0-1.0 OR Apache-2.0 OR Apache-2.0 WITH LLVM-exception +block-buffer MIT OR Apache-2.0 +block-padding MIT OR Apache-2.0 +blocking Apache-2.0 OR MIT +boost Boost +brotli BSD-3-Clause AND MIT +brotli MIT +brotli-decompressor BSD-3-Clause/MIT +bs-thread-pool MIT +bstr MIT OR Apache-2.0 +bumpalo MIT OR Apache-2.0 +by_address MIT OR Apache-2.0 +bytemuck Zlib OR Apache-2.0 OR MIT +byteorder Unlicense OR MIT +bytes MIT +bytesize Apache-2.0 +bzip2 MIT/Apache-2.0 +bzip2 bzip2 +bzip2-sys MIT/Apache-2.0 +c-ares MIT +camino MIT OR Apache-2.0 +cap-fs-ext Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +cap-net-ext Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +cap-primitives Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +cap-std Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +cap-time-ext Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +capnproto MIT +capstone MIT +capstone-sys MIT +cargo-platform MIT OR Apache-2.0 +cargo_metadata MIT +cast MIT OR Apache-2.0 +castaway MIT +cbc MIT OR Apache-2.0 +cbindgen MPL-2.0 +cc MIT OR Apache-2.0 +cccl Apache +cctz Apache +cexpr Apache-2.0/MIT +cfg-if MIT OR Apache-2.0 +cfg-if MIT/Apache-2.0 +cfg_aliases MIT +chacha20 MIT OR Apache-2.0 +chdig MIT +chrono MIT OR Apache-2.0 +chrono-tz MIT OR Apache-2.0 +chrono-tz-build MIT OR Apache-2.0 +chumsky MIT +chunked_transfer MIT OR Apache-2.0 +ciborium Apache-2.0 +ciborium-io Apache-2.0 +ciborium-ll Apache-2.0 +cipher MIT OR Apache-2.0 +cityhash-rs MIT OR Apache-2.0 +cityhash102 MIT +clang-sys Apache-2.0 +clap MIT OR Apache-2.0 +clap_builder MIT OR Apache-2.0 +clap_complete MIT OR Apache-2.0 +clap_derive MIT OR Apache-2.0 +clap_lex MIT OR Apache-2.0 +cld2 Apache +clickhouse MIT OR Apache-2.0 +clickhouse-derive MIT OR Apache-2.0 +clickhouse-rs-1.1.0 MIT +clickhouse-rs-cityhash-sys MIT +clickstack MIT +clipboard-win BSL-1.0 +cmake MIT OR Apache-2.0 +cobs MIT OR Apache-2.0 +codespan-reporting Apache-2.0 +colorchoice MIT OR Apache-2.0 +combine MIT +comfy-table MIT +compact_str MIT +concurrent-queue Apache-2.0 OR MIT +console-api MIT +console-subscriber MIT +const-random MIT OR Apache-2.0 +const-random-macro MIT OR Apache-2.0 +const_for MIT +constant_time_eq CC0-1.0 +constant_time_eq CC0-1.0 OR MIT-0 OR Apache-2.0 +convert_case MIT +core-foundation MIT OR Apache-2.0 +core-foundation-sys MIT OR Apache-2.0 +corrosion MIT +cpp_demangle MIT OR Apache-2.0 +cppkafka BSD 2-clause +cpufeatures MIT OR Apache-2.0 +crc MIT OR Apache-2.0 +crc-catalog MIT OR Apache-2.0 +crc32-s390x Apache +crc32-vpmsum Apache +crc32c BSD 3-clause +crc32fast MIT OR Apache-2.0 +criterion Apache-2.0 OR MIT +criterion-plot Apache-2.0 OR MIT +criterion-plot MIT/Apache-2.0 +critical-section MIT OR Apache-2.0 +croaring Apache +crossbeam MIT OR Apache-2.0 +crossbeam-channel MIT OR Apache-2.0 +crossbeam-deque MIT OR Apache-2.0 +crossbeam-epoch MIT OR Apache-2.0 +crossbeam-queue MIT OR Apache-2.0 +crossbeam-utils MIT OR Apache-2.0 +crossterm MIT +crossterm_winapi MIT +crunchy MIT +crypto-common MIT OR Apache-2.0 +csv Unlicense/MIT +csv-core Unlicense/MIT +ctr MIT OR Apache-2.0 +cty MIT OR Apache-2.0 +cucascade Apache +cuCollections Apache +cudf Apache +custom-labels Apache-2.0 +cxx MIT OR Apache-2.0 +cxx-build MIT OR Apache-2.0 +cxxbridge-cmd MIT OR Apache-2.0 +cxxbridge-flags MIT OR Apache-2.0 +cxxbridge-macro MIT OR Apache-2.0 +cyrus-sasl BSD 2-clause +darling MIT +darling_core MIT +darling_macro MIT +darts-clone BSD +dashmap MIT +datasketches-cpp Apache +datatest-stable MIT OR Apache-2.0 +debugid Apache-2.0 +defer-drop MPL-2.0 +defmt MIT OR Apache-2.0 +defmt-macros MIT OR Apache-2.0 +defmt-parser MIT OR Apache-2.0 +delta-kernel-rs Apache +der Apache-2.0 OR MIT +deranged MIT OR Apache-2.0 +derive_arbitrary MIT/Apache-2.0 +derive_builder MIT OR Apache-2.0 +derive_builder_core MIT OR Apache-2.0 +derive_builder_macro MIT OR Apache-2.0 +derive_more MIT +derive_more-impl MIT +des MIT OR Apache-2.0 +dhat MIT OR Apache-2.0 +diff MIT OR Apache-2.0 +digest MIT OR Apache-2.0 +directories-next MIT OR Apache-2.0 +dirs-next MIT OR Apache-2.0 +dirs-sys-next MIT OR Apache-2.0 +dispatch2 Zlib OR Apache-2.0 OR MIT +displaydoc MIT OR Apache-2.0 +dlmalloc MIT/Apache-2.0 +dlpack Apache +dmlc-core Apache +dns-lookup MIT/Apache-2.0 +document-features MIT OR Apache-2.0 +double-conversion BSD 3-clause +driftsort Apache +dtype_dispatch Apache-2.0 +dunce CC0-1.0 OR MIT-0 OR Apache-2.0 +dyn-clone MIT OR Apache-2.0 +easy-smt MIT OR Apache-2.0 +egg MIT +either MIT OR Apache-2.0 +embedded-io MIT OR Apache-2.0 +encoding_rs (Apache-2.0 OR MIT) AND BSD-3-Clause +enum-as-inner MIT/Apache-2.0 +enum-iterator 0BSD +enum-iterator-derive 0BSD +env_filter MIT OR Apache-2.0 +env_home MIT OR Apache-2.0 +env_logger MIT OR Apache-2.0 +equivalent Apache-2.0 OR MIT +errno MIT OR Apache-2.0 +error-code BSL-1.0 +escape8259 MIT +event-listener Apache-2.0 OR MIT +event-listener-strategy Apache-2.0 OR MIT +ext-trait Zlib OR MIT OR Apache-2.0 +ext-trait-proc_macros Zlib OR MIT OR Apache-2.0 +extension-traits Zlib OR MIT OR Apache-2.0 +fallible-iterator MIT/Apache-2.0 +fancy-regex MIT +fast_float MIT +fastlanes Apache-2.0 +fastops MIT +fastrand Apache-2.0 OR MIT +fd-lock MIT OR Apache-2.0 +file-per-thread-logger Apache-2.0 WITH LLVM-exception +filecheck Apache-2.0 WITH LLVM-exception +filedescriptor MIT +filetime MIT/Apache-2.0 +find-msvc-tools MIT OR Apache-2.0 +fixedbitset MIT/Apache-2.0 +flagset Apache-2.0 +flamelens MIT +flatbuffers Apache +flatbuffers Apache-2.0 +flate2 MIT OR Apache-2.0 +flexi_logger MIT OR Apache-2.0 +fmtlib MIT +fnv Apache-2.0 / MIT +foldhash Zlib +foreign-types MIT/Apache-2.0 +foreign-types-shared MIT/Apache-2.0 +form_urlencoded MIT OR Apache-2.0 +fortanix-sgx-abi MPL-2.0 +FP16 MIT +fs-set-times Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +fs_extra MIT +fslock MIT +fsst-rs Apache-2.0 +funty MIT +futures MIT OR Apache-2.0 +futures-channel MIT OR Apache-2.0 +futures-core MIT OR Apache-2.0 +futures-executor MIT OR Apache-2.0 +futures-io MIT OR Apache-2.0 +futures-lite Apache-2.0 OR MIT +futures-macro MIT OR Apache-2.0 +futures-sink MIT OR Apache-2.0 +futures-task MIT OR Apache-2.0 +futures-timer MIT/Apache-2.0 +futures-util MIT OR Apache-2.0 +fuzzy-matcher MIT +fxprof-processed-profile MIT OR Apache-2.0 +g2gen MIT/Apache-2.0 +g2p MIT/Apache-2.0 +g2poly MIT/Apache-2.0 +gdbstub MIT OR Apache-2.0 +gdbstub_arch MIT OR Apache-2.0 +generic-array MIT +gethostname Apache-2.0 +getopts MIT OR Apache-2.0 +getrandom MIT OR Apache-2.0 +ghash Apache-2.0 OR MIT +gimli MIT OR Apache-2.0 +glob MIT OR Apache-2.0 +google-benchmark Apache +google-cloud-cpp Apache +google-protobuf BSD 3-clause +googletest BSD 3-clause +grpc Apache +gzip-header MIT/Apache-2.0 +h2 MIT +h3 Apache +half MIT OR Apache-2.0 +hashbrown MIT OR Apache-2.0 +hdfs-native Apache-2.0 +hdfs-native-object-store Apache-2.0 +hdrhistogram MIT/Apache-2.0 +heck MIT OR Apache-2.0 +heks MIT +hermit-abi MIT OR Apache-2.0 +hex MIT OR Apache-2.0 +hive-metastore Apache +hmac MIT OR Apache-2.0 +home MIT OR Apache-2.0 +hostname MIT +http MIT OR Apache-2.0 +http-body MIT +http-body-util MIT +httparse MIT OR Apache-2.0 +httpdate MIT OR Apache-2.0 +httpdate MIT/Apache-2.0 +humansize MIT/Apache-2.0 +humantime MIT OR Apache-2.0 +humantime MIT/Apache-2.0 +hyper MIT +hyper-rustls Apache-2.0 OR ISC OR MIT +hyper-timeout MIT OR Apache-2.0 +hyper-tls MIT/Apache-2.0 +hyper-util MIT +iana-time-zone MIT OR Apache-2.0 +iana-time-zone-haiku MIT OR Apache-2.0 +icu Public Domain +icu_collections Unicode-3.0 +icu_locale_core Unicode-3.0 +icu_locid Unicode-3.0 +icu_locid_transform Unicode-3.0 +icu_locid_transform_data Unicode-3.0 +icu_normalizer Unicode-3.0 +icu_normalizer_data Unicode-3.0 +icu_properties Unicode-3.0 +icu_properties_data Unicode-3.0 +icu_provider Unicode-3.0 +icu_provider_macros Unicode-3.0 +icudata Public Domain +id-arena MIT/Apache-2.0 +ident_case MIT/Apache-2.0 +idna MIT +idna MIT OR Apache-2.0 +idna_adapter Apache-2.0 OR MIT +indexmap Apache-2.0 OR MIT +indoc MIT OR Apache-2.0 +inout MIT OR Apache-2.0 +instability MIT +instant BSD-3-Clause +integer-encoding MIT +inventory MIT OR Apache-2.0 +io-extras Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +io-lifetimes Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +io-uring MIT OR Apache-2.0 +ipcrypt2 ISC +ipnet MIT OR Apache-2.0 +ipnsort Apache +is-terminal MIT +is_terminal_polyfill MIT OR Apache-2.0 +isa-l BSD 3-clause +itertools MIT OR Apache-2.0 +itertools MIT/Apache-2.0 +itoa MIT OR Apache-2.0 +ittapi GPL-2.0-only OR BSD-3-Clause +ittapi-sys GPL-2.0-only OR BSD-3-Clause +jemalloc BSD 2-clause +Jieba-CPP MIT +jiff Unlicense OR MIT +jiff-core Unlicense OR MIT +jiff-static Unlicense OR MIT +jiff-tzdb Unlicense OR MIT +jiff-tzdb-platform Unlicense OR MIT +jni MIT OR Apache-2.0 +jni-macros MIT OR Apache-2.0 +jni-sys MIT OR Apache-2.0 +jni-sys-macros MIT OR Apache-2.0 +jobserver MIT OR Apache-2.0 +js-sys MIT OR Apache-2.0 +json-from-wast Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +jwt-cpp MIT +kanal MIT +kasuari MIT OR Apache-2.0 +krb5 BSD +kvikio Apache +lasso MIT OR Apache-2.0 +lazy_static MIT OR Apache-2.0 +lazy_static MIT/Apache-2.0 +lazycell MIT/Apache-2.0 +leb128 Apache-2.0/MIT +leb128fmt MIT OR Apache-2.0 +lemmagen-c LGPL +lending-iterator Zlib OR MIT OR Apache-2.0 +lending-iterator-proc_macros Zlib OR MIT OR Apache-2.0 +lexical-core MIT/Apache-2.0 +lexical-parse-float MIT/Apache-2.0 +lexical-parse-integer MIT/Apache-2.0 +lexical-util MIT/Apache-2.0 +lexical-write-float MIT/Apache-2.0 +lexical-write-integer MIT/Apache-2.0 +libarchive Apache +libbcrypt Public Domain +libc MIT OR Apache-2.0 +libcotp Apache +libcpuid BSD 2-clause +libdeflate MIT +libdivide Boost +libfarmhash MIT +libfiu Public Domain +libfuzzer-sys (MIT OR Apache-2.0) AND NCSA +libgsasl LGPL +libhdfs3 Apache +libloading ISC +libm MIT +libmetrohash Apache +libpqxx BSD 3-clause +libprotobuf-mutator Apache +librdkafka Apache +libredox MIT +librseq BSD +librtcx Apache +libssh LGPL +libstemmer_c BSD 3-clause +libtest-mimic MIT/Apache-2.0 +libucontext ISC +liburing MIT +libuv BSD +libxml2 MIT +line-clipping MIT OR Apache-2.0 +link-cplusplus MIT OR Apache-2.0 +linux-raw-sys Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +listenfd Apache-2.0 +litemap Unicode-3.0 +litrs MIT OR Apache-2.0 +llvm-project Apache +lock_api MIT OR Apache-2.0 +log MIT OR Apache-2.0 +logos MIT OR Apache-2.0 +logos-codegen MIT OR Apache-2.0 +logos-derive MIT OR Apache-2.0 +lru MIT +lru-slab MIT OR Apache-2.0 OR Zlib +lz4 BSD +lz4 MIT +lz4-sys-1.11.1+lz4 MIT +lz4_flex MIT +mach2 BSD-2-Clause OR MIT OR Apache-2.0 +macro-string MIT OR Apache-2.0 +macro_rules_attribute MIT +macro_rules_attribute-proc_macro MIT +magic_enum MIT +managed 0BSD +mapbox-geometry ISC +mariadb-connector-c LGPL +match_cfg MIT/Apache-2.0 +matchers MIT +matchit MIT AND BSD-3-Clause +matrixmultiply MIT/Apache-2.0 +maybe-owned MIT OR Apache-2.0 +md-5 MIT OR Apache-2.0 +MeCab BSD +memchr Unlicense OR MIT +memfd MIT OR Apache-2.0 +memmap2 MIT OR Apache-2.0 +memmap2 MIT/Apache-2.0 +mime MIT OR Apache-2.0 +minimal-lexical MIT/Apache-2.0 +miniselect Boost +miniz_oxide MIT OR Zlib OR Apache-2.0 +minizip-ng zLib +mintex Apache-2.0 +mio MIT +moka (MIT OR Apache-2.0) AND Apache-2.0 +mongo-c-driver Apache +mongo-cxx-driver Apache +morton-nd MIT +moto-rt MIT OR Apache-2.0 +msgpack-c Boost +murmurhash Public Domain +musl Public Domain +mutatis MIT OR Apache-2.0 +mutatis-derive MIT OR Apache-2.0 +nanoarrow Apache +native-tls MIT OR Apache-2.0 +nats-io Apache +ndarray MIT OR Apache-2.0 +never-say-never Zlib OR MIT OR Apache-2.0 +nix MIT +nlohmann-json MIT +nom MIT +nougat Zlib OR MIT OR Apache-2.0 +nougat-proc_macros Zlib OR MIT OR Apache-2.0 +nu-ansi-term MIT +num-bigint MIT OR Apache-2.0 +num-complex MIT OR Apache-2.0 +num-conv MIT OR Apache-2.0 +num-integer MIT OR Apache-2.0 +num-traits MIT OR Apache-2.0 +num_cpus MIT OR Apache-2.0 +num_enum BSD-3-Clause OR MIT OR Apache-2.0 +num_enum_derive BSD-3-Clause OR MIT OR Apache-2.0 +num_threads MIT OR Apache-2.0 +NuRaft Apache +nvtx Apache +objc2 MIT +objc2-app-kit Zlib OR Apache-2.0 OR MIT +objc2-core-foundation Zlib OR Apache-2.0 OR MIT +objc2-core-graphics Zlib OR Apache-2.0 OR MIT +objc2-encode MIT +objc2-foundation MIT +objc2-io-surface Zlib OR Apache-2.0 OR MIT +object Apache-2.0 OR MIT +object_store MIT/Apache-2.0 +ocaml-boxroot-sys MIT +ocaml-interop MIT +ocaml-sys ISC +once_cell MIT OR Apache-2.0 +once_cell_polyfill MIT OR Apache-2.0 +oneshot MIT OR Apache-2.0 +onig MIT +onig_sys MIT +onpair Apache-2.0 +oorandom MIT +opaque-debug MIT OR Apache-2.0 +openldap OpenLDAP Version 2.8 +openssl Apache +openssl Apache-2.0 +openssl-macros MIT/Apache-2.0 +openssl-probe MIT OR Apache-2.0 +openssl-probe MIT/Apache-2.0 +openssl-sys MIT +openvino Apache-2.0 +openvino-finder Apache-2.0 +openvino-sys Apache-2.0 +optimized-routines Apache +ordered-float MIT +ort-2.0.0 MIT OR Apache-2.0 +ort-sys-2.0.0 MIT OR Apache-2.0 +page_size MIT/Apache-2.0 +palette MIT OR Apache-2.0 +palette_derive MIT OR Apache-2.0 +palette_math MIT OR Apache-2.0 +parking Apache-2.0 OR MIT +parking_lot MIT OR Apache-2.0 +parking_lot_core MIT OR Apache-2.0 +parquet Apache-2.0 +parquet-testing Apache +parse-zoneinfo MIT +password-hash MIT OR Apache-2.0 +paste MIT OR Apache-2.0 +pastey MIT OR Apache-2.0 +pbkdf2 MIT OR Apache-2.0 +pco Apache-2.0 +pcodec Apache +pdqsort zLib +pem-rfc7468 Apache-2.0 OR MIT +percent-encoding MIT OR Apache-2.0 +perfetto_protos Apache-2.0 +petgraph MIT OR Apache-2.0 +phf MIT +phf_codegen MIT +phf_generator MIT +phf_shared MIT +pin-project Apache-2.0 OR MIT +pin-project-internal Apache-2.0 OR MIT +pin-project-lite Apache-2.0 OR MIT +pin-utils MIT OR Apache-2.0 +piper MIT OR Apache-2.0 +pkg-config MIT OR Apache-2.0 +plain MIT/Apache-2.0 +plotters MIT +plotters-backend MIT +plotters-svg MIT +pocketfft BSD 3-clause +poco Boost +polling Apache-2.0 OR MIT +polonius-the-crab Zlib OR MIT OR Apache-2.0 +polyglot-sql MIT +polyval Apache-2.0 OR MIT +portable-atomic Apache-2.0 OR MIT +portable-atomic-util Apache-2.0 OR MIT +postcard MIT OR Apache-2.0 +postgres PostgreSQL +potential_utf Unicode-3.0 +powerfmt MIT OR Apache-2.0 +ppv-lite86 MIT OR Apache-2.0 +ppv-lite86 MIT/Apache-2.0 +pretty_assertions MIT OR Apache-2.0 +prettyplease MIT OR Apache-2.0 +proc-macro-crate MIT OR Apache-2.0 +proc-macro2 MIT OR Apache-2.0 +prometheus-protobufs Apache +prometheus-protobufs-gogo BSD 3-clause +proptest MIT OR Apache-2.0 +prost Apache-2.0 +prost-derive Apache-2.0 +prost-types Apache-2.0 +protobuf MIT +protobuf-codegen MIT +protobuf-parse MIT +protobuf-support MIT +protoc-bin-vendored MIT +protoc-bin-vendored-linux-aarch_64 MIT +protoc-bin-vendored-linux-ppcle_64 MIT +protoc-bin-vendored-linux-s390_64 MIT +protoc-bin-vendored-linux-x86_32 MIT +protoc-bin-vendored-linux-x86_64 MIT +protoc-bin-vendored-macos-aarch_64 MIT +protoc-bin-vendored-macos-x86_64 MIT +protoc-bin-vendored-win32 MIT +prqlc Apache-2.0 +prqlc-parser Apache-2.0 +psm MIT OR Apache-2.0 +pulldown-cmark MIT +pybind11 BSD 3-clause +pybind11-upstream BSD 3-clause +quick-error MIT/Apache-2.0 +quick-xml MIT +quinn MIT OR Apache-2.0 +quinn-proto MIT OR Apache-2.0 +quinn-udp MIT OR Apache-2.0 +quote MIT OR Apache-2.0 +r-efi MIT OR Apache-2.0 OR LGPL-2.1-or-later +r-efi-alloc MIT OR Apache-2.0 OR LGPL-2.1-or-later +radium MIT +rand MIT OR Apache-2.0 +rand_chacha MIT OR Apache-2.0 +rand_core MIT OR Apache-2.0 +rand_xorshift MIT OR Apache-2.0 +rand_xoshiro MIT OR Apache-2.0 +rapidjson BSD +rapids-logger Apache +ratatui MIT +ratatui-core MIT +ratatui-crossterm MIT +ratatui-widgets MIT +raw-cpuid MIT +rawpointer MIT/Apache-2.0 +rayon MIT OR Apache-2.0 +rayon-core MIT OR Apache-2.0 +re2 BSD 3-clause +recursive MIT +recursive-proc-macro-impl MIT +redox_syscall MIT +redox_users MIT +ref-cast MIT OR Apache-2.0 +ref-cast-impl MIT OR Apache-2.0 +regalloc2 Apache-2.0 WITH LLVM-exception +regex MIT OR Apache-2.0 +regex-automata MIT OR Apache-2.0 +regex-syntax MIT OR Apache-2.0 +region MIT +relative-path MIT OR Apache-2.0 +replace_with MIT OR Apache-2.0 +replxx BSD 3-clause +reqwest MIT OR Apache-2.0 +ring Apache-2.0 AND ISC +rmm Apache +roaring MIT OR Apache-2.0 +rocksdb BSD 3-clause +roxmltree MIT OR Apache-2.0 +rstest MIT OR Apache-2.0 +rstest_macros MIT OR Apache-2.0 +rstest_reuse MIT OR Apache-2.0 +rustc-demangle MIT/Apache-2.0 +rustc-hash Apache-2.0 OR MIT +rustc-hash Apache-2.0/MIT +rustc-literal-escaper Apache-2.0 OR MIT +rustc_version MIT OR Apache-2.0 +rustix Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +rustix-linux-procfs Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +rustls Apache-2.0 OR ISC OR MIT +rustls-native-certs Apache-2.0 OR ISC OR MIT +rustls-pemfile Apache-2.0 OR ISC OR MIT +rustls-pki-types MIT OR Apache-2.0 +rustls-platform-verifier MIT OR Apache-2.0 +rustls-platform-verifier-android MIT OR Apache-2.0 +rustls-webpki ISC +rustversion MIT OR Apache-2.0 +rusty-fork MIT/Apache-2.0 +ryu Apache-2.0 OR BSL-1.0 +s2geometry Apache +safetensors Apache-2.0 +same-file Unlicense/MIT +schannel MIT +schemars MIT +schemars_derive MIT +scopeguard MIT OR Apache-2.0 +scratch MIT OR Apache-2.0 +sealed MIT OR Apache-2.0 +security-framework MIT OR Apache-2.0 +security-framework-sys MIT OR Apache-2.0 +semver MIT OR Apache-2.0 +seq-macro MIT OR Apache-2.0 +serde MIT OR Apache-2.0 +serde_bytes MIT OR Apache-2.0 +serde_core MIT OR Apache-2.0 +serde_derive MIT OR Apache-2.0 +serde_derive_internals MIT OR Apache-2.0 +serde_json MIT OR Apache-2.0 +serde_spanned MIT OR Apache-2.0 +serde_urlencoded MIT/Apache-2.0 +serde_yaml MIT OR Apache-2.0 +sha1 MIT OR Apache-2.0 +sha2 MIT OR Apache-2.0 +SHA3IUF MIT +sharded-slab MIT +shell-quote Apache-2.0 +shlex MIT OR Apache-2.0 +shuffling-allocator MPL-2.0 +signal-hook Apache-2.0/MIT +signal-hook-mio MIT OR Apache-2.0 +signal-hook-registry Apache-2.0/MIT +signal-hook-registry MIT OR Apache-2.0 +silk Apache +simd-adler32 MIT +simd_cesu8 Apache-2.0 OR MIT +simdcomp BSD 3-clause +simdjson MIT +simdutf MIT +simdutf8 MIT OR Apache-2.0 +similar Apache-2.0 +SimSIMD Apache +siphasher MIT OR Apache-2.0 +siphasher MIT/Apache-2.0 +size MIT +sketches-ddsketch Apache-2.0 +skim MIT +skim-common MIT +skim-tuikit MIT +slab MIT +smallvec MIT OR Apache-2.0 +smallvec-2.0.0 MIT OR Apache-2.0 +smol Apache-2.0 OR MIT +snap BSD-3-Clause +snappy BSD 3-clause +socket2 MIT OR Apache-2.0 +socks MIT/Apache-2.0 +souper-ir MIT OR Apache-2.0 +sparsehash-c11 BSD 3-clause +spdlog MIT +spin MIT +spmc MIT OR Apache-2.0 +sqids-cpp MIT +sql-console Apache +sqlformat MIT OR Apache-2.0 +sqlite-amalgamation BSD 3-clause +sqlparser Apache-2.0 +stable_deref_trait MIT OR Apache-2.0 +stable_deref_trait MIT/Apache-2.0 +stacker MIT OR Apache-2.0 +static_assertions MIT OR Apache-2.0 +strfmt MIT +string-interner MIT/Apache-2.0 +StringZilla Apache +strsim MIT +strum MIT +strum_macros MIT +substrait Apache +subtle BSD-3-Clause +symbolic_expressions MIT/Apache-2.0 +syn MIT OR Apache-2.0 +sync_wrapper Apache-2.0 +synstructure MIT +syntect MIT +system-configuration MIT OR Apache-2.0 +system-configuration-sys MIT OR Apache-2.0 +system-interface Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +sz3 BSD +tagptr MIT/Apache-2.0 +tap MIT +tar MIT OR Apache-2.0 +target-lexicon Apache-2.0 WITH LLVM-exception +target-triple MIT OR Apache-2.0 +tch MIT/Apache-2.0 +tempfile MIT OR Apache-2.0 +term MIT OR Apache-2.0 +term MIT/Apache-2.0 +termcolor Unlicense OR MIT +terminal_size MIT OR Apache-2.0 +termtree MIT +test-case MIT +test-case-core MIT +test-case-macros MIT +test-log Apache-2.0 OR MIT +test-log-core Apache-2.0 OR MIT +test-log-macros Apache-2.0 OR MIT +thiserror MIT OR Apache-2.0 +thiserror-impl MIT OR Apache-2.0 +thousands MIT/Apache-2.0 +thread_local Apache-2.0/MIT +thread_local MIT OR Apache-2.0 +thrift Apache +thrift Apache-2.0 +time MIT OR Apache-2.0 +time-core MIT OR Apache-2.0 +time-macros MIT OR Apache-2.0 +timer MPL-2.0 +tiny-keccak CC0-1.0 +tiny_http MIT OR Apache-2.0 +tinystr Unicode-3.0 +tinytemplate Apache-2.0 OR MIT +tinyvec Zlib OR Apache-2.0 OR MIT +tinyvec_macros MIT OR Apache-2.0 OR Zlib +tokio MIT +tokio-macros MIT +tokio-native-tls MIT +tokio-openssl MIT/Apache-2.0 +tokio-rustls MIT OR Apache-2.0 +tokio-stream MIT +tokio-util MIT +toml MIT OR Apache-2.0 +toml-0.9.12+spec MIT OR Apache-2.0 +toml-1.1.2+spec MIT OR Apache-2.0 +toml_datetime MIT OR Apache-2.0 +toml_datetime-0.7.5+spec MIT OR Apache-2.0 +toml_datetime-1.1.1+spec MIT OR Apache-2.0 +toml_edit-0.25.11+spec MIT OR Apache-2.0 +toml_parser MIT OR Apache-2.0 +toml_parser-1.1.2+spec MIT OR Apache-2.0 +toml_writer MIT OR Apache-2.0 +toml_writer-1.1.1+spec MIT OR Apache-2.0 +tonic MIT +tonic-prost MIT +torch-sys MIT/Apache-2.0 +tower MIT +tower-http MIT +tower-layer MIT +tower-service MIT +tracing MIT +tracing-attributes MIT +tracing-core MIT +tracing-log MIT +tracing-serde MIT +tracing-subscriber MIT +try-lock MIT +trybuild MIT OR Apache-2.0 +tui-input MIT +twox-hash MIT +typenum MIT OR Apache-2.0 +unarray MIT OR Apache-2.0 +unicase MIT OR Apache-2.0 +unicode-ident (MIT OR Apache-2.0) AND Unicode-3.0 +unicode-segmentation MIT OR Apache-2.0 +unicode-segmentation MIT/Apache-2.0 +unicode-truncate MIT OR Apache-2.0 +unicode-width MIT OR Apache-2.0 +unicode-width MIT/Apache-2.0 +unicode-xid MIT OR Apache-2.0 +unicode_categories MIT OR Apache-2.0 +universal-hash MIT OR Apache-2.0 +unsafe-libyaml MIT +untrusted ISC +unwinding MIT OR Apache-2.0 +ureq MIT OR Apache-2.0 +ureq-proto MIT OR Apache-2.0 +url MIT OR Apache-2.0 +usearch Apache +utf-8 MIT OR Apache-2.0 +utf16_iter Apache-2.0 OR MIT +utf8-zero MIT OR Apache-2.0 +utf8_iter Apache-2.0 OR MIT +utf8parse Apache-2.0 OR MIT +utf8proc MIT +uuid Apache-2.0 OR MIT +v8 MIT +valuable MIT +vcpkg MIT/Apache-2.0 +vectorscan Boost +vergen MIT OR Apache-2.0 +vergen-gitcl MIT OR Apache-2.0 +vergen-lib MIT OR Apache-2.0 +version_check MIT/Apache-2.0 +vex-sdk MIT +vortex Apache-2.0 +vortex-alp Apache-2.0 +vortex-array Apache-2.0 +vortex-array-macros Apache-2.0 +vortex-arrow Apache-2.0 +vortex-btrblocks Apache-2.0 +vortex-buffer Apache-2.0 +vortex-bytebool Apache-2.0 +vortex-compressor Apache-2.0 +vortex-compute Apache-2.0 +vortex-datetime-parts Apache-2.0 +vortex-decimal-byte-parts Apache-2.0 +vortex-edition Apache-2.0 +vortex-error Apache-2.0 +vortex-fastlanes Apache-2.0 +vortex-file Apache-2.0 +vortex-flatbuffers Apache-2.0 +vortex-fsst Apache-2.0 +vortex-io Apache-2.0 +vortex-ipc Apache-2.0 +vortex-layout Apache-2.0 +vortex-mask Apache-2.0 +vortex-metrics Apache-2.0 +vortex-onpair Apache-2.0 +vortex-pco Apache-2.0 +vortex-proto Apache-2.0 +vortex-runend Apache-2.0 +vortex-scan Apache-2.0 +vortex-sequence Apache-2.0 +vortex-session Apache-2.0 +vortex-sparse Apache-2.0 +vortex-utils Apache-2.0 +vortex-zigzag Apache-2.0 +vortex-zstd Apache-2.0 +vte Apache-2.0 OR MIT +wagyu Boost +wait-timeout MIT/Apache-2.0 +waker-fn Apache-2.0 OR MIT +walkdir Unlicense/MIT +want MIT +wasi-0.11.0+wasi-snapshot Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasi-0.11.1+wasi-snapshot Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasi-0.13.3+wasi Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasi-0.14.2+wasi Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasi-0.14.4+wasi Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasi-nn Apache-2.0 +wasip1 Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasip2-1.0.0+wasi Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasip2-1.0.3+wasi Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasip2-1.0.4+wasi Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasip3-0.4.0+wasi-0.3.0-rc-2026-01 Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasite Apache-2.0 OR BSL-1.0 OR MIT +wasm-bindgen MIT OR Apache-2.0 +wasm-bindgen-backend MIT OR Apache-2.0 +wasm-bindgen-futures MIT OR Apache-2.0 +wasm-bindgen-macro MIT OR Apache-2.0 +wasm-bindgen-macro-support MIT OR Apache-2.0 +wasm-bindgen-shared MIT OR Apache-2.0 +wasm-compose Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasm-encoder Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasm-metadata Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasm-mutate Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasm-smith Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasm-streams MIT OR Apache-2.0 +wasm-wave Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasmi MIT/Apache-2.0 +wasmi_collections MIT/Apache-2.0 +wasmi_core MIT/Apache-2.0 +wasmi_ir MIT/Apache-2.0 +wasmparser Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasmprinter Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wasmtime Apache +wast Apache-2.0 WITH LLVM-exception +wast Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wat Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +web-sys MIT OR Apache-2.0 +web-time MIT OR Apache-2.0 +webpki-root-certs CDLA-Permissive-2.0 +webpki-roots CDLA-Permissive-2.0 +webpki-roots MPL-2.0 +which MIT +whoami Apache-2.0 OR BSL-1.0 OR MIT +winapi MIT/Apache-2.0 +winapi-i686-pc-windows-gnu MIT/Apache-2.0 +winapi-util Unlicense OR MIT +winapi-util Unlicense/MIT +winapi-x86_64-pc-windows-gnu MIT/Apache-2.0 +windows MIT OR Apache-2.0 +windows-core MIT OR Apache-2.0 +windows-implement MIT OR Apache-2.0 +windows-interface MIT OR Apache-2.0 +windows-link MIT OR Apache-2.0 +windows-registry MIT OR Apache-2.0 +windows-result MIT OR Apache-2.0 +windows-strings MIT OR Apache-2.0 +windows-sys MIT OR Apache-2.0 +windows-targets MIT OR Apache-2.0 +windows_aarch64_gnullvm MIT OR Apache-2.0 +windows_aarch64_msvc MIT OR Apache-2.0 +windows_i686_gnu MIT OR Apache-2.0 +windows_i686_gnullvm MIT OR Apache-2.0 +windows_i686_msvc MIT OR Apache-2.0 +windows_x86_64_gnu MIT OR Apache-2.0 +windows_x86_64_gnullvm MIT OR Apache-2.0 +windows_x86_64_msvc MIT OR Apache-2.0 +winnow MIT +winsafe MIT +winx Apache-2.0 WITH LLVM-exception +wit-bindgen Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wit-bindgen-core Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wit-bindgen-rt Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wit-bindgen-rust Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wit-bindgen-rust-macro Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wit-component Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +wit-parser Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT +witx Apache-2.0 +write16 Apache-2.0 OR MIT +writeable Unicode-3.0 +wstd Apache-2.0 WITH LLVM-exception +wstd-macro Apache-2.0 WITH LLVM-exception +wyhash Public Domain +wyz MIT +x11rb MIT OR Apache-2.0 +x11rb-protocol MIT OR Apache-2.0 +xattr MIT OR Apache-2.0 +xdg Apache-2.0 OR MIT +xgboost Apache +xsimd BSD 3-clause +xxHash BSD +xz LGPL +yaml-cpp MIT +yansi MIT OR Apache-2.0 +yoke Unicode-3.0 +yoke-derive Unicode-3.0 +z85 MIT/Apache-2.0 +zerocopy BSD-2-Clause OR Apache-2.0 OR MIT +zerocopy-derive BSD-2-Clause OR Apache-2.0 OR MIT +zerofrom Unicode-3.0 +zerofrom-derive Unicode-3.0 +zeroize Apache-2.0 OR MIT +zerotrie Unicode-3.0 +zerovec Unicode-3.0 +zerovec-derive Unicode-3.0 +zigzag MIT +zip MIT +zlib-ng zLib +zlib-rs Zlib +zmij MIT +zstd BSD +zstd MIT +zstd-safe MIT OR Apache-2.0 +zstd-safe MIT/Apache-2.0 +zstd-sys MIT/Apache-2.0 +zxc BSD diff --git a/pom.xml b/pom.xml index e078e7d..e0123ce 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.clickhouse.chdb chdb-java-parent - 1.0.0-SNAPSHOT + 1.0.0-rc.3-SNAPSHOT pom chDB Java Binding (parent) @@ -65,12 +65,13 @@ mirrors them for the shell/CMake side. Changing engine.version requires a new Maven version (work plan section 4.3) and a full platform re-test. --> - 26.11.0-rc.2 + 26.11.0-rc.3 1 - 2 + 3 1 + 19.0.0 5.10.2 3.2.5 @@ -80,6 +81,7 @@ chdb-bom chdb-jdbc + chdb-arrow chdb-native-macos-aarch64 chdb-native-macos-x86_64 chdb-native-linux-x86_64-gnu diff --git a/scripts/engine.properties b/scripts/engine.properties index 8b11d37..1110ebc 100644 --- a/scripts/engine.properties +++ b/scripts/engine.properties @@ -6,24 +6,20 @@ # keep this file in sync with in the root pom.xml. # # Checksums are the release asset digests published by GitHub for the tag below -# (gh api repos/chdb-io/chdb-core/releases/tags/v26.11.0-rc.2 --jq '.assets[].digest'), and +# (gh api repos/chdb-io/chdb-core/releases/tags/v26.11.0-rc.3 --jq '.assets[].digest'), and # the release's SHA256SUMS asset agrees with them. # # engine.version is what chdb_version() returns from the shared library, not the tag with # its 'v' stripped and not the version in the wheel file names. The loader compares the two -# and refuses a mismatch, so this is measured rather than derived: dlopen the linux-x86_64 -# asset, call chdb_version(), and it returns "26.11.0-rc.2", which is also the CHDB_VERSION -# in the chdb.h every one of the four tarballs ships. The wheels disagree, as they did on -# v26.7.2-rc.2: they are named 26.11.0, without the qualifier. Agreement is the happy case, -# not the rule; keep measuring it. +# and refuses a mismatch. The macOS arm64 rc.3 tarball's header reports +# CHDB_VERSION "26.11.0-rc.3"; the runtime version and other three platforms must also +# pass the real-JAR matrix before a release claim. Do not infer it from a wheel filename. # -# v26.11.0-rc.2 is a chdb-core PRE-RELEASE (published 2026-11-05, prerelease=true; not on -# PyPI), pinned to validate the ClickHouse 26.11 baseline ahead of the stable v26.11.1. It is -# v26.11.0-rc.1 plus chdb-core #312, on the same ClickHouse baseline (26.11.1.1). Work plan -# section 4.3: a binding built on an engine RC is itself an RC and cannot be V1 GA, so while -# this is the baseline only -rc. versions may be cut (1.0.0-rc.2 is cut from it; 1.0.0-rc.1 -# stays on v26.7.3), and the stable engine has to be re-pinned before 1.0.0. The last stable -# baseline was v26.7.3. +# v26.11.0-rc.3 is a chdb-core PRE-RELEASE (published 2026-11-12), based on commit +# 574f212fb5ee0eca9dec68a92f890eaf9941c262 which includes #323's Arrow +# registration-schema and failed array-registration cleanup fixes. A Java binding +# pinned to this core RC is itself an RC candidate, not 1.0.0 GA. The previous +# 1.0.0-rc.2 binding and v26.11.0-rc.2 core remain separate releases. # # chdb-core #224, the fix for the signal-handler window reported as chdb-core #221 and # tracked here as issue #14, arrived in v26.7.3: scripts/run-signal-window-test.sh measure @@ -34,11 +30,12 @@ # over ~1.08M samples a run, and stress 6 survived 6 of 6 with 0 killed. # Re-run on v26.11.0-rc.2, on linux-x86_64: measure 3 reported hostHandlersAtSigDfl=0/0/0 # over ~1.11-1.25M samples a run, and stress 6 survived 6 of 6 with 0 killed. +# v26.11.0-rc.3 must be retested in the four-platform CI before release. -engine.version=26.11.0-rc.2 -engine.tag=v26.11.0-rc.2 +engine.version=26.11.0-rc.3 +engine.tag=v26.11.0-rc.3 engine.repo=chdb-io/chdb-core -engine.asset.base=https://github.com/chdb-io/chdb-core/releases/download/v26.11.0-rc.2 +engine.asset.base=https://github.com/chdb-io/chdb-core/releases/download/v26.11.0-rc.3 # platform id -> release asset name asset.macos-aarch64=macos-arm64-libchdb.tar.gz @@ -47,10 +44,10 @@ asset.linux-x86_64-gnu=linux-x86_64-libchdb.tar.gz asset.linux-aarch64-gnu=linux-aarch64-libchdb.tar.gz # platform id -> SHA-256 of the release asset -sha256.macos-aarch64=c9c92100e2b2ff95a5e95055cfce46855da99ee082d21f19ccbb15e0a66bd185 -sha256.macos-x86_64=f3aaeabfc79bb43f92b971b5a8eefe59488525ae16145c17bcd6ba028325832a -sha256.linux-x86_64-gnu=f180eb9dfee3536b5017cb01e70369defff19f96ef4c351bf707e80d19ab739a -sha256.linux-aarch64-gnu=065b63064107172317b8d631bfc2ec08e22249695ecc8611def33fee589f26ca +sha256.macos-aarch64=7b0ea049efcf75a851e744c7d74325d7a9b143227f9b72097e5df252564e3d58 +sha256.macos-x86_64=f3be47d0e2d2bef19c65ed6f9837fc5d38888995d4e1fe1750dbb08a64e12cc9 +sha256.linux-x86_64-gnu=51ebb27abf2d7579941932b6ddbda23c1d25b8108be30142c9d99882f60a7ec6 +sha256.linux-aarch64-gnu=a7c1024125f3696b99395c568dcc9f226edf41061f35bcc55cceb348179adcf8 # platform id -> the libchdb file name inside the extracted tarball / the name we ship libname.macos-aarch64=libchdb.so @@ -65,11 +62,8 @@ libname.linux-aarch64-gnu=libchdb.so # (chdb_core-26.11.0-cp39-abi3-macosx_11_0_arm64.whl and ...-macosx_10_15_x86_64.whl), and # build-native.sh fails the build if the shim ends up above the engine's. # -# Re-checked against the v26.11.0-rc.2 binaries rather than trusting the wheel names: the -# LC_BUILD_VERSION load command of each asset's libchdb.so (what `vtool -show-build` prints) -# says minos 11.0 for the arm64 asset and 10.15 for the x86_64 one, so the two values below -# still match what the engine will actually load on. Both assets were downloaded and read for -# this, not inferred from the previous baseline. +# These targets were measured against v26.11.0-rc.2. Build-native.sh checks the rc.3 +# assets on their respective CI runners and refuses a shim whose minimum exceeds its engine. # # Left unset, the target follows the build machine's SDK: an unpinned build on a macOS 26 # host produced a shim demanding macOS 26 beside an engine that runs on 11. @@ -87,11 +81,8 @@ deployment.target.macos-x86_64=10.15 # below it is out of support: RHEL 7 (2.17) ended June 2024 and Amazon Linux 2 (2.26) June # 2026. Everything above it is already covered. # -# Unchanged by the v26.11.0-rc.2 move, checked two ways. Its Linux wheels carry the same -# manylinux2014/manylinux_2_17 tags every baseline since v26.7.0 has, and the released engines -# themselves ask for less than that: the highest GLIBC_* symbol version in the x86_64 asset is -# still GLIBC_2.4, and in the aarch64 asset GLIBC_2.17. So the engine's own floor did not rise -# and the ceiling still has room. +# The last v26.11.0-rc.2 measurements had room under this ceiling. The rc.3 +# engine and newly linked shim are remeasured by build-native.sh in Linux CI. # # What this cannot check from a macOS host is the *shim's* floor, which is the thing the # ceiling actually guards. That is measured in the Linux jobs, where build-native.sh reads it