From 0328db3e520c06a664b968b56208fad5f74ef989 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:25:57 +0000 Subject: [PATCH 1/3] feat(client-v2, jdbc-v2): add QBit data type support QBit(element_type, dimension) is transmitted over RowBinary exactly like Array(element_type) - a var-int length followed by that many element values - so it is read and written as a Java array of its element type (float[] for BFloat16/Float32, double[] for Float64) reusing the existing array codec, while keeping dataType == QBit for metadata fidelity. In jdbc-v2 QBit maps to java.sql.Types.ARRAY and is exposed as a java.sql.Array. Element types: BFloat16, Float32, Float64. The bit-transposed on-disk layout is a server storage detail and is not exposed to the client. Implements: https://github.com/ClickHouse/clickhouse-java/issues/2610 --- CHANGELOG.md | 8 ++ .../com/clickhouse/data/ClickHouseColumn.java | 32 +++++++ .../clickhouse/data/ClickHouseColumnTest.java | 29 ++++++ .../internal/BinaryStreamReader.java | 2 + .../internal/SerializerUtils.java | 2 + .../api/internal/DataTypeConverter.java | 2 + .../client/datatypes/DataTypeTests.java | 93 ++++++++++++++++++- docs/features.md | 3 + .../clickhouse/jdbc/internal/JdbcUtils.java | 2 +- .../java/com/clickhouse/jdbc/types/Array.java | 4 +- .../clickhouse/jdbc/JdbcDataTypeTests.java | 55 +++++++++++ 11 files changed, 229 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4f9aabad..c34909c67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ `getFloat`/`setFloat` and `getObject` accessors, and reported as such by `ResultSetMetaData` and `DatabaseMetaData`. Previously reading or writing a `BFloat16` column failed with an unsupported-data-type error. (https://github.com/ClickHouse/clickhouse-java/issues/2279) +- **[client-v2, jdbc-v2]** Added support for the experimental `QBit(element_type, dimension)` vector data type + (ClickHouse `25.10+`; the `allow_experimental_qbit_type` server setting is required to create a column), with + `BFloat16`, `Float32`, or `Float64` element types. A `QBit` value is transmitted over `RowBinary` exactly like + `Array(element_type)`, so it is read and written as a Java array of the element type (`float[]` for + `BFloat16`/`Float32`, `double[]` for `Float64`) through generic records, binary readers, and POJO binding. In the + JDBC driver (`jdbc-v2`) `QBit` maps to `java.sql.Types.ARRAY` and is returned as a `java.sql.Array` from + `getObject`/`getArray`. Previously `QBit` was an unimplemented type constant and reading or writing such a column + failed. (https://github.com/ClickHouse/clickhouse-java/issues/2610) - **[client-v2, jdbc-v2]** Added TLS cipher suite selection. `Client.Builder.setSSLCipherSuites(String...)` (client-v2) and the comma-separated `ssl_cipher_suites` connection property (client-v2 and jdbc-v2) restrict the cipher suites enabled on secure connections; when unset, the transport defaults are used. Cipher-suite selection is independent of the diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java index 64ed857cc..b4118d7c3 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java @@ -73,6 +73,7 @@ public final class ClickHouseColumn implements Serializable { private static final String KEYWORD_NESTED = ClickHouseDataType.Nested.name(); private static final String KEYWORD_VARIANT = ClickHouseDataType.Variant.name(); private static final String KEYWORD_JSON = ClickHouseDataType.JSON.name(); + private static final String KEYWORD_QBIT = ClickHouseDataType.QBit.name(); private int columnCount; private int columnIndex; @@ -146,6 +147,17 @@ private static ClickHouseColumn update(ClickHouseColumn column) { } } break; + case QBit: + // QBit(element_type, dimension) is a one-level array of its element type on the + // wire; the dimension parameter is kept as the column precision. + if (!column.nested.isEmpty()) { + column.arrayLevel = 1; + column.arrayBaseColumn = column.nested.get(0); + } + if (size > 1) { + column.precision = Integer.parseInt(column.parameters.get(1).trim()); + } + break; case Bool: column.template = ClickHouseBoolValue.ofNull(); break; @@ -567,6 +579,26 @@ protected static int readColumn(String args, int startIndex, int len, String nam fixedLength = false; estimatedLength++; } + } else if (args.startsWith(KEYWORD_QBIT, i)) { + int index = args.indexOf('(', i + KEYWORD_QBIT.length()); + if (index < i) { + throw new IllegalArgumentException(ERROR_MISSING_NESTED_TYPE); + } + List params = new LinkedList<>(); + i = ClickHouseUtils.readParameters(args, index, len, params); + if (params.size() < 2) { + throw new IllegalArgumentException( + "QBit requires an element type and a dimension, e.g. QBit(Float32, 8)"); + } + // QBit(element_type, dimension) is transmitted over RowBinary exactly like + // Array(element_type): a var-int length followed by that many element values. The + // first parameter is the element type and drives the nested (item) column. + List nestedColumns = new LinkedList<>(); + nestedColumns.add(ClickHouseColumn.of("", params.get(0))); + column = new ClickHouseColumn(ClickHouseDataType.QBit, name, args.substring(startIndex, i), + nullable, lowCardinality, params, nestedColumns); + fixedLength = false; + estimatedLength++; } if (column == null) { diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java index 6a3508b11..fb6cd89fe 100644 --- a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java @@ -508,4 +508,33 @@ public Object[][] testJSONBinaryFormat_dp() { {"JSON(max_dynamic_types=3,max_dynamic_paths=3, SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)", 2, Arrays.asList("flags")}, }; } + + @DataProvider(name = "qbitTypesProvider") + private static Object[][] qbitTypesProvider() { + return new Object[][] { + { "QBit(BFloat16, 4)", ClickHouseDataType.BFloat16, 4 }, + { "QBit(Float32, 8)", ClickHouseDataType.Float32, 8 }, + { "QBit(Float64, 1536)", ClickHouseDataType.Float64, 1536 }, + }; + } + + @Test(groups = { "unit" }, dataProvider = "qbitTypesProvider") + public void testParseQBit(String typeName, ClickHouseDataType elementType, int dimension) { + ClickHouseColumn column = ClickHouseColumn.of("vec", typeName); + Assert.assertEquals(column.getDataType(), ClickHouseDataType.QBit); + Assert.assertEquals(column.getOriginalTypeName(), typeName); + // QBit(element_type, dimension) is a one-level array of its element type on the wire. + Assert.assertEquals(column.getArrayNestedLevel(), 1); + Assert.assertNotNull(column.getArrayBaseColumn()); + Assert.assertEquals(column.getArrayBaseColumn().getDataType(), elementType); + Assert.assertEquals(column.getNestedColumns().size(), 1); + Assert.assertEquals(column.getNestedColumns().get(0).getDataType(), elementType); + // The fixed vector dimension is retained as the column precision. + Assert.assertEquals(column.getPrecision(), dimension); + } + + @Test(groups = { "unit" }, expectedExceptions = IllegalArgumentException.class) + public void testParseQBitRequiresDimension() { + ClickHouseColumn.of("vec", "QBit(Float32)"); + } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 782b6d0d1..8f6e2d448 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -234,6 +234,8 @@ public T readValue(ClickHouseColumn column, Class typeHint) throws IOExce return (T) readJsonData(input, actualColumn); } // case Object: // deprecated https://clickhouse.com/docs/en/sql-reference/data-types/object-data-type + case QBit: + // QBit(element_type, dimension) is transmitted like Array(element_type). case Array: if (typeHint == null) { typeHint = arrayDefaultTypeHint;} return convertArray(readArray(actualColumn), typeHint); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index e00438f12..1cb4a7760 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -63,6 +63,8 @@ public class SerializerUtils { public static void serializeData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException { //Serialize the value to the stream based on the data type switch (column.getDataType()) { + case QBit: + // QBit(element_type, dimension) is serialized like Array(element_type). case Array: serializeArrayData(stream, value, column); break; diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/DataTypeConverter.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/DataTypeConverter.java index d7f4a0f34..06eaa5bea 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/DataTypeConverter.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/DataTypeConverter.java @@ -67,6 +67,8 @@ public String convertToString(Object value, ClickHouseColumn column) { case IPv4: case IPv6: return ipvToString(value, column); + case QBit: + // QBit is rendered as an array literal of its element type, like Array(element_type). case Array: return arrayToString(value, column); case Point: diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index b24883ca9..c2d5a8175 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -92,8 +92,17 @@ public void tearDown() { private void writeReadVerify(String table, String tableDef, Class dtoClass, List data, BiConsumer, T> rowVerifier) throws Exception { + writeReadVerify(table, tableDef, dtoClass, data, rowVerifier, null); + } + + private void writeReadVerify(String table, String tableDef, Class dtoClass, List data, + BiConsumer, T> rowVerifier, CommandSettings ddlSettings) throws Exception { client.execute("DROP TABLE IF EXISTS " + table).get(); - client.execute(tableDef); + if (ddlSettings == null) { + client.execute(tableDef); + } else { + client.execute(tableDef, ddlSettings).get(); + } final TableSchema tableSchema = client.getTableSchema(table); client.register(dtoClass, tableSchema); @@ -107,6 +116,88 @@ private void writeReadVerify(String table, String tableDef, Class dtoClas Assert.assertEquals(rowCount.get(), data.size()); } + // QBit was introduced in ClickHouse 25.10. + private static final String QBIT_UNSUPPORTED_VERSIONS = "(,25.9]"; + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class QBitFloat32DTO { + private long rowId; + private float[] vec; + private int tail; + + public static String tblCreateSQL(String table) { + return tableDefinition(table, "rowId Int64", "vec QBit(Float32, 8)", "tail Int32"); + } + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class QBitFloat64DTO { + private long rowId; + private double[] vec; + private int tail; + + public static String tblCreateSQL(String table) { + return tableDefinition(table, "rowId Int64", "vec QBit(Float64, 8)", "tail Int32"); + } + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class QBitBFloat16DTO { + private long rowId; + private float[] vec; + private int tail; + + public static String tblCreateSQL(String table) { + return tableDefinition(table, "rowId Int64", "vec QBit(BFloat16, 8)", "tail Int32"); + } + } + + @Test(groups = {"integration"}) + public void testQBit() throws Exception { + if (isVersionMatch(QBIT_UNSUPPORTED_VERSIONS)) { + throw new SkipException("QBit requires ClickHouse 25.10+"); + } + + // A QBit(element_type, dimension) value is transmitted over RowBinary exactly like an + // Array(element_type): a var-int length followed by that many element values. The full + // client write -> server -> client read round-trip is verified for every supported + // element type. The trailing Int32 column would shift (and the assertion fail) if the + // QBit codec consumed the wrong number of bytes. + CommandSettings ddl = (CommandSettings) new CommandSettings() + .serverSetting("allow_experimental_qbit_type", "1"); + + final float[] f32 = {1f, -2f, 3.5f, 4f, 5f, 6f, 7f, 8f}; + writeReadVerify("test_qbit_f32", QBitFloat32DTO.tblCreateSQL("test_qbit_f32"), + QBitFloat32DTO.class, Arrays.asList(new QBitFloat32DTO(0, f32, 42)), + (all, dto) -> { + Assert.assertEquals(dto.getVec(), f32); + Assert.assertEquals(dto.getTail(), 42); + }, ddl); + + final double[] f64 = {1d, -2d, 3.5d, 4d, 5d, 6d, 7d, 8d}; + writeReadVerify("test_qbit_f64", QBitFloat64DTO.tblCreateSQL("test_qbit_f64"), + QBitFloat64DTO.class, Arrays.asList(new QBitFloat64DTO(0, f64, 42)), + (all, dto) -> { + Assert.assertEquals(dto.getVec(), f64); + Assert.assertEquals(dto.getTail(), 42); + }, ddl); + + // Integers up to 256 are exactly representable in BFloat16, so these round-trip bit-for-bit. + final float[] bf16 = {1f, 2f, 4f, 8f, 16f, 32f, 64f, 128f}; + writeReadVerify("test_qbit_bf16", QBitBFloat16DTO.tblCreateSQL("test_qbit_bf16"), + QBitBFloat16DTO.class, Arrays.asList(new QBitBFloat16DTO(0, bf16, 42)), + (all, dto) -> { + Assert.assertEquals(dto.getVec(), bf16); + Assert.assertEquals(dto.getTail(), 42); + }, ddl); + } + @Test(groups = {"integration"}) public void testNestedDataTypes() throws Exception { final String table = "test_nested_types"; diff --git a/docs/features.md b/docs/features.md index 73b01df59..7126628d2 100644 --- a/docs/features.md +++ b/docs/features.md @@ -21,6 +21,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - JSONEachRow text reader: Can stream `JSONEachRow` responses through a caller-supplied `JsonParser`, with Jackson and Gson parser factory implementations available as optional classpath dependencies, and infers a best-effort schema from the first row. - Data type conversion: Maps ClickHouse types to Java values for binary reads, POJO binding, and SQL parameter formatting, including date/time handling. - BFloat16 type support: For ClickHouse `24.11+`, reads and writes the `BFloat16` type through generic records, binary readers, POJO binding, and `Nullable`/`Dynamic`/`Variant` wrappers. `BFloat16` maps to the Java `float` type: a read widens the stored 16-bit value losslessly, while a write keeps the high 16 bits of the `float`. In `jdbc-v2`, `BFloat16` maps to `java.sql.Types.FLOAT` / `java.lang.Float` and is read and written through the standard `getFloat`/`setFloat` and `getObject` accessors. +- QBit type support: For ClickHouse `25.10+` (requires the `allow_experimental_qbit_type` server setting to create a column), reads and writes the experimental `QBit(element_type, dimension)` vector type, where `element_type` is `BFloat16`, `Float32`, or `Float64`. On the wire a `QBit` value is encoded exactly like `Array(element_type)` (a length-prefixed list of elements), so the client reads and writes it as a Java array of the element type — `float[]` for `BFloat16`/`Float32`, `double[]` for `Float64` — through generic records, binary readers, and POJO binding, using the same code path as `Array(element_type)`. The bit-transposed on-disk layout is a server storage detail and is not exposed to the client. - Geometry type support: For ClickHouse `25.11+`, where `Geometry` changed from a string alias to `Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)`, the client reads and writes `Geometry` values through generic records, binary readers, POJO binding, and SQL parameter formatting, using Java array dimensionality to represent the geometry shape. - Insert APIs: Supports inserting registered POJOs, raw streams, and callback-driven writers, with optional column lists and format selection. - Insert controls: Supports insert-specific settings such as deduplication token, query id, compression behavior, and request headers. @@ -43,6 +44,7 @@ Compatibility-sensitive traits: - Identifier quoting behavior is stable API for helper callers: identifiers are double-quoted, embedded double quotes are doubled, and optional quoting keeps simple identifiers unchanged. - Instant formatting is type-sensitive and should not drift: `Date` formatting depends on an explicit timezone, `DateTime` is serialized as epoch seconds, and higher-precision timestamps preserve up to 9 fractional digits. - `BFloat16` conversion is precision-sensitive and should not drift: a write keeps only the high 16 bits of the `float` (the low mantissa bits are truncated toward zero, matching the server's `Float32` → `BFloat16` conversion), so values that are not exactly representable in `BFloat16` change when written; a read widens the 16-bit value back to `float` losslessly. +- `QBit` is wire-compatible with `Array(element_type)` and should not drift: the client transmits the logical vector as a length-prefixed array of its element type (`float[]` for `BFloat16`/`Float32`, `double[]` for `Float64`) rather than the server's bit-transposed on-disk layout, so a `QBit(E, N)` value read from or written to the server round-trips as an array of `E`. - Timezone conversion helpers preserve nanoseconds and can intentionally shift local date or time when interpreted in a different timezone; this behavior is covered by tests and should not be normalized away. - `Geometry` handling is shape-sensitive: supported values are 1D through 4D Java arrays representing the nested geometry variants, and unsupported shapes or non-array values are rejected during serialization. - `Geometry` write inference is dimension-based rather than fully type-specific: point, ring/line string, polygon/multi-line string, and multi-polygon are selected from array depth, so writing `Geometry` cannot currently distinguish `Ring` from `LineString` or `Polygon` from `MultiLineString`. @@ -76,6 +78,7 @@ Compatibility-sensitive traits: - Database metadata: Implements JDBC `DatabaseMetaData` for ClickHouse catalogs, schemas, tables, columns, and related capability reporting. - Parameter metadata: Reports prepared-statement parameter counts. - Type mapping and conversions: Maps ClickHouse types to JDBC types and Java classes, including date/time handling and `java.time` support. +- QBit type mapping: For ClickHouse `25.10+`, JDBC exposes the experimental `QBit(element_type, dimension)` type as `ARRAY`, returning the vector as a `java.sql.Array` of the element type from `getObject()`/`getArray()`. Supported element types are `BFloat16` and `Float32` (both `java.lang.Float`) and `Float64` (`java.lang.Double`). The `allow_experimental_qbit_type` server setting is required to create a `QBit` column. - Custom result-set type map: `ResultSet#getObject(int|String, Map>)` accepts both ClickHouse type names and JDBC `SQLType` names as map keys. Only unwrapped type names are used — `Nullable(...)` and `LowCardinality(...)` wrappers are stripped before lookup, so a key like `"Int32"` matches both `Int32` and `Nullable(Int32)` columns, and keys like `"Nullable(Int32)"` are not recognized. Lookup order is `ClickHouseColumn#getDataType().name()` (e.g. `"Int32"`, `"String"`, `"DateTime"`) then `SQLType.getName()` (e.g. `"INTEGER"`, `"VARCHAR"`, `"TIMESTAMP"`); a missing entry leaves the value uncoerced (read as-is). The feature is supported for primitive ClickHouse types only — `Array`, `Tuple`, `Map`, `Nested`, and geometry types (`Point`, `Ring`, `LineString`, `Polygon`, `MultiPolygon`, `MultiLineString`, `Geometry`) bypass the type map and are returned in their native form. - Arrays and tuples: Supports JDBC arrays plus ClickHouse tuple values through custom `Array` and `Struct` implementations. - Geometry type mapping: For ClickHouse `25.11+`, where `Geometry` changed from a string alias to `Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)`, JDBC exposes `Geometry` as `ARRAY`, returns nested Java arrays from `getObject()`/`getArray()`, and accepts `Struct` or nested `Array` inputs for prepared-statement inserts depending on the geometry shape. diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java index bf7292428..6c13034c3 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java @@ -117,7 +117,7 @@ private static Map generateTypeMap() { map.put(ClickHouseDataType.AggregateFunction, JDBCType.OTHER); map.put(ClickHouseDataType.Variant, JDBCType.OTHER); map.put(ClickHouseDataType.Dynamic, JDBCType.OTHER); - map.put(ClickHouseDataType.QBit, JDBCType.OTHER); + map.put(ClickHouseDataType.QBit, JDBCType.ARRAY); return ImmutableMap.copyOf(map); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/types/Array.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/types/Array.java index ac82a3fd2..b7b23cc7f 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/types/Array.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/types/Array.java @@ -27,7 +27,9 @@ public class Array implements java.sql.Array { public Array(ClickHouseColumn column, Object[] elements) throws SQLException { this.column = column; this.array = elements; - ClickHouseColumn baseColumn = (this.column.isArray() ? this.column.getArrayBaseColumn() : this.column); + // QBit is not an Array (isArray() == false) but is still a one-level array of its element + // type, so use the array nesting level (set for both Array and QBit) to find the element column. + ClickHouseColumn baseColumn = (this.column.getArrayNestedLevel() > 0 ? this.column.getArrayBaseColumn() : this.column); this.baseDataType = baseColumn.getDataType(); this.elementTypeName = baseColumn.getOriginalTypeName(); this.type = JdbcUtils.CLICKHOUSE_TO_SQL_TYPE_MAP.getOrDefault(baseDataType, JDBCType.OTHER).getVendorTypeNumber(); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcDataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcDataTypeTests.java index a3b46496b..c35a50171 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcDataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcDataTypeTests.java @@ -570,6 +570,61 @@ public void testBFloat16() throws SQLException { } } + // QBit was introduced in ClickHouse 25.10. + private static final String QBIT_UNSUPPORTED_VERSIONS = "(,25.9]"; + + @Test(groups = { "integration" }) + public void testQBit() throws SQLException { + if (ClickHouseVersion.of(getServerVersion()).check(QBIT_UNSUPPORTED_VERSIONS)) { + throw new SkipException("QBit was introduced in ClickHouse 25.10"); + } + + // QBit(element_type, dimension) is exposed through JDBC as an ARRAY of its element type + // (the same way Array/Geometry are), written from and read as a Java array. The + // experimental type flag is only required to create the column, so the insert/read + // connection does not need it. + final String table = "test_qbit_jdbc"; + Properties properties = new Properties(); + properties.setProperty(ClientConfigProperties.serverSetting("allow_experimental_qbit_type"), "1"); + runQuery("DROP TABLE IF EXISTS " + table); + runQuery("CREATE TABLE " + table + + " (rowId Int32, vec QBit(Float32, 8)) ENGINE = MergeTree ORDER BY rowId", properties); + + final float[] expected = { 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f }; + + // Write path: a Java float[] bound to a QBit parameter is rendered as an array literal. + try (Connection conn = getJdbcConnection(); + PreparedStatement ps = conn.prepareStatement("INSERT INTO " + table + " VALUES (?, ?)")) { + ps.setInt(1, 1); + ps.setObject(2, expected); + ps.executeUpdate(); + } + + // Read path: metadata reports ARRAY with the element type, and getArray/getObject return + // a java.sql.Array of the element values. + try (Connection conn = getJdbcConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT rowId, vec FROM " + table + " ORDER BY rowId")) { + ResultSetMetaData meta = rs.getMetaData(); + assertEquals(meta.getColumnType(2), Types.ARRAY, "QBit should map to Types.ARRAY"); + + assertTrue(rs.next()); + java.sql.Array array = rs.getArray("vec"); + assertTrue(array != null, "getArray must return the QBit vector"); + assertEquals(array.getBaseType(), Types.FLOAT, "QBit(Float32) element type should be Types.FLOAT"); + assertEquals(array.getBaseTypeName(), "Float32", "QBit(Float32) element type name should be Float32"); + Object elements = array.getArray(); + assertEquals(java.lang.reflect.Array.getLength(elements), expected.length); + for (int i = 0; i < expected.length; i++) { + assertEquals(((Number) java.lang.reflect.Array.get(elements, i)).floatValue(), expected[i]); + } + + assertTrue(rs.getObject("vec") instanceof java.sql.Array, + "getObject on a QBit column should return java.sql.Array"); + assertFalse(rs.next()); + } + } + @Test(groups = { "integration" }) public void testBFloat16WriteAsFloat() throws SQLException { if (ClickHouseVersion.of(getServerVersion()).check(BFLOAT16_UNSUPPORTED_VERSIONS)) { From 0e1a6bb7c660993ec97c6e2c943a2b54de4303d0 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:58:25 +0000 Subject: [PATCH 2/3] feat(client-v2): validate QBit vector dimension on write QBit(element_type, dimension) is a fixed-size vector: the server requires exactly `dimension` elements and rejects any other length over RowBinary with a late SERIALIZATION_ERROR (verified against server 26.5). The write path reused serializeArrayData, which wrote the actual array/list length without checking it, so a wrong-sized (including empty) vector was serialized like a normal Array and only failed on the server after a round-trip. Split the QBit case out of the Array fall-through into serializeQBitData, which validates the element count against column.getPrecision() (the dimension) and throws a clear IllegalArgumentException naming the column, expected dimension, and actual length before delegating to serializeArrayData. This mirrors the client-side length enforcement already applied to the other fixed-size type, FixedString(N). A correctly-sized QBit is still byte-for-byte identical to Array(element_type). Addresses Cursor Bugbot review on PR #2939. --- .../internal/SerializerUtils.java | 29 ++++++++++++- .../internal/SerializerUtilsTest.java | 41 +++++++++++++++++++ docs/features.md | 2 +- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index 1cb4a7760..f92f1562f 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -64,7 +64,8 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo //Serialize the value to the stream based on the data type switch (column.getDataType()) { case QBit: - // QBit(element_type, dimension) is serialized like Array(element_type). + serializeQBitData(stream, value, column); + break; case Array: serializeArrayData(stream, value, column); break; @@ -472,6 +473,32 @@ public static void serializeArrayData(OutputStream stream, Object value, ClickHo } } + /** + * Serializes a {@code QBit(element_type, dimension)} value. On the wire a {@code QBit} is + * transmitted exactly like {@code Array(element_type)} — a var-int length followed by that many + * element values — but the element count is fixed and must equal the declared dimension. The + * count is validated up-front so a wrong-sized (including empty) vector fails fast on the client + * with a clear message instead of a late server {@code SERIALIZATION_ERROR}, mirroring the + * client-side length enforcement already applied to the other fixed-size type, + * {@code FixedString(N)}. + */ + private static void serializeQBitData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException { + if (value != null) { + int length = -1; + if (value.getClass().isArray()) { + length = Array.getLength(value); + } else if (value instanceof List) { + length = ((List) value).size(); + } + int dimension = column.getPrecision(); + if (length >= 0 && length != dimension) { + throw new IllegalArgumentException("QBit column '" + column.getColumnName() + + "' expects exactly " + dimension + " elements but got " + length); + } + } + serializeArrayData(stream, value, column); + } + /** DO NOT USE - part of internal API that will be changed */ diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java index 9574bed72..87cb9a136 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java @@ -143,6 +143,47 @@ public void testGeometrySerializationRejectsMalformedList() { ClickHouseColumn.of("v", "Geometry"))); } + @Test(dataProvider = "qbitWrongDimension") + public void testQBitSerializationRejectsWrongDimension(String typeName, Object value, int actualLength) { + ClickHouseColumn column = ClickHouseColumn.of("vec", typeName); + + IllegalArgumentException ex = Assert.expectThrows(IllegalArgumentException.class, + () -> SerializerUtils.serializeData(new ByteArrayOutputStream(), value, column)); + String message = ex.getMessage(); + Assert.assertTrue(message.contains("vec"), "Message should name the column: " + message); + Assert.assertTrue(message.contains("8"), "Message should state the expected dimension: " + message); + Assert.assertTrue(message.contains("got " + actualLength), + "Message should state the actual length: " + message); + } + + @DataProvider(name = "qbitWrongDimension") + private Object[][] qbitWrongDimension() { + // A QBit(E, 8) column requires exactly 8 elements: empty, too-short, and too-long vectors are + // all invalid, for both the Java-array and List representations and every element type. + return new Object[][] { + {"QBit(Float32, 8)", new float[0], 0}, + {"QBit(Float32, 8)", new float[] {1f, 2f, 3f, 4f, 5f}, 5}, + {"QBit(Float32, 8)", new float[] {1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f, 10f}, 10}, + {"QBit(Float64, 8)", new double[] {1d, 2d, 3d, 4d, 5d}, 5}, + {"QBit(BFloat16, 8)", new float[] {1f, 2f, 3f}, 3}, + {"QBit(Float32, 8)", Arrays.asList(1f, 2f, 3f), 3}, + }; + } + + @Test + public void testQBitSerializationAcceptsExactDimensionAndMatchesArray() throws Exception { + float[] vec = {1f, -2f, 3.5f, 4f, 5f, 6f, 7f, 8f}; + + ByteArrayOutputStream qbitOut = new ByteArrayOutputStream(); + SerializerUtils.serializeData(qbitOut, vec, ClickHouseColumn.of("vec", "QBit(Float32, 8)")); + + // A correctly-sized QBit passes validation and is serialized byte-for-byte identically to + // Array(element_type), which is the wire contract the reader relies on. + ByteArrayOutputStream arrayOut = new ByteArrayOutputStream(); + SerializerUtils.serializeData(arrayOut, vec, ClickHouseColumn.of("vec", "Array(Float32)")); + Assert.assertEquals(qbitOut.toByteArray(), arrayOut.toByteArray()); + } + @Test(dataProvider = "nonNullableEnumTypes") public void testNullIntoNonNullableEnumThrowsIllegalArgument(String typeName) { ClickHouseColumn column = ClickHouseColumn.of("bs_flag", typeName); diff --git a/docs/features.md b/docs/features.md index 7126628d2..3f3a013f4 100644 --- a/docs/features.md +++ b/docs/features.md @@ -44,7 +44,7 @@ Compatibility-sensitive traits: - Identifier quoting behavior is stable API for helper callers: identifiers are double-quoted, embedded double quotes are doubled, and optional quoting keeps simple identifiers unchanged. - Instant formatting is type-sensitive and should not drift: `Date` formatting depends on an explicit timezone, `DateTime` is serialized as epoch seconds, and higher-precision timestamps preserve up to 9 fractional digits. - `BFloat16` conversion is precision-sensitive and should not drift: a write keeps only the high 16 bits of the `float` (the low mantissa bits are truncated toward zero, matching the server's `Float32` → `BFloat16` conversion), so values that are not exactly representable in `BFloat16` change when written; a read widens the 16-bit value back to `float` losslessly. -- `QBit` is wire-compatible with `Array(element_type)` and should not drift: the client transmits the logical vector as a length-prefixed array of its element type (`float[]` for `BFloat16`/`Float32`, `double[]` for `Float64`) rather than the server's bit-transposed on-disk layout, so a `QBit(E, N)` value read from or written to the server round-trips as an array of `E`. +- `QBit` is wire-compatible with `Array(element_type)` and should not drift: the client transmits the logical vector as a length-prefixed array of its element type (`float[]` for `BFloat16`/`Float32`, `double[]` for `Float64`) rather than the server's bit-transposed on-disk layout, so a `QBit(E, N)` value read from or written to the server round-trips as an array of `E`. A written `QBit(E, N)` value must contain exactly `N` elements: a wrong-sized (including empty) vector is rejected during binary serialization with an `IllegalArgumentException` rather than deferred to a server error, matching the fixed dimension the server enforces. - Timezone conversion helpers preserve nanoseconds and can intentionally shift local date or time when interpreted in a different timezone; this behavior is covered by tests and should not be normalized away. - `Geometry` handling is shape-sensitive: supported values are 1D through 4D Java arrays representing the nested geometry variants, and unsupported shapes or non-array values are rejected during serialization. - `Geometry` write inference is dimension-based rather than fully type-specific: point, ring/line string, polygon/multi-line string, and multi-polygon are selected from array depth, so writing `Geometry` cannot currently distinguish `Ring` from `LineString` or `Polygon` from `MultiLineString`. From e2b23e05d4aa7826b9d6367fd70e874010789bde Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:25:56 +0000 Subject: [PATCH 3/3] fix(client-v2): reject non-array/non-List QBit write values A non-null QBit value that is neither a Java array nor a List fell through serializeQBitData to serializeArrayData, which writes no bytes for such a value, desynchronizing the RowBinary stream so the following columns are misread or rejected by the server. Reject it up-front with a clear IllegalArgumentException, consistent with the existing fixed-dimension validation and the Geometry reject-unsupported path. Addresses Cursor Bugbot review on #2939. --- .../internal/SerializerUtils.java | 13 +++++++-- .../internal/SerializerUtilsTest.java | 29 +++++++++++++++++++ docs/features.md | 2 +- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index f92f1562f..0016beec7 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -480,18 +480,25 @@ public static void serializeArrayData(OutputStream stream, Object value, ClickHo * count is validated up-front so a wrong-sized (including empty) vector fails fast on the client * with a clear message instead of a late server {@code SERIALIZATION_ERROR}, mirroring the * client-side length enforcement already applied to the other fixed-size type, - * {@code FixedString(N)}. + * {@code FixedString(N)}. A non-null value that is neither a Java array nor a {@code List} cannot + * carry a vector and is rejected as well — otherwise it would fall through to + * {@link #serializeArrayData} and write no bytes for the column, desynchronizing the + * {@code RowBinary} stream and corrupting the columns that follow. */ private static void serializeQBitData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException { if (value != null) { - int length = -1; + int length; if (value.getClass().isArray()) { length = Array.getLength(value); } else if (value instanceof List) { length = ((List) value).size(); + } else { + throw new IllegalArgumentException("QBit column '" + column.getColumnName() + + "' expects a Java array or List of its element type but got " + + value.getClass().getName()); } int dimension = column.getPrecision(); - if (length >= 0 && length != dimension) { + if (length != dimension) { throw new IllegalArgumentException("QBit column '" + column.getColumnName() + "' expects exactly " + dimension + " elements but got " + length); } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java index 87cb9a136..5b2a8d587 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java @@ -170,6 +170,35 @@ private Object[][] qbitWrongDimension() { }; } + @Test(dataProvider = "qbitWrongType") + public void testQBitSerializationRejectsNonArrayValue(Object value) { + // A non-null QBit value that is neither a Java array nor a List cannot carry a vector. It must + // be rejected up-front: otherwise it falls through to the Array serializer, which writes no + // bytes for the column, desynchronizing the RowBinary stream and corrupting the following + // columns. Writing into a byte sink so any (wrongly) emitted payload would be observable. + ClickHouseColumn column = ClickHouseColumn.of("vec", "QBit(Float32, 8)"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + IllegalArgumentException ex = Assert.expectThrows(IllegalArgumentException.class, + () -> SerializerUtils.serializeData(out, value, column)); + Assert.assertTrue(ex.getMessage().contains("vec"), + "Message should name the column: " + ex.getMessage()); + Assert.assertEquals(out.size(), 0, + "Nothing should be written to the stream when the value is rejected"); + } + + @DataProvider(name = "qbitWrongType") + private Object[][] qbitWrongType() { + // Values that are neither a Java array nor a List: a String, boxed scalars of the element + // type, and a Map. None of these can represent a QBit(E, N) vector. + return new Object[][] { + {"not-a-vector"}, + {3.14f}, + {42d}, + {newMap("k", "v")}, + }; + } + @Test public void testQBitSerializationAcceptsExactDimensionAndMatchesArray() throws Exception { float[] vec = {1f, -2f, 3.5f, 4f, 5f, 6f, 7f, 8f}; diff --git a/docs/features.md b/docs/features.md index 3f3a013f4..87bd19a42 100644 --- a/docs/features.md +++ b/docs/features.md @@ -44,7 +44,7 @@ Compatibility-sensitive traits: - Identifier quoting behavior is stable API for helper callers: identifiers are double-quoted, embedded double quotes are doubled, and optional quoting keeps simple identifiers unchanged. - Instant formatting is type-sensitive and should not drift: `Date` formatting depends on an explicit timezone, `DateTime` is serialized as epoch seconds, and higher-precision timestamps preserve up to 9 fractional digits. - `BFloat16` conversion is precision-sensitive and should not drift: a write keeps only the high 16 bits of the `float` (the low mantissa bits are truncated toward zero, matching the server's `Float32` → `BFloat16` conversion), so values that are not exactly representable in `BFloat16` change when written; a read widens the 16-bit value back to `float` losslessly. -- `QBit` is wire-compatible with `Array(element_type)` and should not drift: the client transmits the logical vector as a length-prefixed array of its element type (`float[]` for `BFloat16`/`Float32`, `double[]` for `Float64`) rather than the server's bit-transposed on-disk layout, so a `QBit(E, N)` value read from or written to the server round-trips as an array of `E`. A written `QBit(E, N)` value must contain exactly `N` elements: a wrong-sized (including empty) vector is rejected during binary serialization with an `IllegalArgumentException` rather than deferred to a server error, matching the fixed dimension the server enforces. +- `QBit` is wire-compatible with `Array(element_type)` and should not drift: the client transmits the logical vector as a length-prefixed array of its element type (`float[]` for `BFloat16`/`Float32`, `double[]` for `Float64`) rather than the server's bit-transposed on-disk layout, so a `QBit(E, N)` value read from or written to the server round-trips as an array of `E`. A written `QBit(E, N)` value must be a Java array or `List` holding exactly `N` elements: a wrong-sized (including empty) vector, or a non-null value that is neither an array nor a `List`, is rejected during binary serialization with an `IllegalArgumentException` rather than deferred to a server error or silently writing a misaligned stream, matching the fixed dimension the server enforces. - Timezone conversion helpers preserve nanoseconds and can intentionally shift local date or time when interpreted in a different timezone; this behavior is covered by tests and should not be normalized away. - `Geometry` handling is shape-sensitive: supported values are 1D through 4D Java arrays representing the nested geometry variants, and unsupported shapes or non-array values are rejected during serialization. - `Geometry` write inference is dimension-based rather than fully type-specific: point, ring/line string, polygon/multi-line string, and multi-polygon are selected from array depth, so writing `Geometry` cannot currently distinguish `Ring` from `LineString` or `Polygon` from `MultiLineString`.