From 63837105c83700b7415b5a4442c04e7e1d0073fd Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Sun, 19 Jul 2026 19:12:15 +0800 Subject: [PATCH 1/9] docs: document ColumnVariantV2 compute behavior --- .../sql-data-types/semi-structured/VARIANT.md | 45 ++++++++++++++++-- .../variant-functions/element-at.md | 10 +++- .../sql-data-types/semi-structured/VARIANT.md | 47 +++++++++++++++++-- .../variant-functions/element-at.md | 10 +++- 4 files changed, 101 insertions(+), 11 deletions(-) diff --git a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 0dc473d6335dc..01b587c49782b 100644 --- a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -371,7 +371,9 @@ After loading, verify with `SELECT count(*)` or sample with `SELECT * ... LIMIT ## Supported operations and CAST rules -- VARIANT cannot be compared/operated directly with other types; comparisons between two VARIANTs are not supported either. +The default execution path preserves the existing VARIANT guidance: + +- VARIANT cannot be compared or operated on directly with other types; comparisons between two VARIANT values are not supported on the default path. - For comparison, filtering, aggregation, and ordering, CAST subpaths to concrete types (explicitly or implicitly). ```sql @@ -385,22 +387,57 @@ SELECT * FROM tbl WHERE v['bool']; SELECT * FROM tbl WHERE v['str'] MATCH 'Doris'; ``` -- VARIANT itself cannot be used directly in ORDER BY, GROUP BY, as a JOIN KEY, or as an aggregate argument; CAST subpaths instead. +- On the default path, a whole VARIANT value should not be used directly in `ORDER BY`, `GROUP BY`, as a `JOIN KEY`, or as an aggregate argument; CAST the required subpath instead. - Strings can be implicitly converted to VARIANT. +The experimental compute path described below adds canonical hashing and serialization for grouping and set-operation use cases. It does not make root VARIANT comparison predicates or VARIANT join keys available. + | VARIANT | Castable | Coercible | | --------------- | -------- | --------- | -| `ARRAY` | ✔ | ❌ | +| `ARRAY` | ✔ | ❌ | | `BOOLEAN` | ✔ | ✔ | | `DATE/DATETIME` | ✔ | ✔ | | `FLOAT` | ✔ | ✔ | | `IPV4/IPV6` | ✔ | ✔ | | `DECIMAL` | ✔ | ✔ | -| `MAP` | ❌ | ❌ | +| `MAP` | ❌ | ❌ | | `TIMESTAMP` | ✔ | ✔ | | `VARCHAR` | ✔ | ✔ | | `JSON` | ✔ | ✔ | +## Experimental compute path: ColumnVariantV2 + +[PR #65561](https://github.com/apache/doris/pull/65561) adds native `ColumnVariantV2` execution for experimental, compute-only use. It is disabled by default and is controlled by the mutable BE configuration `enable_variant_v2`: + +```text +# BE mutable configuration +enable_variant_v2 = true +``` + +This PR changes the BE execution path only. It does not add V2 table creation, loading, segment storage, readers or writers, statistics, or compaction support. The PR also does not provide V1/V2 mixed-version rolling-upgrade compatibility. + +### What changes when V2 is enabled + +- **Canonical value semantics**: equivalent integral numbers normalize together; decimal trailing zeros are normalized; `+0`, `-0`, and integral zero share the same canonical value; object key order is ignored; array order is preserved. Invalid encodings and violated invariants fail instead of being silently accepted. +- **Hashing and serialization**: canonical hashes and arena serialization enable `GROUP BY`, `DISTINCT`, `COUNT(DISTINCT ...)`, `INTERSECT`, `EXCEPT`, `UNION DISTINCT`, and aggregation grouping keys for supported Variant values. +- **Parsing is explicit**: `parse_to_variant(json_string)` parses a JSON string into a Variant value; `parse_to_variant_error_to_null(json_string)` returns SQL `NULL` when validation fails; `CAST(string AS VARIANT)` creates a typed Variant string and does not parse the string as JSON. +- **Expressions and nested values**: conditional expressions, nested containers, and Variant-aware `explode` can operate on the canonical representation. +- **Null and physical states**: SQL `NULL` remains distinct from Variant/JSON `null`. A `ColumnVariantV2` column uses a whole-column physical state: encoded (`E`) or typed scalar (`T`, with a Variant-null map); it does not mix E and T at row level. + +### Boundaries + +- Root Variant comparison predicates (`=`, `!=`, `<=>`, and ordering comparisons) are not supported. Extract and CAST a comparable path on both sides: + +```sql +SELECT * +FROM tbl +WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); +``` + +- Variant expressions are not supported as join keys. Root Variant values are also not supported as sort or TopN keys, window partition or order keys, or arguments to `MIN`/`MAX`. +- Regression suites that change this process-wide BE configuration should run in a `nonConcurrent` suite. +- Enable V2 only after validating the target workload; it is an experimental execution path and does not change the storage compatibility contract. + ## Wide columns When ingested data contains many distinct JSON keys, the number of subcolumns produced by Subcolumnization can grow rapidly; at scale this may cause metadata bloat, higher write/merge cost, and query slowdowns. To address “wide columns” (too many subcolumns), VARIANT provides two mechanisms: **Sparse columns** and **DOC encoding**. diff --git a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md index a64898c64c814..8d4973d2d76c1 100644 --- a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md +++ b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md @@ -26,7 +26,8 @@ ELEMENT_AT(container, key_or_index) - `key_or_index`: - For `ARRAY`: An integer, with indexing starting from **1**. - For `MAP`: The key type (`K`) of the `MAP`, which can be any supported primitive type. - - For `VARIANT`: A string type. + - For `VARIANT` object access: A string key. + - For a VARIANT array on `ColumnVariantV2`: An integer index; positive indexes are 1-based and negative indexes count backward from the end. ## Return Value @@ -42,6 +43,13 @@ ELEMENT_AT(container, key_or_index) 2. Negative indexes are supported: `-1` represents the last element, `-2` the second-to-last, and so on. 3. The `ELEMENT_AT(container, key_or_index)` function behaves the same as `container[key_or_index]` (see examples for details). +When `enable_variant_v2 = true`, integer access into a VARIANT array uses positive 1-based indexes: `1` is the first element and `-1` is the last. Index `0` and out-of-range indexes return `NULL`. The result is still VARIANT; CAST it before typed comparison or aggregation. + +```sql +SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 1); -- 10 +SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), -1); -- 30 +``` + ## Examples 1. The `ELEMENT_AT` function works the same as `[]`. diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 679fec8621278..d79b0883bd5b7 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -371,8 +371,10 @@ curl --location-trusted -u root: -T gh_2022-11-07-3.json \ ## 支持的运算与 CAST 规则 -- VARIANT 本身不支持与其他类型直接比较/运算，两个 VARIANT 之间也不支持直接比较。 -- 如需比较、过滤、聚合、排序，请对子列显式或隐式 CAST 到确定类型。 +默认执行路径保持现有 VARIANT 使用约束： + +- VARIANT 本身不支持与其他类型直接比较/运算；在默认路径上，两个 VARIANT 之间也不支持直接比较。 +- 如需比较、过滤、聚合、排序，请对子路径显式或隐式 CAST 到确定类型。 ```sql -- 显式 CAST @@ -385,22 +387,57 @@ SELECT * FROM tbl WHERE v['bool']; SELECT * FROM tbl WHERE v['str'] MATCH 'Doris'; ``` -- VARIANT 本身不可直接用于 ORDER BY、GROUP BY、JOIN KEY 或聚合参数；对子列 CAST 后可正常使用。 +- 在默认路径上，不应将整个 VARIANT 直接用于 `ORDER BY`、`GROUP BY`、`JOIN KEY` 或聚合参数；请先对子路径 CAST。 - 字符串类型可隐式转换为 VARIANT。 +下面的实验性计算路径新增了面向分组和集合运算的规范化哈希与序列化，但不会因此开放根 VARIANT 比较谓词或 VARIANT JOIN KEY。 + | VARIANT | Castable | Coercible | | --------------- | -------- | --------- | -| `ARRAY` | ✔ | ❌ | +| `ARRAY` | ✔ | ❌ | | `BOOLEAN` | ✔ | ✔ | | `DATE/DATETIME` | ✔ | ✔ | | `FLOAT` | ✔ | ✔ | | `IPV4/IPV6` | ✔ | ✔ | | `DECIMAL` | ✔ | ✔ | -| `MAP` | ❌ | ❌ | +| `MAP` | ❌ | ❌ | | `TIMESTAMP` | ✔ | ✔ | | `VARCHAR` | ✔ | ✔ | | `JSON` | ✔ | ✔ | +## 实验性计算路径：ColumnVariantV2 + +[PR #65561](https://github.com/apache/doris/pull/65561) 新增了原生 `ColumnVariantV2` 执行能力，当前定位为实验性的仅计算路径。默认关闭，通过可动态修改的 BE 配置 `enable_variant_v2` 控制： + +```text +# BE 动态配置 +enable_variant_v2 = true +``` + +该 PR 只改变 BE 计算路径，不增加 V2 表创建、导入、Segment 存储、读写器、统计信息或 Compaction 支持；同时也没有提供 V1/V2 混部滚动升级兼容能力。 + +### 开启 V2 后的行为变化 + +- **规范化值语义**：等价的整数类型会归一为同一值；Decimal 尾随零会被归一；`+0`、`-0` 与整数零使用同一规范值；对象 key 顺序不影响值；数组顺序仍然保留。非法编码或违反内部不变量时会报错，而不是静默接受。 +- **哈希与序列化**：规范化哈希和 arena 序列化为支持的 Variant 值提供 `GROUP BY`、`DISTINCT`、`COUNT(DISTINCT ...)`、`INTERSECT`、`EXCEPT`、`UNION DISTINCT` 以及聚合分组键能力。 +- **显式解析语义**：`parse_to_variant(json_string)` 将 JSON 字符串解析为 Variant；`parse_to_variant_error_to_null(json_string)` 在校验失败时返回 SQL `NULL`；`CAST(string AS VARIANT)` 创建的是带类型的 Variant 字符串，不会把字符串按 JSON 解析。 +- **表达式与嵌套值**：条件表达式、嵌套容器以及支持 Variant 的 `explode` 可以基于规范化表示执行。 +- **NULL 与物理状态**：SQL `NULL` 仍与 Variant/JSON `null` 不同。`ColumnVariantV2` 列使用整列级物理状态：编码状态（`E`）或带 Variant-null map 的类型化标量状态（`T`），不会在行级混用 E/T。 + +### 使用边界 + +- 根 Variant 比较谓词（`=`、`!=`、`<=>` 以及排序比较）仍不支持。请在两侧提取相同语义的子路径并 CAST 到可比较类型： + +```sql +SELECT * +FROM tbl +WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); +``` + +- 不支持将 Variant 表达式作为 JOIN KEY；根 Variant 也不能作为 Sort/TopN 键、窗口分区键或窗口排序键，也不能作为 `MIN`/`MAX` 参数。 +- 会修改进程级 BE 配置的回归测试应放入 `nonConcurrent` suite。 +- 开启 V2 前请先验证目标 workload；它是实验性计算路径，不改变存储兼容性契约。 + ## 宽列 当导入数据包含大量不同的 JSON key 时，通过子列列式提取（Subcolumnization）生成的子列会迅速增多；当规模达到一定程度，可能出现元数据膨胀、写入/合并开销增大、查询性能下降等问题。为应对“宽列”（子列过多），VARIANT 提供两种机制：**稀疏列** 与 **DOC 编码**。 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md index 64f2d2ef790fa..88f84ec1cd402 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md @@ -26,7 +26,8 @@ ELEMENT_AT(container, key_or_index) - `key_or_index`： - 对于 `ARRAY`：为整数类型，索引从 **1** 开始； - 对于 `MAP`：为 `MAP` 中的键类型（`K`），可为任意支持的基础类型。 - - 对于 `VARIANT`： 为字符串类型 + - 对于 `VARIANT` 对象访问：为字符串类型的 key； + - 对于 `ColumnVariantV2` 中的 VARIANT 数组：为整数索引，正数索引从 1 开始，负数索引从数组末尾倒数。 ## 返回值 @@ -42,6 +43,13 @@ ELEMENT_AT(container, key_or_index) 2. 支持负数索引，`-1` 表示最后一个元素，`-2` 表示倒数第二个，以此类推； 3. `ELEMENT_AT(container, key_or_index)` 函数的功能与 `container[key_or_index]` 作用一致（详细见示例）。 +当 `enable_variant_v2 = true` 时，VARIANT 数组的整数访问遵循正数从 1 开始的规则：`1` 表示第一个元素，`-1` 表示最后一个元素；索引 `0` 和越界索引返回 `NULL`。返回值仍是 VARIANT，如需按确定类型比较或聚合，请先 CAST。 + +```sql +SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 1); -- 10 +SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), -1); -- 30 +``` + ## 示例 1. `ELEMENT_AT` 函数的功能与 `[]` 作用一致。 From ef7240fee42f2e36748cbd2ce9d45a3474fb889a Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Sun, 19 Jul 2026 19:28:56 +0800 Subject: [PATCH 2/9] docs: add Variant parsing function guides --- .../sql-data-types/semi-structured/VARIANT.md | 42 +++++++-- .../variant-functions/element-at.md | 8 +- .../parse-to-variant-error-to-null.md | 71 +++++++++++++++ .../variant-functions/parse-to-variant.md | 87 +++++++++++++++++++ .../sql-data-types/semi-structured/VARIANT.md | 42 +++++++-- .../variant-functions/element-at.md | 8 +- .../parse-to-variant-error-to-null.md | 71 +++++++++++++++ .../variant-functions/parse-to-variant.md | 87 +++++++++++++++++++ 8 files changed, 396 insertions(+), 20 deletions(-) create mode 100644 docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md create mode 100644 docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md create mode 100644 i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md create mode 100644 i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md diff --git a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 01b587c49782b..7bd438d2904b2 100644 --- a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -67,6 +67,35 @@ FROM ${table_name} WHERE ARRAY_CONTAINS(CAST(v['tags'] AS ARRAY), 'Doris'); ``` +## Create and access values + +VARIANT values can be created from JSON text or from typed SQL expressions: + +- Use [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) when a string contains JSON text that should be parsed. +- Use `CAST(expression AS VARIANT)` when the SQL value should become a typed Variant value. A string cast does not parse the string as JSON. + +### Parse JSON text + +```sql +SELECT PARSE_TO_VARIANT('{\"user\": {\"id\": 42}, \"active\": true}'); +SELECT PARSE_TO_VARIANT('[10, 20, 30]'); +``` + +Use [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null) when invalid JSON should return SQL `NULL` instead of failing the query. + +### Access objects and arrays + +Object fields can be accessed with a string key. For `ColumnVariantV2`, VARIANT array indexes are zero-based for non-negative indexes and support negative indexes from the end. The extracted value remains `VARIANT`; CAST it before typed comparison or aggregation. + +```sql +SET enable_variant_v2 = true; + +SELECT CAST(PARSE_TO_VARIANT('{\"user\": {\"id\": 42}}')['user']['id'] AS BIGINT); +SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), 0); -- 10 +SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1); -- 30 +``` + +See [ELEMENT_AT](../../../sql-functions/scalar-functions/variant-functions/element-at) for object and array access details. ## Primitive types VARIANT infers subcolumn types automatically. Supported types include: @@ -407,20 +436,19 @@ The experimental compute path described below adds canonical hashing and seriali ## Experimental compute path: ColumnVariantV2 -[PR #65561](https://github.com/apache/doris/pull/65561) adds native `ColumnVariantV2` execution for experimental, compute-only use. It is disabled by default and is controlled by the mutable BE configuration `enable_variant_v2`: +`ColumnVariantV2` is an experimental, compute-only execution path. It is disabled by default and selected for the current FE session with the session variable `enable_variant_v2`: -```text -# BE mutable configuration -enable_variant_v2 = true +```sql +SET enable_variant_v2 = true; ``` -This PR changes the BE execution path only. It does not add V2 table creation, loading, segment storage, readers or writers, statistics, or compaction support. The PR also does not provide V1/V2 mixed-version rolling-upgrade compatibility. +The session variable changes the FE/BE execution type marker only. It does not add V2 table creation, loading, segment storage, readers or writers, statistics, or compaction support. It also does not provide V1/V2 mixed-version rolling-upgrade compatibility. ### What changes when V2 is enabled - **Canonical value semantics**: equivalent integral numbers normalize together; decimal trailing zeros are normalized; `+0`, `-0`, and integral zero share the same canonical value; object key order is ignored; array order is preserved. Invalid encodings and violated invariants fail instead of being silently accepted. - **Hashing and serialization**: canonical hashes and arena serialization enable `GROUP BY`, `DISTINCT`, `COUNT(DISTINCT ...)`, `INTERSECT`, `EXCEPT`, `UNION DISTINCT`, and aggregation grouping keys for supported Variant values. -- **Parsing is explicit**: `parse_to_variant(json_string)` parses a JSON string into a Variant value; `parse_to_variant_error_to_null(json_string)` returns SQL `NULL` when validation fails; `CAST(string AS VARIANT)` creates a typed Variant string and does not parse the string as JSON. +- **Parsing is explicit**: `parse_to_variant(json_string)` parses a JSON string into a Variant value; `parse_to_variant_error_to_null(json_string)` returns SQL `NULL` when validation fails; `CAST(string AS VARIANT)` creates a typed Variant string and does not parse the string as JSON. See [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) and [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null) for function syntax and examples. - **Expressions and nested values**: conditional expressions, nested containers, and Variant-aware `explode` can operate on the canonical representation. - **Null and physical states**: SQL `NULL` remains distinct from Variant/JSON `null`. A `ColumnVariantV2` column uses a whole-column physical state: encoded (`E`) or typed scalar (`T`, with a Variant-null map); it does not mix E and T at row level. @@ -435,7 +463,7 @@ WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); ``` - Variant expressions are not supported as join keys. Root Variant values are also not supported as sort or TopN keys, window partition or order keys, or arguments to `MIN`/`MAX`. -- Regression suites that change this process-wide BE configuration should run in a `nonConcurrent` suite. +- Regression suites covering V2 remain tagged `nonConcurrent`; the feature selection itself is session-scoped and does not change native Variant storage or compaction. - Enable V2 only after validating the target workload; it is an experimental execution path and does not change the storage compatibility contract. ## Wide columns diff --git a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md index 8d4973d2d76c1..6e05a4b5b227f 100644 --- a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md +++ b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md @@ -27,7 +27,7 @@ ELEMENT_AT(container, key_or_index) - For `ARRAY`: An integer, with indexing starting from **1**. - For `MAP`: The key type (`K`) of the `MAP`, which can be any supported primitive type. - For `VARIANT` object access: A string key. - - For a VARIANT array on `ColumnVariantV2`: An integer index; positive indexes are 1-based and negative indexes count backward from the end. + - For a VARIANT array on `ColumnVariantV2`: An integer index; non-negative indexes are 0-based and negative indexes count backward from the end. ## Return Value @@ -43,10 +43,12 @@ ELEMENT_AT(container, key_or_index) 2. Negative indexes are supported: `-1` represents the last element, `-2` the second-to-last, and so on. 3. The `ELEMENT_AT(container, key_or_index)` function behaves the same as `container[key_or_index]` (see examples for details). -When `enable_variant_v2 = true`, integer access into a VARIANT array uses positive 1-based indexes: `1` is the first element and `-1` is the last. Index `0` and out-of-range indexes return `NULL`. The result is still VARIANT; CAST it before typed comparison or aggregation. +After `SET enable_variant_v2 = true`, integer access into a VARIANT array uses zero-based non-negative indexes: `0` is the first element and `1` is the second; `-1` is the last. Out-of-range indexes return `NULL`. The result is still VARIANT; CAST it before typed comparison or aggregation. ```sql -SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 1); -- 10 +SET enable_variant_v2 = true; +SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 0); -- 10 +SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 1); -- 20 SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), -1); -- 30 ``` diff --git a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md new file mode 100644 index 0000000000000..5547cd201337c --- /dev/null +++ b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md @@ -0,0 +1,71 @@ +--- +{ + "title": "PARSE_TO_VARIANT_ERROR_TO_NULL", + "language": "en", + "description": "Parses JSON text into a VARIANT value and returns SQL NULL for invalid input." +} +--- + +## Function + +`PARSE_TO_VARIANT_ERROR_TO_NULL` parses JSON text into `VARIANT`. Invalid JSON returns SQL `NULL` instead of failing the query. + +## Syntax + +```sql +PARSE_TO_VARIANT_ERROR_TO_NULL(json_text) +``` + +## Parameters + +- `json_text`: A `VARCHAR` expression containing one complete JSON value. + +## Return Value + +- Returns a nullable `VARIANT` value. +- SQL `NULL` input returns SQL `NULL`. +- Invalid JSON returns SQL `NULL`. +- Valid JSON `null` remains Variant/JSON `null`, distinct from SQL `NULL`. + +## Experimental behavior + +`ColumnVariantV2` is an experimental, compute-only execution path. It is disabled by default and selected for the current FE session with: + +```sql +SET enable_variant_v2 = true; +``` + +The session variable changes the execution type of the expression result only. It does not change the physical `VARIANT` type used by table storage, readers, writers, or compaction. + +## Examples + +### Keep valid values and null invalid JSON + +```sql +SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\": 1}') AS valid_value, + PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value, + PARSE_TO_VARIANT_ERROR_TO_NULL(NULL) AS sql_null_value; +``` + +The first expression returns a VARIANT object. The second and third expressions return SQL `NULL`. + +### Parse arrays and access elements + +```sql +SET enable_variant_v2 = true; + +SELECT CAST(ELEMENT_AT( + PARSE_TO_VARIANT_ERROR_TO_NULL('[10, 20, 30]'), + 0 + ) AS INT) AS first_item, + CAST(ELEMENT_AT( + PARSE_TO_VARIANT_ERROR_TO_NULL('[10, 20, 30]'), + -1 + ) AS INT) AS last_item; +``` + +For `ColumnVariantV2`, non-negative array indexes are zero-based and negative indexes count from the end. + +### Choose strict or tolerant parsing + +Use [PARSE_TO_VARIANT](./parse-to-variant) when invalid JSON should stop the query and surface an error. Use this function when invalid JSON should become SQL `NULL` and the remaining rows should continue. diff --git a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md new file mode 100644 index 0000000000000..d90d88a5bbecf --- /dev/null +++ b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md @@ -0,0 +1,87 @@ +--- +{ + "title": "PARSE_TO_VARIANT", + "language": "en", + "description": "Parses JSON text into a VARIANT value." +} +--- + +## Function + +`PARSE_TO_VARIANT` parses one complete JSON value from a `VARCHAR` expression and returns it as `VARIANT`. JSON objects, arrays, strings, numbers, booleans, and the JSON literal `null` are supported. + +## Syntax + +```sql +PARSE_TO_VARIANT(json_text) +``` + +## Parameters + +- `json_text`: A `VARCHAR` expression containing one complete JSON value. + +## Return Value + +- Returns a `VARIANT` value. +- SQL `NULL` input returns SQL `NULL`. +- JSON `null` returns Variant/JSON `null`, which is distinct from SQL `NULL`. + +## Experimental behavior + +`ColumnVariantV2` is an experimental, compute-only execution path. It is disabled by default and selected for the current FE session with: + +```sql +SET enable_variant_v2 = true; +``` + +The session variable changes the execution type of the expression result only. It does not change the physical `VARIANT` type used by table storage, readers, writers, or compaction. + +## Errors + +Invalid JSON causes `PARSE_TO_VARIANT` to return an error. Use [PARSE_TO_VARIANT_ERROR_TO_NULL](./parse-to-variant-error-to-null) when invalid input should become SQL `NULL` instead. + +## Examples + +### Parse JSON values + +```sql +SELECT PARSE_TO_VARIANT('{\"id\": 42, \"tags\": [\"doris\", \"sql\"]}'); +SELECT PARSE_TO_VARIANT('[10, 20, 30]'); +SELECT PARSE_TO_VARIANT('42'); +SELECT PARSE_TO_VARIANT('true'); +SELECT PARSE_TO_VARIANT('\"doris\"'); +SELECT PARSE_TO_VARIANT('null'); +``` + +### Extract and cast a value + +```sql +SET enable_variant_v2 = true; + +SELECT CAST( + PARSE_TO_VARIANT('{\"user\": {\"id\": 42}}')['user']['id'] + AS BIGINT + ) AS user_id; + +SELECT CAST(ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), 0) AS INT) AS first_item, + CAST(ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1) AS INT) AS last_item; +``` + +For `ColumnVariantV2`, non-negative indexes into a VARIANT array are zero-based (`0` is the first element) and negative indexes count from the end. See [ELEMENT_AT](./element-at) for access rules. + +### Distinguish JSON parsing from string casting + +```sql +-- Parses the JSON object into a structured VARIANT value. +SELECT PARSE_TO_VARIANT('{\"a\": 1}'); + +-- Creates a typed VARIANT string; it does not parse the string as JSON. +SELECT CAST('{\"a\": 1}' AS VARIANT); +``` + +### Invalid input + +```sql +-- Returns an error because the JSON object is incomplete. +SELECT PARSE_TO_VARIANT('{\"id\":'); +``` diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index d79b0883bd5b7..686c296b7d66e 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -67,6 +67,35 @@ FROM ${table_name} WHERE ARRAY_CONTAINS(CAST(v['tags'] AS ARRAY), 'Doris'); ``` +## 创建和访问值 + +VARIANT 值可以从 JSON 文本或带确定类型的 SQL 表达式创建： + +- 如果字符串中包含需要解析的 JSON 文本，请使用 [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant)。 +- 如果要将 SQL 值转换为带类型的 Variant 值，请使用 `CAST(expression AS VARIANT)`。字符串 CAST 不会把字符串按 JSON 解析。 + +### 解析 JSON 文本 + +```sql +SELECT PARSE_TO_VARIANT('{\"user\": {\"id\": 42}, \"active\": true}'); +SELECT PARSE_TO_VARIANT('[10, 20, 30]'); +``` + +如果非法 JSON 应该返回 SQL `NULL` 而不是使查询失败，请使用 [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null)。 + +### 访问对象和数组 + +对象字段可以使用字符串 key 访问。对于 `ColumnVariantV2`，VARIANT 数组的非负索引从 0 开始，并支持从数组末尾倒数的负数索引。提取出的值仍是 `VARIANT`，如需按确定类型比较或聚合，请先 CAST。 + +```sql +SET enable_variant_v2 = true; + +SELECT CAST(PARSE_TO_VARIANT('{\"user\": {\"id\": 42}}')['user']['id'] AS BIGINT); +SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), 0); -- 10 +SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1); -- 30 +``` + +对象和数组访问的详细说明请参见 [ELEMENT_AT](../../../sql-functions/scalar-functions/variant-functions/element-at)。 ## 基本类型 VARIANT 自动推断的子列基础类型包括： @@ -407,20 +436,19 @@ SELECT * FROM tbl WHERE v['str'] MATCH 'Doris'; ## 实验性计算路径：ColumnVariantV2 -[PR #65561](https://github.com/apache/doris/pull/65561) 新增了原生 `ColumnVariantV2` 执行能力，当前定位为实验性的仅计算路径。默认关闭，通过可动态修改的 BE 配置 `enable_variant_v2` 控制： +`ColumnVariantV2` 是实验性的、仅用于计算的执行路径。默认关闭，通过当前 FE session 的 session variable `enable_variant_v2` 选择： -```text -# BE 动态配置 -enable_variant_v2 = true +```sql +SET enable_variant_v2 = true; ``` -该 PR 只改变 BE 计算路径，不增加 V2 表创建、导入、Segment 存储、读写器、统计信息或 Compaction 支持；同时也没有提供 V1/V2 混部滚动升级兼容能力。 +该 session variable 只改变 FE/BE 执行类型标记，不增加 V2 表创建、导入、Segment 存储、读写器、统计信息或 Compaction 支持；同时也没有提供 V1/V2 混部滚动升级兼容能力。 ### 开启 V2 后的行为变化 - **规范化值语义**：等价的整数类型会归一为同一值；Decimal 尾随零会被归一；`+0`、`-0` 与整数零使用同一规范值；对象 key 顺序不影响值；数组顺序仍然保留。非法编码或违反内部不变量时会报错，而不是静默接受。 - **哈希与序列化**：规范化哈希和 arena 序列化为支持的 Variant 值提供 `GROUP BY`、`DISTINCT`、`COUNT(DISTINCT ...)`、`INTERSECT`、`EXCEPT`、`UNION DISTINCT` 以及聚合分组键能力。 -- **显式解析语义**：`parse_to_variant(json_string)` 将 JSON 字符串解析为 Variant；`parse_to_variant_error_to_null(json_string)` 在校验失败时返回 SQL `NULL`；`CAST(string AS VARIANT)` 创建的是带类型的 Variant 字符串，不会把字符串按 JSON 解析。 +- **显式解析语义**：`parse_to_variant(json_string)` 将 JSON 字符串解析为 Variant；`parse_to_variant_error_to_null(json_string)` 在校验失败时返回 SQL `NULL`；`CAST(string AS VARIANT)` 创建的是带类型的 Variant 字符串，不会把字符串按 JSON 解析。函数语法和示例请参见 [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) 与 [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null)。 - **表达式与嵌套值**：条件表达式、嵌套容器以及支持 Variant 的 `explode` 可以基于规范化表示执行。 - **NULL 与物理状态**：SQL `NULL` 仍与 Variant/JSON `null` 不同。`ColumnVariantV2` 列使用整列级物理状态：编码状态（`E`）或带 Variant-null map 的类型化标量状态（`T`），不会在行级混用 E/T。 @@ -435,7 +463,7 @@ WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); ``` - 不支持将 Variant 表达式作为 JOIN KEY；根 Variant 也不能作为 Sort/TopN 键、窗口分区键或窗口排序键，也不能作为 `MIN`/`MAX` 参数。 -- 会修改进程级 BE 配置的回归测试应放入 `nonConcurrent` suite。 +- V2 相关回归测试仍标记为 `nonConcurrent`；功能选择本身是 session-scoped，不会改变原生 Variant 的存储或 Compaction。 - 开启 V2 前请先验证目标 workload；它是实验性计算路径，不改变存储兼容性契约。 ## 宽列 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md index 88f84ec1cd402..500e59d0f8840 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md @@ -27,7 +27,7 @@ ELEMENT_AT(container, key_or_index) - 对于 `ARRAY`：为整数类型，索引从 **1** 开始； - 对于 `MAP`：为 `MAP` 中的键类型（`K`），可为任意支持的基础类型。 - 对于 `VARIANT` 对象访问：为字符串类型的 key； - - 对于 `ColumnVariantV2` 中的 VARIANT 数组：为整数索引，正数索引从 1 开始，负数索引从数组末尾倒数。 + - 对于 `ColumnVariantV2` 中的 VARIANT 数组：为整数索引，非负索引从 0 开始，负数索引从数组末尾倒数。 ## 返回值 @@ -43,10 +43,12 @@ ELEMENT_AT(container, key_or_index) 2. 支持负数索引，`-1` 表示最后一个元素，`-2` 表示倒数第二个，以此类推； 3. `ELEMENT_AT(container, key_or_index)` 函数的功能与 `container[key_or_index]` 作用一致（详细见示例）。 -当 `enable_variant_v2 = true` 时，VARIANT 数组的整数访问遵循正数从 1 开始的规则：`1` 表示第一个元素，`-1` 表示最后一个元素；索引 `0` 和越界索引返回 `NULL`。返回值仍是 VARIANT，如需按确定类型比较或聚合，请先 CAST。 +执行 `SET enable_variant_v2 = true` 后，VARIANT 数组的整数访问使用非负索引从 0 开始的规则：`0` 表示第一个元素，`1` 表示第二个元素，`-1` 表示最后一个元素；越界索引返回 `NULL`。返回值仍是 VARIANT，如需按确定类型比较或聚合，请先 CAST。 ```sql -SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 1); -- 10 +SET enable_variant_v2 = true; +SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 0); -- 10 +SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 1); -- 20 SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), -1); -- 30 ``` diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md new file mode 100644 index 0000000000000..11651e3f16f50 --- /dev/null +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md @@ -0,0 +1,71 @@ +--- +{ + "title": "PARSE_TO_VARIANT_ERROR_TO_NULL", + "language": "zh-CN", + "description": "将 JSON 文本解析为 VARIANT 值，非法输入返回 SQL NULL。" +} +--- + +## 功能 + +`PARSE_TO_VARIANT_ERROR_TO_NULL` 将 JSON 文本解析为 `VARIANT`。输入不是合法 JSON 时返回 SQL `NULL`，不会使查询失败。 + +## 语法 + +```sql +PARSE_TO_VARIANT_ERROR_TO_NULL(json_text) +``` + +## 参数 + +- `json_text`：包含一个完整 JSON 值的 `VARCHAR` 表达式。 + +## 返回值 + +- 返回可为 NULL 的 `VARIANT` 值。 +- 输入为 SQL `NULL` 时返回 SQL `NULL`。 +- 输入为非法 JSON 时返回 SQL `NULL`。 +- 合法 JSON `null` 仍然是 Variant/JSON `null`，与 SQL `NULL` 不同。 + +## 实验性行为 + +`ColumnVariantV2` 是实验性的、仅用于计算的执行路径，默认关闭。可以在当前 FE session 中执行以下语句开启： + +```sql +SET enable_variant_v2 = true; +``` + +该 session variable 只改变表达式结果的执行类型，不改变表存储、读写器或 Compaction 使用的物理 `VARIANT` 类型。 + +## 示例 + +### 保留合法值，将非法 JSON 转为 NULL + +```sql +SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\": 1}') AS valid_value, + PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value, + PARSE_TO_VARIANT_ERROR_TO_NULL(NULL) AS sql_null_value; +``` + +第一个表达式返回 VARIANT 对象，第二个和第三个表达式返回 SQL `NULL`。 + +### 解析数组并访问元素 + +```sql +SET enable_variant_v2 = true; + +SELECT CAST(ELEMENT_AT( + PARSE_TO_VARIANT_ERROR_TO_NULL('[10, 20, 30]'), + 0 + ) AS INT) AS first_item, + CAST(ELEMENT_AT( + PARSE_TO_VARIANT_ERROR_TO_NULL('[10, 20, 30]'), + -1 + ) AS INT) AS last_item; +``` + +对于 `ColumnVariantV2`，数组非负索引从 0 开始，负数索引从数组末尾倒数。 + +### 选择严格解析或容错解析 + +如果非法 JSON 应该让查询失败并暴露错误，请使用 [PARSE_TO_VARIANT](./parse-to-variant)；如果非法 JSON 应该转换为 SQL `NULL` 并继续处理其他行，请使用本函数。 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md new file mode 100644 index 0000000000000..f0e99c8f93d99 --- /dev/null +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md @@ -0,0 +1,87 @@ +--- +{ + "title": "PARSE_TO_VARIANT", + "language": "zh-CN", + "description": "将 JSON 文本解析为 VARIANT 值。" +} +--- + +## 功能 + +`PARSE_TO_VARIANT` 将 `VARCHAR` 表达式中的一个完整 JSON 值解析为 `VARIANT`。支持 JSON 对象、数组、字符串、数字、布尔值和 JSON 字面量 `null`。 + +## 语法 + +```sql +PARSE_TO_VARIANT(json_text) +``` + +## 参数 + +- `json_text`：包含一个完整 JSON 值的 `VARCHAR` 表达式。 + +## 返回值 + +- 返回 `VARIANT` 值。 +- 输入为 SQL `NULL` 时返回 SQL `NULL`。 +- 输入为 JSON `null` 时返回 Variant/JSON `null`，它与 SQL `NULL` 不同。 + +## 实验性行为 + +`ColumnVariantV2` 是实验性的、仅用于计算的执行路径，默认关闭。可以在当前 FE session 中执行以下语句开启： + +```sql +SET enable_variant_v2 = true; +``` + +该 session variable 只改变表达式结果的执行类型，不改变表存储、读写器或 Compaction 使用的物理 `VARIANT` 类型。 + +## 错误处理 + +非法 JSON 会使 `PARSE_TO_VARIANT` 返回错误。如果希望将非法输入转换为 SQL `NULL`，请使用 [PARSE_TO_VARIANT_ERROR_TO_NULL](./parse-to-variant-error-to-null)。 + +## 示例 + +### 解析 JSON 值 + +```sql +SELECT PARSE_TO_VARIANT('{\"id\": 42, \"tags\": [\"doris\", \"sql\"]}'); +SELECT PARSE_TO_VARIANT('[10, 20, 30]'); +SELECT PARSE_TO_VARIANT('42'); +SELECT PARSE_TO_VARIANT('true'); +SELECT PARSE_TO_VARIANT('\"doris\"'); +SELECT PARSE_TO_VARIANT('null'); +``` + +### 提取值并 CAST 为确定类型 + +```sql +SET enable_variant_v2 = true; + +SELECT CAST( + PARSE_TO_VARIANT('{\"user\": {\"id\": 42}}')['user']['id'] + AS BIGINT + ) AS user_id; + +SELECT CAST(ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), 0) AS INT) AS first_item, + CAST(ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1) AS INT) AS last_item; +``` + +对于 `ColumnVariantV2`，VARIANT 数组的非负索引从 0 开始（`0` 表示第一个元素），负数索引从数组末尾倒数。访问规则请参见 [ELEMENT_AT](./element-at)。 + +### 区分 JSON 解析与字符串 CAST + +```sql +-- 解析 JSON 对象并返回结构化 VARIANT 值。 +SELECT PARSE_TO_VARIANT('{\"a\": 1}'); + +-- 创建带类型的 VARIANT 字符串，不会把字符串按 JSON 解析。 +SELECT CAST('{\"a\": 1}' AS VARIANT); +``` + +### 非法输入 + +```sql +-- JSON 对象不完整，会返回错误。 +SELECT PARSE_TO_VARIANT('{\"id\":'); +``` From 9fd9aec610f2316b38b7a9c18e619a86d7ccf3e3 Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Sun, 19 Jul 2026 19:32:33 +0800 Subject: [PATCH 3/9] docs: normalize Variant element index notes --- .../scalar-functions/variant-functions/element-at.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md index 3f28a236867c9..12b7645a59d39 100644 --- a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md +++ b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md @@ -42,9 +42,9 @@ ELEMENT_AT(container, key_or_index) ## Notes -1. **ARRAY indexes start from 1**, not 0. -2. When `enable_variant_v2` is enabled, non-negative indexes into a VARIANT array start from `0`; `0` is the first element and `-1` is the last. -3. Negative indexes are supported for ARRAY and VARIANT array access: `-1` represents the last element, `-2` the second-to-last, and so on. +1. **ARRAY indexes start from 1**, not 0. +2. When `enable_variant_v2` is enabled, non-negative indexes into a VARIANT array start from `0`; `0` is the first element and `-1` is the last. +3. Negative indexes are supported for ARRAY and VARIANT array access: `-1` represents the last element, `-2` the second-to-last, and so on. 4. The `ELEMENT_AT(container, key_or_index)` function behaves the same as `container[key_or_index]` (see examples for details). ```sql From 038c006505c458f4a9413e69a993b888f25bbcd7 Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Sun, 19 Jul 2026 19:41:13 +0800 Subject: [PATCH 4/9] docs: add ColumnVariantV2 behavior examples --- .../sql-data-types/semi-structured/VARIANT.md | 74 +++++++++++++++++++ .../sql-data-types/semi-structured/VARIANT.md | 74 +++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 21a4266c3222d..2f3745d34c2e3 100644 --- a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -96,6 +96,47 @@ SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1); -- 30 ``` See [ELEMENT_AT](../../../sql-functions/scalar-functions/variant-functions/element-at) for object and array access details. +## Equality semantics + +For `ColumnVariantV2`, equality used by canonical hashing and serialization is based on the logical value rather than the physical encoded bytes: + +- Equivalent integral numeric representations normalize to the same value. +- Decimal trailing zeros do not change the value. +- `+0`, `-0`, and integral zero normalize together. +- Object key order does not affect equality, while array element order does. +- Variant/JSON `null` is distinct from SQL `NULL`. + +These rules are used by supported hash-based operations such as `GROUP BY`, `DISTINCT`, `COUNT(DISTINCT ...)`, `INTERSECT`, `EXCEPT`, and `UNION DISTINCT`. They do not enable root Variant comparison predicates: direct `VARIANT = VARIANT` and ordering comparisons remain unsupported. + +```sql +SET enable_variant_v2 = true; + +-- 1 and 1.0 have one canonical distinct value. +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('1') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('1.0') AS value +) AS numeric_values; +-- distinct_count: 1 + +-- Object key order is ignored; array order is preserved. +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('{\"a\": 1, \"b\": 2}') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('{\"b\": 2, \"a\": 1}') AS value +) AS object_values; +-- distinct_count: 1 + +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('[1, 2]') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('[2, 1]') AS value +) AS array_values; +-- distinct_count: 2 +``` ## Primitive types VARIANT infers subcolumn types automatically. Supported types include: @@ -470,6 +511,39 @@ The session variable changes the FE/BE execution type marker only. It does not a - **Expressions and nested values**: conditional expressions, nested containers, and Variant-aware `explode` can operate on the canonical representation. - **Null and physical states**: SQL `NULL` remains distinct from Variant/JSON `null`. A `ColumnVariantV2` column uses a whole-column physical state: encoded (`E`) or typed scalar (`T`, with a Variant-null map); it does not mix E and T at row level. +### Examples + +The following examples run in the current FE session after V2 is enabled: + +```sql +SET enable_variant_v2 = true; + +-- JSON text is parsed; CAST(string AS VARIANT) keeps the string as a typed Variant value. +SELECT PARSE_TO_VARIANT('{\"id\": 1, \"items\": [10, 20]}') AS parsed_value, + CAST('{\"id\": 1, \"items\": [10, 20]}' AS VARIANT) AS typed_string; + +-- Canonical hashing treats 1 and 1.0 as one distinct value. +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('1') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('1.0') AS value +) AS numeric_values; +-- distinct_count: 1 + +-- Invalid JSON becomes SQL NULL with the tolerant parser. +SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value; +-- invalid_value: NULL + +-- Nested access can feed a conditional expression after an explicit CAST. +SELECT CASE + WHEN CAST(PARSE_TO_VARIANT('{\"enabled\": true}')['enabled'] AS BOOLEAN) + THEN 'on' + ELSE 'off' + END AS status; +-- status: on +``` + ### Boundaries - Root Variant comparison predicates (`=`, `!=`, `<=>`, and ordering comparisons) are not supported. Extract and CAST a comparable path on both sides: diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 6d1b772dbeee0..9b54fcb6d8d2c 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -96,6 +96,47 @@ SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1); -- 30 ``` 对象和数组访问的详细说明请参见 [ELEMENT_AT](../../../sql-functions/scalar-functions/variant-functions/element-at)。 +## Equality 语义 + +对于 `ColumnVariantV2`，规范化哈希和序列化使用逻辑值而不是物理编码字节判断相等性： + +- 等价的整数数值表示会归一为同一个值。 +- Decimal 尾随零不影响值。 +- `+0`、`-0` 与整数零会归一为同一个值。 +- 对象 key 的顺序不影响相等性，但数组元素顺序会影响相等性。 +- Variant/JSON `null` 与 SQL `NULL` 不同。 + +这些规则用于支持的基于哈希的操作，例如 `GROUP BY`、`DISTINCT`、`COUNT(DISTINCT ...)`、`INTERSECT`、`EXCEPT` 和 `UNION DISTINCT`。这并不意味着根 Variant 比较谓词已经可用：直接执行 `VARIANT = VARIANT` 或排序比较仍不支持。 + +```sql +SET enable_variant_v2 = true; + +-- 1 和 1.0 归一后只有一个 distinct 值。 +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('1') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('1.0') AS value +) AS numeric_values; +-- distinct_count: 1 + +-- 对象 key 顺序被忽略；数组顺序会保留。 +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('{\"a\": 1, \"b\": 2}') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('{\"b\": 2, \"a\": 1}') AS value +) AS object_values; +-- distinct_count: 1 + +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('[1, 2]') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('[2, 1]') AS value +) AS array_values; +-- distinct_count: 2 +``` ## 基本类型 VARIANT 自动推断的子列基础类型包括： @@ -470,6 +511,39 @@ SET enable_variant_v2 = true; - **表达式与嵌套值**：条件表达式、嵌套容器以及支持 Variant 的 `explode` 可以基于规范化表示执行。 - **NULL 与物理状态**：SQL `NULL` 仍与 Variant/JSON `null` 不同。`ColumnVariantV2` 列使用整列级物理状态：编码状态（`E`）或带 Variant-null map 的类型化标量状态（`T`），不会在行级混用 E/T。 +### 示例 + +以下示例均在开启 V2 的当前 FE session 中执行： + +```sql +SET enable_variant_v2 = true; + +-- 解析 JSON 文本；CAST(string AS VARIANT) 则保留为带类型的 Variant 字符串。 +SELECT PARSE_TO_VARIANT('{\"id\": 1, \"items\": [10, 20]}') AS parsed_value, + CAST('{\"id\": 1, \"items\": [10, 20]}' AS VARIANT) AS typed_string; + +-- 规范化哈希将 1 和 1.0 视为一个 distinct 值。 +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('1') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('1.0') AS value +) AS numeric_values; +-- distinct_count: 1 + +-- 使用容错解析函数时，非法 JSON 转换为 SQL NULL。 +SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value; +-- invalid_value: NULL + +-- 嵌套访问显式 CAST 后可以作为条件表达式输入。 +SELECT CASE + WHEN CAST(PARSE_TO_VARIANT('{\"enabled\": true}')['enabled'] AS BOOLEAN) + THEN 'on' + ELSE 'off' + END AS status; +-- status: on +``` + ### 使用边界 - 根 Variant 比较谓词（`=`、`!=`、`<=>` 以及排序比较）仍不支持。请在两侧提取相同语义的子路径并 CAST 到可比较类型： From 7ef9c28337d67e5a84c2ce95ada0dc48f33a1f7d Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Sun, 19 Jul 2026 19:51:50 +0800 Subject: [PATCH 5/9] docs: clarify Variant cast and memory semantics --- .../sql-data-types/semi-structured/VARIANT.md | 40 +++++++++++++++++++ .../sql-data-types/semi-structured/VARIANT.md | 40 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 2f3745d34c2e3..4c804f8ca5046 100644 --- a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -96,6 +96,36 @@ SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1); -- 30 ``` See [ELEMENT_AT](../../../sql-functions/scalar-functions/variant-functions/element-at) for object and array access details. +## CAST semantics + +`CAST` preserves the distinction between a SQL string and JSON text: + +- `CAST(string AS VARIANT)` creates a Variant whose root value is a typed Variant string. It does **not** parse JSON. Therefore, a string such as `'{"id": 1}'` remains one string value, and invalid JSON text is still a valid input to this cast. +- `PARSE_TO_VARIANT(string)` parses the string as JSON. A JSON object or array can then be accessed by path or `ELEMENT_AT`; use `PARSE_TO_VARIANT_ERROR_TO_NULL` if invalid JSON should become SQL `NULL`. +- `CAST(PARSE_TO_VARIANT(...) AS scalar)` converts a parsed Variant value to a concrete SQL type when the value is compatible. The cast is not a JSON parser and can fail for incompatible shapes or ranges. +- `CAST(typed_expression AS VARIANT)` converts a supported typed SQL value to Variant. With `enable_variant_v2`, this only changes the execution representation for the current session; it does not change on-disk Variant storage, readers, writers, or compaction. + +```sql +SET enable_variant_v2 = true; + +SELECT CAST('{"id": 1}' AS VARIANT) AS typed_string, + PARSE_TO_VARIANT('{"id": 1}') AS parsed_object; +-- typed_string: the literal string {"id": 1} +-- parsed_object: {"id": 1} + +SELECT ELEMENT_AT(CAST('{"id": 1}' AS VARIANT), 'id') AS from_string, + ELEMENT_AT(PARSE_TO_VARIANT('{"id": 1}'), 'id') AS from_json; +-- from_string: NULL; from_json: 1 + +SELECT CAST(PARSE_TO_VARIANT('42') AS BIGINT) AS id; +-- id: 42 + +SELECT CAST('{invalid json' AS VARIANT) AS still_a_string; +-- succeeds because CAST does not parse the input as JSON +``` + +Do not use `CAST(string AS VARIANT)` as a JSON validation step. Use `PARSE_TO_VARIANT` for strict parsing, or `PARSE_TO_VARIANT_ERROR_TO_NULL` when malformed JSON should be converted to SQL `NULL`. + ## Equality semantics For `ColumnVariantV2`, equality used by canonical hashing and serialization is based on the logical value rather than the physical encoded bytes: @@ -503,6 +533,16 @@ SET enable_variant_v2 = true; The session variable changes the FE/BE execution type marker only. It does not add V2 table creation, loading, segment storage, readers or writers, statistics, or compaction support. It also does not provide V1/V2 mixed-version rolling-upgrade compatibility. +### Memory encoding and organization + +`ColumnVariantV2` uses a compact, self-describing representation during execution: + +- Nested or heterogeneous values use encoded bytes in an arena. Type and length information in the encoding lets expressions locate child values without requiring a fixed SQL type for every row. +- Simple scalar values can use typed scalar columns. A separate Variant-null map records JSON/Variant `null`, while SQL `NULL` remains represented by the SQL null bitmap. +- `E` and `T` are whole-column physical states: `E` means encoded and `T` means typed scalar. A column does not mix `E` and `T` at row level. + +This is an in-memory, compute-time organization only; it is not a new persisted Variant file format. For an external organization reference, see the official [Apache Parquet File Format](https://parquet.apache.org/docs/file-format/), [Parquet Variant Shredding](https://parquet.apache.org/docs/file-format/types/variantshredding/), and [Parquet Nested Encoding](https://parquet.apache.org/docs/file-format/nestedencoding/). Parquet organizes a Variant around `metadata` and `value`, with optional `typed_value` fields for homogeneous paths so readers can project columns and skip data. This is a design reference for organization, not a claim that `ColumnVariantV2` uses Parquet on disk. + ### What changes when V2 is enabled - **Canonical value semantics**: equivalent integral numbers normalize together; decimal trailing zeros are normalized; `+0`, `-0`, and integral zero share the same canonical value; object key order is ignored; array order is preserved. Invalid encodings and violated invariants fail instead of being silently accepted. diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 9b54fcb6d8d2c..26ad4430f06e2 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -96,6 +96,36 @@ SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1); -- 30 ``` 对象和数组访问的详细说明请参见 [ELEMENT_AT](../../../sql-functions/scalar-functions/variant-functions/element-at)。 +## CAST 语义 + +`CAST` 会保留 SQL 字符串和 JSON 文本之间的区别： + +- `CAST(string AS VARIANT)` 创建根值为“带类型的 Variant 字符串”的 Variant，**不会**解析 JSON。因此，`'{"id": 1}'` 会保持为一个字符串值，非法 JSON 文本对这个 CAST 仍然是合法输入。 +- `PARSE_TO_VARIANT(string)` 才会把字符串按 JSON 解析；解析为对象或数组后，才能继续使用路径访问或 `ELEMENT_AT`。如果非法 JSON 需要返回 SQL `NULL`，请使用 `PARSE_TO_VARIANT_ERROR_TO_NULL`。 +- `CAST(PARSE_TO_VARIANT(...) AS scalar)` 会在值形状和范围兼容时，把已经解析的 Variant 转换为确定的 SQL 标量类型。这个 CAST 不是 JSON 解析器，形状或范围不兼容时可能失败。 +- `CAST(typed_expression AS VARIANT)` 会把受支持的带类型 SQL 值转换为 Variant。开启 `enable_variant_v2` 只会改变当前 session 的执行表示，不会改变磁盘上的 Variant 存储、reader、writer 或 compaction。 + +```sql +SET enable_variant_v2 = true; + +SELECT CAST('{"id": 1}' AS VARIANT) AS typed_string, + PARSE_TO_VARIANT('{"id": 1}') AS parsed_object; +-- typed_string：字面量字符串 {"id": 1} +-- parsed_object：{"id": 1} + +SELECT ELEMENT_AT(CAST('{"id": 1}' AS VARIANT), 'id') AS from_string, + ELEMENT_AT(PARSE_TO_VARIANT('{"id": 1}'), 'id') AS from_json; +-- from_string：NULL；from_json：1 + +SELECT CAST(PARSE_TO_VARIANT('42') AS BIGINT) AS id; +-- id：42 + +SELECT CAST('{invalid json' AS VARIANT) AS still_a_string; +-- 成功，因为 CAST 不会把输入按 JSON 解析 +``` + +不要使用 `CAST(string AS VARIANT)` 做 JSON 校验。需要严格解析时使用 `PARSE_TO_VARIANT`；需要把格式错误转换为 SQL `NULL` 时使用 `PARSE_TO_VARIANT_ERROR_TO_NULL`。 + ## Equality 语义 对于 `ColumnVariantV2`，规范化哈希和序列化使用逻辑值而不是物理编码字节判断相等性： @@ -503,6 +533,16 @@ SET enable_variant_v2 = true; 该 session variable 只改变 FE/BE 执行类型标记，不增加 V2 表创建、导入、Segment 存储、读写器、统计信息或 Compaction 支持；同时也没有提供 V1/V2 混部滚动升级兼容能力。 +### 内存编码与组织方式 + +`ColumnVariantV2` 在执行期间使用紧凑的自描述表示： + +- 嵌套值或异构值使用 arena 中的编码字节保存；编码中携带类型、长度等信息，使表达式可以定位子值，而不要求每一行都使用固定的 SQL 类型。 +- 简单标量可以使用带类型的标量列；独立的 Variant-null map 用来记录 JSON/Variant `null`，而 SQL `NULL` 仍由 SQL null bitmap 表示。 +- `E` 和 `T` 是整列级别的物理状态：`E` 表示 encoded，`T` 表示 typed scalar；同一列不会在行级别混用 `E` 和 `T`。 + +这只是内存中的执行组织方式，不是新的 Variant 持久化文件格式。组织方式可参考官方 [Apache Parquet File Format](https://parquet.apache.org/docs/file-format/)、[Parquet Variant Shredding](https://parquet.apache.org/docs/file-format/types/variantshredding/) 和 [Parquet Nested Encoding](https://parquet.apache.org/docs/file-format/nestedencoding/)。Parquet 以 `metadata` 和 `value` 组织 Variant，并可为同质路径增加 `typed_value`，从而支持列投影和数据跳过。这里是组织方式参考，并不表示 `ColumnVariantV2` 在磁盘上使用 Parquet 格式。 + ### 开启 V2 后的行为变化 - **规范化值语义**：等价的整数类型会归一为同一值；Decimal 尾随零会被归一；`+0`、`-0` 与整数零使用同一规范值；对象 key 顺序不影响值；数组顺序仍然保留。非法编码或违反内部不变量时会报错，而不是静默接受。 From 8b97975101876b0231c8e2ee286a47f28d2433de Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Sun, 19 Jul 2026 22:20:34 +0800 Subject: [PATCH 6/9] docs: document Variant decimal and datetime encoding --- .../sql-data-types/semi-structured/VARIANT.md | 52 +++++++++++++++++++ .../sql-data-types/semi-structured/VARIANT.md | 52 +++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 4c804f8ca5046..ac61d650d4b07 100644 --- a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -541,6 +541,58 @@ The session variable changes the FE/BE execution type marker only. It does not a - Simple scalar values can use typed scalar columns. A separate Variant-null map records JSON/Variant `null`, while SQL `NULL` remains represented by the SQL null bitmap. - `E` and `T` are whole-column physical states: `E` means encoded and `T` means typed scalar. A column does not mix `E` and `T` at row level. +#### Typed scalar materialization to encoded values + +When an operator needs to materialize a typed scalar column (`T`) as encoded values (`E`), `with_typed_scalar()` selects the physical Variant primitive and payload width for each row. The following tables describe this physical mapping; canonical equality is applied separately. + +**Decimal types** + +| Doris type | Unscaled value read from the column | Encoding call | Variant primitive | +| --- | --- | --- | --- | +| `DECIMALV2` | `__int128` from `DecimalV2Value::value()` | `decimal(value, scale, 16)` | `DECIMAL16` | +| `DECIMAL32` | `int32_t` from `Decimal32::value` | `decimal(value, scale, 4)` | `DECIMAL4` | +| `DECIMAL64` | `int64_t` from `Decimal64::value` | `decimal(value, scale, 8)` | `DECIMAL8` | +| `DECIMAL128I` | `__int128` from `Decimal128V3::value` | `decimal(value, scale, 16)` | `DECIMAL16` | + +The encoded decimal layout is `[primitive header][scale][little-endian signed unscaled value]`. Its total size is therefore 6 bytes for `DECIMAL4`, 10 bytes for `DECIMAL8`, and 18 bytes for `DECIMAL16`. + +Decimal limits and invariants: + +- The Variant decimal scale must be in `[0, 38]`. The absolute unscaled value is limited to `10^9 - 1` for `DECIMAL4`, `10^18 - 1` for `DECIMAL8`, and `10^38 - 1` for `DECIMAL16`. +- Doris source-type limits still apply: `DECIMALV2` supports precision up to 27 and scale up to 9; `DECIMAL32`, `DECIMAL64`, and `DECIMAL128I` support precision up to 9, 18, and 38 respectively. +- The typed column's physical scale must exactly match its data-type metadata. A mismatch is treated as an invariant violation rather than being rescaled during materialization. +- The source type determines the encoded width. This path does not shrink a small `DECIMALV2` or `DECIMAL128I` value to `DECIMAL4` or `DECIMAL8`. +- `DECIMAL256` is not a supported `ColumnVariantV2` typed identity. Its precision can reach 76, beyond the Variant decimal precision limit of 38, so it cannot use this typed-to-encoded path. +- Physical width and scale do not define canonical equality. For example, `1.20` and `1.2` can have different physical encodings but normalize to the same canonical numeric value for hashing and grouping. + +**Date and time types** + +The conversion first calculates: + +```text +days = daynr(value) - daynr(1970-01-01) +micros = (days * 86400 + hour * 3600 + minute * 60 + second) * 1000000 + + microsecond +``` + +| Doris type | Normalized payload | Encoding call | Variant primitive | +| --- | --- | --- | --- | +| `DATE` | `int32_t days` | `date(days)` | `DATE` | +| `DATEV2` | `int32_t days` | `date(days)` | `DATE` | +| `DATETIME` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | +| `DATETIMEV2` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | +| `TIMESTAMPTZ` | `int64_t micros` | `timestamp_micros(micros, true)` | `TIMESTAMP_MICROS` | + +`DATE` uses a 4-byte payload plus a 1-byte header. Each timestamp uses an 8-byte payload plus a 1-byte header. The `utc_adjusted` argument is represented by the primitive ID: `false` selects the no-time-zone (`NTZ`) primitive, while `true` selects the UTC-adjusted primitive. + +Date and time limits and invariants: + +- Every source value must pass Doris date validation. An invalid `DATE`, `DATEV2`, `DATETIME`, `DATETIMEV2`, or `TIMESTAMPTZ` value raises `INVALID_ARGUMENT`; this path does not silently repair it or convert it to `NULL`. +- `DATE` and `DATEV2` preserve only a signed day count relative to `1970-01-01`; they carry neither a time of day nor a time-zone adjustment. +- `DATETIME` and `DATETIMEV2` are encoded as wall-clock, no-time-zone timestamps. This materialization path does not apply the session time zone. `TIMESTAMPTZ` is the only mapped Doris type that selects the UTC-adjusted timestamp primitive. +- Typed materialization emits microsecond primitives only. `DATETIMEV2` and `TIMESTAMPTZ` preserve at most their supported six fractional digits; `TIMESTAMP_NANOS` and `TIMESTAMP_NTZ_NANOS` exist in the Variant encoding but are not emitted by this mapping. +- `TIMEV2` is not a supported typed identity. Although the Variant encoding defines `TIME_NTZ_MICROS`, this typed-to-encoded path does not produce it. + This is an in-memory, compute-time organization only; it is not a new persisted Variant file format. For an external organization reference, see the official [Apache Parquet File Format](https://parquet.apache.org/docs/file-format/), [Parquet Variant Shredding](https://parquet.apache.org/docs/file-format/types/variantshredding/), and [Parquet Nested Encoding](https://parquet.apache.org/docs/file-format/nestedencoding/). Parquet organizes a Variant around `metadata` and `value`, with optional `typed_value` fields for homogeneous paths so readers can project columns and skip data. This is a design reference for organization, not a claim that `ColumnVariantV2` uses Parquet on disk. ### What changes when V2 is enabled diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 26ad4430f06e2..65af88c5fe03a 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -541,6 +541,58 @@ SET enable_variant_v2 = true; - 简单标量可以使用带类型的标量列；独立的 Variant-null map 用来记录 JSON/Variant `null`，而 SQL `NULL` 仍由 SQL null bitmap 表示。 - `E` 和 `T` 是整列级别的物理状态：`E` 表示 encoded，`T` 表示 typed scalar；同一列不会在行级别混用 `E` 和 `T`。 +#### Typed scalar 物化为 encoded + +当算子需要把 typed scalar 列（`T`）物化为 encoded 值（`E`）时，`with_typed_scalar()` 会逐行选择物理 Variant primitive 和 payload 宽度。下表描述的是物理编码映射；canonical equality 在另一层单独处理。 + +**Decimal 类型** + +| Doris 类型 | 从列中读取的 unscaled 值 | 编码调用 | Variant primitive | +| --- | --- | --- | --- | +| `DECIMALV2` | `DecimalV2Value::value()` 返回的 `__int128` | `decimal(value, scale, 16)` | `DECIMAL16` | +| `DECIMAL32` | `Decimal32::value` 中的 `int32_t` | `decimal(value, scale, 4)` | `DECIMAL4` | +| `DECIMAL64` | `Decimal64::value` 中的 `int64_t` | `decimal(value, scale, 8)` | `DECIMAL8` | +| `DECIMAL128I` | `Decimal128V3::value` 中的 `__int128` | `decimal(value, scale, 16)` | `DECIMAL16` | + +Decimal 的 encoded 布局是 `[primitive header][scale][小端有符号 unscaled value]`，所以 `DECIMAL4`、`DECIMAL8`、`DECIMAL16` 的总长度分别是 6、10、18 字节。 + +Decimal 限制与不变量： + +- Variant Decimal 的 scale 必须在 `[0, 38]` 内。`DECIMAL4`、`DECIMAL8`、`DECIMAL16` 的 unscaled 绝对值上限分别是 `10^9 - 1`、`10^18 - 1`、`10^38 - 1`。 +- Doris 源类型自身的限制仍然生效：`DECIMALV2` 最大 precision 为 27、最大 scale 为 9；`DECIMAL32`、`DECIMAL64`、`DECIMAL128I` 的最大 precision 分别为 9、18、38。 +- typed column 的物理 scale 必须与 data type 元数据中的 scale 完全一致；不一致会被视为不变量破坏，物化时不会自动 rescale。 +- encoded 宽度由源类型决定。该路径不会因为数值较小，就把 `DECIMALV2` 或 `DECIMAL128I` 缩窄为 `DECIMAL4` 或 `DECIMAL8`。 +- `DECIMAL256` 不是 `ColumnVariantV2` 支持的 typed identity。它的 precision 最高可达 76，超过 Variant Decimal 的 38 位上限，因此不能走这条 typed-to-encoded 路径。 +- 物理宽度和 scale 不决定 canonical equality。例如 `1.20` 与 `1.2` 可以拥有不同物理编码，但在哈希和分组时会归一为同一个 canonical 数值。 + +**日期和时间类型** + +转换会先计算： + +```text +days = daynr(value) - daynr(1970-01-01) +micros = (days * 86400 + hour * 3600 + minute * 60 + second) * 1000000 + + microsecond +``` + +| Doris 类型 | 归一化 payload | 编码调用 | Variant primitive | +| --- | --- | --- | --- | +| `DATE` | `int32_t days` | `date(days)` | `DATE` | +| `DATEV2` | `int32_t days` | `date(days)` | `DATE` | +| `DATETIME` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | +| `DATETIMEV2` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | +| `TIMESTAMPTZ` | `int64_t micros` | `timestamp_micros(micros, true)` | `TIMESTAMP_MICROS` | + +`DATE` 使用 4 字节 payload 加 1 字节 header；每个 timestamp 使用 8 字节 payload 加 1 字节 header。`utc_adjusted` 参数由 primitive ID 表达：`false` 选择无时区（NTZ）primitive，`true` 选择 UTC-adjusted primitive。 + +日期和时间限制与不变量： + +- 所有源值都必须通过 Doris 的日期合法性检查。非法的 `DATE`、`DATEV2`、`DATETIME`、`DATETIMEV2` 或 `TIMESTAMPTZ` 会抛出 `INVALID_ARGUMENT`；该路径不会静默修复，也不会转为 `NULL`。 +- `DATE` 和 `DATEV2` 只保留相对 `1970-01-01` 的有符号天数，不携带日内时间或时区调整信息。 +- `DATETIME` 和 `DATETIMEV2` 编码为 wall-clock、无时区的 timestamp；这条物化路径不会应用 session time zone。只有 `TIMESTAMPTZ` 会选择 UTC-adjusted timestamp primitive。 +- typed materialization 只生成微秒 primitive。`DATETIMEV2` 和 `TIMESTAMPTZ` 最多保留其支持的 6 位小数；Variant 编码虽然定义了 `TIMESTAMP_NANOS` 和 `TIMESTAMP_NTZ_NANOS`，但该映射不会生成它们。 +- `TIMEV2` 不是支持的 typed identity。Variant 编码虽然定义了 `TIME_NTZ_MICROS`，但这条 typed-to-encoded 路径不会生成它。 + 这只是内存中的执行组织方式，不是新的 Variant 持久化文件格式。组织方式可参考官方 [Apache Parquet File Format](https://parquet.apache.org/docs/file-format/)、[Parquet Variant Shredding](https://parquet.apache.org/docs/file-format/types/variantshredding/) 和 [Parquet Nested Encoding](https://parquet.apache.org/docs/file-format/nestedencoding/)。Parquet 以 `metadata` 和 `value` 组织 Variant，并可为同质路径增加 `typed_value`，从而支持列投影和数据跳过。这里是组织方式参考，并不表示 `ColumnVariantV2` 在磁盘上使用 Parquet 格式。 ### 开启 V2 后的行为变化 From 8918b282c158d5a35875e10e61e136358f46b333 Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Sun, 19 Jul 2026 23:00:11 +0800 Subject: [PATCH 7/9] docs: split Variant V2 compute semantics reference --- .../sql-data-types/semi-structured/VARIANT.md | 128 +---------- .../variant-v2-compute-semantics.md | 206 ++++++++++++++++++ .../sql-data-types/semi-structured/VARIANT.md | 128 +---------- .../variant-v2-compute-semantics.md | 206 ++++++++++++++++++ sidebars.ts | 5 + 5 files changed, 437 insertions(+), 236 deletions(-) create mode 100644 docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md create mode 100644 i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md diff --git a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index ac61d650d4b07..e0f8999f1084a 100644 --- a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -167,6 +167,9 @@ FROM ( ) AS array_values; -- distinct_count: 2 ``` + +For the physical encoding, normalization pipeline, and additional boundaries behind these rules, see [VARIANT V2 Compute Semantics and Memory Encoding](./variant-v2-compute-semantics). + ## Primitive types VARIANT infers subcolumn types automatically. Supported types include: @@ -508,7 +511,7 @@ SELECT * FROM tbl WHERE v['str'] MATCH 'Doris'; - On the default path, a whole VARIANT value should not be used directly in `ORDER BY`, `GROUP BY`, as a `JOIN KEY`, or as an aggregate argument; CAST the required subpath instead. - Strings can be implicitly converted to VARIANT. -The experimental compute path described below adds canonical hashing and serialization for grouping and set-operation use cases. It does not make root VARIANT comparison predicates or VARIANT join keys available. +The [experimental V2 compute path](./variant-v2-compute-semantics) adds canonical hashing and serialization for grouping and set-operation use cases. It does not make root VARIANT comparison predicates or Variant join keys available. | VARIANT | Castable | Coercible | | --------------- | -------- | --------- | @@ -525,130 +528,19 @@ The experimental compute path described below adds canonical hashing and seriali ## Experimental compute path: ColumnVariantV2 -`ColumnVariantV2` is an experimental, compute-only execution path. It is disabled by default and selected for the current FE session with the session variable `enable_variant_v2`: - -```sql -SET enable_variant_v2 = true; -``` - -The session variable changes the FE/BE execution type marker only. It does not add V2 table creation, loading, segment storage, readers or writers, statistics, or compaction support. It also does not provide V1/V2 mixed-version rolling-upgrade compatibility. - -### Memory encoding and organization - -`ColumnVariantV2` uses a compact, self-describing representation during execution: - -- Nested or heterogeneous values use encoded bytes in an arena. Type and length information in the encoding lets expressions locate child values without requiring a fixed SQL type for every row. -- Simple scalar values can use typed scalar columns. A separate Variant-null map records JSON/Variant `null`, while SQL `NULL` remains represented by the SQL null bitmap. -- `E` and `T` are whole-column physical states: `E` means encoded and `T` means typed scalar. A column does not mix `E` and `T` at row level. - -#### Typed scalar materialization to encoded values - -When an operator needs to materialize a typed scalar column (`T`) as encoded values (`E`), `with_typed_scalar()` selects the physical Variant primitive and payload width for each row. The following tables describe this physical mapping; canonical equality is applied separately. - -**Decimal types** - -| Doris type | Unscaled value read from the column | Encoding call | Variant primitive | -| --- | --- | --- | --- | -| `DECIMALV2` | `__int128` from `DecimalV2Value::value()` | `decimal(value, scale, 16)` | `DECIMAL16` | -| `DECIMAL32` | `int32_t` from `Decimal32::value` | `decimal(value, scale, 4)` | `DECIMAL4` | -| `DECIMAL64` | `int64_t` from `Decimal64::value` | `decimal(value, scale, 8)` | `DECIMAL8` | -| `DECIMAL128I` | `__int128` from `Decimal128V3::value` | `decimal(value, scale, 16)` | `DECIMAL16` | - -The encoded decimal layout is `[primitive header][scale][little-endian signed unscaled value]`. Its total size is therefore 6 bytes for `DECIMAL4`, 10 bytes for `DECIMAL8`, and 18 bytes for `DECIMAL16`. - -Decimal limits and invariants: - -- The Variant decimal scale must be in `[0, 38]`. The absolute unscaled value is limited to `10^9 - 1` for `DECIMAL4`, `10^18 - 1` for `DECIMAL8`, and `10^38 - 1` for `DECIMAL16`. -- Doris source-type limits still apply: `DECIMALV2` supports precision up to 27 and scale up to 9; `DECIMAL32`, `DECIMAL64`, and `DECIMAL128I` support precision up to 9, 18, and 38 respectively. -- The typed column's physical scale must exactly match its data-type metadata. A mismatch is treated as an invariant violation rather than being rescaled during materialization. -- The source type determines the encoded width. This path does not shrink a small `DECIMALV2` or `DECIMAL128I` value to `DECIMAL4` or `DECIMAL8`. -- `DECIMAL256` is not a supported `ColumnVariantV2` typed identity. Its precision can reach 76, beyond the Variant decimal precision limit of 38, so it cannot use this typed-to-encoded path. -- Physical width and scale do not define canonical equality. For example, `1.20` and `1.2` can have different physical encodings but normalize to the same canonical numeric value for hashing and grouping. - -**Date and time types** - -The conversion first calculates: - -```text -days = daynr(value) - daynr(1970-01-01) -micros = (days * 86400 + hour * 3600 + minute * 60 + second) * 1000000 - + microsecond -``` - -| Doris type | Normalized payload | Encoding call | Variant primitive | -| --- | --- | --- | --- | -| `DATE` | `int32_t days` | `date(days)` | `DATE` | -| `DATEV2` | `int32_t days` | `date(days)` | `DATE` | -| `DATETIME` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | -| `DATETIMEV2` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | -| `TIMESTAMPTZ` | `int64_t micros` | `timestamp_micros(micros, true)` | `TIMESTAMP_MICROS` | - -`DATE` uses a 4-byte payload plus a 1-byte header. Each timestamp uses an 8-byte payload plus a 1-byte header. The `utc_adjusted` argument is represented by the primitive ID: `false` selects the no-time-zone (`NTZ`) primitive, while `true` selects the UTC-adjusted primitive. - -Date and time limits and invariants: - -- Every source value must pass Doris date validation. An invalid `DATE`, `DATEV2`, `DATETIME`, `DATETIMEV2`, or `TIMESTAMPTZ` value raises `INVALID_ARGUMENT`; this path does not silently repair it or convert it to `NULL`. -- `DATE` and `DATEV2` preserve only a signed day count relative to `1970-01-01`; they carry neither a time of day nor a time-zone adjustment. -- `DATETIME` and `DATETIMEV2` are encoded as wall-clock, no-time-zone timestamps. This materialization path does not apply the session time zone. `TIMESTAMPTZ` is the only mapped Doris type that selects the UTC-adjusted timestamp primitive. -- Typed materialization emits microsecond primitives only. `DATETIMEV2` and `TIMESTAMPTZ` preserve at most their supported six fractional digits; `TIMESTAMP_NANOS` and `TIMESTAMP_NTZ_NANOS` exist in the Variant encoding but are not emitted by this mapping. -- `TIMEV2` is not a supported typed identity. Although the Variant encoding defines `TIME_NTZ_MICROS`, this typed-to-encoded path does not produce it. - -This is an in-memory, compute-time organization only; it is not a new persisted Variant file format. For an external organization reference, see the official [Apache Parquet File Format](https://parquet.apache.org/docs/file-format/), [Parquet Variant Shredding](https://parquet.apache.org/docs/file-format/types/variantshredding/), and [Parquet Nested Encoding](https://parquet.apache.org/docs/file-format/nestedencoding/). Parquet organizes a Variant around `metadata` and `value`, with optional `typed_value` fields for homogeneous paths so readers can project columns and skip data. This is a design reference for organization, not a claim that `ColumnVariantV2` uses Parquet on disk. - -### What changes when V2 is enabled - -- **Canonical value semantics**: equivalent integral numbers normalize together; decimal trailing zeros are normalized; `+0`, `-0`, and integral zero share the same canonical value; object key order is ignored; array order is preserved. Invalid encodings and violated invariants fail instead of being silently accepted. -- **Hashing and serialization**: canonical hashes and arena serialization enable `GROUP BY`, `DISTINCT`, `COUNT(DISTINCT ...)`, `INTERSECT`, `EXCEPT`, `UNION DISTINCT`, and aggregation grouping keys for supported Variant values. -- **Parsing is explicit**: `parse_to_variant(json_string)` parses a JSON string into a Variant value; `parse_to_variant_error_to_null(json_string)` returns SQL `NULL` when validation fails; `CAST(string AS VARIANT)` creates a typed Variant string and does not parse the string as JSON. See [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) and [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null) for function syntax and examples. -- **Expressions and nested values**: conditional expressions, nested containers, and Variant-aware `explode` can operate on the canonical representation. -- **Null and physical states**: SQL `NULL` remains distinct from Variant/JSON `null`. A `ColumnVariantV2` column uses a whole-column physical state: encoded (`E`) or typed scalar (`T`, with a Variant-null map); it does not mix E and T at row level. - -### Examples +:::caution Experimental feature +`ColumnVariantV2` is disabled by default and changes only the compute path selected for the current FE session. It does not change the persisted Variant format or add V2 storage, loading, statistics, or compaction support. +::: -The following examples run in the current FE session after V2 is enabled: +Select it for the current FE session with the session variable `enable_variant_v2`: ```sql SET enable_variant_v2 = true; - --- JSON text is parsed; CAST(string AS VARIANT) keeps the string as a typed Variant value. -SELECT PARSE_TO_VARIANT('{\"id\": 1, \"items\": [10, 20]}') AS parsed_value, - CAST('{\"id\": 1, \"items\": [10, 20]}' AS VARIANT) AS typed_string; - --- Canonical hashing treats 1 and 1.0 as one distinct value. -SELECT COUNT(DISTINCT value) AS distinct_count -FROM ( - SELECT PARSE_TO_VARIANT('1') AS value - UNION ALL - SELECT PARSE_TO_VARIANT('1.0') AS value -) AS numeric_values; --- distinct_count: 1 - --- Invalid JSON becomes SQL NULL with the tolerant parser. -SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value; --- invalid_value: NULL - --- Nested access can feed a conditional expression after an explicit CAST. -SELECT CASE - WHEN CAST(PARSE_TO_VARIANT('{\"enabled\": true}')['enabled'] AS BOOLEAN) - THEN 'on' - ELSE 'off' - END AS status; --- status: on ``` -### Boundaries - -- Root Variant comparison predicates (`=`, `!=`, `<=>`, and ordering comparisons) are not supported. Extract and CAST a comparable path on both sides: - -```sql -SELECT * -FROM tbl -WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); -``` +When enabled, V2 adds canonical hashing and serialization for supported grouping and set operations, makes JSON parsing explicit, and uses whole-column typed (`T`) or encoded (`E`) in-memory states. Root Variant comparison predicates, Variant join keys, Sort/TopN keys, and `MIN`/`MAX` arguments remain unsupported. -- Variant expressions are not supported as join keys. Root Variant values are also not supported as sort or TopN keys, window partition or order keys, or arguments to `MIN`/`MAX`. -- Regression suites covering V2 remain tagged `nonConcurrent`; the feature selection itself is session-scoped and does not change native Variant storage or compaction. -- Enable V2 only after validating the target workload; it is an experimental execution path and does not change the storage compatibility contract. +For the component architecture, materialization flow, memory encoding, Decimal and date/time mappings, canonical equality, CAST behavior, examples, and complete limits, see [VARIANT V2 Compute Semantics and Memory Encoding](./variant-v2-compute-semantics). ## Wide columns diff --git a/docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md b/docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md new file mode 100644 index 0000000000000..391d8e9f04b11 --- /dev/null +++ b/docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md @@ -0,0 +1,206 @@ +--- +{ + "title": "VARIANT V2 Compute Semantics and Memory Encoding", + "language": "en-US", + "description": "Architecture and behavioral reference for the experimental ColumnVariantV2 compute path, including memory states, scalar encoding, canonical equality, CAST and parsing semantics, and current limitations." +} +--- + +## Overview + +`ColumnVariantV2` is an experimental, compute-only execution path for `VARIANT`. It adds a self-describing in-memory representation and canonical value semantics for supported expressions, grouping, and set operations. + +Use the [VARIANT type reference](./VARIANT) for SQL syntax, type rules, indexes, storage configuration, and general limitations. Use the [VARIANT Workload Guide](./variant-workload-guide) when choosing between default storage behavior, sparse columns, DOC mode, and Schema Template. This page focuses on the V2 execution architecture and behavior. + +:::caution Experimental feature +`ColumnVariantV2` is disabled by default. It changes only the compute path selected for the current FE session. It does not introduce a V2 table format, loading path, segment reader or writer, statistics format, or compaction path, and it does not provide V1/V2 mixed-version rolling-upgrade compatibility. +::: + +## Enable the compute path + +Enable V2 for the current FE session with the session variable `enable_variant_v2`: + +```sql +SET enable_variant_v2 = true; +``` + +The session variable changes the FE/BE execution type marker only. Existing persisted `VARIANT` data and its storage compatibility contract remain unchanged. + +## Execution architecture + +### Whole-column state model + +A `ColumnVariantV2` column has one physical state for the whole column: + +- **Typed scalar (`T`)**: homogeneous scalar rows borrow a concrete Doris scalar column and its data type. A separate Variant-null map represents Variant/JSON `null`. +- **Encoded (`E`)**: nested, heterogeneous, or materialized values are stored as self-describing bytes in an arena. +- A column does not mix `T` and `E` at row granularity. SQL `NULL` remains in the SQL nullable bitmap and is distinct from Variant/JSON `null` in either state. + +This distinction lets simple scalar expressions avoid immediate encoding while allowing nested or heterogeneous operators to consume one uniform representation when required. + +### Main components + +| Component | Responsibility | +| --- | --- | +| `ColumnVariantV2` | Owns the whole-column `T` or `E` state, null semantics, read views, and materialization entry points. | +| `VariantBlockBuilder` | Builds encoded scalar, array, and object rows and appends their self-describing bytes to the arena. | +| `VariantScalarEncodingPlan` | Selects a primitive ID, payload width, scale or length metadata, validates limits, and writes the encoded scalar bytes. | +| `with_typed_scalar()` | Central typed-scalar mapping matrix. It creates both the physical encoding plan and the canonical scalar view for a Doris value. | +| `VariantCanonicalScalarRef` | Represents the logical scalar used by canonical hashing and serialization, independently of physical width or source type. | +| Variant V2 CAST and parser functions | Define the boundary between typed SQL values, Variant strings, and parsed JSON values. | + +### Typed-to-encoded materialization flow + +1. A supported homogeneous scalar can enter the compute path in `T` state with an exact Doris scalar type. +2. Operators that only need a typed read view can borrow that state without encoding the column. +3. When an operator requires self-describing values, `ensure_encoded()` dispatches once on the Doris primitive type and visits the rows. +4. `with_typed_scalar()` produces a physical encoding factory and a canonical-value factory for each non-null row. +5. `VariantScalarEncodingPlan` writes the physical value to the arena, after which the column is represented in `E` state for subsequent encoded operations. + +This centralized mapping keeps physical serialization and canonical equality aligned while avoiding a virtual or type switch for every row. + +## Memory encoding and organization + +The encoded representation carries the primitive type and any required scale or length metadata next to the payload. Arrays and objects additionally carry enough structural information to locate child values. This representation is designed for compute-time traversal rather than persisted table storage. + +The organization is conceptually similar to separating an encoded Variant value from optional typed projections. For external format references, see the official [Apache Parquet File Format](https://parquet.apache.org/docs/file-format/), [Parquet Variant Shredding](https://parquet.apache.org/docs/file-format/types/variantshredding/), and [Parquet Nested Encoding](https://parquet.apache.org/docs/file-format/nestedencoding/). Parquet organizes a Variant around `metadata` and `value`, with optional `typed_value` fields for homogeneous paths. This is an organizational reference; `ColumnVariantV2` does not use Parquet as its in-memory or Doris on-disk format. + +## Typed scalar materialization mappings + +### Decimal types + +When a typed Decimal is materialized from `T` to `E`, `with_typed_scalar()` preserves its unscaled integer and scale and selects the payload width from the Doris source type: + +| Doris type | Unscaled value read from the column | Encoding call | Variant primitive | +| --- | --- | --- | --- | +| `DECIMALV2` | `__int128` from `DecimalV2Value::value()` | `decimal(value, scale, 16)` | `DECIMAL16` | +| `DECIMAL32` | `int32_t` from `Decimal32::value` | `decimal(value, scale, 4)` | `DECIMAL4` | +| `DECIMAL64` | `int64_t` from `Decimal64::value` | `decimal(value, scale, 8)` | `DECIMAL8` | +| `DECIMAL128I` | `__int128` from `Decimal128V3::value` | `decimal(value, scale, 16)` | `DECIMAL16` | + +The encoded decimal layout is `[primitive header][scale][little-endian signed unscaled value]`. Its total size is 6 bytes for `DECIMAL4`, 10 bytes for `DECIMAL8`, and 18 bytes for `DECIMAL16`. + +Decimal limits and invariants: + +- The Variant decimal scale must be in `[0, 38]`. The absolute unscaled value is limited to `10^9 - 1` for `DECIMAL4`, `10^18 - 1` for `DECIMAL8`, and `10^38 - 1` for `DECIMAL16`. +- Doris source-type limits still apply: `DECIMALV2` supports precision up to 27 and scale up to 9; `DECIMAL32`, `DECIMAL64`, and `DECIMAL128I` support precision up to 9, 18, and 38 respectively. +- The typed column's physical scale must exactly match its data-type metadata. A mismatch is an invariant violation; materialization does not rescale the value. +- The source type determines the encoded width. A small `DECIMALV2` or `DECIMAL128I` value is not narrowed to `DECIMAL4` or `DECIMAL8` on this path. +- `DECIMAL256` is not a supported `ColumnVariantV2` typed identity. Its precision can reach 76, beyond the Variant decimal precision limit of 38. +- Physical width and scale do not define canonical equality. For example, `1.20` and `1.2` can have different physical encodings but normalize to the same canonical numeric value. + +### Date and time types + +The date and timestamp mapping first calculates: + +```text +days = daynr(value) - daynr(1970-01-01) +micros = (days * 86400 + hour * 3600 + minute * 60 + second) * 1000000 + + microsecond +``` + +| Doris type | Normalized payload | Encoding call | Variant primitive | +| --- | --- | --- | --- | +| `DATE` | `int32_t days` | `date(days)` | `DATE` | +| `DATEV2` | `int32_t days` | `date(days)` | `DATE` | +| `DATETIME` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | +| `DATETIMEV2` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | +| `TIMESTAMPTZ` | `int64_t micros` | `timestamp_micros(micros, true)` | `TIMESTAMP_MICROS` | + +`DATE` uses a 4-byte payload plus a 1-byte header. Each timestamp uses an 8-byte payload plus a 1-byte header. The `utc_adjusted` argument is represented by the primitive ID: `false` selects the no-time-zone (`NTZ`) primitive, while `true` selects the UTC-adjusted primitive. + +Date and time limits and invariants: + +- Every source value must pass Doris date validation. An invalid `DATE`, `DATEV2`, `DATETIME`, `DATETIMEV2`, or `TIMESTAMPTZ` raises `INVALID_ARGUMENT`; this path does not repair it or convert it to `NULL`. +- `DATE` and `DATEV2` preserve only a signed day count relative to `1970-01-01`; they carry neither a time of day nor a time-zone adjustment. +- `DATETIME` and `DATETIMEV2` are encoded as wall-clock, no-time-zone timestamps. This path does not apply the session time zone. `TIMESTAMPTZ` is the only mapped Doris type that selects the UTC-adjusted timestamp primitive. +- Typed materialization emits microsecond primitives only. `DATETIMEV2` and `TIMESTAMPTZ` preserve at most their supported six fractional digits. `TIMESTAMP_NANOS` and `TIMESTAMP_NTZ_NANOS` exist in the encoding but are not emitted by this mapping. +- `TIMEV2` is not a supported typed identity. Although the Variant encoding defines `TIME_NTZ_MICROS`, this typed-to-encoded path does not produce it. + +## Canonical value semantics + +Physical encoded bytes are not used directly as the equality identity. Canonicalization produces a logical representation for hashing and arena serialization: + +- Equivalent integral numeric representations normalize to the same value. +- Decimal trailing zeros do not change the value. +- `+0`, `-0`, and integral zero normalize together. +- Object key order does not affect equality, while array element order does. +- Variant/JSON `null` remains distinct from SQL `NULL`. +- Invalid encodings and violated internal invariants fail instead of being silently accepted. + +These rules enable supported hash-based operations including `GROUP BY`, `DISTINCT`, `COUNT(DISTINCT ...)`, `INTERSECT`, `EXCEPT`, and `UNION DISTINCT`. + +```sql +SET enable_variant_v2 = true; + +-- 1 and 1.0 have one canonical distinct value. +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('1') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('1.0') AS value +) AS numeric_values; +-- distinct_count: 1 + +-- Object key order is ignored. +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('{\"a\": 1, \"b\": 2}') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('{\"b\": 2, \"a\": 1}') AS value +) AS object_values; +-- distinct_count: 1 + +-- Array order is preserved. +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('[1, 2]') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('[2, 1]') AS value +) AS array_values; +-- distinct_count: 2 +``` + +Canonical equality does not enable root Variant comparison predicates. Direct `VARIANT = VARIANT`, null-safe equality, and ordering comparisons remain unsupported. + +## CAST and parsing behavior + +V2 separates creating a typed Variant string from parsing JSON text: + +- `CAST(string AS VARIANT)` creates a typed Variant string. It does not parse or validate the string as JSON. +- `parse_to_variant(json_string)` strictly parses JSON text into a Variant value. +- `parse_to_variant_error_to_null(json_string)` returns SQL `NULL` when parsing or validation fails. +- Extracted Variant values should be cast to concrete SQL types before typed comparison, filtering, ordering, or arithmetic. + +See [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) and [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null) for full function syntax. + +```sql +SET enable_variant_v2 = true; + +SELECT PARSE_TO_VARIANT('{\"id\": 1}') AS parsed_object, + CAST('{\"id\": 1}' AS VARIANT) AS typed_string; + +SELECT ELEMENT_AT(CAST('{\"id\": 1}' AS VARIANT), 'id') AS from_string, + ELEMENT_AT(PARSE_TO_VARIANT('{\"id\": 1}'), 'id') AS from_json; +-- from_string: NULL; from_json: 1 + +SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value; +-- invalid_value: NULL +``` + +Conditional expressions, nested containers, and Variant-aware `explode` can consume the V2 representation where their signatures support Variant values. + +## Boundaries and limitations + +- Root Variant comparison predicates (`=`, `!=`, `<=>`, and ordering comparisons) are not supported. Extract and cast a comparable path on both sides. +- Variant expressions are not supported as join keys. +- Root Variant values are not supported as sort or TopN keys, window partition or order keys, or arguments to `MIN` and `MAX`. +- Feature selection is session-scoped and does not change native Variant storage, loading, statistics, or compaction. +- V2 regression suites remain tagged `nonConcurrent`. +- Enable V2 only after validating the target workload; the execution path remains experimental. + +```sql +SELECT * +FROM tbl +WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); +``` diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 65af88c5fe03a..ab2ba20b871d7 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -167,6 +167,9 @@ FROM ( ) AS array_values; -- distinct_count: 2 ``` + +这些规则背后的物理编码、归一化流程和其他使用边界，请参见 [VARIANT V2 计算语义与内存编码](./variant-v2-compute-semantics)。 + ## 基本类型 VARIANT 自动推断的子列基础类型包括： @@ -508,7 +511,7 @@ SELECT * FROM tbl WHERE v['str'] MATCH 'Doris'; - 在默认路径上，不应将整个 VARIANT 直接用于 `ORDER BY`、`GROUP BY`、`JOIN KEY` 或聚合参数；请先对子路径 CAST。 - 字符串类型可隐式转换为 VARIANT。 -下面的实验性计算路径新增了面向分组和集合运算的规范化哈希与序列化，但不会因此开放根 VARIANT 比较谓词或 VARIANT JOIN KEY。 +[实验性 V2 计算路径](./variant-v2-compute-semantics)新增了面向分组和集合运算的规范化哈希与序列化，但不会因此开放根 VARIANT 比较谓词或 Variant Join Key。 | VARIANT | Castable | Coercible | | --------------- | -------- | --------- | @@ -525,130 +528,19 @@ SELECT * FROM tbl WHERE v['str'] MATCH 'Doris'; ## 实验性计算路径：ColumnVariantV2 -`ColumnVariantV2` 是实验性的、仅用于计算的执行路径。默认关闭，通过当前 FE session 的 session variable `enable_variant_v2` 选择： - -```sql -SET enable_variant_v2 = true; -``` - -该 session variable 只改变 FE/BE 执行类型标记，不增加 V2 表创建、导入、Segment 存储、读写器、统计信息或 Compaction 支持；同时也没有提供 V1/V2 混部滚动升级兼容能力。 - -### 内存编码与组织方式 - -`ColumnVariantV2` 在执行期间使用紧凑的自描述表示： - -- 嵌套值或异构值使用 arena 中的编码字节保存；编码中携带类型、长度等信息，使表达式可以定位子值，而不要求每一行都使用固定的 SQL 类型。 -- 简单标量可以使用带类型的标量列；独立的 Variant-null map 用来记录 JSON/Variant `null`，而 SQL `NULL` 仍由 SQL null bitmap 表示。 -- `E` 和 `T` 是整列级别的物理状态：`E` 表示 encoded，`T` 表示 typed scalar；同一列不会在行级别混用 `E` 和 `T`。 - -#### Typed scalar 物化为 encoded - -当算子需要把 typed scalar 列（`T`）物化为 encoded 值（`E`）时，`with_typed_scalar()` 会逐行选择物理 Variant primitive 和 payload 宽度。下表描述的是物理编码映射；canonical equality 在另一层单独处理。 - -**Decimal 类型** - -| Doris 类型 | 从列中读取的 unscaled 值 | 编码调用 | Variant primitive | -| --- | --- | --- | --- | -| `DECIMALV2` | `DecimalV2Value::value()` 返回的 `__int128` | `decimal(value, scale, 16)` | `DECIMAL16` | -| `DECIMAL32` | `Decimal32::value` 中的 `int32_t` | `decimal(value, scale, 4)` | `DECIMAL4` | -| `DECIMAL64` | `Decimal64::value` 中的 `int64_t` | `decimal(value, scale, 8)` | `DECIMAL8` | -| `DECIMAL128I` | `Decimal128V3::value` 中的 `__int128` | `decimal(value, scale, 16)` | `DECIMAL16` | - -Decimal 的 encoded 布局是 `[primitive header][scale][小端有符号 unscaled value]`，所以 `DECIMAL4`、`DECIMAL8`、`DECIMAL16` 的总长度分别是 6、10、18 字节。 - -Decimal 限制与不变量： - -- Variant Decimal 的 scale 必须在 `[0, 38]` 内。`DECIMAL4`、`DECIMAL8`、`DECIMAL16` 的 unscaled 绝对值上限分别是 `10^9 - 1`、`10^18 - 1`、`10^38 - 1`。 -- Doris 源类型自身的限制仍然生效：`DECIMALV2` 最大 precision 为 27、最大 scale 为 9；`DECIMAL32`、`DECIMAL64`、`DECIMAL128I` 的最大 precision 分别为 9、18、38。 -- typed column 的物理 scale 必须与 data type 元数据中的 scale 完全一致；不一致会被视为不变量破坏，物化时不会自动 rescale。 -- encoded 宽度由源类型决定。该路径不会因为数值较小，就把 `DECIMALV2` 或 `DECIMAL128I` 缩窄为 `DECIMAL4` 或 `DECIMAL8`。 -- `DECIMAL256` 不是 `ColumnVariantV2` 支持的 typed identity。它的 precision 最高可达 76，超过 Variant Decimal 的 38 位上限，因此不能走这条 typed-to-encoded 路径。 -- 物理宽度和 scale 不决定 canonical equality。例如 `1.20` 与 `1.2` 可以拥有不同物理编码，但在哈希和分组时会归一为同一个 canonical 数值。 - -**日期和时间类型** - -转换会先计算： - -```text -days = daynr(value) - daynr(1970-01-01) -micros = (days * 86400 + hour * 3600 + minute * 60 + second) * 1000000 - + microsecond -``` - -| Doris 类型 | 归一化 payload | 编码调用 | Variant primitive | -| --- | --- | --- | --- | -| `DATE` | `int32_t days` | `date(days)` | `DATE` | -| `DATEV2` | `int32_t days` | `date(days)` | `DATE` | -| `DATETIME` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | -| `DATETIMEV2` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | -| `TIMESTAMPTZ` | `int64_t micros` | `timestamp_micros(micros, true)` | `TIMESTAMP_MICROS` | - -`DATE` 使用 4 字节 payload 加 1 字节 header；每个 timestamp 使用 8 字节 payload 加 1 字节 header。`utc_adjusted` 参数由 primitive ID 表达：`false` 选择无时区（NTZ）primitive，`true` 选择 UTC-adjusted primitive。 - -日期和时间限制与不变量： - -- 所有源值都必须通过 Doris 的日期合法性检查。非法的 `DATE`、`DATEV2`、`DATETIME`、`DATETIMEV2` 或 `TIMESTAMPTZ` 会抛出 `INVALID_ARGUMENT`；该路径不会静默修复，也不会转为 `NULL`。 -- `DATE` 和 `DATEV2` 只保留相对 `1970-01-01` 的有符号天数，不携带日内时间或时区调整信息。 -- `DATETIME` 和 `DATETIMEV2` 编码为 wall-clock、无时区的 timestamp；这条物化路径不会应用 session time zone。只有 `TIMESTAMPTZ` 会选择 UTC-adjusted timestamp primitive。 -- typed materialization 只生成微秒 primitive。`DATETIMEV2` 和 `TIMESTAMPTZ` 最多保留其支持的 6 位小数；Variant 编码虽然定义了 `TIMESTAMP_NANOS` 和 `TIMESTAMP_NTZ_NANOS`，但该映射不会生成它们。 -- `TIMEV2` 不是支持的 typed identity。Variant 编码虽然定义了 `TIME_NTZ_MICROS`，但这条 typed-to-encoded 路径不会生成它。 - -这只是内存中的执行组织方式，不是新的 Variant 持久化文件格式。组织方式可参考官方 [Apache Parquet File Format](https://parquet.apache.org/docs/file-format/)、[Parquet Variant Shredding](https://parquet.apache.org/docs/file-format/types/variantshredding/) 和 [Parquet Nested Encoding](https://parquet.apache.org/docs/file-format/nestedencoding/)。Parquet 以 `metadata` 和 `value` 组织 Variant，并可为同质路径增加 `typed_value`，从而支持列投影和数据跳过。这里是组织方式参考，并不表示 `ColumnVariantV2` 在磁盘上使用 Parquet 格式。 - -### 开启 V2 后的行为变化 - -- **规范化值语义**：等价的整数类型会归一为同一值；Decimal 尾随零会被归一；`+0`、`-0` 与整数零使用同一规范值；对象 key 顺序不影响值；数组顺序仍然保留。非法编码或违反内部不变量时会报错，而不是静默接受。 -- **哈希与序列化**：规范化哈希和 arena 序列化为支持的 Variant 值提供 `GROUP BY`、`DISTINCT`、`COUNT(DISTINCT ...)`、`INTERSECT`、`EXCEPT`、`UNION DISTINCT` 以及聚合分组键能力。 -- **显式解析语义**：`parse_to_variant(json_string)` 将 JSON 字符串解析为 Variant；`parse_to_variant_error_to_null(json_string)` 在校验失败时返回 SQL `NULL`；`CAST(string AS VARIANT)` 创建的是带类型的 Variant 字符串，不会把字符串按 JSON 解析。函数语法和示例请参见 [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) 与 [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null)。 -- **表达式与嵌套值**：条件表达式、嵌套容器以及支持 Variant 的 `explode` 可以基于规范化表示执行。 -- **NULL 与物理状态**：SQL `NULL` 仍与 Variant/JSON `null` 不同。`ColumnVariantV2` 列使用整列级物理状态：编码状态（`E`）或带 Variant-null map 的类型化标量状态（`T`），不会在行级混用 E/T。 - -### 示例 +:::caution 实验特性 +`ColumnVariantV2` 默认关闭，只改变当前 FE session 选择的计算路径。它不会改变 Variant 持久化格式，也不会增加 V2 存储、导入、统计信息或 Compaction 支持。 +::: -以下示例均在开启 V2 的当前 FE session 中执行： +通过当前 FE session 的 session variable `enable_variant_v2` 选择该路径： ```sql SET enable_variant_v2 = true; - --- 解析 JSON 文本；CAST(string AS VARIANT) 则保留为带类型的 Variant 字符串。 -SELECT PARSE_TO_VARIANT('{\"id\": 1, \"items\": [10, 20]}') AS parsed_value, - CAST('{\"id\": 1, \"items\": [10, 20]}' AS VARIANT) AS typed_string; - --- 规范化哈希将 1 和 1.0 视为一个 distinct 值。 -SELECT COUNT(DISTINCT value) AS distinct_count -FROM ( - SELECT PARSE_TO_VARIANT('1') AS value - UNION ALL - SELECT PARSE_TO_VARIANT('1.0') AS value -) AS numeric_values; --- distinct_count: 1 - --- 使用容错解析函数时，非法 JSON 转换为 SQL NULL。 -SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value; --- invalid_value: NULL - --- 嵌套访问显式 CAST 后可以作为条件表达式输入。 -SELECT CASE - WHEN CAST(PARSE_TO_VARIANT('{\"enabled\": true}')['enabled'] AS BOOLEAN) - THEN 'on' - ELSE 'off' - END AS status; --- status: on ``` -### 使用边界 - -- 根 Variant 比较谓词（`=`、`!=`、`<=>` 以及排序比较）仍不支持。请在两侧提取相同语义的子路径并 CAST 到可比较类型： - -```sql -SELECT * -FROM tbl -WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); -``` +开启后，V2 会为支持的分组和集合运算增加 canonical hashing 与序列化，明确区分 JSON 解析和字符串 CAST，并使用整列级 typed（`T`）或 encoded（`E`）内存状态。根 Variant 比较谓词、Variant Join Key、Sort/TopN Key 以及 `MIN`/`MAX` 参数仍然不受支持。 -- 不支持将 Variant 表达式作为 JOIN KEY；根 Variant 也不能作为 Sort/TopN 键、窗口分区键或窗口排序键，也不能作为 `MIN`/`MAX` 参数。 -- V2 相关回归测试仍标记为 `nonConcurrent`；功能选择本身是 session-scoped，不会改变原生 Variant 的存储或 Compaction。 -- 开启 V2 前请先验证目标 workload；它是实验性计算路径，不改变存储兼容性契约。 +组件架构、物化流程、内存编码、Decimal 与日期时间映射、canonical equality、CAST 行为、示例和完整限制，请参见 [VARIANT V2 计算语义与内存编码](./variant-v2-compute-semantics)。 ## 宽列 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md new file mode 100644 index 0000000000000..939afbc56dda7 --- /dev/null +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md @@ -0,0 +1,206 @@ +--- +{ + "title": "VARIANT V2 计算语义与内存编码", + "language": "zh-CN", + "description": "介绍实验性 ColumnVariantV2 计算路径的架构与行为，包括内存状态、标量编码、canonical equality、CAST 与解析语义，以及当前限制。" +} +--- + +## 概览 + +`ColumnVariantV2` 是 `VARIANT` 的实验性、仅用于计算的执行路径。它为支持的表达式、分组和集合运算提供自描述的内存表示及 canonical value 语义。 + +SQL 语法、类型规则、索引、存储配置和通用限制请参见 [VARIANT 类型参考](./VARIANT)；在默认存储行为、Sparse、DOC mode 和 Schema Template 之间选型时，请参见 [VARIANT 使用与配置指南](./variant-workload-guide)。本文集中说明 V2 的执行架构和行为。 + +:::caution 实验特性 +`ColumnVariantV2` 默认关闭，只改变当前 FE session 选择的计算路径。它不会引入 V2 表格式、导入路径、Segment 读写器、统计信息格式或 Compaction 路径，也不提供 V1/V2 混部滚动升级兼容能力。 +::: + +## 开启计算路径 + +通过 session variable `enable_variant_v2` 为当前 FE session 开启 V2： + +```sql +SET enable_variant_v2 = true; +``` + +该 session variable 只改变 FE/BE 执行类型标记。已经持久化的 `VARIANT` 数据及其存储兼容性契约不会改变。 + +## 执行架构 + +### 整列状态模型 + +一个 `ColumnVariantV2` 列在任意时刻只有一种整列级物理状态： + +- **Typed scalar（`T`）**：同质标量行借用具体的 Doris 标量列及其 data type；独立的 Variant-null map 表示 Variant/JSON `null`。 +- **Encoded（`E`）**：嵌套值、异构值或已经物化的值，以自描述字节的形式存放在 arena 中。 +- 同一列不会在行级混用 `T` 和 `E`。SQL `NULL` 仍保存在 SQL nullable bitmap 中，在两种状态下都与 Variant/JSON `null` 不同。 + +这种区分使简单标量表达式不必立即编码，同时让嵌套或异构算子在需要时可以消费统一的自描述表示。 + +### 核心组件 + +| 组件 | 职责 | +| --- | --- | +| `ColumnVariantV2` | 管理整列 `T`/`E` 状态、NULL 语义、read view 和物化入口。 | +| `VariantBlockBuilder` | 构建 encoded scalar、array 和 object 行，并把自描述字节追加到 arena。 | +| `VariantScalarEncodingPlan` | 选择 primitive ID、payload 宽度、scale 或 length 元数据，校验边界并写入标量编码。 | +| `with_typed_scalar()` | 集中维护 typed scalar 映射矩阵，同时为 Doris 值创建物理编码计划和 canonical scalar view。 | +| `VariantCanonicalScalarRef` | 表示 canonical 哈希和序列化使用的逻辑标量，不依赖物理宽度或源类型。 | +| Variant V2 CAST 与解析函数 | 定义 typed SQL 值、Variant 字符串和已解析 JSON 值之间的边界。 | + +### Typed-to-encoded 物化流程 + +1. 支持的同质标量可以使用精确的 Doris 标量类型，以 `T` 状态进入计算路径。 +2. 只需要 typed read view 的算子可以直接借用该状态，不必编码整列。 +3. 当算子需要自描述值时，`ensure_encoded()` 按 Doris primitive type 做一次分派，然后访问各行。 +4. `with_typed_scalar()` 为每个非空行同时生成物理 encoding factory 和 canonical-value factory。 +5. `VariantScalarEncodingPlan` 把物理值写入 arena；完成物化后，后续 encoded 运算看到的是 `E` 状态。 + +集中式映射让物理序列化和 canonical equality 保持一致，同时避免每一行都进行虚调用或类型分派。 + +## 内存编码与组织方式 + +Encoded 表示会把 primitive type 以及必要的 scale、length 元数据与 payload 一起保存。Array 和 object 还会携带足以定位子值的结构信息。该表示面向计算期遍历，而不是新的表持久化格式。 + +这种组织方式在概念上类似于把 Variant encoded value 与可选的 typed projection 分开。外部格式可参考官方 [Apache Parquet File Format](https://parquet.apache.org/docs/file-format/)、[Parquet Variant Shredding](https://parquet.apache.org/docs/file-format/types/variantshredding/) 和 [Parquet Nested Encoding](https://parquet.apache.org/docs/file-format/nestedencoding/)。Parquet 以 `metadata` 和 `value` 组织 Variant，并可为同质路径增加 `typed_value`。这里仅用于说明组织思路；`ColumnVariantV2` 的内存格式和 Doris 磁盘格式都不是 Parquet。 + +## Typed scalar 物化映射 + +### Decimal 类型 + +Typed Decimal 从 `T` 物化为 `E` 时，`with_typed_scalar()` 保留 unscaled integer 和 scale，并根据 Doris 源类型选择 payload 宽度： + +| Doris 类型 | 从列中读取的 unscaled 值 | 编码调用 | Variant primitive | +| --- | --- | --- | --- | +| `DECIMALV2` | `DecimalV2Value::value()` 返回的 `__int128` | `decimal(value, scale, 16)` | `DECIMAL16` | +| `DECIMAL32` | `Decimal32::value` 中的 `int32_t` | `decimal(value, scale, 4)` | `DECIMAL4` | +| `DECIMAL64` | `Decimal64::value` 中的 `int64_t` | `decimal(value, scale, 8)` | `DECIMAL8` | +| `DECIMAL128I` | `Decimal128V3::value` 中的 `__int128` | `decimal(value, scale, 16)` | `DECIMAL16` | + +Decimal 的 encoded 布局是 `[primitive header][scale][小端有符号 unscaled value]`，所以 `DECIMAL4`、`DECIMAL8`、`DECIMAL16` 的总长度分别是 6、10、18 字节。 + +Decimal 限制与不变量： + +- Variant Decimal 的 scale 必须在 `[0, 38]` 内。`DECIMAL4`、`DECIMAL8`、`DECIMAL16` 的 unscaled 绝对值上限分别是 `10^9 - 1`、`10^18 - 1`、`10^38 - 1`。 +- Doris 源类型自身的限制仍然生效：`DECIMALV2` 最大 precision 为 27、最大 scale 为 9；`DECIMAL32`、`DECIMAL64`、`DECIMAL128I` 的最大 precision 分别为 9、18、38。 +- typed column 的物理 scale 必须与 data type 元数据中的 scale 完全一致；不一致属于不变量破坏，物化时不会自动 rescale。 +- encoded 宽度由源类型决定。数值较小的 `DECIMALV2` 或 `DECIMAL128I` 不会在该路径上缩窄为 `DECIMAL4` 或 `DECIMAL8`。 +- `DECIMAL256` 不是 `ColumnVariantV2` 支持的 typed identity。它的 precision 最高可达 76，超过 Variant Decimal 的 38 位上限。 +- 物理宽度和 scale 不决定 canonical equality。例如 `1.20` 与 `1.2` 可以拥有不同物理编码，但会归一为同一个 canonical 数值。 + +### 日期和时间类型 + +日期和 timestamp 映射会先计算： + +```text +days = daynr(value) - daynr(1970-01-01) +micros = (days * 86400 + hour * 3600 + minute * 60 + second) * 1000000 + + microsecond +``` + +| Doris 类型 | 归一化 payload | 编码调用 | Variant primitive | +| --- | --- | --- | --- | +| `DATE` | `int32_t days` | `date(days)` | `DATE` | +| `DATEV2` | `int32_t days` | `date(days)` | `DATE` | +| `DATETIME` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | +| `DATETIMEV2` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | +| `TIMESTAMPTZ` | `int64_t micros` | `timestamp_micros(micros, true)` | `TIMESTAMP_MICROS` | + +`DATE` 使用 4 字节 payload 加 1 字节 header；每个 timestamp 使用 8 字节 payload 加 1 字节 header。`utc_adjusted` 参数由 primitive ID 表达：`false` 选择无时区（NTZ）primitive，`true` 选择 UTC-adjusted primitive。 + +日期和时间限制与不变量： + +- 所有源值都必须通过 Doris 的日期合法性检查。非法的 `DATE`、`DATEV2`、`DATETIME`、`DATETIMEV2` 或 `TIMESTAMPTZ` 会抛出 `INVALID_ARGUMENT`；该路径不会修复，也不会转为 `NULL`。 +- `DATE` 和 `DATEV2` 只保留相对 `1970-01-01` 的有符号天数，不携带日内时间或时区调整信息。 +- `DATETIME` 和 `DATETIMEV2` 编码为 wall-clock、无时区的 timestamp；该路径不会应用 session time zone。只有 `TIMESTAMPTZ` 会选择 UTC-adjusted timestamp primitive。 +- Typed materialization 只生成微秒 primitive。`DATETIMEV2` 和 `TIMESTAMPTZ` 最多保留其支持的 6 位小数。Variant 编码虽然定义了 `TIMESTAMP_NANOS` 和 `TIMESTAMP_NTZ_NANOS`，但该映射不会生成它们。 +- `TIMEV2` 不是支持的 typed identity。Variant 编码虽然定义了 `TIME_NTZ_MICROS`，但这条 typed-to-encoded 路径不会生成它。 + +## Canonical value 语义 + +物理 encoded 字节不会被直接当作 equality identity。Canonicalization 会为哈希和 arena 序列化生成逻辑表示： + +- 等价的整数数值表示会归一为同一个值。 +- Decimal 尾随零不改变数值。 +- `+0`、`-0` 和整数零会归一为同一个值。 +- Object key 顺序不影响相等性，array 元素顺序会影响相等性。 +- Variant/JSON `null` 与 SQL `NULL` 仍然不同。 +- 非法编码或违反内部不变量时会报错，而不是静默接受。 + +这些规则为支持的值提供 `GROUP BY`、`DISTINCT`、`COUNT(DISTINCT ...)`、`INTERSECT`、`EXCEPT` 和 `UNION DISTINCT` 等哈希运算能力。 + +```sql +SET enable_variant_v2 = true; + +-- 1 和 1.0 只有一个 canonical distinct 值。 +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('1') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('1.0') AS value +) AS numeric_values; +-- distinct_count: 1 + +-- Object key 顺序不影响 equality。 +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('{\"a\": 1, \"b\": 2}') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('{\"b\": 2, \"a\": 1}') AS value +) AS object_values; +-- distinct_count: 1 + +-- Array 顺序会保留。 +SELECT COUNT(DISTINCT value) AS distinct_count +FROM ( + SELECT PARSE_TO_VARIANT('[1, 2]') AS value + UNION ALL + SELECT PARSE_TO_VARIANT('[2, 1]') AS value +) AS array_values; +-- distinct_count: 2 +``` + +Canonical equality 不会开放根 Variant 比较谓词。直接执行 `VARIANT = VARIANT`、null-safe equality 或排序比较仍然不受支持。 + +## CAST 与解析行为 + +V2 明确区分“创建 typed Variant 字符串”和“解析 JSON 文本”： + +- `CAST(string AS VARIANT)` 创建 typed Variant 字符串，不会按 JSON 解析或校验该字符串。 +- `parse_to_variant(json_string)` 将 JSON 文本严格解析为 Variant 值。 +- `parse_to_variant_error_to_null(json_string)` 在解析或校验失败时返回 SQL `NULL`。 +- 提取出的 Variant 值在进行确定类型的比较、过滤、排序或算术运算前，应先 CAST 到具体 SQL 类型。 + +完整函数语法请参见 [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) 和 [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null)。 + +```sql +SET enable_variant_v2 = true; + +SELECT PARSE_TO_VARIANT('{\"id\": 1}') AS parsed_object, + CAST('{\"id\": 1}' AS VARIANT) AS typed_string; + +SELECT ELEMENT_AT(CAST('{\"id\": 1}' AS VARIANT), 'id') AS from_string, + ELEMENT_AT(PARSE_TO_VARIANT('{\"id\": 1}'), 'id') AS from_json; +-- from_string: NULL; from_json: 1 + +SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value; +-- invalid_value: NULL +``` + +当函数签名支持 Variant 值时，条件表达式、嵌套容器以及支持 Variant 的 `explode` 可以直接消费 V2 表示。 + +## 使用边界与限制 + +- 不支持根 Variant 比较谓词（`=`、`!=`、`<=>` 和排序比较）；应在两侧提取可比较的路径并 CAST。 +- 不支持把 Variant 表达式作为 Join Key。 +- 根 Variant 不能作为 Sort/TopN 键、窗口分区键或排序键，也不能作为 `MIN` 和 `MAX` 的参数。 +- 功能选择只作用于 session，不会改变原生 Variant 存储、导入、统计信息或 Compaction。 +- V2 回归测试仍标记为 `nonConcurrent`。 +- 仅在验证目标 workload 后开启 V2；该执行路径仍是实验特性。 + +```sql +SELECT * +FROM tbl +WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); +``` diff --git a/sidebars.ts b/sidebars.ts index 0a53243f2dd9a..d3bec64eb5c9a 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -1264,6 +1264,11 @@ const sidebars: SidebarsConfig = { id: 'sql-manual/basic-element/sql-data-types/semi-structured/VARIANT', label: 'VARIANT Reference', }, + { + type: 'doc', + id: 'sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics', + label: 'V2 Compute Semantics', + }, ], }, ], From a5774d453daf8765cc64ac3dabe54267ede8f55e Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Sun, 19 Jul 2026 23:49:14 +0800 Subject: [PATCH 8/9] docs: focus Variant V2 reference on user behavior --- .../sql-data-types/semi-structured/VARIANT.md | 26 +-- .../variant-v2-compute-semantics.md | 217 ++++++++---------- .../sql-data-types/semi-structured/VARIANT.md | 26 +-- .../variant-v2-compute-semantics.md | 217 ++++++++---------- sidebars.ts | 2 +- 5 files changed, 215 insertions(+), 273 deletions(-) diff --git a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index e0f8999f1084a..5b7aba2e7bbbe 100644 --- a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -85,7 +85,7 @@ Use [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/var ### Access objects and arrays -Object fields can be accessed with a string key. For `ColumnVariantV2`, VARIANT array indexes are zero-based for non-negative indexes and support negative indexes from the end. The extracted value remains `VARIANT`; CAST it before typed comparison or aggregation. +Object fields can be accessed with a string key. With VARIANT V2 enabled, VARIANT array indexes are zero-based for non-negative indexes and support negative indexes from the end. The extracted value remains `VARIANT`; CAST it before typed comparison or aggregation. ```sql SET enable_variant_v2 = true; @@ -128,20 +128,20 @@ Do not use `CAST(string AS VARIANT)` as a JSON validation step. Use `PARSE_TO_VA ## Equality semantics -For `ColumnVariantV2`, equality used by canonical hashing and serialization is based on the logical value rather than the physical encoded bytes: +With VARIANT V2 enabled, supported grouping, deduplication, and set operations compare logical values rather than source SQL types or representations: -- Equivalent integral numeric representations normalize to the same value. +- Equivalent integral numeric representations compare as equal. - Decimal trailing zeros do not change the value. -- `+0`, `-0`, and integral zero normalize together. +- `+0`, `-0`, and integral zero compare as equal. - Object key order does not affect equality, while array element order does. - Variant/JSON `null` is distinct from SQL `NULL`. -These rules are used by supported hash-based operations such as `GROUP BY`, `DISTINCT`, `COUNT(DISTINCT ...)`, `INTERSECT`, `EXCEPT`, and `UNION DISTINCT`. They do not enable root Variant comparison predicates: direct `VARIANT = VARIANT` and ordering comparisons remain unsupported. +These rules apply to supported operations such as `GROUP BY`, `DISTINCT`, `COUNT(DISTINCT ...)`, `INTERSECT`, `EXCEPT`, and `UNION DISTINCT`. They do not enable root Variant comparison predicates: direct `VARIANT = VARIANT` and ordering comparisons remain unsupported. ```sql SET enable_variant_v2 = true; --- 1 and 1.0 have one canonical distinct value. +-- 1 and 1.0 have one distinct logical value. SELECT COUNT(DISTINCT value) AS distinct_count FROM ( SELECT PARSE_TO_VARIANT('1') AS value @@ -168,7 +168,7 @@ FROM ( -- distinct_count: 2 ``` -For the physical encoding, normalization pipeline, and additional boundaries behind these rules, see [VARIANT V2 Compute Semantics and Memory Encoding](./variant-v2-compute-semantics). +For more examples, supported operations, data type behavior, and current limits, see [VARIANT V2 Behavior and Semantics](./variant-v2-compute-semantics). ## Primitive types @@ -511,7 +511,7 @@ SELECT * FROM tbl WHERE v['str'] MATCH 'Doris'; - On the default path, a whole VARIANT value should not be used directly in `ORDER BY`, `GROUP BY`, as a `JOIN KEY`, or as an aggregate argument; CAST the required subpath instead. - Strings can be implicitly converted to VARIANT. -The [experimental V2 compute path](./variant-v2-compute-semantics) adds canonical hashing and serialization for grouping and set-operation use cases. It does not make root VARIANT comparison predicates or Variant join keys available. +The [experimental V2 behavior](./variant-v2-compute-semantics) adds logical-value semantics for grouping and set-operation use cases. It does not make root VARIANT comparison predicates or Variant join keys available. | VARIANT | Castable | Coercible | | --------------- | -------- | --------- | @@ -526,21 +526,21 @@ The [experimental V2 compute path](./variant-v2-compute-semantics) adds canonica | `VARCHAR` | ✔ | ✔ | | `JSON` | ✔ | ✔ | -## Experimental compute path: ColumnVariantV2 +## Experimental VARIANT V2 behavior :::caution Experimental feature -`ColumnVariantV2` is disabled by default and changes only the compute path selected for the current FE session. It does not change the persisted Variant format or add V2 storage, loading, statistics, or compaction support. +VARIANT V2 is disabled by default and affects only execution behavior in the current session. Existing data and the persisted Variant format remain unchanged. ::: -Select it for the current FE session with the session variable `enable_variant_v2`: +Enable it for the current session with the session variable `enable_variant_v2`: ```sql SET enable_variant_v2 = true; ``` -When enabled, V2 adds canonical hashing and serialization for supported grouping and set operations, makes JSON parsing explicit, and uses whole-column typed (`T`) or encoded (`E`) in-memory states. Root Variant comparison predicates, Variant join keys, Sort/TopN keys, and `MIN`/`MAX` arguments remain unsupported. +When enabled, V2 supports logical-value grouping, deduplication, and set operations for supported Variant values, and makes JSON parsing explicit. Root Variant comparison predicates, Variant join keys, Sort/TopN keys, and `MIN`/`MAX` arguments remain unsupported. -For the component architecture, materialization flow, memory encoding, Decimal and date/time mappings, canonical equality, CAST behavior, examples, and complete limits, see [VARIANT V2 Compute Semantics and Memory Encoding](./variant-v2-compute-semantics). +For behavior changes, equality, CAST and parsing, Decimal and date/time semantics, examples, and complete limits, see [VARIANT V2 Behavior and Semantics](./variant-v2-compute-semantics). ## Wide columns diff --git a/docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md b/docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md index 391d8e9f04b11..e2a6b11ed7c95 100644 --- a/docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md +++ b/docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md @@ -1,139 +1,49 @@ --- { - "title": "VARIANT V2 Compute Semantics and Memory Encoding", + "title": "VARIANT V2 Behavior and Semantics", "language": "en-US", - "description": "Architecture and behavioral reference for the experimental ColumnVariantV2 compute path, including memory states, scalar encoding, canonical equality, CAST and parsing semantics, and current limitations." + "description": "User-facing reference for experimental VARIANT V2 behavior, including equality, CAST, JSON parsing, Decimal and date/time semantics, examples, and limits." } --- ## Overview -`ColumnVariantV2` is an experimental, compute-only execution path for `VARIANT`. It adds a self-describing in-memory representation and canonical value semantics for supported expressions, grouping, and set operations. +Experimental VARIANT V2 extends the operations that can consume `VARIANT` values and defines consistent logical-value semantics for grouping, deduplication, and set operations. -Use the [VARIANT type reference](./VARIANT) for SQL syntax, type rules, indexes, storage configuration, and general limitations. Use the [VARIANT Workload Guide](./variant-workload-guide) when choosing between default storage behavior, sparse columns, DOC mode, and Schema Template. This page focuses on the V2 execution architecture and behavior. +Use the [VARIANT type reference](./VARIANT) for SQL syntax, indexes, storage configuration, and general type rules. Use the [VARIANT Workload Guide](./variant-workload-guide) when choosing between default storage behavior, sparse columns, DOC mode, and Schema Template. This page describes only the behavior users can observe after enabling V2. :::caution Experimental feature -`ColumnVariantV2` is disabled by default. It changes only the compute path selected for the current FE session. It does not introduce a V2 table format, loading path, segment reader or writer, statistics format, or compaction path, and it does not provide V1/V2 mixed-version rolling-upgrade compatibility. +V2 is disabled by default and applies only to the current session. Existing data and the persisted Variant format remain unchanged. Enable it only after validating the target workload. ::: -## Enable the compute path +## Enable V2 -Enable V2 for the current FE session with the session variable `enable_variant_v2`: +Set the session variable before running the target statements: ```sql SET enable_variant_v2 = true; ``` -The session variable changes the FE/BE execution type marker only. Existing persisted `VARIANT` data and its storage compatibility contract remain unchanged. +The setting affects only the current session. Existing persisted `VARIANT` data remains unchanged. -## Execution architecture +## Behavior after enabling V2 -### Whole-column state model - -A `ColumnVariantV2` column has one physical state for the whole column: - -- **Typed scalar (`T`)**: homogeneous scalar rows borrow a concrete Doris scalar column and its data type. A separate Variant-null map represents Variant/JSON `null`. -- **Encoded (`E`)**: nested, heterogeneous, or materialized values are stored as self-describing bytes in an arena. -- A column does not mix `T` and `E` at row granularity. SQL `NULL` remains in the SQL nullable bitmap and is distinct from Variant/JSON `null` in either state. - -This distinction lets simple scalar expressions avoid immediate encoding while allowing nested or heterogeneous operators to consume one uniform representation when required. - -### Main components - -| Component | Responsibility | +| Area | V2 behavior | | --- | --- | -| `ColumnVariantV2` | Owns the whole-column `T` or `E` state, null semantics, read views, and materialization entry points. | -| `VariantBlockBuilder` | Builds encoded scalar, array, and object rows and appends their self-describing bytes to the arena. | -| `VariantScalarEncodingPlan` | Selects a primitive ID, payload width, scale or length metadata, validates limits, and writes the encoded scalar bytes. | -| `with_typed_scalar()` | Central typed-scalar mapping matrix. It creates both the physical encoding plan and the canonical scalar view for a Doris value. | -| `VariantCanonicalScalarRef` | Represents the logical scalar used by canonical hashing and serialization, independently of physical width or source type. | -| Variant V2 CAST and parser functions | Define the boundary between typed SQL values, Variant strings, and parsed JSON values. | - -### Typed-to-encoded materialization flow - -1. A supported homogeneous scalar can enter the compute path in `T` state with an exact Doris scalar type. -2. Operators that only need a typed read view can borrow that state without encoding the column. -3. When an operator requires self-describing values, `ensure_encoded()` dispatches once on the Doris primitive type and visits the rows. -4. `with_typed_scalar()` produces a physical encoding factory and a canonical-value factory for each non-null row. -5. `VariantScalarEncodingPlan` writes the physical value to the arena, after which the column is represented in `E` state for subsequent encoded operations. - -This centralized mapping keeps physical serialization and canonical equality aligned while avoiding a virtual or type switch for every row. - -## Memory encoding and organization - -The encoded representation carries the primitive type and any required scale or length metadata next to the payload. Arrays and objects additionally carry enough structural information to locate child values. This representation is designed for compute-time traversal rather than persisted table storage. - -The organization is conceptually similar to separating an encoded Variant value from optional typed projections. For external format references, see the official [Apache Parquet File Format](https://parquet.apache.org/docs/file-format/), [Parquet Variant Shredding](https://parquet.apache.org/docs/file-format/types/variantshredding/), and [Parquet Nested Encoding](https://parquet.apache.org/docs/file-format/nestedencoding/). Parquet organizes a Variant around `metadata` and `value`, with optional `typed_value` fields for homogeneous paths. This is an organizational reference; `ColumnVariantV2` does not use Parquet as its in-memory or Doris on-disk format. - -## Typed scalar materialization mappings - -### Decimal types - -When a typed Decimal is materialized from `T` to `E`, `with_typed_scalar()` preserves its unscaled integer and scale and selects the payload width from the Doris source type: - -| Doris type | Unscaled value read from the column | Encoding call | Variant primitive | -| --- | --- | --- | --- | -| `DECIMALV2` | `__int128` from `DecimalV2Value::value()` | `decimal(value, scale, 16)` | `DECIMAL16` | -| `DECIMAL32` | `int32_t` from `Decimal32::value` | `decimal(value, scale, 4)` | `DECIMAL4` | -| `DECIMAL64` | `int64_t` from `Decimal64::value` | `decimal(value, scale, 8)` | `DECIMAL8` | -| `DECIMAL128I` | `__int128` from `Decimal128V3::value` | `decimal(value, scale, 16)` | `DECIMAL16` | - -The encoded decimal layout is `[primitive header][scale][little-endian signed unscaled value]`. Its total size is 6 bytes for `DECIMAL4`, 10 bytes for `DECIMAL8`, and 18 bytes for `DECIMAL16`. - -Decimal limits and invariants: - -- The Variant decimal scale must be in `[0, 38]`. The absolute unscaled value is limited to `10^9 - 1` for `DECIMAL4`, `10^18 - 1` for `DECIMAL8`, and `10^38 - 1` for `DECIMAL16`. -- Doris source-type limits still apply: `DECIMALV2` supports precision up to 27 and scale up to 9; `DECIMAL32`, `DECIMAL64`, and `DECIMAL128I` support precision up to 9, 18, and 38 respectively. -- The typed column's physical scale must exactly match its data-type metadata. A mismatch is an invariant violation; materialization does not rescale the value. -- The source type determines the encoded width. A small `DECIMALV2` or `DECIMAL128I` value is not narrowed to `DECIMAL4` or `DECIMAL8` on this path. -- `DECIMAL256` is not a supported `ColumnVariantV2` typed identity. Its precision can reach 76, beyond the Variant decimal precision limit of 38. -- Physical width and scale do not define canonical equality. For example, `1.20` and `1.2` can have different physical encodings but normalize to the same canonical numeric value. - -### Date and time types - -The date and timestamp mapping first calculates: - -```text -days = daynr(value) - daynr(1970-01-01) -micros = (days * 86400 + hour * 3600 + minute * 60 + second) * 1000000 - + microsecond -``` - -| Doris type | Normalized payload | Encoding call | Variant primitive | -| --- | --- | --- | --- | -| `DATE` | `int32_t days` | `date(days)` | `DATE` | -| `DATEV2` | `int32_t days` | `date(days)` | `DATE` | -| `DATETIME` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | -| `DATETIMEV2` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | -| `TIMESTAMPTZ` | `int64_t micros` | `timestamp_micros(micros, true)` | `TIMESTAMP_MICROS` | +| Grouping and deduplication | Supported Variant values can be used by `GROUP BY`, `DISTINCT`, and `COUNT(DISTINCT ...)` with logical-value equality. | +| Set operations | Supported Variant values can participate in `INTERSECT`, `EXCEPT`, and `UNION DISTINCT`. | +| JSON parsing | JSON text is parsed explicitly with `parse_to_variant` or `parse_to_variant_error_to_null`. | +| String conversion | `CAST(string AS VARIANT)` creates a Variant string and does not parse the string as JSON. | +| Nested expressions | Supported conditional expressions, nested containers, and Variant-aware `explode` can consume Variant values. | +| Invalid values | Malformed or unsupported Variant values return an error instead of being silently accepted. | -`DATE` uses a 4-byte payload plus a 1-byte header. Each timestamp uses an 8-byte payload plus a 1-byte header. The `utc_adjusted` argument is represented by the primitive ID: `false` selects the no-time-zone (`NTZ`) primitive, while `true` selects the UTC-adjusted primitive. +V2 does not enable every operation on a whole (root) Variant value. In particular, direct comparison predicates, join keys, sorting, and `MIN`/`MAX` remain restricted as described in [Current limitations](#current-limitations). -Date and time limits and invariants: - -- Every source value must pass Doris date validation. An invalid `DATE`, `DATEV2`, `DATETIME`, `DATETIMEV2`, or `TIMESTAMPTZ` raises `INVALID_ARGUMENT`; this path does not repair it or convert it to `NULL`. -- `DATE` and `DATEV2` preserve only a signed day count relative to `1970-01-01`; they carry neither a time of day nor a time-zone adjustment. -- `DATETIME` and `DATETIMEV2` are encoded as wall-clock, no-time-zone timestamps. This path does not apply the session time zone. `TIMESTAMPTZ` is the only mapped Doris type that selects the UTC-adjusted timestamp primitive. -- Typed materialization emits microsecond primitives only. `DATETIMEV2` and `TIMESTAMPTZ` preserve at most their supported six fractional digits. `TIMESTAMP_NANOS` and `TIMESTAMP_NTZ_NANOS` exist in the encoding but are not emitted by this mapping. -- `TIMEV2` is not a supported typed identity. Although the Variant encoding defines `TIME_NTZ_MICROS`, this typed-to-encoded path does not produce it. - -## Canonical value semantics - -Physical encoded bytes are not used directly as the equality identity. Canonicalization produces a logical representation for hashing and arena serialization: - -- Equivalent integral numeric representations normalize to the same value. -- Decimal trailing zeros do not change the value. -- `+0`, `-0`, and integral zero normalize together. -- Object key order does not affect equality, while array element order does. -- Variant/JSON `null` remains distinct from SQL `NULL`. -- Invalid encodings and violated internal invariants fail instead of being silently accepted. - -These rules enable supported hash-based operations including `GROUP BY`, `DISTINCT`, `COUNT(DISTINCT ...)`, `INTERSECT`, `EXCEPT`, and `UNION DISTINCT`. +For example, grouping or deduplicating a root Variant value is not supported on the default path. After V2 is enabled, the same values can be grouped by logical equality: ```sql SET enable_variant_v2 = true; --- 1 and 1.0 have one canonical distinct value. SELECT COUNT(DISTINCT value) AS distinct_count FROM ( SELECT PARSE_TO_VARIANT('1') AS value @@ -141,7 +51,20 @@ FROM ( SELECT PARSE_TO_VARIANT('1.0') AS value ) AS numeric_values; -- distinct_count: 1 +``` +## Equality semantics + +Grouping, deduplication, and set operations compare the logical value rather than its source SQL type or representation: + +- Equivalent integral numeric representations are equal. +- Decimal trailing zeros do not change the value, so `1.20` and `1.2` are equal. +- `+0`, `-0`, and integral zero are equal. +- Object key order does not affect equality. +- Array element order affects equality. +- Variant/JSON `null` is distinct from SQL `NULL`. + +```sql -- Object key order is ignored. SELECT COUNT(DISTINCT value) AS distinct_count FROM ( @@ -161,16 +84,18 @@ FROM ( -- distinct_count: 2 ``` -Canonical equality does not enable root Variant comparison predicates. Direct `VARIANT = VARIANT`, null-safe equality, and ordering comparisons remain unsupported. +These equality rules apply to supported hash-based and set operations. They do not enable direct `VARIANT = VARIANT`, null-safe equality, or ordering comparisons. -## CAST and parsing behavior +## CAST and JSON parsing -V2 separates creating a typed Variant string from parsing JSON text: +V2 distinguishes a Variant string from a parsed JSON value: -- `CAST(string AS VARIANT)` creates a typed Variant string. It does not parse or validate the string as JSON. -- `parse_to_variant(json_string)` strictly parses JSON text into a Variant value. -- `parse_to_variant_error_to_null(json_string)` returns SQL `NULL` when parsing or validation fails. -- Extracted Variant values should be cast to concrete SQL types before typed comparison, filtering, ordering, or arithmetic. +| Operation | Result | Invalid JSON behavior | +| --- | --- | --- | +| `CAST(string AS VARIANT)` | A Variant string containing the original text | No JSON validation is performed | +| `parse_to_variant(string)` | A Variant value parsed from JSON text | Returns an error | +| `parse_to_variant_error_to_null(string)` | A Variant value parsed from JSON text | Returns SQL `NULL` | +| `CAST(variant_value AS T)` | A concrete SQL value of type `T` | Follows the target type's cast rules | See [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) and [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null) for full function syntax. @@ -178,7 +103,7 @@ See [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions SET enable_variant_v2 = true; SELECT PARSE_TO_VARIANT('{\"id\": 1}') AS parsed_object, - CAST('{\"id\": 1}' AS VARIANT) AS typed_string; + CAST('{\"id\": 1}' AS VARIANT) AS variant_string; SELECT ELEMENT_AT(CAST('{\"id\": 1}' AS VARIANT), 'id') AS from_string, ELEMENT_AT(PARSE_TO_VARIANT('{\"id\": 1}'), 'id') AS from_json; @@ -186,20 +111,66 @@ SELECT ELEMENT_AT(CAST('{\"id\": 1}' AS VARIANT), 'id') AS from_string, SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value; -- invalid_value: NULL + +SELECT CAST(PARSE_TO_VARIANT('42') AS BIGINT) AS id; +-- id: 42 ``` -Conditional expressions, nested containers, and Variant-aware `explode` can consume the V2 representation where their signatures support Variant values. +Do not use `CAST(string AS VARIANT)` to validate JSON. Extracted Variant values should be cast to a concrete SQL type before typed comparison, filtering, ordering, or arithmetic. + +## Decimal semantics and limits + +V2 preserves supported Decimal values exactly, including their scale. Equality semantics then ignore insignificant trailing zeros. + +| Doris input type | Supported range | V2 behavior | +| --- | --- | --- | +| Legacy `DECIMALV2` | Precision up to 27; scale up to 9 | Preserved as an exact Variant Decimal | +| `DECIMAL(p, s)` | `1 <= p <= 38`, `0 <= s <= p` | Preserved as an exact Variant Decimal | +| `DECIMAL(p, s)` | `39 <= p <= 76` | Not supported by V2; cast to precision 38 or lower first | + +Current Decimal limits: + +- Variant Decimal supports precision and scale up to 38. +- The source value must fit both its Doris Decimal type and the Variant precision limit. +- Values such as `1.20` and `1.2` retain their Decimal value and compare as one logical value in supported grouping and set operations. + +## Date and time semantics and limits + +| Doris input type | Variant semantics | Precision and time-zone behavior | +| --- | --- | --- | +| `DATE` | Calendar date | Day precision; no time or time zone | +| Legacy `DATETIME` | Timestamp without time zone | Whole-second precision; no session time-zone adjustment | +| `DATETIME(p)` | Timestamp without time zone | `0 <= p <= 6`; no session time-zone adjustment | +| `TIMESTAMPTZ(p)` | Time-zone-adjusted timestamp | `0 <= p <= 6` | +| `TIME` | — | Not supported by V2 | + +Current date and time limits: + +- Every source value must be a valid Doris date or timestamp. Invalid values return `INVALID_ARGUMENT`; they are not repaired or converted to `NULL`. +- `DATETIME` and `DATETIMEV2` keep wall-clock, no-time-zone semantics. This conversion does not apply the session time zone. +- `TIMESTAMPTZ` is the supported input that keeps time-zone-adjusted timestamp semantics. +- V2 date and time conversion supports microsecond precision, not nanosecond precision. + +## NULL semantics + +SQL `NULL` and Variant/JSON `null` are different values: + +- SQL `NULL` represents the absence of a SQL value and participates in normal SQL null propagation. +- Variant/JSON `null` is a Variant value produced, for example, by `parse_to_variant('null')`. +- `parse_to_variant_error_to_null` returns SQL `NULL` for malformed input; this is different from successfully parsing the JSON literal `null`. -## Boundaries and limitations +## Current limitations -- Root Variant comparison predicates (`=`, `!=`, `<=>`, and ordering comparisons) are not supported. Extract and cast a comparable path on both sides. +- Root Variant comparison predicates (`=`, `!=`, `<=>`, `<`, `<=`, `>`, and `>=`) are not supported. Extract and cast a comparable path on both sides. - Variant expressions are not supported as join keys. -- Root Variant values are not supported as sort or TopN keys, window partition or order keys, or arguments to `MIN` and `MAX`. -- Feature selection is session-scoped and does not change native Variant storage, loading, statistics, or compaction. -- V2 regression suites remain tagged `nonConcurrent`. -- Enable V2 only after validating the target workload; the execution path remains experimental. +- Root Variant values are not supported as Sort/TopN keys or as window partition or order keys. +- Root Variant values are not supported as arguments to `MIN` or `MAX`. +- Decimal values with precision greater than 38 and `TIME` values are not supported V2 inputs. +- Feature selection is session-scoped and does not change persisted Variant data. +- Enable V2 only after validating the target workload; the feature remains experimental. ```sql +-- Compare concrete subpaths rather than root Variant values. SELECT * FROM tbl WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index ab2ba20b871d7..456c6b5b84750 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -85,7 +85,7 @@ SELECT PARSE_TO_VARIANT('[10, 20, 30]'); ### 访问对象和数组 -对象字段可以使用字符串 key 访问。对于 `ColumnVariantV2`，VARIANT 数组的非负索引从 0 开始，并支持从数组末尾倒数的负数索引。提取出的值仍是 `VARIANT`，如需按确定类型比较或聚合，请先 CAST。 +对象字段可以使用字符串 key 访问。开启 VARIANT V2 后，VARIANT 数组的非负索引从 0 开始，并支持从数组末尾倒数的负数索引。提取出的值仍是 `VARIANT`，如需按确定类型比较或聚合，请先 CAST。 ```sql SET enable_variant_v2 = true; @@ -128,20 +128,20 @@ SELECT CAST('{invalid json' AS VARIANT) AS still_a_string; ## Equality 语义 -对于 `ColumnVariantV2`，规范化哈希和序列化使用逻辑值而不是物理编码字节判断相等性： +开启 VARIANT V2 后，支持的分组、去重和集合运算会按 logical value 判断相等性，而不是按来源 SQL 类型或表示形式： -- 等价的整数数值表示会归一为同一个值。 +- 等价的整数数值表示相等。 - Decimal 尾随零不影响值。 -- `+0`、`-0` 与整数零会归一为同一个值。 +- `+0`、`-0` 与整数零相等。 - 对象 key 的顺序不影响相等性，但数组元素顺序会影响相等性。 - Variant/JSON `null` 与 SQL `NULL` 不同。 -这些规则用于支持的基于哈希的操作，例如 `GROUP BY`、`DISTINCT`、`COUNT(DISTINCT ...)`、`INTERSECT`、`EXCEPT` 和 `UNION DISTINCT`。这并不意味着根 Variant 比较谓词已经可用：直接执行 `VARIANT = VARIANT` 或排序比较仍不支持。 +这些规则适用于 `GROUP BY`、`DISTINCT`、`COUNT(DISTINCT ...)`、`INTERSECT`、`EXCEPT` 和 `UNION DISTINCT` 等已支持的操作。这并不意味着根 Variant 比较谓词已经可用：直接执行 `VARIANT = VARIANT` 或排序比较仍不支持。 ```sql SET enable_variant_v2 = true; --- 1 和 1.0 归一后只有一个 distinct 值。 +-- 1 和 1.0 只有一个 distinct logical value。 SELECT COUNT(DISTINCT value) AS distinct_count FROM ( SELECT PARSE_TO_VARIANT('1') AS value @@ -168,7 +168,7 @@ FROM ( -- distinct_count: 2 ``` -这些规则背后的物理编码、归一化流程和其他使用边界，请参见 [VARIANT V2 计算语义与内存编码](./variant-v2-compute-semantics)。 +更多示例、支持的运算、数据类型行为和当前限制，请参见 [VARIANT V2 行为与语义](./variant-v2-compute-semantics)。 ## 基本类型 @@ -511,7 +511,7 @@ SELECT * FROM tbl WHERE v['str'] MATCH 'Doris'; - 在默认路径上，不应将整个 VARIANT 直接用于 `ORDER BY`、`GROUP BY`、`JOIN KEY` 或聚合参数；请先对子路径 CAST。 - 字符串类型可隐式转换为 VARIANT。 -[实验性 V2 计算路径](./variant-v2-compute-semantics)新增了面向分组和集合运算的规范化哈希与序列化，但不会因此开放根 VARIANT 比较谓词或 Variant Join Key。 +[实验性 V2 行为](./variant-v2-compute-semantics)为分组和集合运算增加 logical-value 语义，但不会因此开放根 VARIANT 比较谓词或 Variant Join Key。 | VARIANT | Castable | Coercible | | --------------- | -------- | --------- | @@ -526,21 +526,21 @@ SELECT * FROM tbl WHERE v['str'] MATCH 'Doris'; | `VARCHAR` | ✔ | ✔ | | `JSON` | ✔ | ✔ | -## 实验性计算路径：ColumnVariantV2 +## 实验性 VARIANT V2 行为 :::caution 实验特性 -`ColumnVariantV2` 默认关闭，只改变当前 FE session 选择的计算路径。它不会改变 Variant 持久化格式，也不会增加 V2 存储、导入、统计信息或 Compaction 支持。 +VARIANT V2 默认关闭，只影响当前 session 的执行行为。现有数据和 Variant 持久化格式不会改变。 ::: -通过当前 FE session 的 session variable `enable_variant_v2` 选择该路径： +通过 session variable `enable_variant_v2` 为当前 session 开启该功能： ```sql SET enable_variant_v2 = true; ``` -开启后，V2 会为支持的分组和集合运算增加 canonical hashing 与序列化，明确区分 JSON 解析和字符串 CAST，并使用整列级 typed（`T`）或 encoded（`E`）内存状态。根 Variant 比较谓词、Variant Join Key、Sort/TopN Key 以及 `MIN`/`MAX` 参数仍然不受支持。 +开启后，V2 会为支持的 Variant 值提供 logical-value 分组、去重和集合运算，并明确区分 JSON 解析和字符串 CAST。根 Variant 比较谓词、Variant Join Key、Sort/TopN Key 以及 `MIN`/`MAX` 参数仍然不受支持。 -组件架构、物化流程、内存编码、Decimal 与日期时间映射、canonical equality、CAST 行为、示例和完整限制，请参见 [VARIANT V2 计算语义与内存编码](./variant-v2-compute-semantics)。 +行为变化、Equality、CAST 与解析、Decimal 和日期时间语义、示例及完整限制，请参见 [VARIANT V2 行为与语义](./variant-v2-compute-semantics)。 ## 宽列 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md index 939afbc56dda7..a4b7a311cbf7c 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md @@ -1,139 +1,49 @@ --- { - "title": "VARIANT V2 计算语义与内存编码", + "title": "VARIANT V2 行为与语义", "language": "zh-CN", - "description": "介绍实验性 ColumnVariantV2 计算路径的架构与行为，包括内存状态、标量编码、canonical equality、CAST 与解析语义，以及当前限制。" + "description": "面向用户介绍实验性 VARIANT V2 的功能、行为和语义，包括开启方式、Equality、CAST 与 JSON 解析、Decimal 和日期时间处理、SQL 示例及当前限制。" } --- ## 概览 -`ColumnVariantV2` 是 `VARIANT` 的实验性、仅用于计算的执行路径。它为支持的表达式、分组和集合运算提供自描述的内存表示及 canonical value 语义。 +实验性 VARIANT V2 扩展了可以消费 `VARIANT` 值的运算，并为分组、去重和集合运算定义了一致的 logical-value 语义。 -SQL 语法、类型规则、索引、存储配置和通用限制请参见 [VARIANT 类型参考](./VARIANT)；在默认存储行为、Sparse、DOC mode 和 Schema Template 之间选型时，请参见 [VARIANT 使用与配置指南](./variant-workload-guide)。本文集中说明 V2 的执行架构和行为。 +SQL 语法、索引、存储配置和通用类型规则请参见 [VARIANT 类型参考](./VARIANT)；在默认存储行为、Sparse、DOC mode 和 Schema Template 之间选型时，请参见 [VARIANT 使用与配置指南](./variant-workload-guide)。本文只说明开启 V2 后用户可以观察到的行为。 :::caution 实验特性 -`ColumnVariantV2` 默认关闭，只改变当前 FE session 选择的计算路径。它不会引入 V2 表格式、导入路径、Segment 读写器、统计信息格式或 Compaction 路径，也不提供 V1/V2 混部滚动升级兼容能力。 +V2 默认关闭，只作用于当前 session。现有数据和 Variant 持久化格式不会改变。请在验证目标 workload 后再开启。 ::: -## 开启计算路径 +## 开启 V2 -通过 session variable `enable_variant_v2` 为当前 FE session 开启 V2： +执行目标 SQL 前设置 session variable： ```sql SET enable_variant_v2 = true; ``` -该 session variable 只改变 FE/BE 执行类型标记。已经持久化的 `VARIANT` 数据及其存储兼容性契约不会改变。 +该设置只影响当前 session，已经持久化的 `VARIANT` 数据不会改变。 -## 执行架构 +## 开启 V2 后的行为 -### 整列状态模型 - -一个 `ColumnVariantV2` 列在任意时刻只有一种整列级物理状态： - -- **Typed scalar（`T`）**：同质标量行借用具体的 Doris 标量列及其 data type；独立的 Variant-null map 表示 Variant/JSON `null`。 -- **Encoded（`E`）**：嵌套值、异构值或已经物化的值，以自描述字节的形式存放在 arena 中。 -- 同一列不会在行级混用 `T` 和 `E`。SQL `NULL` 仍保存在 SQL nullable bitmap 中，在两种状态下都与 Variant/JSON `null` 不同。 - -这种区分使简单标量表达式不必立即编码，同时让嵌套或异构算子在需要时可以消费统一的自描述表示。 - -### 核心组件 - -| 组件 | 职责 | +| 场景 | V2 行为 | | --- | --- | -| `ColumnVariantV2` | 管理整列 `T`/`E` 状态、NULL 语义、read view 和物化入口。 | -| `VariantBlockBuilder` | 构建 encoded scalar、array 和 object 行，并把自描述字节追加到 arena。 | -| `VariantScalarEncodingPlan` | 选择 primitive ID、payload 宽度、scale 或 length 元数据，校验边界并写入标量编码。 | -| `with_typed_scalar()` | 集中维护 typed scalar 映射矩阵，同时为 Doris 值创建物理编码计划和 canonical scalar view。 | -| `VariantCanonicalScalarRef` | 表示 canonical 哈希和序列化使用的逻辑标量，不依赖物理宽度或源类型。 | -| Variant V2 CAST 与解析函数 | 定义 typed SQL 值、Variant 字符串和已解析 JSON 值之间的边界。 | - -### Typed-to-encoded 物化流程 - -1. 支持的同质标量可以使用精确的 Doris 标量类型，以 `T` 状态进入计算路径。 -2. 只需要 typed read view 的算子可以直接借用该状态，不必编码整列。 -3. 当算子需要自描述值时，`ensure_encoded()` 按 Doris primitive type 做一次分派，然后访问各行。 -4. `with_typed_scalar()` 为每个非空行同时生成物理 encoding factory 和 canonical-value factory。 -5. `VariantScalarEncodingPlan` 把物理值写入 arena；完成物化后，后续 encoded 运算看到的是 `E` 状态。 - -集中式映射让物理序列化和 canonical equality 保持一致，同时避免每一行都进行虚调用或类型分派。 - -## 内存编码与组织方式 - -Encoded 表示会把 primitive type 以及必要的 scale、length 元数据与 payload 一起保存。Array 和 object 还会携带足以定位子值的结构信息。该表示面向计算期遍历，而不是新的表持久化格式。 - -这种组织方式在概念上类似于把 Variant encoded value 与可选的 typed projection 分开。外部格式可参考官方 [Apache Parquet File Format](https://parquet.apache.org/docs/file-format/)、[Parquet Variant Shredding](https://parquet.apache.org/docs/file-format/types/variantshredding/) 和 [Parquet Nested Encoding](https://parquet.apache.org/docs/file-format/nestedencoding/)。Parquet 以 `metadata` 和 `value` 组织 Variant，并可为同质路径增加 `typed_value`。这里仅用于说明组织思路；`ColumnVariantV2` 的内存格式和 Doris 磁盘格式都不是 Parquet。 - -## Typed scalar 物化映射 - -### Decimal 类型 - -Typed Decimal 从 `T` 物化为 `E` 时，`with_typed_scalar()` 保留 unscaled integer 和 scale，并根据 Doris 源类型选择 payload 宽度： - -| Doris 类型 | 从列中读取的 unscaled 值 | 编码调用 | Variant primitive | -| --- | --- | --- | --- | -| `DECIMALV2` | `DecimalV2Value::value()` 返回的 `__int128` | `decimal(value, scale, 16)` | `DECIMAL16` | -| `DECIMAL32` | `Decimal32::value` 中的 `int32_t` | `decimal(value, scale, 4)` | `DECIMAL4` | -| `DECIMAL64` | `Decimal64::value` 中的 `int64_t` | `decimal(value, scale, 8)` | `DECIMAL8` | -| `DECIMAL128I` | `Decimal128V3::value` 中的 `__int128` | `decimal(value, scale, 16)` | `DECIMAL16` | - -Decimal 的 encoded 布局是 `[primitive header][scale][小端有符号 unscaled value]`，所以 `DECIMAL4`、`DECIMAL8`、`DECIMAL16` 的总长度分别是 6、10、18 字节。 - -Decimal 限制与不变量： - -- Variant Decimal 的 scale 必须在 `[0, 38]` 内。`DECIMAL4`、`DECIMAL8`、`DECIMAL16` 的 unscaled 绝对值上限分别是 `10^9 - 1`、`10^18 - 1`、`10^38 - 1`。 -- Doris 源类型自身的限制仍然生效：`DECIMALV2` 最大 precision 为 27、最大 scale 为 9；`DECIMAL32`、`DECIMAL64`、`DECIMAL128I` 的最大 precision 分别为 9、18、38。 -- typed column 的物理 scale 必须与 data type 元数据中的 scale 完全一致；不一致属于不变量破坏，物化时不会自动 rescale。 -- encoded 宽度由源类型决定。数值较小的 `DECIMALV2` 或 `DECIMAL128I` 不会在该路径上缩窄为 `DECIMAL4` 或 `DECIMAL8`。 -- `DECIMAL256` 不是 `ColumnVariantV2` 支持的 typed identity。它的 precision 最高可达 76，超过 Variant Decimal 的 38 位上限。 -- 物理宽度和 scale 不决定 canonical equality。例如 `1.20` 与 `1.2` 可以拥有不同物理编码，但会归一为同一个 canonical 数值。 - -### 日期和时间类型 - -日期和 timestamp 映射会先计算： - -```text -days = daynr(value) - daynr(1970-01-01) -micros = (days * 86400 + hour * 3600 + minute * 60 + second) * 1000000 - + microsecond -``` - -| Doris 类型 | 归一化 payload | 编码调用 | Variant primitive | -| --- | --- | --- | --- | -| `DATE` | `int32_t days` | `date(days)` | `DATE` | -| `DATEV2` | `int32_t days` | `date(days)` | `DATE` | -| `DATETIME` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | -| `DATETIMEV2` | `int64_t micros` | `timestamp_micros(micros, false)` | `TIMESTAMP_NTZ_MICROS` | -| `TIMESTAMPTZ` | `int64_t micros` | `timestamp_micros(micros, true)` | `TIMESTAMP_MICROS` | +| 分组与去重 | 支持的 Variant 值可以基于 logical-value equality 用于 `GROUP BY`、`DISTINCT` 和 `COUNT(DISTINCT ...)`。 | +| 集合运算 | 支持的 Variant 值可以参与 `INTERSECT`、`EXCEPT` 和 `UNION DISTINCT`。 | +| JSON 解析 | 使用 `parse_to_variant` 或 `parse_to_variant_error_to_null` 显式解析 JSON 文本。 | +| 字符串转换 | `CAST(string AS VARIANT)` 创建 Variant 字符串，不会把字符串按 JSON 解析。 | +| 嵌套表达式 | 支持的条件表达式、嵌套容器和支持 Variant 的 `explode` 可以消费 Variant 值。 | +| 非法值 | 格式错误或不受支持的 Variant 值会报错，而不是静默接受。 | -`DATE` 使用 4 字节 payload 加 1 字节 header；每个 timestamp 使用 8 字节 payload 加 1 字节 header。`utc_adjusted` 参数由 primitive ID 表达：`false` 选择无时区（NTZ）primitive，`true` 选择 UTC-adjusted primitive。 +V2 不会开放整个（根）Variant 值的全部运算。直接比较谓词、Join Key、排序和 `MIN`/`MAX` 等能力仍受限制，详见[当前限制](#当前限制)。 -日期和时间限制与不变量： - -- 所有源值都必须通过 Doris 的日期合法性检查。非法的 `DATE`、`DATEV2`、`DATETIME`、`DATETIMEV2` 或 `TIMESTAMPTZ` 会抛出 `INVALID_ARGUMENT`；该路径不会修复，也不会转为 `NULL`。 -- `DATE` 和 `DATEV2` 只保留相对 `1970-01-01` 的有符号天数，不携带日内时间或时区调整信息。 -- `DATETIME` 和 `DATETIMEV2` 编码为 wall-clock、无时区的 timestamp；该路径不会应用 session time zone。只有 `TIMESTAMPTZ` 会选择 UTC-adjusted timestamp primitive。 -- Typed materialization 只生成微秒 primitive。`DATETIMEV2` 和 `TIMESTAMPTZ` 最多保留其支持的 6 位小数。Variant 编码虽然定义了 `TIMESTAMP_NANOS` 和 `TIMESTAMP_NTZ_NANOS`，但该映射不会生成它们。 -- `TIMEV2` 不是支持的 typed identity。Variant 编码虽然定义了 `TIME_NTZ_MICROS`，但这条 typed-to-encoded 路径不会生成它。 - -## Canonical value 语义 - -物理 encoded 字节不会被直接当作 equality identity。Canonicalization 会为哈希和 arena 序列化生成逻辑表示： - -- 等价的整数数值表示会归一为同一个值。 -- Decimal 尾随零不改变数值。 -- `+0`、`-0` 和整数零会归一为同一个值。 -- Object key 顺序不影响相等性，array 元素顺序会影响相等性。 -- Variant/JSON `null` 与 SQL `NULL` 仍然不同。 -- 非法编码或违反内部不变量时会报错，而不是静默接受。 - -这些规则为支持的值提供 `GROUP BY`、`DISTINCT`、`COUNT(DISTINCT ...)`、`INTERSECT`、`EXCEPT` 和 `UNION DISTINCT` 等哈希运算能力。 +例如，在默认路径上不能对根 Variant 值直接分组或去重；开启 V2 后，相同的值可以按 logical equality 分组： ```sql SET enable_variant_v2 = true; --- 1 和 1.0 只有一个 canonical distinct 值。 SELECT COUNT(DISTINCT value) AS distinct_count FROM ( SELECT PARSE_TO_VARIANT('1') AS value @@ -141,7 +51,20 @@ FROM ( SELECT PARSE_TO_VARIANT('1.0') AS value ) AS numeric_values; -- distinct_count: 1 +``` +## Equality 语义 + +分组、去重和集合运算比较的是 logical value，而不是来源 SQL 类型或表示形式： + +- 等价的整数数值表示相等。 +- Decimal 尾随零不改变值，因此 `1.20` 与 `1.2` 相等。 +- `+0`、`-0` 和整数零相等。 +- Object key 顺序不影响相等性。 +- Array 元素顺序影响相等性。 +- Variant/JSON `null` 与 SQL `NULL` 不同。 + +```sql -- Object key 顺序不影响 equality。 SELECT COUNT(DISTINCT value) AS distinct_count FROM ( @@ -161,16 +84,18 @@ FROM ( -- distinct_count: 2 ``` -Canonical equality 不会开放根 Variant 比较谓词。直接执行 `VARIANT = VARIANT`、null-safe equality 或排序比较仍然不受支持。 +这些 Equality 规则只用于支持的哈希和集合运算，不会开放直接 `VARIANT = VARIANT`、null-safe equality 或排序比较。 -## CAST 与解析行为 +## CAST 与 JSON 解析 -V2 明确区分“创建 typed Variant 字符串”和“解析 JSON 文本”： +V2 明确区分 Variant 字符串和已经解析的 JSON 值： -- `CAST(string AS VARIANT)` 创建 typed Variant 字符串，不会按 JSON 解析或校验该字符串。 -- `parse_to_variant(json_string)` 将 JSON 文本严格解析为 Variant 值。 -- `parse_to_variant_error_to_null(json_string)` 在解析或校验失败时返回 SQL `NULL`。 -- 提取出的 Variant 值在进行确定类型的比较、过滤、排序或算术运算前，应先 CAST 到具体 SQL 类型。 +| 操作 | 结果 | 非法 JSON 行为 | +| --- | --- | --- | +| `CAST(string AS VARIANT)` | 保留原始文本的 Variant 字符串 | 不执行 JSON 校验 | +| `parse_to_variant(string)` | 从 JSON 文本解析出的 Variant 值 | 返回错误 | +| `parse_to_variant_error_to_null(string)` | 从 JSON 文本解析出的 Variant 值 | 返回 SQL `NULL` | +| `CAST(variant_value AS T)` | 类型为 `T` 的具体 SQL 值 | 遵循目标类型的 CAST 规则 | 完整函数语法请参见 [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) 和 [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null)。 @@ -178,7 +103,7 @@ V2 明确区分“创建 typed Variant 字符串”和“解析 JSON 文本” SET enable_variant_v2 = true; SELECT PARSE_TO_VARIANT('{\"id\": 1}') AS parsed_object, - CAST('{\"id\": 1}' AS VARIANT) AS typed_string; + CAST('{\"id\": 1}' AS VARIANT) AS variant_string; SELECT ELEMENT_AT(CAST('{\"id\": 1}' AS VARIANT), 'id') AS from_string, ELEMENT_AT(PARSE_TO_VARIANT('{\"id\": 1}'), 'id') AS from_json; @@ -186,20 +111,66 @@ SELECT ELEMENT_AT(CAST('{\"id\": 1}' AS VARIANT), 'id') AS from_string, SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value; -- invalid_value: NULL + +SELECT CAST(PARSE_TO_VARIANT('42') AS BIGINT) AS id; +-- id: 42 ``` -当函数签名支持 Variant 值时，条件表达式、嵌套容器以及支持 Variant 的 `explode` 可以直接消费 V2 表示。 +不要使用 `CAST(string AS VARIANT)` 校验 JSON。提取出的 Variant 值在进行确定类型的比较、过滤、排序或算术运算前，应先 CAST 到具体 SQL 类型。 + +## Decimal 语义与限制 + +V2 会精确保留受支持的 Decimal 值及其 scale；Equality 语义会忽略不影响数值的尾随零。 + +| Doris 输入类型 | 支持范围 | V2 行为 | +| --- | --- | --- | +| 旧版 `DECIMALV2` | Precision 最大 27；scale 最大 9 | 精确保留为 Variant Decimal | +| `DECIMAL(p, s)` | `1 <= p <= 38`，`0 <= s <= p` | 精确保留为 Variant Decimal | +| `DECIMAL(p, s)` | `39 <= p <= 76` | V2 不支持；请先 CAST 到 precision 38 或更低 | + +当前 Decimal 限制： + +- Variant Decimal 支持的 precision 和 scale 上限均为 38。 +- 源值必须同时满足 Doris Decimal 类型和 Variant precision 上限。 +- `1.20` 与 `1.2` 等值会保留 Decimal 数值，并在支持的分组和集合运算中视为同一个 logical value。 + +## 日期和时间语义与限制 + +| Doris 输入类型 | Variant 语义 | 精度与时区行为 | +| --- | --- | --- | +| `DATE` | 日历日期 | 天级精度；不包含时间和时区 | +| 旧版 `DATETIME` | 无时区 timestamp | 秒级精度；不进行 session time zone 调整 | +| `DATETIME(p)` | 无时区 timestamp | `0 <= p <= 6`；不进行 session time zone 调整 | +| `TIMESTAMPTZ(p)` | 带时区调整语义的 timestamp | `0 <= p <= 6` | +| `TIME` | — | V2 不支持 | + +当前日期和时间限制： + +- 所有源值都必须是合法的 Doris 日期或 timestamp。非法值返回 `INVALID_ARGUMENT`，不会被修复或转换为 `NULL`。 +- `DATETIME` 和 `DATETIMEV2` 保持 wall-clock、无时区语义；转换过程不会应用 session time zone。 +- `TIMESTAMPTZ` 是当前支持并保留时区调整 timestamp 语义的输入类型。 +- V2 日期和时间转换支持微秒精度，不支持纳秒精度。 + +## NULL 语义 + +SQL `NULL` 与 Variant/JSON `null` 是不同的值： + +- SQL `NULL` 表示 SQL 值缺失，并遵循普通 SQL NULL 传播规则。 +- Variant/JSON `null` 是一个 Variant 值，例如由 `parse_to_variant('null')` 产生。 +- `parse_to_variant_error_to_null` 遇到非法输入时返回 SQL `NULL`；这与成功解析 JSON literal `null` 不同。 -## 使用边界与限制 +## 当前限制 -- 不支持根 Variant 比较谓词（`=`、`!=`、`<=>` 和排序比较）；应在两侧提取可比较的路径并 CAST。 +- 不支持根 Variant 比较谓词（`=`、`!=`、`<=>`、`<`、`<=`、`>` 和 `>=`）；应在两侧提取可比较的路径并 CAST。 - 不支持把 Variant 表达式作为 Join Key。 -- 根 Variant 不能作为 Sort/TopN 键、窗口分区键或排序键，也不能作为 `MIN` 和 `MAX` 的参数。 -- 功能选择只作用于 session，不会改变原生 Variant 存储、导入、统计信息或 Compaction。 -- V2 回归测试仍标记为 `nonConcurrent`。 -- 仅在验证目标 workload 后开启 V2；该执行路径仍是实验特性。 +- 根 Variant 不能作为 Sort/TopN Key、窗口分区键或窗口排序键。 +- 根 Variant 不能作为 `MIN` 或 `MAX` 的参数。 +- Precision 大于 38 的 Decimal 值和 `TIME` 值不支持作为 V2 输入。 +- 功能选择只作用于 session，不会改变已经持久化的 Variant 数据。 +- 仅在验证目标 workload 后开启 V2；该功能仍是实验特性。 ```sql +-- 比较具体子路径，而不是根 Variant 值。 SELECT * FROM tbl WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); diff --git a/sidebars.ts b/sidebars.ts index d3bec64eb5c9a..32078d869d3bc 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -1267,7 +1267,7 @@ const sidebars: SidebarsConfig = { { type: 'doc', id: 'sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics', - label: 'V2 Compute Semantics', + label: 'V2 Behavior and Semantics', }, ], }, From 05552018c09813db6fea82e2596a5a4ee417cf86 Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Tue, 21 Jul 2026 14:03:16 +0800 Subject: [PATCH 9/9] docs: address VARIANT review feedback --- .../sql-data-types/semi-structured/VARIANT.md | 185 +++++++++++------- .../variant-v2-compute-semantics.md | 177 ----------------- .../variant-functions/element-at.md | 5 +- .../parse-to-variant-error-to-null.md | 87 ++++---- .../variant-functions/parse-to-variant.md | 106 ++++++---- .../sql-data-types/semi-structured/VARIANT.md | 181 ++++++++++------- .../variant-v2-compute-semantics.md | 177 ----------------- .../variant-functions/element-at.md | 9 +- .../parse-to-variant-error-to-null.md | 85 ++++---- .../variant-functions/parse-to-variant.md | 104 ++++++---- sidebars.ts | 7 +- 11 files changed, 459 insertions(+), 664 deletions(-) delete mode 100644 docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md delete mode 100644 i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md diff --git a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 5b7aba2e7bbbe..47f36be65313c 100644 --- a/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/docs/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -2,7 +2,7 @@ { "title": "VARIANT", "language": "en-US", - "description": "The VARIANT type stores semi-structured JSON data. It can contain different primitive types (integers, strings, booleans, etc.), one-dimensional arrays, and nested objects. On write, Doris infers the structure and type of sub-paths based on JSON paths and performs Subcolumnization on frequent paths, exposing them as independent columnar subcolumns for both flexibility and performance." + "description": "VARIANT stores semi-structured JSON data and supports typed access, casts, canonical equality, grouping, hashing, and selected SQL operations." } --- @@ -69,78 +69,125 @@ WHERE ARRAY_CONTAINS(CAST(v['tags'] AS ARRAY), 'Doris'); ## Create and access values -VARIANT values can be created from JSON text or from typed SQL expressions: +:::info Version availability +The behavior in this section applies to Doris 4.2 and later. +::: + +VARIANT values can be created from JSON text, JSON/JSONB values, or typed SQL expressions: -- Use [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) when a string contains JSON text that should be parsed. -- Use `CAST(expression AS VARIANT)` when the SQL value should become a typed Variant value. A string cast does not parse the string as JSON. +- Use [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) when a string or JSON/JSONB expression should be parsed as a structured VARIANT value. +- Use `CAST(expression AS VARIANT)` to convert a supported SQL value to VARIANT. String parsing for this CAST is controlled by `enable_variant_string_cast_parse`, as described in [CAST rules](#cast-rules). ### Parse JSON text ```sql -SELECT PARSE_TO_VARIANT('{\"user\": {\"id\": 42}, \"active\": true}'); +SELECT PARSE_TO_VARIANT('{"user": {"id": 42}, "active": true}'); SELECT PARSE_TO_VARIANT('[10, 20, 30]'); +SELECT PARSE_TO_VARIANT(CAST('{"user": {"id": 42}}' AS JSON)); ``` Use [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null) when invalid JSON should return SQL `NULL` instead of failing the query. ### Access objects and arrays -Object fields can be accessed with a string key. With VARIANT V2 enabled, VARIANT array indexes are zero-based for non-negative indexes and support negative indexes from the end. The extracted value remains `VARIANT`; CAST it before typed comparison or aggregation. +Object fields can be accessed with a string key. In Doris 4.2 and later, VARIANT array indexes are zero-based for non-negative indexes and support negative indexes from the end. The extracted value remains `VARIANT`; CAST it before typed comparison, arithmetic, or aggregation. ```sql -SET enable_variant_v2 = true; - -SELECT CAST(PARSE_TO_VARIANT('{\"user\": {\"id\": 42}}')['user']['id'] AS BIGINT); +SELECT CAST(PARSE_TO_VARIANT('{"user": {"id": 42}}')['user']['id'] AS BIGINT); SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), 0); -- 10 SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1); -- 30 ``` See [ELEMENT_AT](../../../sql-functions/scalar-functions/variant-functions/element-at) for object and array access details. -## CAST semantics -`CAST` preserves the distinction between a SQL string and JSON text: +## CAST rules + +CAST involving VARIANT has two directions: converting a supported SQL value to VARIANT and extracting a compatible SQL value from VARIANT. -- `CAST(string AS VARIANT)` creates a Variant whose root value is a typed Variant string. It does **not** parse JSON. Therefore, a string such as `'{"id": 1}'` remains one string value, and invalid JSON text is still a valid input to this cast. -- `PARSE_TO_VARIANT(string)` parses the string as JSON. A JSON object or array can then be accessed by path or `ELEMENT_AT`; use `PARSE_TO_VARIANT_ERROR_TO_NULL` if invalid JSON should become SQL `NULL`. -- `CAST(PARSE_TO_VARIANT(...) AS scalar)` converts a parsed Variant value to a concrete SQL type when the value is compatible. The cast is not a JSON parser and can fail for incompatible shapes or ranges. -- `CAST(typed_expression AS VARIANT)` converts a supported typed SQL value to Variant. With `enable_variant_v2`, this only changes the execution representation for the current session; it does not change on-disk Variant storage, readers, writers, or compaction. +### CAST other types to VARIANT + +| Source type | Behavior | +| --- | --- | +| `CHAR`, `VARCHAR`, `STRING` | In query sessions, parses the input as JSON by default. Set `enable_variant_string_cast_parse` to `false` to preserve the input as a VARIANT string instead. | +| `JSON` / `JSONB` | Converts the structured JSON value directly to VARIANT. | +| Boolean, integer, floating-point, and Decimal types | Preserves the typed scalar value, subject to the Decimal limits below. | +| `DATE`, `DATETIME`, `TIMESTAMPTZ`, `IPV4`, `IPV6` | Preserves the typed logical value. | +| `ARRAY` | Converts each supported element recursively and preserves SQL NULL elements. | +| `MAP` | Not supported. | + +`enable_variant_string_cast_parse` is `true` by default in query sessions. Use `PARSE_TO_VARIANT` when the SQL text should always make the JSON-parsing intent explicit, independent of the session setting. ```sql -SET enable_variant_v2 = true; +-- Default behavior: parse the string as JSON. +SELECT CAST('{"id": 1}' AS VARIANT) AS parsed_object; +-- {"id":1} + +-- Preserve the same input as a VARIANT string root. +SET enable_variant_string_cast_parse = false; +SELECT CAST(CAST('{"id": 1}' AS VARIANT) AS STRING) AS string_value, + VARIANT_TYPE(CAST('{"id": 1}' AS VARIANT)) AS root_type; +-- string_value: {"id": 1}; root_type: string + +-- JSON/JSONB input remains structured. +SET enable_variant_string_cast_parse = true; +SELECT CAST(CAST('{"id": 1}' AS JSON) AS VARIANT) AS parsed_object; +-- {"id":1} +``` + +When string parsing is enabled, invalid JSON causes the CAST to fail. Use `PARSE_TO_VARIANT_ERROR_TO_NULL` if malformed input should become SQL `NULL`. -SELECT CAST('{"id": 1}' AS VARIANT) AS typed_string, - PARSE_TO_VARIANT('{"id": 1}') AS parsed_object; --- typed_string: the literal string {"id": 1} --- parsed_object: {"id": 1} +### CAST VARIANT to other types -SELECT ELEMENT_AT(CAST('{"id": 1}' AS VARIANT), 'id') AS from_string, - ELEMENT_AT(PARSE_TO_VARIANT('{"id": 1}'), 'id') AS from_json; --- from_string: NULL; from_json: 1 +VARIANT can be cast to a compatible scalar, JSON/JSONB, or array target: +| Target type | Behavior | +| --- | --- | +| Boolean, integer, floating-point, Decimal, date/time, and IP types | Converts compatible scalar roots; incompatible shapes, invalid text, or out-of-range values return an error or SQL `NULL` according to the applicable CAST mode. | +| `STRING` | Returns scalar text for scalar roots and JSON text for objects and arrays. Variant/JSON `null` becomes the string `null`; outer SQL `NULL` remains SQL `NULL`. | +| `JSON` / `JSONB` | Converts a representable VARIANT value to structured JSON/JSONB. | +| `ARRAY` | Converts a VARIANT array element by element to `T`; incompatible elements follow the target CAST rules. | +| `MAP` | Not supported. | + +```sql SELECT CAST(PARSE_TO_VARIANT('42') AS BIGINT) AS id; --- id: 42 +-- 42 -SELECT CAST('{invalid json' AS VARIANT) AS still_a_string; --- succeeds because CAST does not parse the input as JSON +SELECT CAST(PARSE_TO_VARIANT('[1, null, 3]') AS ARRAY) AS values; +-- [1, NULL, 3] + +SELECT CAST(PARSE_TO_VARIANT('{"id": 1}') AS JSON) AS json_value; +-- {"id":1} ``` -Do not use `CAST(string AS VARIANT)` as a JSON validation step. Use `PARSE_TO_VARIANT` for strict parsing, or `PARSE_TO_VARIANT_ERROR_TO_NULL` when malformed JSON should be converted to SQL `NULL`. +### Decimal and date/time conversion limits + +| Doris input type | Supported VARIANT behavior | +| --- | --- | +| Legacy `DECIMALV2` | Precision up to 27 and scale up to 9 are preserved exactly. | +| `DECIMAL(p, s)` | `1 <= p <= 38` and `0 <= s <= p` are preserved exactly. Values that require precision greater than 38 are not supported. | +| `DATE` | Preserved as a calendar date with no time or time zone. | +| Legacy `DATETIME` | Preserved with whole-second precision and no time-zone adjustment. | +| `DATETIME(p)` | Supports `0 <= p <= 6` with no time-zone adjustment. | +| `TIMESTAMPTZ(p)` | Supports `0 <= p <= 6` with time-zone-adjusted timestamp semantics. | +| `TIME` | Not supported as input to VARIANT. | -## Equality semantics +Every source value must also be valid for its Doris source type. Unsupported precision, invalid dates, or incompatible values return an error instead of being repaired. -With VARIANT V2 enabled, supported grouping, deduplication, and set operations compare logical values rather than source SQL types or representations: +## Equality and hash semantics + +In Doris 4.2 and later, grouping, deduplication, and set operations compare logical values rather than source SQL types or physical representations: - Equivalent integral numeric representations compare as equal. -- Decimal trailing zeros do not change the value. +- Decimal trailing zeros do not change the value, so `1.20` and `1.2` compare as equal. - `+0`, `-0`, and integral zero compare as equal. - Object key order does not affect equality, while array element order does. - Variant/JSON `null` is distinct from SQL `NULL`. -These rules apply to supported operations such as `GROUP BY`, `DISTINCT`, `COUNT(DISTINCT ...)`, `INTERSECT`, `EXCEPT`, and `UNION DISTINCT`. They do not enable root Variant comparison predicates: direct `VARIANT = VARIANT` and ordering comparisons remain unsupported. +Hash-based operators calculate their keys from the same canonical logical representation. Values that compare as equal under the rules above therefore produce the same internal hash key, regardless of whether they came from parsed JSON or a typed CAST. This hash is an implementation detail, not a stable user-facing checksum. -```sql -SET enable_variant_v2 = true; +These rules apply to supported operations such as `GROUP BY`, `DISTINCT`, `COUNT(DISTINCT ...)`, `INTERSECT`, `EXCEPT`, and `UNION DISTINCT`. They do not enable root VARIANT comparison predicates: direct `VARIANT = VARIANT` and ordering comparisons remain unsupported. +```sql -- 1 and 1.0 have one distinct logical value. SELECT COUNT(DISTINCT value) AS distinct_count FROM ( @@ -153,9 +200,9 @@ FROM ( -- Object key order is ignored; array order is preserved. SELECT COUNT(DISTINCT value) AS distinct_count FROM ( - SELECT PARSE_TO_VARIANT('{\"a\": 1, \"b\": 2}') AS value + SELECT PARSE_TO_VARIANT('{"a": 1, "b": 2}') AS value UNION ALL - SELECT PARSE_TO_VARIANT('{\"b\": 2, \"a\": 1}') AS value + SELECT PARSE_TO_VARIANT('{"b": 2, "a": 1}') AS value ) AS object_values; -- distinct_count: 1 @@ -168,7 +215,13 @@ FROM ( -- distinct_count: 2 ``` -For more examples, supported operations, data type behavior, and current limits, see [VARIANT V2 Behavior and Semantics](./variant-v2-compute-semantics). +## NULL semantics + +SQL `NULL` and Variant/JSON `null` are different values: + +- SQL `NULL` represents the absence of a SQL value and follows normal SQL NULL propagation. +- Variant/JSON `null` is a VARIANT value, for example the result of `PARSE_TO_VARIANT('null')`. +- `PARSE_TO_VARIANT_ERROR_TO_NULL` returns SQL `NULL` for malformed input. This differs from successfully parsing the JSON literal `null`. ## Primitive types @@ -490,12 +543,26 @@ SELECT v FROM variant_tbl WHERE k = 2; Sorting applies at every level — top-level keys become `a`, `b`, `c`, and the nested object's keys become `x`, `y`. -## Supported operations and CAST rules +## Supported operations + +In Doris 4.2 and later, whole VARIANT values support hash-based grouping and deduplication, but they do not support comparison, join-key, or ordering semantics. -The default execution path preserves the existing VARIANT guidance: +| Operation on a whole VARIANT value | Support | Notes | +| --- | --- | --- | +| `GROUP BY` | Supported | Uses the logical equality and canonical hash rules described above. | +| `DISTINCT`, `COUNT(DISTINCT ...)` | Supported | Logically equivalent values are deduplicated. | +| `INTERSECT`, `EXCEPT`, `UNION DISTINCT` | Supported | Uses the same logical equality and hash semantics. | +| `COUNT(*)`, `COUNT(variant)` | Supported | `COUNT(variant)` excludes outer SQL `NULL` in the normal SQL manner. | +| `IF`, `CASE`, `IFNULL`, `COALESCE` | Supported | Conditional expressions can return and consume VARIANT values. | +| VARIANT in transient `ARRAY`, `MAP`, or `STRUCT` expressions | Supported | This does not allow these nested types in persisted table schemas. | +| `EXPLODE_VARIANT_ARRAY`, `EXPLODE`/`EXPLODE_OUTER` on `ARRAY` | Supported | Emits VARIANT elements and preserves SQL NULL versus Variant/JSON `null`. | +| `=`, `!=`, `<=>`, `<`, `<=`, `>`, `>=` | Not supported | Extract and CAST a comparable subpath on both sides. | +| Join key | Not supported | CAST the required subpath to the same concrete type on both inputs. | +| `ORDER BY`, Sort/TopN key | Not supported | CAST the required subpath before ordering. | +| Window partition/order key | Not supported | A whole VARIANT value cannot be a window key. | +| `MIN(variant)`, `MAX(variant)` | Not supported | CAST a scalar subpath before aggregation. | -- VARIANT cannot be compared or operated on directly with other types; comparisons between two VARIANT values are not supported on the default path. -- For comparison, filtering, aggregation, and ordering, CAST subpaths to concrete types (explicitly or implicitly). +For comparison, filtering, arithmetic, and ordering, extract the required subpath and CAST it to a concrete type explicitly or implicitly: ```sql -- Explicit CAST @@ -508,40 +575,6 @@ SELECT * FROM tbl WHERE v['bool']; SELECT * FROM tbl WHERE v['str'] MATCH 'Doris'; ``` -- On the default path, a whole VARIANT value should not be used directly in `ORDER BY`, `GROUP BY`, as a `JOIN KEY`, or as an aggregate argument; CAST the required subpath instead. -- Strings can be implicitly converted to VARIANT. - -The [experimental V2 behavior](./variant-v2-compute-semantics) adds logical-value semantics for grouping and set-operation use cases. It does not make root VARIANT comparison predicates or Variant join keys available. - -| VARIANT | Castable | Coercible | -| --------------- | -------- | --------- | -| `ARRAY` | ✔ | ❌ | -| `BOOLEAN` | ✔ | ✔ | -| `DATE/DATETIME` | ✔ | ✔ | -| `FLOAT` | ✔ | ✔ | -| `IPV4/IPV6` | ✔ | ✔ | -| `DECIMAL` | ✔ | ✔ | -| `MAP` | ❌ | ❌ | -| `TIMESTAMP` | ✔ | ✔ | -| `VARCHAR` | ✔ | ✔ | -| `JSON` | ✔ | ✔ | - -## Experimental VARIANT V2 behavior - -:::caution Experimental feature -VARIANT V2 is disabled by default and affects only execution behavior in the current session. Existing data and the persisted Variant format remain unchanged. -::: - -Enable it for the current session with the session variable `enable_variant_v2`: - -```sql -SET enable_variant_v2 = true; -``` - -When enabled, V2 supports logical-value grouping, deduplication, and set operations for supported Variant values, and makes JSON parsing explicit. Root Variant comparison predicates, Variant join keys, Sort/TopN keys, and `MIN`/`MAX` arguments remain unsupported. - -For behavior changes, equality, CAST and parsing, Decimal and date/time semantics, examples, and complete limits, see [VARIANT V2 Behavior and Semantics](./variant-v2-compute-semantics). - ## Wide columns When ingested data contains many distinct JSON keys, the number of subcolumns produced by Subcolumnization can grow rapidly; at scale this may cause metadata bloat, higher write/merge cost, and query slowdowns. To address “wide columns” (too many subcolumns), VARIANT provides two mechanisms: **Sparse columns** and **DOC encoding**. @@ -587,7 +620,7 @@ See the “Configuration” section below for the full property list. - **Wide tables optimization**: For wide tables with a large number of dynamic sub-columns (e.g., more than 2000 columns) generated by the `VARIANT` type, it is highly recommended to enable **Storage Format V3** by specifying `"storage_format" = "V3"` in the table `PROPERTIES`. This decouples column metadata from the Segment Footer, speeding up file opening and reducing memory overhead. - JSON key length ≤ 255. - Cannot be a primary key or sort key. -- Cannot be nested within other types (e.g., `Array`, `Struct`). +- Persisted table schemas cannot nest VARIANT within other types (for example, `ARRAY` or `STRUCT`). Transient expression results can use the supported nested-container operations listed above. - Outside DOC mode, reading the entire VARIANT column scans all subpaths. For very wide columns, direct `SELECT variant_col` is generally not recommended unless DOC mode is enabled. If a column has many subpaths, consider storing the original JSON string in an extra STRING/JSONB column for whole-object searches like `LIKE`: ```sql @@ -682,7 +715,7 @@ SET describe_extend_variant_column = true; DESC variant_tbl; ``` -``` sql +```sql DESCRIBE ${table_name} PARTITION ($partition_name); ``` diff --git a/docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md b/docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md deleted file mode 100644 index e2a6b11ed7c95..0000000000000 --- a/docs/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md +++ /dev/null @@ -1,177 +0,0 @@ ---- -{ - "title": "VARIANT V2 Behavior and Semantics", - "language": "en-US", - "description": "User-facing reference for experimental VARIANT V2 behavior, including equality, CAST, JSON parsing, Decimal and date/time semantics, examples, and limits." -} ---- - -## Overview - -Experimental VARIANT V2 extends the operations that can consume `VARIANT` values and defines consistent logical-value semantics for grouping, deduplication, and set operations. - -Use the [VARIANT type reference](./VARIANT) for SQL syntax, indexes, storage configuration, and general type rules. Use the [VARIANT Workload Guide](./variant-workload-guide) when choosing between default storage behavior, sparse columns, DOC mode, and Schema Template. This page describes only the behavior users can observe after enabling V2. - -:::caution Experimental feature -V2 is disabled by default and applies only to the current session. Existing data and the persisted Variant format remain unchanged. Enable it only after validating the target workload. -::: - -## Enable V2 - -Set the session variable before running the target statements: - -```sql -SET enable_variant_v2 = true; -``` - -The setting affects only the current session. Existing persisted `VARIANT` data remains unchanged. - -## Behavior after enabling V2 - -| Area | V2 behavior | -| --- | --- | -| Grouping and deduplication | Supported Variant values can be used by `GROUP BY`, `DISTINCT`, and `COUNT(DISTINCT ...)` with logical-value equality. | -| Set operations | Supported Variant values can participate in `INTERSECT`, `EXCEPT`, and `UNION DISTINCT`. | -| JSON parsing | JSON text is parsed explicitly with `parse_to_variant` or `parse_to_variant_error_to_null`. | -| String conversion | `CAST(string AS VARIANT)` creates a Variant string and does not parse the string as JSON. | -| Nested expressions | Supported conditional expressions, nested containers, and Variant-aware `explode` can consume Variant values. | -| Invalid values | Malformed or unsupported Variant values return an error instead of being silently accepted. | - -V2 does not enable every operation on a whole (root) Variant value. In particular, direct comparison predicates, join keys, sorting, and `MIN`/`MAX` remain restricted as described in [Current limitations](#current-limitations). - -For example, grouping or deduplicating a root Variant value is not supported on the default path. After V2 is enabled, the same values can be grouped by logical equality: - -```sql -SET enable_variant_v2 = true; - -SELECT COUNT(DISTINCT value) AS distinct_count -FROM ( - SELECT PARSE_TO_VARIANT('1') AS value - UNION ALL - SELECT PARSE_TO_VARIANT('1.0') AS value -) AS numeric_values; --- distinct_count: 1 -``` - -## Equality semantics - -Grouping, deduplication, and set operations compare the logical value rather than its source SQL type or representation: - -- Equivalent integral numeric representations are equal. -- Decimal trailing zeros do not change the value, so `1.20` and `1.2` are equal. -- `+0`, `-0`, and integral zero are equal. -- Object key order does not affect equality. -- Array element order affects equality. -- Variant/JSON `null` is distinct from SQL `NULL`. - -```sql --- Object key order is ignored. -SELECT COUNT(DISTINCT value) AS distinct_count -FROM ( - SELECT PARSE_TO_VARIANT('{\"a\": 1, \"b\": 2}') AS value - UNION ALL - SELECT PARSE_TO_VARIANT('{\"b\": 2, \"a\": 1}') AS value -) AS object_values; --- distinct_count: 1 - --- Array order is preserved. -SELECT COUNT(DISTINCT value) AS distinct_count -FROM ( - SELECT PARSE_TO_VARIANT('[1, 2]') AS value - UNION ALL - SELECT PARSE_TO_VARIANT('[2, 1]') AS value -) AS array_values; --- distinct_count: 2 -``` - -These equality rules apply to supported hash-based and set operations. They do not enable direct `VARIANT = VARIANT`, null-safe equality, or ordering comparisons. - -## CAST and JSON parsing - -V2 distinguishes a Variant string from a parsed JSON value: - -| Operation | Result | Invalid JSON behavior | -| --- | --- | --- | -| `CAST(string AS VARIANT)` | A Variant string containing the original text | No JSON validation is performed | -| `parse_to_variant(string)` | A Variant value parsed from JSON text | Returns an error | -| `parse_to_variant_error_to_null(string)` | A Variant value parsed from JSON text | Returns SQL `NULL` | -| `CAST(variant_value AS T)` | A concrete SQL value of type `T` | Follows the target type's cast rules | - -See [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) and [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null) for full function syntax. - -```sql -SET enable_variant_v2 = true; - -SELECT PARSE_TO_VARIANT('{\"id\": 1}') AS parsed_object, - CAST('{\"id\": 1}' AS VARIANT) AS variant_string; - -SELECT ELEMENT_AT(CAST('{\"id\": 1}' AS VARIANT), 'id') AS from_string, - ELEMENT_AT(PARSE_TO_VARIANT('{\"id\": 1}'), 'id') AS from_json; --- from_string: NULL; from_json: 1 - -SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value; --- invalid_value: NULL - -SELECT CAST(PARSE_TO_VARIANT('42') AS BIGINT) AS id; --- id: 42 -``` - -Do not use `CAST(string AS VARIANT)` to validate JSON. Extracted Variant values should be cast to a concrete SQL type before typed comparison, filtering, ordering, or arithmetic. - -## Decimal semantics and limits - -V2 preserves supported Decimal values exactly, including their scale. Equality semantics then ignore insignificant trailing zeros. - -| Doris input type | Supported range | V2 behavior | -| --- | --- | --- | -| Legacy `DECIMALV2` | Precision up to 27; scale up to 9 | Preserved as an exact Variant Decimal | -| `DECIMAL(p, s)` | `1 <= p <= 38`, `0 <= s <= p` | Preserved as an exact Variant Decimal | -| `DECIMAL(p, s)` | `39 <= p <= 76` | Not supported by V2; cast to precision 38 or lower first | - -Current Decimal limits: - -- Variant Decimal supports precision and scale up to 38. -- The source value must fit both its Doris Decimal type and the Variant precision limit. -- Values such as `1.20` and `1.2` retain their Decimal value and compare as one logical value in supported grouping and set operations. - -## Date and time semantics and limits - -| Doris input type | Variant semantics | Precision and time-zone behavior | -| --- | --- | --- | -| `DATE` | Calendar date | Day precision; no time or time zone | -| Legacy `DATETIME` | Timestamp without time zone | Whole-second precision; no session time-zone adjustment | -| `DATETIME(p)` | Timestamp without time zone | `0 <= p <= 6`; no session time-zone adjustment | -| `TIMESTAMPTZ(p)` | Time-zone-adjusted timestamp | `0 <= p <= 6` | -| `TIME` | — | Not supported by V2 | - -Current date and time limits: - -- Every source value must be a valid Doris date or timestamp. Invalid values return `INVALID_ARGUMENT`; they are not repaired or converted to `NULL`. -- `DATETIME` and `DATETIMEV2` keep wall-clock, no-time-zone semantics. This conversion does not apply the session time zone. -- `TIMESTAMPTZ` is the supported input that keeps time-zone-adjusted timestamp semantics. -- V2 date and time conversion supports microsecond precision, not nanosecond precision. - -## NULL semantics - -SQL `NULL` and Variant/JSON `null` are different values: - -- SQL `NULL` represents the absence of a SQL value and participates in normal SQL null propagation. -- Variant/JSON `null` is a Variant value produced, for example, by `parse_to_variant('null')`. -- `parse_to_variant_error_to_null` returns SQL `NULL` for malformed input; this is different from successfully parsing the JSON literal `null`. - -## Current limitations - -- Root Variant comparison predicates (`=`, `!=`, `<=>`, `<`, `<=`, `>`, and `>=`) are not supported. Extract and cast a comparable path on both sides. -- Variant expressions are not supported as join keys. -- Root Variant values are not supported as Sort/TopN keys or as window partition or order keys. -- Root Variant values are not supported as arguments to `MIN` or `MAX`. -- Decimal values with precision greater than 38 and `TIME` values are not supported V2 inputs. -- Feature selection is session-scoped and does not change persisted Variant data. -- Enable V2 only after validating the target workload; the feature remains experimental. - -```sql --- Compare concrete subpaths rather than root Variant values. -SELECT * -FROM tbl -WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); -``` diff --git a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md index 12b7645a59d39..3c2e688f5d888 100644 --- a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md +++ b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md @@ -29,7 +29,7 @@ ELEMENT_AT(container, key_or_index) - For `MAP`: The key type (`K`) of the `MAP`, which can be any supported primitive type. - For `STRUCT`: A constant integer field position (starting from **1**) or a constant string field name (matched **case-insensitively**). - For `VARIANT` object access: A string key. - - For a VARIANT array on `ColumnVariantV2`: An integer index; non-negative indexes are 0-based and negative indexes count backward from the end. + - For a VARIANT array in Doris 4.2 and later: An integer index; non-negative indexes are 0-based and negative indexes count backward from the end. ## Return Value @@ -43,12 +43,11 @@ ELEMENT_AT(container, key_or_index) ## Notes 1. **ARRAY indexes start from 1**, not 0. -2. When `enable_variant_v2` is enabled, non-negative indexes into a VARIANT array start from `0`; `0` is the first element and `-1` is the last. +2. In Doris 4.2 and later, non-negative indexes into a VARIANT array start from `0`; `0` is the first element and `-1` is the last. 3. Negative indexes are supported for ARRAY and VARIANT array access: `-1` represents the last element, `-2` the second-to-last, and so on. 4. The `ELEMENT_AT(container, key_or_index)` function behaves the same as `container[key_or_index]` (see examples for details). ```sql -SET enable_variant_v2 = true; SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 0); -- 10 SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 1); -- 20 SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), -1); -- 30 diff --git a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md index 5547cd201337c..2ec019310e406 100644 --- a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md +++ b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md @@ -1,71 +1,90 @@ --- { - "title": "PARSE_TO_VARIANT_ERROR_TO_NULL", + "title": "PARSE_TO_VARIANT_ERROR_TO_NULL (Return NULL on Error)", "language": "en", - "description": "Parses JSON text into a VARIANT value and returns SQL NULL for invalid input." + "description": "Parses one complete JSON value into VARIANT and returns SQL NULL when parsing or validation fails." } --- -## Function +## Description -`PARSE_TO_VARIANT_ERROR_TO_NULL` parses JSON text into `VARIANT`. Invalid JSON returns SQL `NULL` instead of failing the query. +`PARSE_TO_VARIANT_ERROR_TO_NULL` parses one complete JSON value into `VARIANT`. The `ERROR_TO_NULL` suffix means that a parsing or validation error returns SQL `NULL` instead of failing the query. This function is available in Doris 4.2 and later. ## Syntax ```sql -PARSE_TO_VARIANT_ERROR_TO_NULL(json_text) +PARSE_TO_VARIANT_ERROR_TO_NULL() ``` ## Parameters -- `json_text`: A `VARCHAR` expression containing one complete JSON value. +| Parameter | Description | +| --- | --- | +| `` | A `CHAR`, `VARCHAR`, or `STRING` expression containing one complete JSON value, or a `JSON`/`JSONB` expression. JSON/JSONB input is converted to JSON text and then parsed as VARIANT. | ## Return Value -- Returns a nullable `VARIANT` value. +Returns a nullable `VARIANT` value. + +- Valid input returns the parsed VARIANT value. +- Invalid JSON or another parsing or validation error returns SQL `NULL`. - SQL `NULL` input returns SQL `NULL`. -- Invalid JSON returns SQL `NULL`. -- Valid JSON `null` remains Variant/JSON `null`, distinct from SQL `NULL`. +- The valid JSON literal `null` returns Variant/JSON `null`, not SQL `NULL`. -## Experimental behavior +## Example -`ColumnVariantV2` is an experimental, compute-only execution path. It is disabled by default and selected for the current FE session with: +Keep valid values and convert invalid JSON to SQL `NULL`: ```sql -SET enable_variant_v2 = true; +SELECT CAST( + PARSE_TO_VARIANT_ERROR_TO_NULL('{"id": 1}') + AS STRING + ) AS valid_value, + PARSE_TO_VARIANT_ERROR_TO_NULL('{"id":') IS NULL AS invalid_is_null, + PARSE_TO_VARIANT_ERROR_TO_NULL(NULL) IS NULL AS input_is_null; ``` -The session variable changes the execution type of the expression result only. It does not change the physical `VARIANT` type used by table storage, readers, writers, or compaction. - -## Examples +```text ++-------------+-----------------+---------------+ +| valid_value | invalid_is_null | input_is_null | ++-------------+-----------------+---------------+ +| {"id":1} | 1 | 1 | ++-------------+-----------------+---------------+ +``` -### Keep valid values and null invalid JSON +Parse JSON/JSONB input: ```sql -SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\": 1}') AS valid_value, - PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value, - PARSE_TO_VARIANT_ERROR_TO_NULL(NULL) AS sql_null_value; +SELECT CAST( + PARSE_TO_VARIANT_ERROR_TO_NULL(CAST('[10, 20, 30]' AS JSON)) + AS STRING + ) AS value; ``` -The first expression returns a VARIANT object. The second and third expressions return SQL `NULL`. +```text ++------------+ +| value | ++------------+ +| [10,20,30] | ++------------+ +``` -### Parse arrays and access elements +JSON `null` and SQL `NULL` remain distinct: ```sql -SET enable_variant_v2 = true; - -SELECT CAST(ELEMENT_AT( - PARSE_TO_VARIANT_ERROR_TO_NULL('[10, 20, 30]'), - 0 - ) AS INT) AS first_item, - CAST(ELEMENT_AT( - PARSE_TO_VARIANT_ERROR_TO_NULL('[10, 20, 30]'), - -1 - ) AS INT) AS last_item; +SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('null') IS NULL AS json_null_is_sql_null, + PARSE_TO_VARIANT_ERROR_TO_NULL('{') IS NULL AS error_is_sql_null; ``` -For `ColumnVariantV2`, non-negative array indexes are zero-based and negative indexes count from the end. +```text ++-----------------------+-------------------+ +| json_null_is_sql_null | error_is_sql_null | ++-----------------------+-------------------+ +| 0 | 1 | ++-----------------------+-------------------+ +``` -### Choose strict or tolerant parsing +## Usage Notes -Use [PARSE_TO_VARIANT](./parse-to-variant) when invalid JSON should stop the query and surface an error. Use this function when invalid JSON should become SQL `NULL` and the remaining rows should continue. +- Use [PARSE_TO_VARIANT](./parse-to-variant) when invalid input should fail the query and expose the data-quality error. +- This function converts only parsing and validation failures to SQL `NULL`; it does not change the meaning of a valid JSON `null` value. diff --git a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md index d90d88a5bbecf..3b2ad53c343f2 100644 --- a/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md +++ b/docs/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md @@ -2,86 +2,112 @@ { "title": "PARSE_TO_VARIANT", "language": "en", - "description": "Parses JSON text into a VARIANT value." + "description": "Parses one complete JSON value from text or a JSON/JSONB expression into a typed VARIANT value." } --- -## Function +## Description -`PARSE_TO_VARIANT` parses one complete JSON value from a `VARCHAR` expression and returns it as `VARIANT`. JSON objects, arrays, strings, numbers, booleans, and the JSON literal `null` are supported. +`PARSE_TO_VARIANT` parses one complete JSON value into `VARIANT`. It accepts JSON objects, arrays, strings, numbers, booleans, and the JSON literal `null`. This function is available in Doris 4.2 and later. ## Syntax ```sql -PARSE_TO_VARIANT(json_text) +PARSE_TO_VARIANT() ``` ## Parameters -- `json_text`: A `VARCHAR` expression containing one complete JSON value. +| Parameter | Description | +| --- | --- | +| `` | A `CHAR`, `VARCHAR`, or `STRING` expression containing one complete JSON value, or a `JSON`/`JSONB` expression. JSON/JSONB input is converted to JSON text and then parsed as VARIANT. | ## Return Value -- Returns a `VARIANT` value. +Returns a `VARIANT` value. + - SQL `NULL` input returns SQL `NULL`. -- JSON `null` returns Variant/JSON `null`, which is distinct from SQL `NULL`. +- The JSON literal `null` returns Variant/JSON `null`, which is different from SQL `NULL`. +- Invalid JSON, duplicate object keys rejected by the current validation settings, unsupported depth, or another validation error causes the query to fail. -## Experimental behavior +## Example -`ColumnVariantV2` is an experimental, compute-only execution path. It is disabled by default and selected for the current FE session with: +Parse JSON text: ```sql -SET enable_variant_v2 = true; +SELECT CAST( + PARSE_TO_VARIANT('{"id": 42, "tags": ["doris", "sql"]}') + AS STRING + ) AS value; ``` -The session variable changes the execution type of the expression result only. It does not change the physical `VARIANT` type used by table storage, readers, writers, or compaction. - -## Errors - -Invalid JSON causes `PARSE_TO_VARIANT` to return an error. Use [PARSE_TO_VARIANT_ERROR_TO_NULL](./parse-to-variant-error-to-null) when invalid input should become SQL `NULL` instead. - -## Examples +```text ++----------------------------------------+ +| value | ++----------------------------------------+ +| {"id":42,"tags":["doris","sql"]} | ++----------------------------------------+ +``` -### Parse JSON values +Parse a JSON/JSONB expression: ```sql -SELECT PARSE_TO_VARIANT('{\"id\": 42, \"tags\": [\"doris\", \"sql\"]}'); -SELECT PARSE_TO_VARIANT('[10, 20, 30]'); -SELECT PARSE_TO_VARIANT('42'); -SELECT PARSE_TO_VARIANT('true'); -SELECT PARSE_TO_VARIANT('\"doris\"'); -SELECT PARSE_TO_VARIANT('null'); +SELECT CAST( + PARSE_TO_VARIANT(CAST('{"id": 42}' AS JSON)) + AS STRING + ) AS value; ``` -### Extract and cast a value +```text ++-----------+ +| value | ++-----------+ +| {"id":42} | ++-----------+ +``` -```sql -SET enable_variant_v2 = true; +Extract a value and CAST it to a concrete SQL type: +```sql SELECT CAST( - PARSE_TO_VARIANT('{\"user\": {\"id\": 42}}')['user']['id'] + PARSE_TO_VARIANT('{"user": {"id": 42}}')['user']['id'] AS BIGINT ) AS user_id; - -SELECT CAST(ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), 0) AS INT) AS first_item, - CAST(ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1) AS INT) AS last_item; ``` -For `ColumnVariantV2`, non-negative indexes into a VARIANT array are zero-based (`0` is the first element) and negative indexes count from the end. See [ELEMENT_AT](./element-at) for access rules. +```text ++---------+ +| user_id | ++---------+ +| 42 | ++---------+ +``` -### Distinguish JSON parsing from string casting +SQL `NULL` remains SQL `NULL`: ```sql --- Parses the JSON object into a structured VARIANT value. -SELECT PARSE_TO_VARIANT('{\"a\": 1}'); +SELECT PARSE_TO_VARIANT(NULL) IS NULL AS is_sql_null; +``` --- Creates a typed VARIANT string; it does not parse the string as JSON. -SELECT CAST('{\"a\": 1}' AS VARIANT); +```text ++-------------+ +| is_sql_null | ++-------------+ +| 1 | ++-------------+ ``` -### Invalid input +Invalid JSON returns an error: ```sql --- Returns an error because the JSON object is incomplete. -SELECT PARSE_TO_VARIANT('{\"id\":'); +SELECT PARSE_TO_VARIANT('{"id":'); ``` + +```text +ERROR: Parse json document failed +``` + +## Usage Notes + +- Use [PARSE_TO_VARIANT_ERROR_TO_NULL](./parse-to-variant-error-to-null) when invalid input should become SQL `NULL` instead of failing the query. +- `PARSE_TO_VARIANT` always expresses JSON-parsing intent. By contrast, the behavior of `CAST(string AS VARIANT)` depends on `enable_variant_string_cast_parse`; see the [VARIANT CAST rules](../../../basic-element/sql-data-types/semi-structured/VARIANT#cast-rules). diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md index 456c6b5b84750..3ffd9f14efef0 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/VARIANT.md @@ -69,78 +69,125 @@ WHERE ARRAY_CONTAINS(CAST(v['tags'] AS ARRAY), 'Doris'); ## 创建和访问值 -VARIANT 值可以从 JSON 文本或带确定类型的 SQL 表达式创建： +:::info 版本说明 +本节所述行为适用于 Doris 4.2 及后续版本。 +::: + +VARIANT 值可以从 JSON 文本、JSON/JSONB 值或带确定类型的 SQL 表达式创建： -- 如果字符串中包含需要解析的 JSON 文本，请使用 [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant)。 -- 如果要将 SQL 值转换为带类型的 Variant 值，请使用 `CAST(expression AS VARIANT)`。字符串 CAST 不会把字符串按 JSON 解析。 +- 如果要将字符串或 JSON/JSONB 表达式解析为结构化 VARIANT 值，请使用 [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant)。 +- 如果要将受支持的 SQL 值转换为 VARIANT，请使用 `CAST(expression AS VARIANT)`。字符串 CAST 是否解析 JSON 由 `enable_variant_string_cast_parse` 控制，详见 [CAST 规则](#cast-规则)。 ### 解析 JSON 文本 ```sql -SELECT PARSE_TO_VARIANT('{\"user\": {\"id\": 42}, \"active\": true}'); +SELECT PARSE_TO_VARIANT('{"user": {"id": 42}, "active": true}'); SELECT PARSE_TO_VARIANT('[10, 20, 30]'); +SELECT PARSE_TO_VARIANT(CAST('{"user": {"id": 42}}' AS JSON)); ``` 如果非法 JSON 应该返回 SQL `NULL` 而不是使查询失败，请使用 [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null)。 ### 访问对象和数组 -对象字段可以使用字符串 key 访问。开启 VARIANT V2 后，VARIANT 数组的非负索引从 0 开始，并支持从数组末尾倒数的负数索引。提取出的值仍是 `VARIANT`，如需按确定类型比较或聚合，请先 CAST。 +对象字段可以使用字符串 key 访问。在 Doris 4.2 及后续版本中，VARIANT 数组的非负索引从 0 开始，并支持从数组末尾倒数的负数索引。提取出的值仍是 `VARIANT`，如需按确定类型比较、计算或聚合，请先 CAST。 ```sql -SET enable_variant_v2 = true; - -SELECT CAST(PARSE_TO_VARIANT('{\"user\": {\"id\": 42}}')['user']['id'] AS BIGINT); +SELECT CAST(PARSE_TO_VARIANT('{"user": {"id": 42}}')['user']['id'] AS BIGINT); SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), 0); -- 10 SELECT ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1); -- 30 ``` 对象和数组访问的详细说明请参见 [ELEMENT_AT](../../../sql-functions/scalar-functions/variant-functions/element-at)。 -## CAST 语义 -`CAST` 会保留 SQL 字符串和 JSON 文本之间的区别： +## CAST 规则 + +VARIANT 的 CAST 包括两个方向：把受支持的 SQL 值转换为 VARIANT，以及把 VARIANT 中兼容的值转换为具体 SQL 类型。 -- `CAST(string AS VARIANT)` 创建根值为“带类型的 Variant 字符串”的 Variant，**不会**解析 JSON。因此，`'{"id": 1}'` 会保持为一个字符串值，非法 JSON 文本对这个 CAST 仍然是合法输入。 -- `PARSE_TO_VARIANT(string)` 才会把字符串按 JSON 解析；解析为对象或数组后，才能继续使用路径访问或 `ELEMENT_AT`。如果非法 JSON 需要返回 SQL `NULL`，请使用 `PARSE_TO_VARIANT_ERROR_TO_NULL`。 -- `CAST(PARSE_TO_VARIANT(...) AS scalar)` 会在值形状和范围兼容时，把已经解析的 Variant 转换为确定的 SQL 标量类型。这个 CAST 不是 JSON 解析器，形状或范围不兼容时可能失败。 -- `CAST(typed_expression AS VARIANT)` 会把受支持的带类型 SQL 值转换为 Variant。开启 `enable_variant_v2` 只会改变当前 session 的执行表示，不会改变磁盘上的 Variant 存储、reader、writer 或 compaction。 +### 其他类型 CAST 为 VARIANT + +| 源类型 | 行为 | +| --- | --- | +| `CHAR`、`VARCHAR`、`STRING` | 查询 session 默认按 JSON 解析输入。将 `enable_variant_string_cast_parse` 设为 `false` 后，输入会保留为 VARIANT 字符串。 | +| `JSON` / `JSONB` | 将结构化 JSON 值直接转换为 VARIANT。 | +| Boolean、整数、浮点数和 Decimal | 保留带类型的标量值，但需满足下文的 Decimal 限制。 | +| `DATE`、`DATETIME`、`TIMESTAMPTZ`、`IPV4`、`IPV6` | 保留对应的逻辑类型和值。 | +| `ARRAY` | 递归转换每个受支持的元素，并保留 SQL NULL 元素。 | +| `MAP` | 不支持。 | + +查询 session 中，`enable_variant_string_cast_parse` 默认为 `true`。如果 SQL 文本需要明确表达“始终解析 JSON”的意图，建议直接使用 `PARSE_TO_VARIANT`，避免依赖 session 设置。 ```sql -SET enable_variant_v2 = true; +-- 默认行为：把字符串按 JSON 解析。 +SELECT CAST('{"id": 1}' AS VARIANT) AS parsed_object; +-- {"id":1} + +-- 将相同输入保留为 VARIANT 字符串根值。 +SET enable_variant_string_cast_parse = false; +SELECT CAST(CAST('{"id": 1}' AS VARIANT) AS STRING) AS string_value, + VARIANT_TYPE(CAST('{"id": 1}' AS VARIANT)) AS root_type; +-- string_value：{"id": 1}；root_type：string + +-- JSON/JSONB 输入仍保留结构。 +SET enable_variant_string_cast_parse = true; +SELECT CAST(CAST('{"id": 1}' AS JSON) AS VARIANT) AS parsed_object; +-- {"id":1} +``` + +开启字符串解析时，非法 JSON 会使 CAST 失败。如果希望将非法输入转换为 SQL `NULL`，请使用 `PARSE_TO_VARIANT_ERROR_TO_NULL`。 -SELECT CAST('{"id": 1}' AS VARIANT) AS typed_string, - PARSE_TO_VARIANT('{"id": 1}') AS parsed_object; --- typed_string：字面量字符串 {"id": 1} --- parsed_object：{"id": 1} +### VARIANT CAST 为其他类型 -SELECT ELEMENT_AT(CAST('{"id": 1}' AS VARIANT), 'id') AS from_string, - ELEMENT_AT(PARSE_TO_VARIANT('{"id": 1}'), 'id') AS from_json; --- from_string：NULL；from_json：1 +VARIANT 可以 CAST 为兼容的标量、JSON/JSONB 或数组类型： +| 目标类型 | 行为 | +| --- | --- | +| Boolean、整数、浮点数、Decimal、日期时间和 IP 类型 | 转换兼容的标量根值；对象形状不兼容、文本非法或数值越界时，按照相应 CAST 模式报错或返回 SQL `NULL`。 | +| `STRING` | 标量根值返回对应文本，对象和数组返回 JSON 文本。Variant/JSON `null` 返回字符串 `null`，外层 SQL `NULL` 仍是 SQL `NULL`。 | +| `JSON` / `JSONB` | 将可表示的 VARIANT 值转换为结构化 JSON/JSONB。 | +| `ARRAY` | 将 VARIANT 数组逐元素转换为 `T`；不兼容元素遵循目标类型的 CAST 规则。 | +| `MAP` | 不支持。 | + +```sql SELECT CAST(PARSE_TO_VARIANT('42') AS BIGINT) AS id; --- id：42 +-- 42 -SELECT CAST('{invalid json' AS VARIANT) AS still_a_string; --- 成功，因为 CAST 不会把输入按 JSON 解析 +SELECT CAST(PARSE_TO_VARIANT('[1, null, 3]') AS ARRAY) AS values; +-- [1, NULL, 3] + +SELECT CAST(PARSE_TO_VARIANT('{"id": 1}') AS JSON) AS json_value; +-- {"id":1} ``` -不要使用 `CAST(string AS VARIANT)` 做 JSON 校验。需要严格解析时使用 `PARSE_TO_VARIANT`；需要把格式错误转换为 SQL `NULL` 时使用 `PARSE_TO_VARIANT_ERROR_TO_NULL`。 +### Decimal 与日期时间转换限制 + +| Doris 输入类型 | VARIANT 支持情况 | +| --- | --- | +| 旧版 `DECIMALV2` | 精确保留 precision 不超过 27、scale 不超过 9 的值。 | +| `DECIMAL(p, s)` | 精确保留 `1 <= p <= 38` 且 `0 <= s <= p` 的值；不支持需要超过 38 位 precision 的值。 | +| `DATE` | 保留为不含时间和时区的日历日期。 | +| 旧版 `DATETIME` | 保留到秒，不进行时区调整。 | +| `DATETIME(p)` | 支持 `0 <= p <= 6`，不进行时区调整。 | +| `TIMESTAMPTZ(p)` | 支持 `0 <= p <= 6`，保留带时区调整的 timestamp 语义。 | +| `TIME` | 不支持作为 VARIANT 输入。 | -## Equality 语义 +源值还必须满足对应 Doris 类型本身的合法性要求。precision 超限、日期非法或值不兼容时会报错，不会自动修复。 -开启 VARIANT V2 后，支持的分组、去重和集合运算会按 logical value 判断相等性，而不是按来源 SQL 类型或表示形式： +## 相等性与 Hash 语义 + +在 Doris 4.2 及后续版本中，分组、去重和集合运算会按逻辑值判断相等性，而不是按来源 SQL 类型或物理表示： - 等价的整数数值表示相等。 -- Decimal 尾随零不影响值。 +- Decimal 尾随零不影响值，因此 `1.20` 与 `1.2` 相等。 - `+0`、`-0` 与整数零相等。 - 对象 key 的顺序不影响相等性，但数组元素顺序会影响相等性。 - Variant/JSON `null` 与 SQL `NULL` 不同。 -这些规则适用于 `GROUP BY`、`DISTINCT`、`COUNT(DISTINCT ...)`、`INTERSECT`、`EXCEPT` 和 `UNION DISTINCT` 等已支持的操作。这并不意味着根 Variant 比较谓词已经可用：直接执行 `VARIANT = VARIANT` 或排序比较仍不支持。 +Hash 类算子使用同一套 canonical 逻辑表示计算 key。因此，只要两个值按上述规则相等，无论来源是解析后的 JSON 还是带类型的 CAST，都会得到相同的内部 Hash Key。该 Hash 只用于 Doris 内部执行，不是稳定的用户侧校验值。 -```sql -SET enable_variant_v2 = true; +这些规则适用于 `GROUP BY`、`DISTINCT`、`COUNT(DISTINCT ...)`、`INTERSECT`、`EXCEPT` 和 `UNION DISTINCT` 等已支持的操作，但不会开放根 VARIANT 比较谓词：直接执行 `VARIANT = VARIANT` 或排序比较仍不支持。 +```sql -- 1 和 1.0 只有一个 distinct logical value。 SELECT COUNT(DISTINCT value) AS distinct_count FROM ( @@ -153,9 +200,9 @@ FROM ( -- 对象 key 顺序被忽略；数组顺序会保留。 SELECT COUNT(DISTINCT value) AS distinct_count FROM ( - SELECT PARSE_TO_VARIANT('{\"a\": 1, \"b\": 2}') AS value + SELECT PARSE_TO_VARIANT('{"a": 1, "b": 2}') AS value UNION ALL - SELECT PARSE_TO_VARIANT('{\"b\": 2, \"a\": 1}') AS value + SELECT PARSE_TO_VARIANT('{"b": 2, "a": 1}') AS value ) AS object_values; -- distinct_count: 1 @@ -168,7 +215,13 @@ FROM ( -- distinct_count: 2 ``` -更多示例、支持的运算、数据类型行为和当前限制，请参见 [VARIANT V2 行为与语义](./variant-v2-compute-semantics)。 +## NULL 语义 + +SQL `NULL` 与 Variant/JSON `null` 是不同的值： + +- SQL `NULL` 表示 SQL 值缺失，并遵循普通 SQL NULL 传播规则。 +- Variant/JSON `null` 是一个 VARIANT 值，例如 `PARSE_TO_VARIANT('null')` 的返回值。 +- `PARSE_TO_VARIANT_ERROR_TO_NULL` 遇到非法输入时返回 SQL `NULL`，这与成功解析 JSON 字面量 `null` 不同。 ## 基本类型 @@ -490,12 +543,26 @@ SELECT v FROM variant_tbl WHERE k = 2; 排序在每一层都会生效——顶层 key 输出为 `a`、`b`、`c`，嵌套 object 内 key 输出为 `x`、`y`。 -## 支持的运算与 CAST 规则 +## 支持的运算 -默认执行路径保持现有 VARIANT 使用约束： +在 Doris 4.2 及后续版本中，整个 VARIANT 值支持基于 Hash 的分组和去重，但不支持比较、Join Key 或排序语义。 -- VARIANT 本身不支持与其他类型直接比较/运算；在默认路径上，两个 VARIANT 之间也不支持直接比较。 -- 如需比较、过滤、聚合、排序，请对子路径显式或隐式 CAST 到确定类型。 +| 对整个 VARIANT 值执行的操作 | 支持情况 | 说明 | +| --- | --- | --- | +| `GROUP BY` | 支持 | 使用上文所述的逻辑相等与 canonical Hash 规则。 | +| `DISTINCT`、`COUNT(DISTINCT ...)` | 支持 | 逻辑等价的值会被去重。 | +| `INTERSECT`、`EXCEPT`、`UNION DISTINCT` | 支持 | 使用相同的逻辑相等与 Hash 语义。 | +| `COUNT(*)`、`COUNT(variant)` | 支持 | `COUNT(variant)` 按普通 SQL 规则排除外层 SQL `NULL`。 | +| `IF`、`CASE`、`IFNULL`、`COALESCE` | 支持 | 条件表达式可以返回和消费 VARIANT 值。 | +| 临时 `ARRAY`、`MAP` 或 `STRUCT` 表达式中包含 VARIANT | 支持 | 这不表示持久化表结构可以使用这些嵌套类型。 | +| `EXPLODE_VARIANT_ARRAY`、对 `ARRAY` 使用 `EXPLODE`/`EXPLODE_OUTER` | 支持 | 输出 VARIANT 元素，并保留 SQL `NULL` 与 Variant/JSON `null` 的区别。 | +| `=`、`!=`、`<=>`、`<`、`<=`、`>`、`>=` | 不支持 | 请在两侧提取可比较的子路径，并 CAST 为具体类型。 | +| Join Key | 不支持 | 请把两侧所需子路径 CAST 为相同的具体类型。 | +| `ORDER BY`、Sort/TopN Key | 不支持 | 排序前请先 CAST 所需子路径。 | +| 窗口分区键或排序键 | 不支持 | 整个 VARIANT 值不能作为窗口 Key。 | +| `MIN(variant)`、`MAX(variant)` | 不支持 | 聚合前请先 CAST 标量子路径。 | + +如需比较、过滤、计算或排序，请提取所需子路径，再显式或隐式 CAST 为具体类型： ```sql -- 显式 CAST @@ -508,40 +575,6 @@ SELECT * FROM tbl WHERE v['bool']; SELECT * FROM tbl WHERE v['str'] MATCH 'Doris'; ``` -- 在默认路径上，不应将整个 VARIANT 直接用于 `ORDER BY`、`GROUP BY`、`JOIN KEY` 或聚合参数；请先对子路径 CAST。 -- 字符串类型可隐式转换为 VARIANT。 - -[实验性 V2 行为](./variant-v2-compute-semantics)为分组和集合运算增加 logical-value 语义，但不会因此开放根 VARIANT 比较谓词或 Variant Join Key。 - -| VARIANT | Castable | Coercible | -| --------------- | -------- | --------- | -| `ARRAY` | ✔ | ❌ | -| `BOOLEAN` | ✔ | ✔ | -| `DATE/DATETIME` | ✔ | ✔ | -| `FLOAT` | ✔ | ✔ | -| `IPV4/IPV6` | ✔ | ✔ | -| `DECIMAL` | ✔ | ✔ | -| `MAP` | ❌ | ❌ | -| `TIMESTAMP` | ✔ | ✔ | -| `VARCHAR` | ✔ | ✔ | -| `JSON` | ✔ | ✔ | - -## 实验性 VARIANT V2 行为 - -:::caution 实验特性 -VARIANT V2 默认关闭，只影响当前 session 的执行行为。现有数据和 Variant 持久化格式不会改变。 -::: - -通过 session variable `enable_variant_v2` 为当前 session 开启该功能： - -```sql -SET enable_variant_v2 = true; -``` - -开启后，V2 会为支持的 Variant 值提供 logical-value 分组、去重和集合运算，并明确区分 JSON 解析和字符串 CAST。根 Variant 比较谓词、Variant Join Key、Sort/TopN Key 以及 `MIN`/`MAX` 参数仍然不受支持。 - -行为变化、Equality、CAST 与解析、Decimal 和日期时间语义、示例及完整限制，请参见 [VARIANT V2 行为与语义](./variant-v2-compute-semantics)。 - ## 宽列 当导入数据包含大量不同的 JSON key 时，通过子列列式提取（Subcolumnization）生成的子列会迅速增多；当规模达到一定程度，可能出现元数据膨胀、写入/合并开销增大、查询性能下降等问题。为应对“宽列”（子列过多），VARIANT 提供两种机制：**稀疏列** 与 **DOC 编码**。 @@ -587,7 +620,7 @@ SET enable_variant_v2 = true; - **大宽表优化**：对于会通过子列列式提取（Subcolumnization）生成大量独立子列的宽表场景（例如超过 2000 列），强烈建议开启 **V3 存储格式**。通过在建表 `PROPERTIES` 中指定 `"storage_format" = "V3"`，可以将列元数据与 Segment Footer 解耦，加快文件打开速度并降低内存占用。 - JSON key 长度 ≤ 255。 - 不支持作为主键或排序键。 -- 不支持与其他类型嵌套（如 `Array`、`Struct`）。 +- 持久化表结构不能把 VARIANT 嵌套在其他类型中（如 `ARRAY`、`STRUCT`）；临时表达式结果可以使用上表列出的嵌套容器能力。 - 在未启用 DOC mode 时，读取整个 VARIANT 列会扫描所有子字段。对于超宽列，一般不建议直接 `SELECT variant_col`；如果整列读取是主要查询模式，建议优先使用 DOC mode。若列包含大量子字段，也可额外存储原始 JSON 的 STRING/JSONB 列，以优化如 `LIKE` 等整体匹配： ```sql diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md deleted file mode 100644 index a4b7a311cbf7c..0000000000000 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics.md +++ /dev/null @@ -1,177 +0,0 @@ ---- -{ - "title": "VARIANT V2 行为与语义", - "language": "zh-CN", - "description": "面向用户介绍实验性 VARIANT V2 的功能、行为和语义，包括开启方式、Equality、CAST 与 JSON 解析、Decimal 和日期时间处理、SQL 示例及当前限制。" -} ---- - -## 概览 - -实验性 VARIANT V2 扩展了可以消费 `VARIANT` 值的运算，并为分组、去重和集合运算定义了一致的 logical-value 语义。 - -SQL 语法、索引、存储配置和通用类型规则请参见 [VARIANT 类型参考](./VARIANT)；在默认存储行为、Sparse、DOC mode 和 Schema Template 之间选型时，请参见 [VARIANT 使用与配置指南](./variant-workload-guide)。本文只说明开启 V2 后用户可以观察到的行为。 - -:::caution 实验特性 -V2 默认关闭，只作用于当前 session。现有数据和 Variant 持久化格式不会改变。请在验证目标 workload 后再开启。 -::: - -## 开启 V2 - -执行目标 SQL 前设置 session variable： - -```sql -SET enable_variant_v2 = true; -``` - -该设置只影响当前 session，已经持久化的 `VARIANT` 数据不会改变。 - -## 开启 V2 后的行为 - -| 场景 | V2 行为 | -| --- | --- | -| 分组与去重 | 支持的 Variant 值可以基于 logical-value equality 用于 `GROUP BY`、`DISTINCT` 和 `COUNT(DISTINCT ...)`。 | -| 集合运算 | 支持的 Variant 值可以参与 `INTERSECT`、`EXCEPT` 和 `UNION DISTINCT`。 | -| JSON 解析 | 使用 `parse_to_variant` 或 `parse_to_variant_error_to_null` 显式解析 JSON 文本。 | -| 字符串转换 | `CAST(string AS VARIANT)` 创建 Variant 字符串，不会把字符串按 JSON 解析。 | -| 嵌套表达式 | 支持的条件表达式、嵌套容器和支持 Variant 的 `explode` 可以消费 Variant 值。 | -| 非法值 | 格式错误或不受支持的 Variant 值会报错，而不是静默接受。 | - -V2 不会开放整个（根）Variant 值的全部运算。直接比较谓词、Join Key、排序和 `MIN`/`MAX` 等能力仍受限制，详见[当前限制](#当前限制)。 - -例如，在默认路径上不能对根 Variant 值直接分组或去重；开启 V2 后，相同的值可以按 logical equality 分组： - -```sql -SET enable_variant_v2 = true; - -SELECT COUNT(DISTINCT value) AS distinct_count -FROM ( - SELECT PARSE_TO_VARIANT('1') AS value - UNION ALL - SELECT PARSE_TO_VARIANT('1.0') AS value -) AS numeric_values; --- distinct_count: 1 -``` - -## Equality 语义 - -分组、去重和集合运算比较的是 logical value，而不是来源 SQL 类型或表示形式： - -- 等价的整数数值表示相等。 -- Decimal 尾随零不改变值，因此 `1.20` 与 `1.2` 相等。 -- `+0`、`-0` 和整数零相等。 -- Object key 顺序不影响相等性。 -- Array 元素顺序影响相等性。 -- Variant/JSON `null` 与 SQL `NULL` 不同。 - -```sql --- Object key 顺序不影响 equality。 -SELECT COUNT(DISTINCT value) AS distinct_count -FROM ( - SELECT PARSE_TO_VARIANT('{\"a\": 1, \"b\": 2}') AS value - UNION ALL - SELECT PARSE_TO_VARIANT('{\"b\": 2, \"a\": 1}') AS value -) AS object_values; --- distinct_count: 1 - --- Array 顺序会保留。 -SELECT COUNT(DISTINCT value) AS distinct_count -FROM ( - SELECT PARSE_TO_VARIANT('[1, 2]') AS value - UNION ALL - SELECT PARSE_TO_VARIANT('[2, 1]') AS value -) AS array_values; --- distinct_count: 2 -``` - -这些 Equality 规则只用于支持的哈希和集合运算，不会开放直接 `VARIANT = VARIANT`、null-safe equality 或排序比较。 - -## CAST 与 JSON 解析 - -V2 明确区分 Variant 字符串和已经解析的 JSON 值： - -| 操作 | 结果 | 非法 JSON 行为 | -| --- | --- | --- | -| `CAST(string AS VARIANT)` | 保留原始文本的 Variant 字符串 | 不执行 JSON 校验 | -| `parse_to_variant(string)` | 从 JSON 文本解析出的 Variant 值 | 返回错误 | -| `parse_to_variant_error_to_null(string)` | 从 JSON 文本解析出的 Variant 值 | 返回 SQL `NULL` | -| `CAST(variant_value AS T)` | 类型为 `T` 的具体 SQL 值 | 遵循目标类型的 CAST 规则 | - -完整函数语法请参见 [PARSE_TO_VARIANT](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant) 和 [PARSE_TO_VARIANT_ERROR_TO_NULL](../../../sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null)。 - -```sql -SET enable_variant_v2 = true; - -SELECT PARSE_TO_VARIANT('{\"id\": 1}') AS parsed_object, - CAST('{\"id\": 1}' AS VARIANT) AS variant_string; - -SELECT ELEMENT_AT(CAST('{\"id\": 1}' AS VARIANT), 'id') AS from_string, - ELEMENT_AT(PARSE_TO_VARIANT('{\"id\": 1}'), 'id') AS from_json; --- from_string: NULL; from_json: 1 - -SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value; --- invalid_value: NULL - -SELECT CAST(PARSE_TO_VARIANT('42') AS BIGINT) AS id; --- id: 42 -``` - -不要使用 `CAST(string AS VARIANT)` 校验 JSON。提取出的 Variant 值在进行确定类型的比较、过滤、排序或算术运算前，应先 CAST 到具体 SQL 类型。 - -## Decimal 语义与限制 - -V2 会精确保留受支持的 Decimal 值及其 scale；Equality 语义会忽略不影响数值的尾随零。 - -| Doris 输入类型 | 支持范围 | V2 行为 | -| --- | --- | --- | -| 旧版 `DECIMALV2` | Precision 最大 27；scale 最大 9 | 精确保留为 Variant Decimal | -| `DECIMAL(p, s)` | `1 <= p <= 38`，`0 <= s <= p` | 精确保留为 Variant Decimal | -| `DECIMAL(p, s)` | `39 <= p <= 76` | V2 不支持；请先 CAST 到 precision 38 或更低 | - -当前 Decimal 限制： - -- Variant Decimal 支持的 precision 和 scale 上限均为 38。 -- 源值必须同时满足 Doris Decimal 类型和 Variant precision 上限。 -- `1.20` 与 `1.2` 等值会保留 Decimal 数值，并在支持的分组和集合运算中视为同一个 logical value。 - -## 日期和时间语义与限制 - -| Doris 输入类型 | Variant 语义 | 精度与时区行为 | -| --- | --- | --- | -| `DATE` | 日历日期 | 天级精度；不包含时间和时区 | -| 旧版 `DATETIME` | 无时区 timestamp | 秒级精度；不进行 session time zone 调整 | -| `DATETIME(p)` | 无时区 timestamp | `0 <= p <= 6`；不进行 session time zone 调整 | -| `TIMESTAMPTZ(p)` | 带时区调整语义的 timestamp | `0 <= p <= 6` | -| `TIME` | — | V2 不支持 | - -当前日期和时间限制： - -- 所有源值都必须是合法的 Doris 日期或 timestamp。非法值返回 `INVALID_ARGUMENT`，不会被修复或转换为 `NULL`。 -- `DATETIME` 和 `DATETIMEV2` 保持 wall-clock、无时区语义；转换过程不会应用 session time zone。 -- `TIMESTAMPTZ` 是当前支持并保留时区调整 timestamp 语义的输入类型。 -- V2 日期和时间转换支持微秒精度，不支持纳秒精度。 - -## NULL 语义 - -SQL `NULL` 与 Variant/JSON `null` 是不同的值： - -- SQL `NULL` 表示 SQL 值缺失，并遵循普通 SQL NULL 传播规则。 -- Variant/JSON `null` 是一个 Variant 值，例如由 `parse_to_variant('null')` 产生。 -- `parse_to_variant_error_to_null` 遇到非法输入时返回 SQL `NULL`；这与成功解析 JSON literal `null` 不同。 - -## 当前限制 - -- 不支持根 Variant 比较谓词（`=`、`!=`、`<=>`、`<`、`<=`、`>` 和 `>=`）；应在两侧提取可比较的路径并 CAST。 -- 不支持把 Variant 表达式作为 Join Key。 -- 根 Variant 不能作为 Sort/TopN Key、窗口分区键或窗口排序键。 -- 根 Variant 不能作为 `MIN` 或 `MAX` 的参数。 -- Precision 大于 38 的 Decimal 值和 `TIME` 值不支持作为 V2 输入。 -- 功能选择只作用于 session，不会改变已经持久化的 Variant 数据。 -- 仅在验证目标 workload 后开启 V2；该功能仍是实验特性。 - -```sql --- 比较具体子路径，而不是根 Variant 值。 -SELECT * -FROM tbl -WHERE CAST(v['id'] AS BIGINT) = CAST(other_v['id'] AS BIGINT); -``` diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md index 67fc6921751eb..2bf4a09c67412 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/element-at.md @@ -2,7 +2,7 @@ { "title": "ELEMENT_AT", "language": "zh-CN", - "description": "ELEMENTAT 函数用于从数组或 map 中按指定的索引或键提取对应的元素值。" + "description": "ELEMENT_AT 函数按索引、键或字段名从 ARRAY、MAP、STRUCT 或 VARIANT 中提取元素，并说明各类型的索引起点、负索引语义与越界行为。" } --- @@ -29,7 +29,7 @@ ELEMENT_AT(container, key_or_index) - 对于 `MAP`：为 `MAP` 中的键类型（`K`），可为任意支持的基础类型。 - 对于 `STRUCT`：为常量整数（字段位置，从 **1** 开始）或常量字符串（字段名，按**大小写不敏感**匹配）。 - 对于 `VARIANT` 对象访问：为字符串类型的 key； - - 对于 `ColumnVariantV2` 中的 VARIANT 数组：为整数索引，非负索引从 0 开始，负数索引从数组末尾倒数。 + - 对于 Doris 4.2 及后续版本中的 VARIANT 数组：为整数索引，非负索引从 0 开始，负数索引从数组末尾倒数。 ## 返回值 @@ -43,12 +43,11 @@ ELEMENT_AT(container, key_or_index) ## 使用说明 1. **ARRAY 数组索引从 1 开始**，不是从 0 开始； -2. 开启 `enable_variant_v2` 后，VARIANT 数组的非负索引从 `0` 开始，`0` 表示第一个元素，`-1` 表示最后一个元素； +2. 在 Doris 4.2 及后续版本中，VARIANT 数组的非负索引从 `0` 开始，`0` 表示第一个元素，`-1` 表示最后一个元素； 3. ARRAY 和 VARIANT 数组都支持负数索引，`-1` 表示最后一个元素，`-2` 表示倒数第二个，以此类推； 4. `ELEMENT_AT(container, key_or_index)` 函数的功能与 `container[key_or_index]` 作用一致（详细见示例）。 ```sql -SET enable_variant_v2 = true; SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 0); -- 10 SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), 1); -- 20 SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), -1); -- 30 @@ -138,5 +137,3 @@ SELECT ELEMENT_AT(parse_to_variant('[10, 20, 30]'), -1); -- 30 | | +----------------------------------------+ ``` - - diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md index 11651e3f16f50..3ce8e9bdea81d 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null.md @@ -1,71 +1,90 @@ --- { - "title": "PARSE_TO_VARIANT_ERROR_TO_NULL", + "title": "PARSE_TO_VARIANT_ERROR_TO_NULL（解析失败返回 NULL）", "language": "zh-CN", - "description": "将 JSON 文本解析为 VARIANT 值，非法输入返回 SQL NULL。" + "description": "PARSE_TO_VARIANT_ERROR_TO_NULL 将完整 JSON 值解析为 VARIANT，并在解析或校验失败时返回 SQL NULL，而不是使当前查询失败。" } --- ## 功能 -`PARSE_TO_VARIANT_ERROR_TO_NULL` 将 JSON 文本解析为 `VARIANT`。输入不是合法 JSON 时返回 SQL `NULL`，不会使查询失败。 +`PARSE_TO_VARIANT_ERROR_TO_NULL` 将一个完整 JSON 值解析为 `VARIANT`。函数名中的 `ERROR_TO_NULL` 表示：发生解析或校验错误时返回 SQL `NULL`，而不是使查询失败。该函数自 Doris 4.2 起支持。 ## 语法 ```sql -PARSE_TO_VARIANT_ERROR_TO_NULL(json_text) +PARSE_TO_VARIANT_ERROR_TO_NULL() ``` ## 参数 -- `json_text`：包含一个完整 JSON 值的 `VARCHAR` 表达式。 +| 参数 | 说明 | +| --- | --- | +| `` | 包含一个完整 JSON 值的 `CHAR`、`VARCHAR` 或 `STRING` 表达式，也可以是 `JSON`/`JSONB` 表达式。JSON/JSONB 输入会先转换为 JSON 文本，再解析为 VARIANT。 | ## 返回值 -- 返回可为 NULL 的 `VARIANT` 值。 +返回可为 NULL 的 `VARIANT` 值。 + +- 合法输入返回解析后的 VARIANT 值。 +- 非法 JSON 或其他解析、校验错误返回 SQL `NULL`。 - 输入为 SQL `NULL` 时返回 SQL `NULL`。 -- 输入为非法 JSON 时返回 SQL `NULL`。 -- 合法 JSON `null` 仍然是 Variant/JSON `null`，与 SQL `NULL` 不同。 +- 合法的 JSON 字面量 `null` 返回 Variant/JSON `null`，不是 SQL `NULL`。 -## 实验性行为 +## 示例 -`ColumnVariantV2` 是实验性的、仅用于计算的执行路径，默认关闭。可以在当前 FE session 中执行以下语句开启： +保留合法值，并把非法 JSON 转换为 SQL `NULL`： ```sql -SET enable_variant_v2 = true; +SELECT CAST( + PARSE_TO_VARIANT_ERROR_TO_NULL('{"id": 1}') + AS STRING + ) AS valid_value, + PARSE_TO_VARIANT_ERROR_TO_NULL('{"id":') IS NULL AS invalid_is_null, + PARSE_TO_VARIANT_ERROR_TO_NULL(NULL) IS NULL AS input_is_null; ``` -该 session variable 只改变表达式结果的执行类型，不改变表存储、读写器或 Compaction 使用的物理 `VARIANT` 类型。 - -## 示例 +```text ++-------------+-----------------+---------------+ +| valid_value | invalid_is_null | input_is_null | ++-------------+-----------------+---------------+ +| {"id":1} | 1 | 1 | ++-------------+-----------------+---------------+ +``` -### 保留合法值，将非法 JSON 转为 NULL +解析 JSON/JSONB 输入： ```sql -SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\": 1}') AS valid_value, - PARSE_TO_VARIANT_ERROR_TO_NULL('{\"id\":') AS invalid_value, - PARSE_TO_VARIANT_ERROR_TO_NULL(NULL) AS sql_null_value; +SELECT CAST( + PARSE_TO_VARIANT_ERROR_TO_NULL(CAST('[10, 20, 30]' AS JSON)) + AS STRING + ) AS value; ``` -第一个表达式返回 VARIANT 对象，第二个和第三个表达式返回 SQL `NULL`。 +```text ++------------+ +| value | ++------------+ +| [10,20,30] | ++------------+ +``` -### 解析数组并访问元素 +JSON `null` 与 SQL `NULL` 仍然不同： ```sql -SET enable_variant_v2 = true; - -SELECT CAST(ELEMENT_AT( - PARSE_TO_VARIANT_ERROR_TO_NULL('[10, 20, 30]'), - 0 - ) AS INT) AS first_item, - CAST(ELEMENT_AT( - PARSE_TO_VARIANT_ERROR_TO_NULL('[10, 20, 30]'), - -1 - ) AS INT) AS last_item; +SELECT PARSE_TO_VARIANT_ERROR_TO_NULL('null') IS NULL AS json_null_is_sql_null, + PARSE_TO_VARIANT_ERROR_TO_NULL('{') IS NULL AS error_is_sql_null; ``` -对于 `ColumnVariantV2`，数组非负索引从 0 开始，负数索引从数组末尾倒数。 +```text ++-----------------------+-------------------+ +| json_null_is_sql_null | error_is_sql_null | ++-----------------------+-------------------+ +| 0 | 1 | ++-----------------------+-------------------+ +``` -### 选择严格解析或容错解析 +## 使用说明 -如果非法 JSON 应该让查询失败并暴露错误，请使用 [PARSE_TO_VARIANT](./parse-to-variant)；如果非法 JSON 应该转换为 SQL `NULL` 并继续处理其他行，请使用本函数。 +- 如果非法输入应该使查询失败并暴露数据质量问题，请使用 [PARSE_TO_VARIANT](./parse-to-variant)。 +- 本函数只会把解析和校验错误转换为 SQL `NULL`，不会改变合法 JSON `null` 的含义。 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md index f0e99c8f93d99..19509d90418b2 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant.md @@ -2,86 +2,112 @@ { "title": "PARSE_TO_VARIANT", "language": "zh-CN", - "description": "将 JSON 文本解析为 VARIANT 值。" + "description": "PARSE_TO_VARIANT 将 JSON 文本或 JSON/JSONB 表达式中的完整 JSON 值解析为带类型信息的 VARIANT，并说明输入、返回值、错误行为与示例。" } --- ## 功能 -`PARSE_TO_VARIANT` 将 `VARCHAR` 表达式中的一个完整 JSON 值解析为 `VARIANT`。支持 JSON 对象、数组、字符串、数字、布尔值和 JSON 字面量 `null`。 +`PARSE_TO_VARIANT` 将一个完整 JSON 值解析为 `VARIANT`，支持 JSON 对象、数组、字符串、数字、布尔值和 JSON 字面量 `null`。该函数自 Doris 4.2 起支持。 ## 语法 ```sql -PARSE_TO_VARIANT(json_text) +PARSE_TO_VARIANT() ``` ## 参数 -- `json_text`：包含一个完整 JSON 值的 `VARCHAR` 表达式。 +| 参数 | 说明 | +| --- | --- | +| `` | 包含一个完整 JSON 值的 `CHAR`、`VARCHAR` 或 `STRING` 表达式，也可以是 `JSON`/`JSONB` 表达式。JSON/JSONB 输入会先转换为 JSON 文本，再解析为 VARIANT。 | ## 返回值 -- 返回 `VARIANT` 值。 +返回 `VARIANT` 值。 + - 输入为 SQL `NULL` 时返回 SQL `NULL`。 -- 输入为 JSON `null` 时返回 Variant/JSON `null`，它与 SQL `NULL` 不同。 +- 输入为 JSON 字面量 `null` 时返回 Variant/JSON `null`，它与 SQL `NULL` 不同。 +- JSON 非法、对象 key 重复且当前校验设置不允许、嵌套深度超限或发生其他校验错误时，查询失败。 -## 实验性行为 +## 示例 -`ColumnVariantV2` 是实验性的、仅用于计算的执行路径，默认关闭。可以在当前 FE session 中执行以下语句开启： +解析 JSON 文本： ```sql -SET enable_variant_v2 = true; +SELECT CAST( + PARSE_TO_VARIANT('{"id": 42, "tags": ["doris", "sql"]}') + AS STRING + ) AS value; ``` -该 session variable 只改变表达式结果的执行类型，不改变表存储、读写器或 Compaction 使用的物理 `VARIANT` 类型。 - -## 错误处理 - -非法 JSON 会使 `PARSE_TO_VARIANT` 返回错误。如果希望将非法输入转换为 SQL `NULL`，请使用 [PARSE_TO_VARIANT_ERROR_TO_NULL](./parse-to-variant-error-to-null)。 - -## 示例 +```text ++----------------------------------------+ +| value | ++----------------------------------------+ +| {"id":42,"tags":["doris","sql"]} | ++----------------------------------------+ +``` -### 解析 JSON 值 +解析 JSON/JSONB 表达式： ```sql -SELECT PARSE_TO_VARIANT('{\"id\": 42, \"tags\": [\"doris\", \"sql\"]}'); -SELECT PARSE_TO_VARIANT('[10, 20, 30]'); -SELECT PARSE_TO_VARIANT('42'); -SELECT PARSE_TO_VARIANT('true'); -SELECT PARSE_TO_VARIANT('\"doris\"'); -SELECT PARSE_TO_VARIANT('null'); +SELECT CAST( + PARSE_TO_VARIANT(CAST('{"id": 42}' AS JSON)) + AS STRING + ) AS value; ``` -### 提取值并 CAST 为确定类型 +```text ++-----------+ +| value | ++-----------+ +| {"id":42} | ++-----------+ +``` -```sql -SET enable_variant_v2 = true; +提取值并 CAST 为具体 SQL 类型： +```sql SELECT CAST( - PARSE_TO_VARIANT('{\"user\": {\"id\": 42}}')['user']['id'] + PARSE_TO_VARIANT('{"user": {"id": 42}}')['user']['id'] AS BIGINT ) AS user_id; - -SELECT CAST(ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), 0) AS INT) AS first_item, - CAST(ELEMENT_AT(PARSE_TO_VARIANT('[10, 20, 30]'), -1) AS INT) AS last_item; ``` -对于 `ColumnVariantV2`，VARIANT 数组的非负索引从 0 开始（`0` 表示第一个元素），负数索引从数组末尾倒数。访问规则请参见 [ELEMENT_AT](./element-at)。 +```text ++---------+ +| user_id | ++---------+ +| 42 | ++---------+ +``` -### 区分 JSON 解析与字符串 CAST +SQL `NULL` 仍返回 SQL `NULL`： ```sql --- 解析 JSON 对象并返回结构化 VARIANT 值。 -SELECT PARSE_TO_VARIANT('{\"a\": 1}'); +SELECT PARSE_TO_VARIANT(NULL) IS NULL AS is_sql_null; +``` --- 创建带类型的 VARIANT 字符串，不会把字符串按 JSON 解析。 -SELECT CAST('{\"a\": 1}' AS VARIANT); +```text ++-------------+ +| is_sql_null | ++-------------+ +| 1 | ++-------------+ ``` -### 非法输入 +非法 JSON 会报错： ```sql --- JSON 对象不完整，会返回错误。 -SELECT PARSE_TO_VARIANT('{\"id\":'); +SELECT PARSE_TO_VARIANT('{"id":'); ``` + +```text +ERROR: Parse json document failed +``` + +## 使用说明 + +- 如果希望把非法输入转换为 SQL `NULL`，请使用 [PARSE_TO_VARIANT_ERROR_TO_NULL](./parse-to-variant-error-to-null)。 +- `PARSE_TO_VARIANT` 始终明确表示“解析 JSON”。相比之下，`CAST(string AS VARIANT)` 的行为取决于 `enable_variant_string_cast_parse`，详见 [VARIANT CAST 规则](../../../basic-element/sql-data-types/semi-structured/VARIANT#cast-规则)。 diff --git a/sidebars.ts b/sidebars.ts index 32078d869d3bc..0365d3a578b60 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -1264,11 +1264,6 @@ const sidebars: SidebarsConfig = { id: 'sql-manual/basic-element/sql-data-types/semi-structured/VARIANT', label: 'VARIANT Reference', }, - { - type: 'doc', - id: 'sql-manual/basic-element/sql-data-types/semi-structured/variant-v2-compute-semantics', - label: 'V2 Behavior and Semantics', - }, ], }, ], @@ -1872,6 +1867,8 @@ const sidebars: SidebarsConfig = { label: 'Variant Functions', items: [ 'sql-manual/sql-functions/scalar-functions/variant-functions/element-at', + 'sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant', + 'sql-manual/sql-functions/scalar-functions/variant-functions/parse-to-variant-error-to-null', 'sql-manual/sql-functions/scalar-functions/variant-functions/variant-type', ], },