diff --git a/ci/journey.sh b/ci/journey.sh index 75bac696..5bfebcbf 100755 --- a/ci/journey.sh +++ b/ci/journey.sh @@ -489,6 +489,23 @@ EOSQL assert_eq "decoupled multi-row INSERT (3 rows)" "3" "$(extract MULTI "$O")" assert_eq "decoupled UPDATE visible" "upd" "$(extract UPD "$O")" assert_eq "decoupled DELETE visible (0 of id=2)" "0" "$(extract DEL "$O")" + # The json/jsonb operators DuckDB lacks on an iceberg-only view: a path into an + # array, and a key and containment test through the ::jsonb cast, the key also + # as a bound parameter. + O=$(qf "$HOST" <<'EOSQL' +INSERT INTO iceonly VALUES (50,date_trunc('month',now()) + interval '10 hours 4 minutes 0 seconds','jsonops','{"k":1,"a":[1,2]}'); +SELECT 'OPS_PATH:'||(data #> '{a,0}')::text FROM iceonly WHERE id=50; +SELECT 'OPS_HASKEY:'||count(*) FROM iceonly WHERE id=50 AND (data::jsonb) ? 'k'; +SELECT 'OPS_CONTAIN:'||((data::jsonb) @> '{"a":[2]}')::text FROM iceonly WHERE id=50; +PREPARE ops_key(text) AS SELECT 'OPS_PKEY:'||count(*) FROM iceonly WHERE id=50 AND (data::jsonb) ? $1; +EXECUTE ops_key('k'); +DELETE FROM iceonly WHERE id=50; +EOSQL +) + assert_eq "decoupled #> into an array" "1" "$(extract OPS_PATH "$O")" + assert_eq "decoupled ? through the ::jsonb cast" "1" "$(extract OPS_HASKEY "$O")" + assert_eq "decoupled @> through the ::jsonb cast" "true" "$(extract OPS_CONTAIN "$O")" + assert_eq "decoupled ? with a bound key" "1" "$(extract OPS_PKEY "$O")" # PostgreSQL 17 and later allow MERGE into a view. if [ "$(q "$HOST" "SHOW server_version_num;")" -ge 170000 ]; then O=$(qf "$HOST" <<'EOSQL' @@ -958,6 +975,19 @@ SELECT 'JSONB_HOT_M:' || (data->>'m') FROM events WHERE ts >= date_trunc('month SELECT 'CAST_COLD:' || ((data::jsonb)->>'m') FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND status='ok' ORDER BY ts LIMIT 1; SELECT 'CAST_HOT:' || ((data::jsonb)->>'m') FROM events WHERE ts >= date_trunc('month',now()) - interval '2 months' AND status='ok' ORDER BY ts LIMIT 1; SELECT 'ALEN_COLD:' || jsonb_array_length('[10,20,30]'::jsonb) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND status='ok' ORDER BY ts LIMIT 1; +-- The json/jsonb operators DuckDB lacks, rewritten into its JSON functions, on +-- rows read from Iceberg. +SELECT 'OP_PATH_COLD:' || (data #>> '{m}') FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND status='ok' ORDER BY ts LIMIT 1; +SELECT 'OP_HASKEY_COLD:' || count(*) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND (data::jsonb) ? 'm'; +SELECT 'OP_ANYKEY_COLD:' || count(*) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND (data::jsonb) ?| '{zz,m}'; +SELECT 'OP_CONTAIN_COLD:' || count(*) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND (data::jsonb) @> '{"m":"m4"}'; +-- The same with the path, key or document as a bound parameter. +PREPARE op_path(text[]) AS SELECT 'OP_PPATH_COLD:' || (data #>> $1) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND status='ok' ORDER BY ts LIMIT 1; +PREPARE op_key(text) AS SELECT 'OP_PKEY_COLD:' || count(*) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND (data::jsonb) ? $1; +PREPARE op_doc(jsonb) AS SELECT 'OP_PDOC_COLD:' || count(*) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND (data::jsonb) @> $1; +EXECUTE op_path('{m}'); +EXECUTE op_key('m'); +EXECUTE op_doc('{"m":"m4"}'); EOSQL ) assert_eq "total rows (hot+cold via view)" "280" "$(extract RO_TOTAL "$O")" @@ -969,6 +999,13 @@ EOSQL assert_eq "::jsonb cast translated, cold" "m4" "$(extract CAST_COLD "$O")" assert_eq "::jsonb cast translated, hot" "m2" "$(extract CAST_HOT "$O")" assert_eq "jsonb_array_length→json_array_length" "3" "$(extract ALEN_COLD "$O")" + assert_eq "#>> on cold rows" "m4" "$(extract OP_PATH_COLD "$O")" + assert_eq "? on cold rows (m4+m3)" "180" "$(extract OP_HASKEY_COLD "$O")" + assert_eq "?| on cold rows (m4+m3)" "180" "$(extract OP_ANYKEY_COLD "$O")" + assert_eq "@> on cold rows (m4 only)" "100" "$(extract OP_CONTAIN_COLD "$O")" + assert_eq "#>> with a bound path on cold rows" "m4" "$(extract OP_PPATH_COLD "$O")" + assert_eq "? with a bound key on cold rows" "180" "$(extract OP_PKEY_COLD "$O")" + assert_eq "@> with a bound document on cold rows" "100" "$(extract OP_PDOC_COLD "$O")" # Hot-tier read routing: a read whose WHERE provably restricts to the hot tier # (ts >= the watermark) is rewritten to the hot heap and runs in plain PostgreSQL, diff --git a/docs/architecture_guides/index.md b/docs/architecture_guides/index.md index 4a93bc98..38d1536e 100644 --- a/docs/architecture_guides/index.md +++ b/docs/architecture_guides/index.md @@ -98,7 +98,7 @@ The following table describes each component, its role, and its license: |-----------|------|---------| | PostgreSQL 16+ | Provides heap storage and range partitioning for the tiered hot tier. ColdFront works uniformly on PG 16, 17, and 18, because the cold-tier secret is a DuckDB persistent secret loaded at instance init, with no version-gated mechanism. | PostgreSQL | | pg_duckdb | Runs DuckDB in-process for Iceberg reads, writes, and analytics; the build is pg_duckdb at commit c04e6a2 (PR #1025) on DuckDB 1.5.4. The bundled `duckdb-iceberg` includes the bakery-aware commit-refresh patch (async parquet overlap, no 409); see [Cold-Write Strategy](#cold-write-strategy-stock-vs-patched-duckdb-iceberg). | MIT | -| coldfront | This PGXS C extension's `post_parse_analyze_hook` rewrites INSERT/UPDATE/DELETE/MERGE on registered views to the correct tier and, on a SELECT DuckDB will run, the spellings DuckDB lacks (`date_bin`, `::jsonb`, the JSON builders); `planner_hook` folds bound parameters into such a read; `ProcessUtility_hook` handles DDL; the hook lazily ATTACHes the Iceberg catalog on the first query touching a tiered view. | PostgreSQL | +| coldfront | This PGXS C extension's `post_parse_analyze_hook` rewrites INSERT/UPDATE/DELETE/MERGE on registered views to the correct tier and, on a SELECT DuckDB will run, the spellings DuckDB lacks (`date_bin`, `::jsonb`, the JSON builders, the JSON operators); `planner_hook` folds bound parameters into such a read; `ProcessUtility_hook` handles DDL; the hook lazily ATTACHes the Iceberg catalog on the first query touching a tiered view. | PostgreSQL | | Lakekeeper | Provides the Iceberg REST catalog as a single Rust binary. | Apache 2.0 | | S3-compatible store or Azure ADLS Gen2 | Stores the cold data; any S3-compatible store works, including SeaweedFS, MinIO, AWS S3, and GCS, and Azure ADLS Gen2 works through `set_storage_secret_azure`. | Varies | | Archiver (tiered mode) | Moves rows from hot to cold; this Go binary is a thin SQL orchestrator that cron invokes. | PostgreSQL | @@ -258,13 +258,19 @@ in the statement the view is named (a CTE, a sub-select, a set-operation branch): `date_bin`, the `::jsonb` cast, `jsonb_array_length` and the JSON builders (`jsonb_build_object`, `jsonb_agg` and their `json_` twins) are rewritten into spellings both engines accept (see -[Supported Column Types](../supported_types.md)). A -`planner_hook` folds bound parameters into such a read before pg_duckdb plans -it when a parameter sits where DuckDB cannot type a placeholder (a direct -argument of a pg_duckdb function, any argument of a table function); the plan -cache's generic-plan build, which has no values, gets a PostgreSQL plan priced -above any custom plan, so under the plan cache's cost-based selection the read -is planned from its values on every execution. +[Supported Column Types](../supported_types.md)). The hook also rewrites the +JSON operators DuckDB lacks (`#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`) into +DuckDB's JSON functions. A `planner_hook` folds bound parameters into such a +read before pg_duckdb plans it when a parameter sits where DuckDB cannot type a +placeholder (a direct argument of a pg_duckdb function, any argument of a table +function); the plan cache's generic-plan build, which has no values, gets a +PostgreSQL plan priced above any custom plan, so under the plan cache's +cost-based selection the read is planned from its values on every execution. +When an operand of those JSON operators holds a bound parameter, the +parse-analyze hook leaves the rewrite to the `planner_hook`. The parse-analyze +hook keeps `#>` and `#>>`, and turns the other operators into +`coldfront._jsonb_*` stand-ins that PostgreSQL can parse. The `planner_hook` +then folds the read's bound parameters and rewrites those operators. `plan_cache_mode = force_generic_plan` bypasses that selection and picks the value-less generic plan, which fails with `only works with DuckDB execution` (see [Supported Column Types](../supported_types.md)). @@ -545,11 +551,12 @@ The cross-cutting limitations are: that references a DuckDB read (all-or-nothing plan takeover), so the cold branch cannot produce a PG `jsonb` directly. The view generator casts `jsonb` columns to DuckDB-safe `json` on both sides: hot emits `"col"::json`, cold - emits `r['col']::json`, the UNION unifies on `json`. Standard JSON access - (`->>`, `->`, `#>`) works without any caller-side cast; jsonb-only operators - (`?`, `@>`, containment, de-dup) need an explicit `data::jsonb` in the - caller's query. Storage in Iceberg remains VARCHAR (Iceberg has no JSON type - either). The cold sink sends the incoming value as its text form. + emits `r['col']::json`, the UNION unifies on `json`. The `->>` and `->` + operators reach DuckDB unchanged. ColdFront's read rewrite translates the + JSON operators that DuckDB lacks. For the operator list and the `data::jsonb` + cast rule, see [Supported Column Types](../supported_types.md). Storage in + Iceberg remains VARCHAR (Iceberg has no JSON type either). The cold sink + sends the incoming value as its text form. - Lakekeeper remote signing may not work with all S3-compatible stores; the workaround is `ACCESS_DELEGATION_MODE NONE` with a direct DuckDB S3 secret. diff --git a/docs/caveats.md b/docs/caveats.md index 9d12dbb8..32be3ca6 100644 --- a/docs/caveats.md +++ b/docs/caveats.md @@ -10,9 +10,13 @@ tables: - `jsonb` reads come back as `json`. The `->`/`->>` operators work, and ColdFront translates the `::jsonb` cast, `jsonb_array_length`, - `jsonb_build_object` / `jsonb_agg` and `date_bin`; other jsonb operators and - functions are unsupported on cold or cross-tier reads. A hot-only read of the - view runs in PostgreSQL with full jsonb (see + `jsonb_build_object` / `jsonb_agg` and `date_bin`. ColdFront also rewrites + the `#>`, `#>>`, `?`, `?|`, `?&`, `@>` and `<@` operators and the functions + behind them. A path or key for these operators must be a constant or a bound + parameter. The `@>` and `<@` operators never match an integer against the + same value written as a decimal. Other jsonb operators and functions are + unsupported on cold or cross-tier reads. A hot-only read of the view runs in + PostgreSQL with full jsonb (see [Supported Column Types](supported_types.md)). - Cross-tier isolation differs from PG-native isolation. The cold tier keeps one Iceberg snapshot for the whole transaction, even at READ COMMITTED, so a diff --git a/docs/changelog.md b/docs/changelog.md index 8e6ab8c8..fefde380 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -355,6 +355,18 @@ this project adheres to it, since DuckDB has `<=>` and `<->` as operators of its own and no `<#>`. The read now spells it as the function DuckDB has under the same name, `list_negative_inner_product`; `<=>` and `<->` are unchanged. +- The JSON operators `#>`, `#>>`, `?`, `?|`, `?&`, `@>` and `<@` failed + whenever DuckDB ran a tiered or decoupled read. DuckDB has no JSON form of + these operators, so `#>` failed with "syntax error at or near #". The read + rewrite turned the documented `data::jsonb` cast into `json`, so `?` failed + with "operator does not exist: json ? text". The read rewrite now turns each + operator, and the function behind each operator, into DuckDB's JSON + functions. A path or key must be a constant or a bound parameter. ColdFront + plans a read from its bound values on every execution when a parameter is an + operand of these operators. The rewritten reads return PostgreSQL's answers + with two exceptions. DuckDB writes a returned container or number in its own + form. The `@>` and `<@` operators never match an integer against the same + value written as a decimal. - Once a `DROP COLUMN` returned a tiered table to an earlier schema, the next column change failed in the Iceberg mirror with "Attempted to add schema with id N, but this already exists in the table!" and rolled back the diff --git a/docs/supported_types.md b/docs/supported_types.md index 67c122f6..5b265a24 100644 --- a/docs/supported_types.md +++ b/docs/supported_types.md @@ -102,12 +102,31 @@ also lacks) and `date_bin` on read: the builders become the `concat` / `to_json` / `array_agg` form both engines evaluate identically (the result is JSON-equal to jsonb's rendering, in compact form and in argument order), and `date_bin` becomes DuckDB's `time_bucket`, which takes the same -arguments and agrees on every fixed-width bucket. The jsonb-only operators -(`?`, `@>`, `<@`, `#>`, `#>>`) and most jsonb functions (`jsonb_typeof`, -`jsonb_extract_path`, `jsonb_extract_path_text`, `jsonb_set`, `jsonb_path_*`, -`jsonb_each`, `jsonb_object_keys`) are not supported on tiered or decoupled -data: DuckDB either lacks them or its same-named function differs in -signature or result. Reach into the document with `->`/`->>`. +arguments and agrees on every fixed-width bucket. + +ColdFront also rewrites the JSON operators that DuckDB lacks into DuckDB's JSON +functions, which return PostgreSQL's answers. The rewrite covers: + +- the `#>`, `#>>`, `?`, `?|`, `?&`, `@>` and `<@` operators. +- the `json_extract_path` and `jsonb_extract_path` functions and their `_text` + forms. +- the `jsonb_exists`, `jsonb_exists_any` and `jsonb_exists_all` functions. +- the `jsonb_contains` and `jsonb_contained` functions. + +A path or key for these operators and functions must be a constant or a bound +parameter, not a column. A path can hold at most eight negative, signed or +zero-padded index elements, such as `-1`, `+1` or `01`. PostgreSQL has no +`json` form of `?`, `?|`, `?&`, `@>` and `<@`, only a `jsonb` one. Cast the +view's `json` column to `jsonb` first, as in `data::jsonb ? 'k'`. Two kinds of +result differ from PostgreSQL's. DuckDB writes a returned container or number +in its own form, such as `{"b":"c"}`, or `1.5` for `1.50`. The `->>` operator +returns values in the same form. The `@>` and `<@` operators never match an +integer against the same value written as a decimal. For example, a stored +`{"p": 10.0}` does not contain `{"p": 10}`, because DuckDB compares JSON +numbers by kind. ColdFront does not support most other jsonb functions on +tiered or decoupled data, such as `jsonb_typeof`, `jsonb_set`, `jsonb_path_*`, +`jsonb_each` and `jsonb_object_keys`. DuckDB either lacks these functions, or +its function of the same name differs in signature or result. These limits apply only to reads that go through a tiered or iceberg-only view, which pg_duckdb plans entirely in DuckDB. Ordinary (non-tiered) PostgreSQL @@ -120,9 +139,11 @@ their context (`ts > $1`); one that is a direct argument of a DuckDB function with several overloads (`time_bucket`'s origin, so `date_bin`'s) or any argument of a table function (`generate_series`) it cannot, so ColdFront plans such a read from the bound values on every execution instead of caching a -generic plan. Under `plan_cache_mode = force_generic_plan` those reads cannot -run: the forced generic plan has no values and fails with -`only works with DuckDB execution`. +generic plan. The rewrite of `#>`, `#>>`, `?`, `?|`, `?&`, `@>` and `<@` also +needs the value of any parameter in an operand. ColdFront plans such a read +from the bound values on every execution as well. Under +`plan_cache_mode = force_generic_plan` those reads cannot run: the forced +generic plan has no values and fails with `only works with DuckDB execution`. A hot-only read is the exception. When a `SELECT` reads a tiered view directly (no join, CTE, or sub-query) and its `WHERE` provably restricts to the hot diff --git a/extension/coldfront/Makefile b/extension/coldfront/Makefile index 3a104c1f..d4bdecc6 100644 --- a/extension/coldfront/Makefile +++ b/extension/coldfront/Makefile @@ -29,7 +29,7 @@ REGRESS = load_order update_unregistered_view update_heap_table \ type_map hot_guard adopt_type_map adopt_read_only adopt_iceberg_table \ cold_write_json_agg duckdb_temp_dir \ vector_centroids vector_type_map vector_cold_render array_cold_render vector_param_render vector_ops vector_probe vector_status vector_multicolumn \ - duckdb_keyword_name partition_clause failed_txn_block tiered_insert_single_pass tiered_view_no_trigger copy_tiered_view param_in_subquery merge_on_view delete_without_where update_from_without_alias settings_registered ensure_replicated read_inner_product \ + duckdb_keyword_name partition_clause failed_txn_block tiered_insert_single_pass tiered_view_no_trigger copy_tiered_view param_in_subquery merge_on_view delete_without_where update_from_without_alias settings_registered ensure_replicated read_inner_product read_json_operators \ async_write_before_claim tiered_insert_stream cold_render_canonical tiered_insert_isolation clustered_append_generation \ whitespace_in_value REGRESS_OPTS = --inputdir=test --outputdir=test diff --git a/extension/coldfront/coldfront--1.0.sql b/extension/coldfront/coldfront--1.0.sql index ba2ad52e..253cb8de 100644 --- a/extension/coldfront/coldfront--1.0.sql +++ b/extension/coldfront/coldfront--1.0.sql @@ -2671,6 +2671,22 @@ BEGIN END; $$; +-- Stand-ins for the jsonb operators ?, ?|, ?& and @> on a read DuckDB runs when an +-- operand holds a bound parameter. The read rewrite turns each operator into +-- DuckDB's JSON functions and needs the operand's value to do so, which only the +-- planner has; until then the stand-in keeps the rewritten read valid for +-- PostgreSQL, which has these operators on jsonb alone where the read has json, +-- and the planner hook rewrites it once the value is bound. Each body gives the +-- operator's own answer. +CREATE OR REPLACE FUNCTION coldfront._jsonb_exists(json, text) +RETURNS boolean LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT $1::jsonb ? $2 $$; +CREATE OR REPLACE FUNCTION coldfront._jsonb_exists_any(json, text[]) +RETURNS boolean LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT $1::jsonb ?| $2 $$; +CREATE OR REPLACE FUNCTION coldfront._jsonb_exists_all(json, text[]) +RETURNS boolean LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT $1::jsonb ?& $2 $$; +CREATE OR REPLACE FUNCTION coldfront._jsonb_contains(json, json) +RETURNS boolean LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT $1::jsonb @> $2::jsonb $$; + -- ============================================================================ -- Canonical text. A value that leaves the caller's session as text (a cold -- write's rendered parameter, a streamed row's text column, a cold VARCHAR diff --git a/extension/coldfront/src/coldfront.c b/extension/coldfront/src/coldfront.c index 61bffd28..9c69c46c 100644 --- a/extension/coldfront/src/coldfront.c +++ b/extension/coldfront/src/coldfront.c @@ -76,7 +76,9 @@ #include "optimizer/optimizer.h" #include "optimizer/planner.h" #include "parser/analyze.h" +#include "parser/parse_collate.h" #include "parser/parse_func.h" +#include "parser/parse_oper.h" #include "parser/parse_type.h" #include "parser/parsetree.h" #include "rewrite/rewriteHandler.h" @@ -90,6 +92,7 @@ #include "utils/builtins.h" #include "utils/datum.h" #include "utils/guc.h" +#include "utils/json.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/snapmgr.h" @@ -288,6 +291,8 @@ static bool coldfront_registry_present(void); /* Defined with it: run a registry read as the extension's owner. */ static int as_coldfront_owner(Oid *save_uid, int *save_sec); static void as_caller(Oid save_uid, int save_sec, int nest); +/* Defined with the planner hook: does a tree hold a bound parameter? */ +static bool has_extern_param_walker(Node *node, void *ctx); /* * Re-entrancy guard for the DDL hook. _rebuild_tiered_view issues CREATE VIEW / @@ -909,8 +914,9 @@ static const CfSubst cf_write_subst[] = { * arguments, pg_duckdb declares it PG-side, and the two agree on every fixed-width * bucket (a month or year width, which date_bin rejects, time_bucket accepts). * The JSON builders (jsonb_build_object, jsonb_agg) need more than a spelling and - * are rewritten on the node tree instead, as is the <#> operator, which DuckDB - * lacks and whose function it has: see cf_json_builder_mutator. + * are rewritten on the node tree instead, as are the <#> operator, which DuckDB + * lacks and whose function it has, and the json/jsonb operators DuckDB lacks (#>, + * #>>, ?, ?|, ?&, @>, <@): see cf_json_builder_mutator. */ static const CfSubst cf_read_subst[] = { { "::jsonb", "::json", false, false }, @@ -1097,7 +1103,9 @@ normalize_for_read(const char *sql) * argument List; the emitted function OIDs come from the catalog, once per backend. * A VARIADIC array argument cannot be paired and is left alone. */ -typedef struct { bool changed; } JsonBuilderCtx; +/* defer: keep a json operator whose operand holds a bound parameter for the planner + * hook (cf_json_operator); false when the planner hook itself rewrites. */ +typedef struct { bool changed; bool defer; } JsonBuilderCtx; static Oid cf_to_json_oid = InvalidOid; static Oid cf_concat_oid = InvalidOid; @@ -1227,6 +1235,298 @@ cf_json_agg(Aggref *agg, Oid elemtype, Oid arrtype) InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL); } +/* ---------- read-path JSON operators ----------------------------------- */ + +/* + * DuckDB has no #>, #>>, ?, ?| or ?& (its parser has no '#' or '?' operator + * character) and no jsonb containment, so each of these operators, and the + * json_/jsonb_ function behind it, becomes calls of the JSON functions pg_duckdb + * declares and hands to DuckDB by name, answering as PostgreSQL does: + * d #> '{a,0}' json_extract(d, '/a/0'): a JSON Pointer token is an array index + * or an object key by what it meets, as a path element is. An + * element PostgreSQL also takes as an index and the pointer does + * not (-1, 01, +1) tries both, COALESCE(json_extract(d, -1), + * json_extract(d, '/-1')). + * d #>> path the same, with json_extract_string as the last step. + * d ? 'k' a top-level key, a top-level string element or the bare string: + * json_exists(d, '/k') OR json_contains(W(d), '{"\u0000":["k"]}') + * OR json_contains(W(d), '{"\u0000":"k"}'). + * d ?| / ?& ks the ? test of each non-NULL key, ORed or ANDed. + * l @> r json_contains(W(l), W(r)), or W(l) holding [r] for a scalar r. + * W(x) is ('{"\u0000":' || x::text || '}')::json. json_contains matches its needle + * at any depth, and under a key no jsonb can hold, only at x's root. It compares + * numbers by kind, so an integer never matches the same value written as a + * decimal (documented). A key the pointer would take as an array index ("0", "-") + * is a quoted JSONPath key instead, which only an object has. The path or keys must + * be a constant when the rewrite runs, so an operator with a bound parameter in an + * operand is rewritten by the planner hook, which has the value (cf_json_operator). + */ +#define CF_JSON_ANCHOR "{\"\\u0000\":" + +/* Most path elements one #> or #>> may hold that PostgreSQL reads as an array index + * and a JSON Pointer does not (-1, 01, +1): the rewrite tries both for each, which + * doubles the expression it builds. */ +#define CF_JSON_EITHER_MAX 8 + +/* A call of nsp.name, resolved as the parser resolves it. */ +static Node * +cf_call(const char *nsp, const char *name, List *args) +{ + List *fname = list_make2(makeString(pstrdup(nsp)), makeString(pstrdup(name))); + + return ParseFuncOrColumn(make_parsestate(NULL), fname, args, NULL, + makeFuncCall(fname, args, COERCE_EXPLICIT_CALL, -1), false, -1); +} + +/* A call of pg_duckdb's function `name`. */ +static Node * +cf_duck_call(const char *name, List *args) +{ + return cf_call(get_namespace_name(get_extension_schema(get_extension_oid("pg_duckdb", false))), + name, args); +} + +/* x as json: a jsonb value is cast. */ +static Node * +cf_as_json(Node *x) +{ + return exprType(x) == JSONOID ? x : cf_cast_via_io(x, JSONOID); +} + +/* l op r, resolved as the parser resolves it. */ +static Node * +cf_op(const char *op, Node *l, Node *r) +{ + return (Node *) make_op(make_parsestate(NULL), + list_make2(makeString("pg_catalog"), makeString(pstrdup(op))), + l, r, NULL, -1); +} + +/* (open || x::text || close)::json */ +static Node * +cf_json_wrap(const char *open, Node *x, const char *close) +{ + return cf_cast_via_io(cf_op("||", cf_op("||", cf_text_const(open), cf_cast_via_io(x, TEXTOID)), + cf_text_const(close)), + JSONOID); +} + +/* True when a JSON Pointer token addresses an array element: "-", "0", "12". */ +static bool +cf_pointer_index(const char *s) +{ + return strcmp(s, "-") == 0 || strcmp(s, "0") == 0 || /* nosemgrep */ + (*s >= '1' && *s <= '9' && strspn(s, "0123456789") == strlen(s)); /* nosemgrep */ +} + +/* Append s as a JSON Pointer token, '~' and '/' escaped. */ +static void +cf_pointer_token(StringInfo buf, const char *s) +{ + appendStringInfoChar(buf, '/'); + for (; *s; s++) + { + if (*s == '~') + appendStringInfoString(buf, "~0"); + else if (*s == '/') + appendStringInfoString(buf, "~1"); + else + appendStringInfoChar(buf, *s); + } +} + +/* d #> path, or d #>> path when as_text, for a path without a NULL element. */ +static Node * +cf_json_path(Node *d, ArrayType *path, bool as_text) +{ + Datum *elems; + bool *nulls; + int n; + int i; + int either = 0; + StringInfoData ptr; + Node *cur = d; + + deconstruct_array_builtin(path, TEXTOID, &elems, &nulls, &n); + initStringInfo(&ptr); + for (i = 0; i < n; i++) + { + char *s = TextDatumGetCString(elems[i]); + const char *fn = (as_text && i == n - 1) ? "json_extract_string" : "json_extract"; + char *end; + int idx; + CoalesceExpr *c; + + errno = 0; + idx = strtoint(s, &end, 10); + if (end == s || *end != '\0' || errno != 0 || cf_pointer_index(s)) + { + cf_pointer_token(&ptr, s); + continue; + } + if (++either > CF_JSON_EITHER_MAX) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("coldfront: JSON path holds more than %d elements that may be an " + "array index or an object key", CF_JSON_EITHER_MAX))); + if (ptr.len > 0) + cur = cf_duck_call("json_extract", list_make2(cur, cf_text_const(ptr.data))); + resetStringInfo(&ptr); + cf_pointer_token(&ptr, s); + c = makeNode(CoalesceExpr); + c->args = list_make2(cf_duck_call(fn, list_make2(cur, makeConst(INT4OID, -1, InvalidOid, + sizeof(int32), Int32GetDatum(idx), + false, true))), + cf_duck_call(fn, list_make2(copyObject(cur), cf_text_const(ptr.data)))); + c->coalescetype = exprType(linitial(c->args)); + c->coalescecollid = exprCollation(linitial(c->args)); + c->location = -1; + cur = (Node *) c; + resetStringInfo(&ptr); + } + if (ptr.len > 0 || cur == d) + cur = cf_duck_call(as_text ? "json_extract_string" : "json_extract", + list_make2(cur, cf_text_const(ptr.data))); + return as_text ? cf_cast_via_io(cur, TEXTOID) : cur; +} + +/* d ? k */ +static Node * +cf_json_has_key(Node *d, const char *k) +{ + StringInfoData path; + StringInfoData key; + + initStringInfo(&path); + if (cf_pointer_index(k)) + appendStringInfo(&path, "$.\"%s\"", k); + else + cf_pointer_token(&path, k); + initStringInfo(&key); + escape_json(&key, k); + return (Node *) makeBoolExpr(OR_EXPR, list_make3( + cf_duck_call("json_exists", list_make2(d, cf_text_const(path.data))), + cf_duck_call("json_contains", + list_make2(cf_json_wrap(CF_JSON_ANCHOR, copyObject(d), "}"), + cf_cast_via_io(cf_text_const(psprintf(CF_JSON_ANCHOR "[%s]}", key.data)), + JSONOID))), + cf_duck_call("json_contains", + list_make2(cf_json_wrap(CF_JSON_ANCHOR, copyObject(d), "}"), + cf_cast_via_io(cf_text_const(psprintf(CF_JSON_ANCHOR "%s}", key.data)), + JSONOID)))), -1); +} + +/* d ?| keys, or d ?& keys when all. */ +static Node * +cf_json_has_keys(Node *d, ArrayType *keys, bool all) +{ + Datum *elems; + bool *nulls; + int n; + int i; + List *tests = NIL; + NullTest *t; + + deconstruct_array_builtin(keys, TEXTOID, &elems, &nulls, &n); + for (i = 0; i < n; i++) + if (!nulls[i]) + tests = lappend(tests, cf_json_has_key(copyObject(d), TextDatumGetCString(elems[i]))); + if (tests != NIL) + return (Node *) makeBoolExpr(all ? AND_EXPR : OR_EXPR, tests, -1); + /* No key: false for ?| (d IS NULL AND NULL), true for ?& (d IS NOT NULL OR + * NULL), and NULL for a NULL document. */ + t = makeNode(NullTest); + t->arg = (Expr *) d; + t->nulltesttype = all ? IS_NOT_NULL : IS_NULL; + t->location = -1; + return (Node *) makeBoolExpr(all ? OR_EXPR : AND_EXPR, + list_make2(t, makeBoolConst(false, true)), -1); +} + +/* l @> r */ +static Node * +cf_json_contains(Node *l, Node *r) +{ + Node *in_array = cf_duck_call("json_contains", + list_make2(cf_json_wrap(CF_JSON_ANCHOR, copyObject(l), "}"), + cf_json_wrap(CF_JSON_ANCHOR "[", copyObject(r), "]}"))); + List *scalar = list_make3(cf_op("<>", cf_duck_call("json_type", list_make1(copyObject(r))), + cf_text_const("OBJECT")), + cf_op("<>", cf_duck_call("json_type", list_make1(copyObject(r))), + cf_text_const("ARRAY")), + in_array); + + return (Node *) makeBoolExpr(OR_EXPR, list_make2( + cf_duck_call("json_contains", list_make2(cf_json_wrap(CF_JSON_ANCHOR, l, "}"), + cf_json_wrap(CF_JSON_ANCHOR, r, "}"))), + makeBoolExpr(AND_EXPR, scalar, -1)), -1); +} + +/* + * The rewrite of a call of the function behind a json/jsonb operator DuckDB lacks, + * or NULL. opno is the operator the call came from, if any, to name in an error. + * The rewrite needs the value of an operand that holds a bound parameter. With + * defer, such a call becomes the json operator (#>, #>>) or a coldfront._jsonb_* + * stand-in, which survive the reparse, and the planner hook, which has the value, + * brings it back here (cf_json_waits). + */ +static Node * +cf_json_operator(Oid funcid, Oid opno, List *args, Oid restype, bool defer) +{ + Oid ns = get_func_namespace(funcid); + const char *f = get_func_name(funcid); + bool path; + bool as_text; + bool contains; + Const *c; + + if (list_length(args) != 2) + return NULL; + if (ns == get_namespace_oid("coldfront", true) && *f == '_') + f++; /* a stand-in for the function it names */ + else if (ns != PG_CATALOG_NAMESPACE) + return NULL; + path = strcmp(f, "json_extract_path") == 0 || strcmp(f, "jsonb_extract_path") == 0; /* nosemgrep */ + as_text = strcmp(f, "json_extract_path_text") == 0 || /* nosemgrep */ + strcmp(f, "jsonb_extract_path_text") == 0; /* nosemgrep */ + contains = strcmp(f, "jsonb_contains") == 0 || strcmp(f, "jsonb_contained") == 0; /* nosemgrep */ + if (!path && !as_text && !contains && strcmp(f, "jsonb_exists") != 0 && /* nosemgrep */ + strcmp(f, "jsonb_exists_any") != 0 && strcmp(f, "jsonb_exists_all") != 0) /* nosemgrep */ + return NULL; + if (strcmp(f, "jsonb_contained") == 0) /* nosemgrep */ + { + args = list_make2(lsecond(args), linitial(args)); + f = "jsonb_contains"; + } + if (defer && has_extern_param_walker((Node *) args, NULL)) + { + if (path || as_text) + return cf_op(as_text ? "#>>" : "#>", cf_as_json(linitial(args)), lsecond(args)); + return cf_call("coldfront", psprintf("_%s", f), + list_make2(cf_as_json(linitial(args)), + contains ? cf_as_json(lsecond(args)) : lsecond(args))); + } + args = (List *) eval_const_expressions(NULL, (Node *) copyObject(args)); + if (contains) + return cf_json_contains(linitial(args), lsecond(args)); + c = (Const *) lsecond(args); + if (!IsA(c, Const)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("%s on a tiered or decoupled view needs its path or key as a constant " + "or a bound parameter", + OidIsValid(opno) ? get_opname(opno) : f))); + if (c->constisnull || + ((path || as_text) && array_contains_nulls(DatumGetArrayTypeP(c->constvalue)))) + return (Node *) makeNullConst(restype, -1, InvalidOid); + if (path || as_text) + return cf_json_path(linitial(args), DatumGetArrayTypeP(c->constvalue), as_text); + if (strcmp(f, "jsonb_exists") == 0) /* nosemgrep */ + return cf_json_has_key(linitial(args), TextDatumGetCString(c->constvalue)); + return cf_json_has_keys(linitial(args), DatumGetArrayTypeP(c->constvalue), + strcmp(f, "jsonb_exists_all") == 0); /* nosemgrep */ +} + static Node * cf_json_builder_mutator(Node *node, void *ctx) { @@ -1239,6 +1539,7 @@ cf_json_builder_mutator(Node *node, void *ctx) if (IsA(node, FuncExpr)) { FuncExpr *f = (FuncExpr *) expression_tree_mutator(node, cf_json_builder_mutator, ctx); + Node *rw; if (!f->funcvariadic && cf_is_pg_catalog_func(f->funcid, "jsonb_build_object", "json_build_object")) @@ -1247,6 +1548,13 @@ cf_json_builder_mutator(Node *node, void *ctx) jc->changed = true; return cf_json_object(f->args); } + if ((rw = cf_json_operator(f->funcid, InvalidOid, f->args, f->funcresulttype, + jc->defer)) != NULL) + { + assign_expr_collations(make_parsestate(NULL), rw); + jc->changed = true; + return rw; + } return (Node *) f; } if (IsA(node, Aggref)) @@ -1272,6 +1580,7 @@ cf_json_builder_mutator(Node *node, void *ctx) { OpExpr *op = (OpExpr *) expression_tree_mutator(node, cf_json_builder_mutator, ctx); char *fname = get_func_name(op->opfuncid); + Node *rw; /* pg_duckdb hands an operator to DuckDB by its symbol. DuckDB has <=> and * <-> as aliases of its own list_cosine_distance and list_distance and has @@ -1285,6 +1594,13 @@ cf_json_builder_mutator(Node *node, void *ctx) op->opcollid, op->inputcollid, COERCE_EXPLICIT_CALL); } + if ((rw = cf_json_operator(op->opfuncid, op->opno, op->args, op->opresulttype, + jc->defer)) != NULL) + { + assign_expr_collations(make_parsestate(NULL), rw); + jc->changed = true; + return rw; + } return (Node *) op; } return expression_tree_mutator(node, cf_json_builder_mutator, ctx); @@ -3251,10 +3567,11 @@ cf_try_reroute_hot_read(Query *query) /* * Make a read DuckDB will run acceptable to it. A query against a tiered / * iceberg-only view runs entirely in DuckDB (the view body reads the Iceberg - * table), which has no jsonb type, no date_bin, no JSON builders and no <#> - * operator. Two passes over the analysed tree: the JSON builders and the <#> - * operator are rewritten on the node tree (cf_json_builder_mutator, where - * key/value pairing is exact and the operator's function is known), then the deparsed + * table), which has no jsonb type, no date_bin, no JSON builders, no <#> and none + * of the json/jsonb operators #>, #>>, ?, ?|, ?&, @>, <@. Two passes over the + * analysed tree: the JSON builders and those operators are rewritten on the node + * tree (cf_json_builder_mutator, where key/value pairing is exact and the + * operator's function and constant operand are known), then the deparsed * text gets the read whitelist (normalize_for_read: the ::jsonb cast and the * functions verified equivalent in both engines), and the result is reparsed in * place. Nothing to rewrite ⇒ the query is left untouched (the common case: ->>/-> @@ -3264,7 +3581,7 @@ cf_try_reroute_hot_read(Query *query) static void cf_normalize_read(Query *query) { - JsonBuilderCtx jc = { false }; + JsonBuilderCtx jc = { false, true }; Query *tree = query_tree_mutator(query, cf_json_builder_mutator, &jc, 0); char *sql = pg_get_querydef(tree, false); char *norm = normalize_for_read(sql); @@ -6283,17 +6600,20 @@ register_gucs(void) * placeholders from their context (ts > $1), but not one that is a direct argument * of a DuckDB function with several overloads (time_bucket's origin) or any argument * of a table function (generate_series), so such a prepared tiered read fails to - * plan. The values are known here: the plan cache hands the planner bound_params for - * every custom plan, and a custom plan serves exactly that execution, so when a - * parameter sits in one of those two places every parameter is folded to a Const - * before pg_duckdb plans (what eval_const_expressions would do later for a - * PostgreSQL plan). The generic-plan build carries no values; it is answered with a - * PostgreSQL plan priced above any custom plan, so the cache keeps choosing the - * value-bearing custom plans and never runs the generic one (plan_cache_mode = - * force_generic_plan does run it, and pg_duckdb's read functions then refuse - * PostgreSQL execution). A read whose parameters DuckDB types on its own is left - * alone and keeps its generic plan. A DuckDB function is recognised as one the - * pg_duckdb extension owns: every function it declares stands for a DuckDB one. + * plan, and a json operator the read rewrite kept for a parameter's value + * (cf_json_operator) cannot reach DuckDB until it is rewritten. The values are known + * here: the plan cache hands the planner bound_params for every custom plan, and a + * custom plan serves exactly that execution, so when a parameter sits in one of + * those three places every parameter is folded to a Const before pg_duckdb plans + * (what eval_const_expressions would do later for a PostgreSQL plan), and the read + * rewrite runs on the json operators it kept. The generic-plan build carries no + * values; it is answered with a PostgreSQL plan priced above any custom plan, so + * the cache keeps choosing the value-bearing custom plans and never runs the + * generic one (plan_cache_mode = force_generic_plan does run it, and pg_duckdb's + * read functions then refuse PostgreSQL execution). A read whose parameters DuckDB + * types on its own is left alone and keeps its generic plan. A DuckDB function is + * recognised as one the pg_duckdb extension owns: every function it declares stands + * for a DuckDB one. */ #define CF_GENERIC_PLAN_COST 1.0e10 @@ -6319,7 +6639,26 @@ has_extern_param_walker(Node *node, void *ctx) return expression_tree_walker(node, has_extern_param_walker, ctx); } -/* True if a PARAM_EXTERN sits where DuckDB cannot type a placeholder. */ +/* True for a call the read rewrite kept for its parameter's value: a + * coldfront._jsonb_* stand-in, or #> / #>> with an operand that holds a bound + * parameter (cf_json_operator). */ +static bool +cf_json_waits(Oid funcid, List *args) +{ + Oid ns = get_func_namespace(funcid); + const char *f = get_func_name(funcid); + + if (f == NULL) + return false; + if (ns == get_namespace_oid("coldfront", true)) + return strncmp(f, "_jsonb_", 7) == 0; /* nosemgrep */ + return ns == PG_CATALOG_NAMESPACE && + (strcmp(f, "json_extract_path") == 0 || strcmp(f, "json_extract_path_text") == 0) && /* nosemgrep */ + has_extern_param_walker((Node *) args, NULL); +} + +/* True if a PARAM_EXTERN sits where DuckDB cannot type a placeholder, or where the + * read rewrite needs its value. */ static bool param_needs_value_walker(Node *node, void *ctx) { @@ -6329,6 +6668,9 @@ param_needs_value_walker(Node *node, void *ctx) return false; if (IsA(node, Param)) return nv->in_table_func && is_extern_param(node); + if ((IsA(node, OpExpr) && cf_json_waits(((OpExpr *) node)->opfuncid, ((OpExpr *) node)->args)) || + (IsA(node, FuncExpr) && cf_json_waits(((FuncExpr *) node)->funcid, ((FuncExpr *) node)->args))) + return true; if (IsA(node, RangeTblEntry)) { RangeTblEntry *rte = (RangeTblEntry *) node; @@ -6404,7 +6746,8 @@ coldfront_planner(Query *parse, const char *query_string, int cursor_options, if (OidIsValid(nv.duckdb_ext) && query_tree_walker(parse, param_needs_value_walker, &nv, QTW_EXAMINE_RTES_BEFORE)) { - FoldParamsCtx fc = { bound_params }; + FoldParamsCtx fc = { bound_params }; + JsonBuilderCtx jc = { false, false }; if (bound_params == NULL) { @@ -6414,6 +6757,7 @@ coldfront_planner(Query *parse, const char *query_string, int cursor_options, return generic; } parse = query_tree_mutator(parse, fold_params_mutator, &fc, 0); + parse = query_tree_mutator(parse, cf_json_builder_mutator, &jc, 0); } } if (prev_planner_hook) diff --git a/extension/coldfront/test/expected/read_json_operators.out b/extension/coldfront/test/expected/read_json_operators.out new file mode 100644 index 00000000..c0f9c04e --- /dev/null +++ b/extension/coldfront/test/expected/read_json_operators.out @@ -0,0 +1,697 @@ +-- The json/jsonb operators DuckDB lacks (#>, #>>, ?, ?|, ?&, @>, <@) on a read +-- DuckDB runs. DuckDB's parser has no # or ? operator character and DuckDB has no +-- containment operator, so the read rewrite turns each into the JSON functions +-- pg_duckdb hands to DuckDB (json_extract, json_extract_string, json_exists, +-- json_contains, json_type). White-box: DuckDB executes the rewritten read against +-- the heap (duckdb.force_execution) through a view that casts jsonb to json, as a +-- tiered view does, and PostgreSQL runs the same operators on the heap; no +-- Iceberg I/O. +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; +SET TIME ZONE 'UTC'; +-- White-box: checks the generated SQL, not Iceberg I/O. +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +CREATE TABLE public._docs (id int, ts timestamptz, data jsonb); +CREATE VIEW public.docs AS SELECT id, ts, data::json AS data FROM public._docs; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'docs', 'public._docs', 'ice.default.docs', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'docs', '2026-03-01'::timestamptz); +-- An object whose keys a JSON Pointer or a JSONPath would misread ("-1", "01", +-- "", "a/b", "t~", "*", a quote, a backslash), an array, a bare string, a bare +-- number and SQL NULL. +INSERT INTO public._docs VALUES + (1, '2026-01-01', '{"k": 1, "a": [1, 2, 3], "o": {"b": "c"}, "-1": "neg", "01": "pad", "": "empty", "a/b": "slash", "t~": "tilde", "*": "star", "q\"k": "quote", "b\\s": "bslash", "n": null}'), + (2, '2026-01-02', '["k", ["k"], 1, "0", {"k": 1}, null]'), + (3, '2026-01-03', '"k"'), + (4, '2026-01-04', '5'), + (5, '2026-01-05', NULL); +-- (A) A path into an array, and a key test through the ::jsonb cast. +SET duckdb.force_execution = true; +SELECT data #> '{a,0}' AS a0 FROM public.docs WHERE id = 1; + a0 +---- + 1 +(1 row) + +SELECT id FROM public.docs WHERE (data::jsonb) ? 'k' ORDER BY id; + id +---- + 1 + 2 + 3 +(3 rows) + +RESET duckdb.force_execution; +-- (B) #> and #>>. A path element reaches an array element or an object key, as +-- PostgreSQL's does: a negative, zero-padded or signed index counts on an array +-- and names a key on an object. +SET duckdb.force_execution = true; +SELECT id, data #>> '{a,0}' AS a0, data #>> '{a,-1}' AS alast, data #>> '{a,01}' AS apad, + data #>> '{a,+1}' AS asign, data #>> '{a,-4}' AS aout, data #>> '{-1}' AS neg, + data #>> '{01}' AS pad, data #>> '{""}' AS empty, data #>> '{a/b}' AS slash, + data #>> '{t~}' AS tilde, data #>> '{*}' AS star, data #>> ARRAY['q"k'] AS quote, + data #>> ARRAY['b\s'] AS bslash, data #>> '{o,b}' AS ob, data #>> '{4,k}' AS nested, + data #>> '{n}' AS jnull, data #>> '{a,NULL}' AS nullelem, data #> '{0}' AS first + FROM public.docs ORDER BY id; + id | a0 | alast | apad | asign | aout | neg | pad | empty | slash | tilde | star | quote | bslash | ob | nested | jnull | nullelem | first +----+----+-------+------+-------+------+-----+-------+-------+-------+-------+------+-------+--------+----+--------+-------+----------+------- + 1 | 1 | 3 | 2 | 2 | | neg | pad | empty | slash | tilde | star | quote | bslash | c | | | | + 2 | | | | | | | ["k"] | | | | | | | | 1 | | | "k" + 3 | | | | | | | | | | | | | | | | | | + 4 | | | | | | | | | | | | | | | | | | + 5 | | | | | | | | | | | | | | | | | | +(5 rows) + +RESET duckdb.force_execution; +SELECT id, data #>> '{a,0}' AS a0, data #>> '{a,-1}' AS alast, data #>> '{a,01}' AS apad, + data #>> '{a,+1}' AS asign, data #>> '{a,-4}' AS aout, data #>> '{-1}' AS neg, + data #>> '{01}' AS pad, data #>> '{""}' AS empty, data #>> '{a/b}' AS slash, + data #>> '{t~}' AS tilde, data #>> '{*}' AS star, data #>> ARRAY['q"k'] AS quote, + data #>> ARRAY['b\s'] AS bslash, data #>> '{o,b}' AS ob, data #>> '{4,k}' AS nested, + data #>> '{n}' AS jnull, data #>> '{a,NULL}' AS nullelem, data #> '{0}' AS first + FROM public._docs ORDER BY id; + id | a0 | alast | apad | asign | aout | neg | pad | empty | slash | tilde | star | quote | bslash | ob | nested | jnull | nullelem | first +----+----+-------+------+-------+------+-----+-------+-------+-------+-------+------+-------+--------+----+--------+-------+----------+------- + 1 | 1 | 3 | 2 | 2 | | neg | pad | empty | slash | tilde | star | quote | bslash | c | | | | + 2 | | | | | | | ["k"] | | | | | | | | 1 | | | "k" + 3 | | | | | | | | | | | | | | | | | | + 4 | | | | | | | | | | | | | | | | | | + 5 | | | | | | | | | | | | | | | | | | +(5 rows) + +-- An empty path is the whole value; DuckDB renders a container compactly, as it +-- does for ->. +SET duckdb.force_execution = true; +SELECT id, data #>> '{}' AS whole, data #> '{o}' AS o FROM public.docs WHERE id IN (1, 3) ORDER BY id; + id | whole | o +----+--------------------------------------------------------------------------------------------------------------------------------------------------+----------- + 1 | {"":"empty","*":"star","a":[1,2,3],"k":1,"n":null,"o":{"b":"c"},"-1":"neg","01":"pad","t~":"tilde","a/b":"slash","b\\s":"bslash","q\"k":"quote"} | {"b":"c"} + 3 | k | +(2 rows) + +RESET duckdb.force_execution; +SELECT id, data #>> '{}' AS whole, data #> '{o}' AS o FROM public._docs WHERE id IN (1, 3) ORDER BY id; + id | whole | o +----+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------ + 1 | {"": "empty", "*": "star", "a": [1, 2, 3], "k": 1, "n": null, "o": {"b": "c"}, "-1": "neg", "01": "pad", "t~": "tilde", "a/b": "slash", "b\\s": "bslash", "q\"k": "quote"} | {"b": "c"} + 3 | k | +(2 rows) + +-- (C) ?, ?| and ?&: a top-level key, a top-level string element or the bare +-- string itself, never a number and never a nested key. NULL keys are skipped. +SET duckdb.force_execution = true; +SELECT id, (data::jsonb) ? 'k' AS k, (data::jsonb) ? '0' AS zero, (data::jsonb) ? '-1' AS neg, + (data::jsonb) ? '' AS empty, (data::jsonb) ? '*' AS star, (data::jsonb) ? 'a/b' AS slash, + (data::jsonb) ? 'q"k' AS quote, (data::jsonb) ? 'b\s' AS bslash, (data::jsonb) ? '5' AS five, + (data::jsonb) ?| '{zz,k}' AS any_k, (data::jsonb) ?| '{zz}' AS any_zz, + (data::jsonb) ?| '{}' AS any_none, (data::jsonb) ?& ARRAY['k', 'a'] AS all_ka, + (data::jsonb) ?& '{k,NULL}' AS all_knull, (data::jsonb) ?& '{}' AS all_none + FROM public.docs ORDER BY id; + id | k | zero | neg | empty | star | slash | quote | bslash | five | any_k | any_zz | any_none | all_ka | all_knull | all_none +----+---+------+-----+-------+------+-------+-------+--------+------+-------+--------+----------+--------+-----------+---------- + 1 | t | f | t | t | t | t | t | t | f | t | f | f | t | t | t + 2 | t | t | f | f | f | f | f | f | f | t | f | f | f | t | t + 3 | t | f | f | f | f | f | f | f | f | t | f | f | f | t | t + 4 | f | f | f | f | f | f | f | f | f | f | f | f | f | f | t + 5 | | | | | | | | | | | | | | | +(5 rows) + +RESET duckdb.force_execution; +SELECT id, (data::jsonb) ? 'k' AS k, (data::jsonb) ? '0' AS zero, (data::jsonb) ? '-1' AS neg, + (data::jsonb) ? '' AS empty, (data::jsonb) ? '*' AS star, (data::jsonb) ? 'a/b' AS slash, + (data::jsonb) ? 'q"k' AS quote, (data::jsonb) ? 'b\s' AS bslash, (data::jsonb) ? '5' AS five, + (data::jsonb) ?| '{zz,k}' AS any_k, (data::jsonb) ?| '{zz}' AS any_zz, + (data::jsonb) ?| '{}' AS any_none, (data::jsonb) ?& ARRAY['k', 'a'] AS all_ka, + (data::jsonb) ?& '{k,NULL}' AS all_knull, (data::jsonb) ?& '{}' AS all_none + FROM public._docs ORDER BY id; + id | k | zero | neg | empty | star | slash | quote | bslash | five | any_k | any_zz | any_none | all_ka | all_knull | all_none +----+---+------+-----+-------+------+-------+-------+--------+------+-------+--------+----------+--------+-----------+---------- + 1 | t | f | t | t | t | t | t | t | f | t | f | f | t | t | t + 2 | t | t | f | f | f | f | f | f | f | t | f | f | f | t | t + 3 | t | f | f | f | f | f | f | f | f | t | f | f | f | t | t + 4 | f | f | f | f | f | f | f | f | f | f | f | f | f | f | t + 5 | | | | | | | | | | | | | | | +(5 rows) + +-- (D) @> and <@: containment from the top level down, an array containing a +-- bare scalar, and nothing found deeper than the top. +SET duckdb.force_execution = true; +SELECT id, (data::jsonb) @> '{"o": {"b": "c"}}' AS nested, (data::jsonb) @> '{"b": "c"}' AS deeper, + (data::jsonb) @> '"k"' AS scalar, (data::jsonb) @> '["k"]' AS arr, + (data::jsonb) @> '[["k"]]' AS arrarr, (data::jsonb) @> '{"a": [3, 1]}' AS subset, + (data::jsonb) @> '{}' AS empty_obj, (data::jsonb) @> '[]' AS empty_arr, + (data::jsonb) <@ '["k", ["k"], 1, "0", {"k": 1}, null, "extra"]' AS within + FROM public.docs ORDER BY id; + id | nested | deeper | scalar | arr | arrarr | subset | empty_obj | empty_arr | within +----+--------+--------+--------+-----+--------+--------+-----------+-----------+-------- + 1 | t | f | f | f | f | t | t | f | f + 2 | f | f | t | t | t | f | f | t | t + 3 | f | f | t | f | f | f | f | f | t + 4 | f | f | f | f | f | f | f | f | f + 5 | | | | | | | | | +(5 rows) + +RESET duckdb.force_execution; +SELECT id, (data::jsonb) @> '{"o": {"b": "c"}}' AS nested, (data::jsonb) @> '{"b": "c"}' AS deeper, + (data::jsonb) @> '"k"' AS scalar, (data::jsonb) @> '["k"]' AS arr, + (data::jsonb) @> '[["k"]]' AS arrarr, (data::jsonb) @> '{"a": [3, 1]}' AS subset, + (data::jsonb) @> '{}' AS empty_obj, (data::jsonb) @> '[]' AS empty_arr, + (data::jsonb) <@ '["k", ["k"], 1, "0", {"k": 1}, null, "extra"]' AS within + FROM public._docs ORDER BY id; + id | nested | deeper | scalar | arr | arrarr | subset | empty_obj | empty_arr | within +----+--------+--------+--------+-----+--------+--------+-----------+-----------+-------- + 1 | t | f | f | f | f | t | t | f | f + 2 | f | f | t | t | t | f | f | t | t + 3 | f | f | t | f | f | f | f | f | t + 4 | f | f | f | f | f | f | f | f | f + 5 | | | | | | | | | +(5 rows) + +-- DuckDB compares JSON numbers by kind, so an integer never contains or matches +-- the same value written as a decimal, which PostgreSQL's numeric comparison +-- does (a documented difference). +INSERT INTO public._docs VALUES (6, '2026-01-06', '{"p": 10.0}'); +SET duckdb.force_execution = true; +SELECT (data::jsonb) @> '{"p": 10.0}' AS same_kind, (data::jsonb) @> '{"p": 10}' AS int_in_decimal + FROM public.docs WHERE id = 6; + same_kind | int_in_decimal +-----------+---------------- + t | f +(1 row) + +RESET duckdb.force_execution; +SELECT (data::jsonb) @> '{"p": 10.0}' AS same_kind, (data::jsonb) @> '{"p": 10}' AS int_in_decimal + FROM public._docs WHERE id = 6; + same_kind | int_in_decimal +-----------+---------------- + t | t +(1 row) + +DELETE FROM public._docs WHERE id = 6; +-- (E) The function spellings of the same operators. +SET duckdb.force_execution = true; +SELECT id, json_extract_path_text(data, 'a', '-1') AS f1, jsonb_extract_path_text(data::jsonb, 'o', 'b') AS f2, + jsonb_exists(data::jsonb, 'k') AS f3 + FROM public.docs ORDER BY id; + id | f1 | f2 | f3 +----+----+----+---- + 1 | 3 | c | t + 2 | | | t + 3 | | | t + 4 | | | f + 5 | | | +(5 rows) + +RESET duckdb.force_execution; +SELECT id, json_extract_path_text(data::json, 'a', '-1') AS f1, jsonb_extract_path_text(data, 'o', 'b') AS f2, + jsonb_exists(data, 'k') AS f3 + FROM public._docs ORDER BY id; + id | f1 | f2 | f3 +----+----+----+---- + 1 | 3 | c | t + 2 | | | t + 3 | | | t + 4 | | | f + 5 | | | +(5 rows) + +-- (F) A path or key set taken from a column cannot be rewritten, and says so. +SELECT data #> ARRAY[id::text] FROM public.docs; +ERROR: #> on a tiered or decoupled view needs its path or key as a constant or a bound parameter +SELECT (data::jsonb) ?| ARRAY[id::text] FROM public.docs; +ERROR: ?| on a tiered or decoupled view needs its path or key as a constant or a bound parameter +-- (G) A read the hot heap can answer keeps the operator: it never reaches DuckDB. +EXPLAIN (COSTS OFF, VERBOSE) +SELECT data #> '{a,0}' FROM public.docs WHERE ts >= '2026-04-01'::timestamptz; + QUERY PLAN +---------------------------------------------------------------------------------- + Seq Scan on public._docs + Output: (data #> '{a,0}'::text[]) + Filter: (_docs.ts >= 'Wed Apr 01 00:00:00 2026 UTC'::timestamp with time zone) +(3 rows) + +-- (H) A bound parameter as the path or keys. The rewrite needs its value, so the +-- read is planned from the value on every execution; seven executions cross the +-- plan cache's switch from custom to generic planning after the fifth. The heap +-- answers the same statements. +SET duckdb.force_execution = true; +PREPARE jpath(text[]) AS + SELECT string_agg(coalesce(data #>> $1, '-'), ',' ORDER BY id) AS v FROM public.docs; +PREPARE jkey(text) AS + SELECT string_agg(coalesce(((data::jsonb) ? $1)::text, '-'), ',' ORDER BY id) AS v FROM public.docs; +PREPARE jkeys(text[]) AS + SELECT string_agg(coalesce(((data::jsonb) ?| $1)::text, '-') || '/' || + coalesce(((data::jsonb) ?& $1)::text, '-'), ',' ORDER BY id) AS v FROM public.docs; +PREPARE jdoc(jsonb) AS + SELECT string_agg(coalesce(((data::jsonb) @> $1)::text, '-'), ',' ORDER BY id) AS v FROM public.docs; +EXECUTE jpath('{a,0}'); + v +----------- + 1,-,-,-,- +(1 row) + +EXECUTE jpath('{a,-1}'); + v +----------- + 3,-,-,-,- +(1 row) + +EXECUTE jpath('{a,01}'); + v +----------- + 2,-,-,-,- +(1 row) + +EXECUTE jpath('{-1}'); + v +------------- + neg,-,-,-,- +(1 row) + +EXECUTE jpath('{o,b}'); + v +----------- + c,-,-,-,- +(1 row) + +EXECUTE jpath('{4,k}'); + v +----------- + -,1,-,-,- +(1 row) + +EXECUTE jpath('{""}'); + v +--------------- + empty,-,-,-,- +(1 row) + +EXECUTE jkey('k'); + v +------------------------ + true,true,true,false,- +(1 row) + +EXECUTE jkey('0'); + v +-------------------------- + false,true,false,false,- +(1 row) + +EXECUTE jkey('-1'); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE jkey(''); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE jkey('*'); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE jkey('q"k'); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE jkey('zz'); + v +--------------------------- + false,false,false,false,- +(1 row) + +EXECUTE jkeys('{zz,k}'); + v +-------------------------------------------------- + true/false,true/false,true/false,false/false,-/- +(1 row) + +EXECUTE jkeys('{k,a}'); + v +------------------------------------------------- + true/true,true/false,true/false,false/false,-/- +(1 row) + +EXECUTE jkeys('{}'); + v +------------------------------------------------- + false/true,false/true,false/true,false/true,-/- +(1 row) + +EXECUTE jkeys('{k,NULL}'); + v +----------------------------------------------- + true/true,true/true,true/true,false/false,-/- +(1 row) + +EXECUTE jkeys('{zz}'); + v +----------------------------------------------------- + false/false,false/false,false/false,false/false,-/- +(1 row) + +EXECUTE jkeys('{a}'); + v +--------------------------------------------------- + true/true,false/false,false/false,false/false,-/- +(1 row) + +EXECUTE jkeys('{k}'); + v +----------------------------------------------- + true/true,true/true,true/true,false/false,-/- +(1 row) + +EXECUTE jdoc('{"o": {"b": "c"}}'); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE jdoc('"k"'); + v +------------------------- + false,true,true,false,- +(1 row) + +EXECUTE jdoc('[["k"]]'); + v +-------------------------- + false,true,false,false,- +(1 row) + +EXECUTE jdoc('{"b": "c"}'); + v +--------------------------- + false,false,false,false,- +(1 row) + +EXECUTE jdoc('{}'); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE jdoc('[]'); + v +-------------------------- + false,true,false,false,- +(1 row) + +EXECUTE jdoc('{"a": [3, 1]}'); + v +-------------------------- + true,false,false,false,- +(1 row) + +RESET duckdb.force_execution; +PREPARE hpath(text[]) AS + SELECT string_agg(coalesce(data #>> $1, '-'), ',' ORDER BY id) AS v FROM public._docs; +PREPARE hkey(text) AS + SELECT string_agg(coalesce(((data::jsonb) ? $1)::text, '-'), ',' ORDER BY id) AS v FROM public._docs; +PREPARE hkeys(text[]) AS + SELECT string_agg(coalesce(((data::jsonb) ?| $1)::text, '-') || '/' || + coalesce(((data::jsonb) ?& $1)::text, '-'), ',' ORDER BY id) AS v FROM public._docs; +PREPARE hdoc(jsonb) AS + SELECT string_agg(coalesce(((data::jsonb) @> $1)::text, '-'), ',' ORDER BY id) AS v FROM public._docs; +EXECUTE hpath('{a,0}'); + v +----------- + 1,-,-,-,- +(1 row) + +EXECUTE hpath('{a,-1}'); + v +----------- + 3,-,-,-,- +(1 row) + +EXECUTE hpath('{a,01}'); + v +----------- + 2,-,-,-,- +(1 row) + +EXECUTE hpath('{-1}'); + v +------------- + neg,-,-,-,- +(1 row) + +EXECUTE hpath('{o,b}'); + v +----------- + c,-,-,-,- +(1 row) + +EXECUTE hpath('{4,k}'); + v +----------- + -,1,-,-,- +(1 row) + +EXECUTE hpath('{""}'); + v +--------------- + empty,-,-,-,- +(1 row) + +EXECUTE hkey('k'); + v +------------------------ + true,true,true,false,- +(1 row) + +EXECUTE hkey('0'); + v +-------------------------- + false,true,false,false,- +(1 row) + +EXECUTE hkey('-1'); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE hkey(''); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE hkey('*'); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE hkey('q"k'); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE hkey('zz'); + v +--------------------------- + false,false,false,false,- +(1 row) + +EXECUTE hkeys('{zz,k}'); + v +-------------------------------------------------- + true/false,true/false,true/false,false/false,-/- +(1 row) + +EXECUTE hkeys('{k,a}'); + v +------------------------------------------------- + true/true,true/false,true/false,false/false,-/- +(1 row) + +EXECUTE hkeys('{}'); + v +------------------------------------------------- + false/true,false/true,false/true,false/true,-/- +(1 row) + +EXECUTE hkeys('{k,NULL}'); + v +----------------------------------------------- + true/true,true/true,true/true,false/false,-/- +(1 row) + +EXECUTE hkeys('{zz}'); + v +----------------------------------------------------- + false/false,false/false,false/false,false/false,-/- +(1 row) + +EXECUTE hkeys('{a}'); + v +--------------------------------------------------- + true/true,false/false,false/false,false/false,-/- +(1 row) + +EXECUTE hkeys('{k}'); + v +----------------------------------------------- + true/true,true/true,true/true,false/false,-/- +(1 row) + +EXECUTE hdoc('{"o": {"b": "c"}}'); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE hdoc('"k"'); + v +------------------------- + false,true,true,false,- +(1 row) + +EXECUTE hdoc('[["k"]]'); + v +-------------------------- + false,true,false,false,- +(1 row) + +EXECUTE hdoc('{"b": "c"}'); + v +--------------------------- + false,false,false,false,- +(1 row) + +EXECUTE hdoc('{}'); + v +-------------------------- + true,false,false,false,- +(1 row) + +EXECUTE hdoc('[]'); + v +-------------------------- + false,true,false,false,- +(1 row) + +EXECUTE hdoc('{"a": [3, 1]}'); + v +-------------------------- + true,false,false,false,- +(1 row) + +-- (I) plpgsql variables as the path or key reach the planner through the +-- parameter-fetch hook; each loop crosses the same custom-to-generic switch. The +-- function reads the view, through DuckDB, or the heap. +SET duckdb.unsafe_allow_execution_inside_functions = true; +CREATE FUNCTION public.json_ops_by_var(heap bool) RETURNS text LANGUAGE plpgsql AS $$ +DECLARE + out text := ''; + k text; + p text; +BEGIN + FOREACH k IN ARRAY ARRAY['k', '0', '-1', '', '*', 'zz', 'k'] LOOP + IF heap THEN + out := out || (SELECT count(*) FROM public._docs WHERE (data::jsonb) ? k) || ' '; + ELSE + out := out || (SELECT count(*) FROM public.docs WHERE (data::jsonb) ? k) || ' '; + END IF; + END LOOP; + FOREACH p IN ARRAY ARRAY['a,0', 'a,-1', 'a,01', '-1', 'o,b', '4,k', 'a,9'] LOOP + IF heap THEN + out := out || (SELECT string_agg(coalesce(data #>> string_to_array(p, ','), '-'), ',' ORDER BY id) + FROM public._docs) || ' '; + ELSE + out := out || (SELECT string_agg(coalesce(data #>> string_to_array(p, ','), '-'), ',' ORDER BY id) + FROM public.docs) || ' '; + END IF; + END LOOP; + RETURN out; +END $$; +SET duckdb.force_execution = true; +SELECT public.json_ops_by_var(false) AS duckdb; + duckdb +---------------------------------------------------------------------------------------- + 3 1 1 1 1 0 3 1,-,-,-,- 3,-,-,-,- 2,-,-,-,- neg,-,-,-,- c,-,-,-,- -,1,-,-,- -,-,-,-,- +(1 row) + +RESET duckdb.force_execution; +SELECT public.json_ops_by_var(true) AS postgres; + postgres +---------------------------------------------------------------------------------------- + 3 1 1 1 1 0 3 1,-,-,-,- 3,-,-,-,- 2,-,-,-,- neg,-,-,-,- c,-,-,-,- -,1,-,-,- -,-,-,-,- +(1 row) + +-- (J) A jsonb document that is itself a bound parameter, not a ::jsonb cast the +-- rewrite turns into json, reaches the planner hook the same way. The plan counts +-- show every prepared read above ran on custom plans built from its values. +SET duckdb.force_execution = true; +PREPARE jdocpath(jsonb, text[]) AS + SELECT string_agg(coalesce($1 #>> $2, '-'), ',' ORDER BY id) AS v FROM public.docs; +EXECUTE jdocpath('{"a": [7, 8, 9]}', '{a,1}'); + v +----------- + 8,8,8,8,8 +(1 row) + +EXECUTE jdocpath('{"a": [7, 8, 9]}', '{a,-1}'); + v +----------- + 9,9,9,9,9 +(1 row) + +RESET duckdb.force_execution; +SELECT name, generic_plans, custom_plans FROM pg_prepared_statements + WHERE name IN ('jpath', 'jkey', 'jkeys', 'jdoc', 'jdocpath') ORDER BY name; + name | generic_plans | custom_plans +----------+---------------+-------------- + jdoc | 0 | 7 + jdocpath | 0 | 2 + jkey | 0 | 7 + jkeys | 0 | 7 + jpath | 0 | 7 +(5 rows) + +-- (K) A path element PostgreSQL may read as an array index or as an object key +-- (-1, 01, +1) doubles the rewritten expression, so a path holds at most eight. +SET duckdb.force_execution = true; +SELECT count(data #> '{-1,-1,-1,-1,-1,-1,-1,-1}') AS eight FROM public.docs; + eight +------- + 0 +(1 row) + +SELECT count(data #> '{-1,-1,-1,-1,-1,-1,-1,-1,-1}') AS nine FROM public.docs; +ERROR: coldfront: JSON path holds more than 8 elements that may be an array index or an object key +RESET duckdb.force_execution; +-- Cleanup. +DEALLOCATE jdocpath; +DEALLOCATE jpath; +DEALLOCATE jkey; +DEALLOCATE jkeys; +DEALLOCATE jdoc; +DEALLOCATE hpath; +DEALLOCATE hkey; +DEALLOCATE hkeys; +DEALLOCATE hdoc; +DROP FUNCTION public.json_ops_by_var(bool); +DELETE FROM coldfront.tiered_views; +DELETE FROM coldfront.archive_watermark; +DROP VIEW public.docs; +DROP TABLE public._docs; diff --git a/extension/coldfront/test/sql/read_json_operators.sql b/extension/coldfront/test/sql/read_json_operators.sql new file mode 100644 index 00000000..77f5d257 --- /dev/null +++ b/extension/coldfront/test/sql/read_json_operators.sql @@ -0,0 +1,282 @@ +-- The json/jsonb operators DuckDB lacks (#>, #>>, ?, ?|, ?&, @>, <@) on a read +-- DuckDB runs. DuckDB's parser has no # or ? operator character and DuckDB has no +-- containment operator, so the read rewrite turns each into the JSON functions +-- pg_duckdb hands to DuckDB (json_extract, json_extract_string, json_exists, +-- json_contains, json_type). White-box: DuckDB executes the rewritten read against +-- the heap (duckdb.force_execution) through a view that casts jsonb to json, as a +-- tiered view does, and PostgreSQL runs the same operators on the heap; no +-- Iceberg I/O. +-- Suppress the run-order-dependent "already exists" NOTICE: in the shared regress +-- db an earlier test may have created the extensions, standalone not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; +SET TIME ZONE 'UTC'; +-- White-box: checks the generated SQL, not Iceberg I/O. +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; + +CREATE TABLE public._docs (id int, ts timestamptz, data jsonb); +CREATE VIEW public.docs AS SELECT id, ts, data::json AS data FROM public._docs; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'docs', 'public._docs', 'ice.default.docs', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'docs', '2026-03-01'::timestamptz); +-- An object whose keys a JSON Pointer or a JSONPath would misread ("-1", "01", +-- "", "a/b", "t~", "*", a quote, a backslash), an array, a bare string, a bare +-- number and SQL NULL. +INSERT INTO public._docs VALUES + (1, '2026-01-01', '{"k": 1, "a": [1, 2, 3], "o": {"b": "c"}, "-1": "neg", "01": "pad", "": "empty", "a/b": "slash", "t~": "tilde", "*": "star", "q\"k": "quote", "b\\s": "bslash", "n": null}'), + (2, '2026-01-02', '["k", ["k"], 1, "0", {"k": 1}, null]'), + (3, '2026-01-03', '"k"'), + (4, '2026-01-04', '5'), + (5, '2026-01-05', NULL); + +-- (A) A path into an array, and a key test through the ::jsonb cast. +SET duckdb.force_execution = true; +SELECT data #> '{a,0}' AS a0 FROM public.docs WHERE id = 1; +SELECT id FROM public.docs WHERE (data::jsonb) ? 'k' ORDER BY id; +RESET duckdb.force_execution; + +-- (B) #> and #>>. A path element reaches an array element or an object key, as +-- PostgreSQL's does: a negative, zero-padded or signed index counts on an array +-- and names a key on an object. +SET duckdb.force_execution = true; +SELECT id, data #>> '{a,0}' AS a0, data #>> '{a,-1}' AS alast, data #>> '{a,01}' AS apad, + data #>> '{a,+1}' AS asign, data #>> '{a,-4}' AS aout, data #>> '{-1}' AS neg, + data #>> '{01}' AS pad, data #>> '{""}' AS empty, data #>> '{a/b}' AS slash, + data #>> '{t~}' AS tilde, data #>> '{*}' AS star, data #>> ARRAY['q"k'] AS quote, + data #>> ARRAY['b\s'] AS bslash, data #>> '{o,b}' AS ob, data #>> '{4,k}' AS nested, + data #>> '{n}' AS jnull, data #>> '{a,NULL}' AS nullelem, data #> '{0}' AS first + FROM public.docs ORDER BY id; +RESET duckdb.force_execution; +SELECT id, data #>> '{a,0}' AS a0, data #>> '{a,-1}' AS alast, data #>> '{a,01}' AS apad, + data #>> '{a,+1}' AS asign, data #>> '{a,-4}' AS aout, data #>> '{-1}' AS neg, + data #>> '{01}' AS pad, data #>> '{""}' AS empty, data #>> '{a/b}' AS slash, + data #>> '{t~}' AS tilde, data #>> '{*}' AS star, data #>> ARRAY['q"k'] AS quote, + data #>> ARRAY['b\s'] AS bslash, data #>> '{o,b}' AS ob, data #>> '{4,k}' AS nested, + data #>> '{n}' AS jnull, data #>> '{a,NULL}' AS nullelem, data #> '{0}' AS first + FROM public._docs ORDER BY id; + +-- An empty path is the whole value; DuckDB renders a container compactly, as it +-- does for ->. +SET duckdb.force_execution = true; +SELECT id, data #>> '{}' AS whole, data #> '{o}' AS o FROM public.docs WHERE id IN (1, 3) ORDER BY id; +RESET duckdb.force_execution; +SELECT id, data #>> '{}' AS whole, data #> '{o}' AS o FROM public._docs WHERE id IN (1, 3) ORDER BY id; + +-- (C) ?, ?| and ?&: a top-level key, a top-level string element or the bare +-- string itself, never a number and never a nested key. NULL keys are skipped. +SET duckdb.force_execution = true; +SELECT id, (data::jsonb) ? 'k' AS k, (data::jsonb) ? '0' AS zero, (data::jsonb) ? '-1' AS neg, + (data::jsonb) ? '' AS empty, (data::jsonb) ? '*' AS star, (data::jsonb) ? 'a/b' AS slash, + (data::jsonb) ? 'q"k' AS quote, (data::jsonb) ? 'b\s' AS bslash, (data::jsonb) ? '5' AS five, + (data::jsonb) ?| '{zz,k}' AS any_k, (data::jsonb) ?| '{zz}' AS any_zz, + (data::jsonb) ?| '{}' AS any_none, (data::jsonb) ?& ARRAY['k', 'a'] AS all_ka, + (data::jsonb) ?& '{k,NULL}' AS all_knull, (data::jsonb) ?& '{}' AS all_none + FROM public.docs ORDER BY id; +RESET duckdb.force_execution; +SELECT id, (data::jsonb) ? 'k' AS k, (data::jsonb) ? '0' AS zero, (data::jsonb) ? '-1' AS neg, + (data::jsonb) ? '' AS empty, (data::jsonb) ? '*' AS star, (data::jsonb) ? 'a/b' AS slash, + (data::jsonb) ? 'q"k' AS quote, (data::jsonb) ? 'b\s' AS bslash, (data::jsonb) ? '5' AS five, + (data::jsonb) ?| '{zz,k}' AS any_k, (data::jsonb) ?| '{zz}' AS any_zz, + (data::jsonb) ?| '{}' AS any_none, (data::jsonb) ?& ARRAY['k', 'a'] AS all_ka, + (data::jsonb) ?& '{k,NULL}' AS all_knull, (data::jsonb) ?& '{}' AS all_none + FROM public._docs ORDER BY id; + +-- (D) @> and <@: containment from the top level down, an array containing a +-- bare scalar, and nothing found deeper than the top. +SET duckdb.force_execution = true; +SELECT id, (data::jsonb) @> '{"o": {"b": "c"}}' AS nested, (data::jsonb) @> '{"b": "c"}' AS deeper, + (data::jsonb) @> '"k"' AS scalar, (data::jsonb) @> '["k"]' AS arr, + (data::jsonb) @> '[["k"]]' AS arrarr, (data::jsonb) @> '{"a": [3, 1]}' AS subset, + (data::jsonb) @> '{}' AS empty_obj, (data::jsonb) @> '[]' AS empty_arr, + (data::jsonb) <@ '["k", ["k"], 1, "0", {"k": 1}, null, "extra"]' AS within + FROM public.docs ORDER BY id; +RESET duckdb.force_execution; +SELECT id, (data::jsonb) @> '{"o": {"b": "c"}}' AS nested, (data::jsonb) @> '{"b": "c"}' AS deeper, + (data::jsonb) @> '"k"' AS scalar, (data::jsonb) @> '["k"]' AS arr, + (data::jsonb) @> '[["k"]]' AS arrarr, (data::jsonb) @> '{"a": [3, 1]}' AS subset, + (data::jsonb) @> '{}' AS empty_obj, (data::jsonb) @> '[]' AS empty_arr, + (data::jsonb) <@ '["k", ["k"], 1, "0", {"k": 1}, null, "extra"]' AS within + FROM public._docs ORDER BY id; + +-- DuckDB compares JSON numbers by kind, so an integer never contains or matches +-- the same value written as a decimal, which PostgreSQL's numeric comparison +-- does (a documented difference). +INSERT INTO public._docs VALUES (6, '2026-01-06', '{"p": 10.0}'); +SET duckdb.force_execution = true; +SELECT (data::jsonb) @> '{"p": 10.0}' AS same_kind, (data::jsonb) @> '{"p": 10}' AS int_in_decimal + FROM public.docs WHERE id = 6; +RESET duckdb.force_execution; +SELECT (data::jsonb) @> '{"p": 10.0}' AS same_kind, (data::jsonb) @> '{"p": 10}' AS int_in_decimal + FROM public._docs WHERE id = 6; +DELETE FROM public._docs WHERE id = 6; + +-- (E) The function spellings of the same operators. +SET duckdb.force_execution = true; +SELECT id, json_extract_path_text(data, 'a', '-1') AS f1, jsonb_extract_path_text(data::jsonb, 'o', 'b') AS f2, + jsonb_exists(data::jsonb, 'k') AS f3 + FROM public.docs ORDER BY id; +RESET duckdb.force_execution; +SELECT id, json_extract_path_text(data::json, 'a', '-1') AS f1, jsonb_extract_path_text(data, 'o', 'b') AS f2, + jsonb_exists(data, 'k') AS f3 + FROM public._docs ORDER BY id; + +-- (F) A path or key set taken from a column cannot be rewritten, and says so. +SELECT data #> ARRAY[id::text] FROM public.docs; +SELECT (data::jsonb) ?| ARRAY[id::text] FROM public.docs; + +-- (G) A read the hot heap can answer keeps the operator: it never reaches DuckDB. +EXPLAIN (COSTS OFF, VERBOSE) +SELECT data #> '{a,0}' FROM public.docs WHERE ts >= '2026-04-01'::timestamptz; + +-- (H) A bound parameter as the path or keys. The rewrite needs its value, so the +-- read is planned from the value on every execution; seven executions cross the +-- plan cache's switch from custom to generic planning after the fifth. The heap +-- answers the same statements. +SET duckdb.force_execution = true; +PREPARE jpath(text[]) AS + SELECT string_agg(coalesce(data #>> $1, '-'), ',' ORDER BY id) AS v FROM public.docs; +PREPARE jkey(text) AS + SELECT string_agg(coalesce(((data::jsonb) ? $1)::text, '-'), ',' ORDER BY id) AS v FROM public.docs; +PREPARE jkeys(text[]) AS + SELECT string_agg(coalesce(((data::jsonb) ?| $1)::text, '-') || '/' || + coalesce(((data::jsonb) ?& $1)::text, '-'), ',' ORDER BY id) AS v FROM public.docs; +PREPARE jdoc(jsonb) AS + SELECT string_agg(coalesce(((data::jsonb) @> $1)::text, '-'), ',' ORDER BY id) AS v FROM public.docs; +EXECUTE jpath('{a,0}'); +EXECUTE jpath('{a,-1}'); +EXECUTE jpath('{a,01}'); +EXECUTE jpath('{-1}'); +EXECUTE jpath('{o,b}'); +EXECUTE jpath('{4,k}'); +EXECUTE jpath('{""}'); +EXECUTE jkey('k'); +EXECUTE jkey('0'); +EXECUTE jkey('-1'); +EXECUTE jkey(''); +EXECUTE jkey('*'); +EXECUTE jkey('q"k'); +EXECUTE jkey('zz'); +EXECUTE jkeys('{zz,k}'); +EXECUTE jkeys('{k,a}'); +EXECUTE jkeys('{}'); +EXECUTE jkeys('{k,NULL}'); +EXECUTE jkeys('{zz}'); +EXECUTE jkeys('{a}'); +EXECUTE jkeys('{k}'); +EXECUTE jdoc('{"o": {"b": "c"}}'); +EXECUTE jdoc('"k"'); +EXECUTE jdoc('[["k"]]'); +EXECUTE jdoc('{"b": "c"}'); +EXECUTE jdoc('{}'); +EXECUTE jdoc('[]'); +EXECUTE jdoc('{"a": [3, 1]}'); +RESET duckdb.force_execution; +PREPARE hpath(text[]) AS + SELECT string_agg(coalesce(data #>> $1, '-'), ',' ORDER BY id) AS v FROM public._docs; +PREPARE hkey(text) AS + SELECT string_agg(coalesce(((data::jsonb) ? $1)::text, '-'), ',' ORDER BY id) AS v FROM public._docs; +PREPARE hkeys(text[]) AS + SELECT string_agg(coalesce(((data::jsonb) ?| $1)::text, '-') || '/' || + coalesce(((data::jsonb) ?& $1)::text, '-'), ',' ORDER BY id) AS v FROM public._docs; +PREPARE hdoc(jsonb) AS + SELECT string_agg(coalesce(((data::jsonb) @> $1)::text, '-'), ',' ORDER BY id) AS v FROM public._docs; +EXECUTE hpath('{a,0}'); +EXECUTE hpath('{a,-1}'); +EXECUTE hpath('{a,01}'); +EXECUTE hpath('{-1}'); +EXECUTE hpath('{o,b}'); +EXECUTE hpath('{4,k}'); +EXECUTE hpath('{""}'); +EXECUTE hkey('k'); +EXECUTE hkey('0'); +EXECUTE hkey('-1'); +EXECUTE hkey(''); +EXECUTE hkey('*'); +EXECUTE hkey('q"k'); +EXECUTE hkey('zz'); +EXECUTE hkeys('{zz,k}'); +EXECUTE hkeys('{k,a}'); +EXECUTE hkeys('{}'); +EXECUTE hkeys('{k,NULL}'); +EXECUTE hkeys('{zz}'); +EXECUTE hkeys('{a}'); +EXECUTE hkeys('{k}'); +EXECUTE hdoc('{"o": {"b": "c"}}'); +EXECUTE hdoc('"k"'); +EXECUTE hdoc('[["k"]]'); +EXECUTE hdoc('{"b": "c"}'); +EXECUTE hdoc('{}'); +EXECUTE hdoc('[]'); +EXECUTE hdoc('{"a": [3, 1]}'); + +-- (I) plpgsql variables as the path or key reach the planner through the +-- parameter-fetch hook; each loop crosses the same custom-to-generic switch. The +-- function reads the view, through DuckDB, or the heap. +SET duckdb.unsafe_allow_execution_inside_functions = true; +CREATE FUNCTION public.json_ops_by_var(heap bool) RETURNS text LANGUAGE plpgsql AS $$ +DECLARE + out text := ''; + k text; + p text; +BEGIN + FOREACH k IN ARRAY ARRAY['k', '0', '-1', '', '*', 'zz', 'k'] LOOP + IF heap THEN + out := out || (SELECT count(*) FROM public._docs WHERE (data::jsonb) ? k) || ' '; + ELSE + out := out || (SELECT count(*) FROM public.docs WHERE (data::jsonb) ? k) || ' '; + END IF; + END LOOP; + FOREACH p IN ARRAY ARRAY['a,0', 'a,-1', 'a,01', '-1', 'o,b', '4,k', 'a,9'] LOOP + IF heap THEN + out := out || (SELECT string_agg(coalesce(data #>> string_to_array(p, ','), '-'), ',' ORDER BY id) + FROM public._docs) || ' '; + ELSE + out := out || (SELECT string_agg(coalesce(data #>> string_to_array(p, ','), '-'), ',' ORDER BY id) + FROM public.docs) || ' '; + END IF; + END LOOP; + RETURN out; +END $$; +SET duckdb.force_execution = true; +SELECT public.json_ops_by_var(false) AS duckdb; +RESET duckdb.force_execution; +SELECT public.json_ops_by_var(true) AS postgres; + +-- (J) A jsonb document that is itself a bound parameter, not a ::jsonb cast the +-- rewrite turns into json, reaches the planner hook the same way. The plan counts +-- show every prepared read above ran on custom plans built from its values. +SET duckdb.force_execution = true; +PREPARE jdocpath(jsonb, text[]) AS + SELECT string_agg(coalesce($1 #>> $2, '-'), ',' ORDER BY id) AS v FROM public.docs; +EXECUTE jdocpath('{"a": [7, 8, 9]}', '{a,1}'); +EXECUTE jdocpath('{"a": [7, 8, 9]}', '{a,-1}'); +RESET duckdb.force_execution; +SELECT name, generic_plans, custom_plans FROM pg_prepared_statements + WHERE name IN ('jpath', 'jkey', 'jkeys', 'jdoc', 'jdocpath') ORDER BY name; + +-- (K) A path element PostgreSQL may read as an array index or as an object key +-- (-1, 01, +1) doubles the rewritten expression, so a path holds at most eight. +SET duckdb.force_execution = true; +SELECT count(data #> '{-1,-1,-1,-1,-1,-1,-1,-1}') AS eight FROM public.docs; +SELECT count(data #> '{-1,-1,-1,-1,-1,-1,-1,-1,-1}') AS nine FROM public.docs; +RESET duckdb.force_execution; + +-- Cleanup. +DEALLOCATE jdocpath; +DEALLOCATE jpath; +DEALLOCATE jkey; +DEALLOCATE jkeys; +DEALLOCATE jdoc; +DEALLOCATE hpath; +DEALLOCATE hkey; +DEALLOCATE hkeys; +DEALLOCATE hdoc; +DROP FUNCTION public.json_ops_by_var(bool); +DELETE FROM coldfront.tiered_views; +DELETE FROM coldfront.archive_watermark; +DROP VIEW public.docs; +DROP TABLE public._docs;