fix: rewrite the JSON operators DuckDB lacks on tiered and decoupled reads

ci/journey.sh

@@ -489,6 +489,23 @@ EOSQL
     assert_eq "decoupled multi-row INSERT (3 rows)"     "3"    "$(extract MULTI "$O")"
     assert_eq "decoupled UPDATE visible"                "upd"  "$(extract UPD "$O")"
     assert_eq "decoupled DELETE visible (0 of id=2)"    "0"    "$(extract DEL "$O")"
+    # The json/jsonb operators DuckDB lacks on an iceberg-only view: a path into an
+    # array, and a key and containment test through the ::jsonb cast, the key also
+    # as a bound parameter.
+    O=$(qf "$HOST" <<'EOSQL'
+INSERT INTO iceonly VALUES (50,date_trunc('month',now()) + interval '10 hours 4 minutes 0 seconds','jsonops','{"k":1,"a":[1,2]}');
+SELECT 'OPS_PATH:'||(data #> '{a,0}')::text FROM iceonly WHERE id=50;
+SELECT 'OPS_HASKEY:'||count(*) FROM iceonly WHERE id=50 AND (data::jsonb) ? 'k';
+SELECT 'OPS_CONTAIN:'||((data::jsonb) @> '{"a":[2]}')::text FROM iceonly WHERE id=50;
+PREPARE ops_key(text) AS SELECT 'OPS_PKEY:'||count(*) FROM iceonly WHERE id=50 AND (data::jsonb) ? $1;
+EXECUTE ops_key('k');
+DELETE FROM iceonly WHERE id=50;
+EOSQL
+)
+    assert_eq "decoupled #> into an array"             "1"    "$(extract OPS_PATH "$O")"
+    assert_eq "decoupled ? through the ::jsonb cast"   "1"    "$(extract OPS_HASKEY "$O")"
+    assert_eq "decoupled @> through the ::jsonb cast"  "true" "$(extract OPS_CONTAIN "$O")"
+    assert_eq "decoupled ? with a bound key"           "1"    "$(extract OPS_PKEY "$O")"
     # PostgreSQL 17 and later allow MERGE into a view.
     if [ "$(q "$HOST" "SHOW server_version_num;")" -ge 170000 ]; then
         O=$(qf "$HOST" <<'EOSQL'
@@ -958,6 +975,19 @@ SELECT 'JSONB_HOT_M:'  || (data->>'m') FROM events WHERE ts >= date_trunc('month
 SELECT 'CAST_COLD:' || ((data::jsonb)->>'m') FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND status='ok' ORDER BY ts LIMIT 1;
 SELECT 'CAST_HOT:'  || ((data::jsonb)->>'m') FROM events WHERE ts >= date_trunc('month',now()) - interval '2 months' AND status='ok' ORDER BY ts LIMIT 1;
 SELECT 'ALEN_COLD:' || jsonb_array_length('[10,20,30]'::jsonb) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND status='ok' ORDER BY ts LIMIT 1;
+-- The json/jsonb operators DuckDB lacks, rewritten into its JSON functions, on
+-- rows read from Iceberg.
+SELECT 'OP_PATH_COLD:'    || (data #>> '{m}') FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND status='ok' ORDER BY ts LIMIT 1;
+SELECT 'OP_HASKEY_COLD:'  || count(*) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND (data::jsonb) ? 'm';
+SELECT 'OP_ANYKEY_COLD:'  || count(*) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND (data::jsonb) ?| '{zz,m}';
+SELECT 'OP_CONTAIN_COLD:' || count(*) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND (data::jsonb) @> '{"m":"m4"}';
+-- The same with the path, key or document as a bound parameter.
+PREPARE op_path(text[]) AS SELECT 'OP_PPATH_COLD:' || (data #>> $1) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND status='ok' ORDER BY ts LIMIT 1;
+PREPARE op_key(text) AS SELECT 'OP_PKEY_COLD:' || count(*) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND (data::jsonb) ? $1;
+PREPARE op_doc(jsonb) AS SELECT 'OP_PDOC_COLD:' || count(*) FROM events WHERE ts < date_trunc('month',now()) - interval '2 months' AND (data::jsonb) @> $1;
+EXECUTE op_path('{m}');
+EXECUTE op_key('m');
+EXECUTE op_doc('{"m":"m4"}');
 EOSQL
 )
     assert_eq "total rows (hot+cold via view)" "280"  "$(extract RO_TOTAL "$O")"
@@ -969,6 +999,13 @@ EOSQL
     assert_eq "::jsonb cast translated, cold" "m4" "$(extract CAST_COLD "$O")"
     assert_eq "::jsonb cast translated, hot"  "m2" "$(extract CAST_HOT "$O")"
     assert_eq "jsonb_array_length→json_array_length" "3" "$(extract ALEN_COLD "$O")"
+    assert_eq "#>> on cold rows"                       "m4"  "$(extract OP_PATH_COLD "$O")"
+    assert_eq "? on cold rows (m4+m3)"                 "180" "$(extract OP_HASKEY_COLD "$O")"
+    assert_eq "?| on cold rows (m4+m3)"                "180" "$(extract OP_ANYKEY_COLD "$O")"
+    assert_eq "@> on cold rows (m4 only)"              "100" "$(extract OP_CONTAIN_COLD "$O")"
+    assert_eq "#>> with a bound path on cold rows"     "m4"  "$(extract OP_PPATH_COLD "$O")"
+    assert_eq "? with a bound key on cold rows"        "180" "$(extract OP_PKEY_COLD "$O")"
+    assert_eq "@> with a bound document on cold rows"  "100" "$(extract OP_PDOC_COLD "$O")"
 
     # Hot-tier read routing: a read whose WHERE provably restricts to the hot tier
     # (ts >= the watermark) is rewritten to the hot heap and runs in plain PostgreSQL,

docs/architecture_guides/index.md

@@ -98,7 +98,7 @@ The following table describes each component, its role, and its license:
 |-----------|------|---------|
 | PostgreSQL 16+ | Provides heap storage and range partitioning for the tiered hot tier. ColdFront works uniformly on PG 16, 17, and 18, because the cold-tier secret is a DuckDB persistent secret loaded at instance init, with no version-gated mechanism. | PostgreSQL |
 | pg_duckdb | Runs DuckDB in-process for Iceberg reads, writes, and analytics; the build is pg_duckdb at commit c04e6a2 (PR #1025) on DuckDB 1.5.4. The bundled `duckdb-iceberg` includes the bakery-aware commit-refresh patch (async parquet overlap, no 409); see [Cold-Write Strategy](#cold-write-strategy-stock-vs-patched-duckdb-iceberg). | MIT |
-| coldfront | This PGXS C extension's `post_parse_analyze_hook` rewrites INSERT/UPDATE/DELETE/MERGE on registered views to the correct tier and, on a SELECT DuckDB will run, the spellings DuckDB lacks (`date_bin`, `::jsonb`, the JSON builders); `planner_hook` folds bound parameters into such a read; `ProcessUtility_hook` handles DDL; the hook lazily ATTACHes the Iceberg catalog on the first query touching a tiered view. | PostgreSQL |
+| coldfront | This PGXS C extension's `post_parse_analyze_hook` rewrites INSERT/UPDATE/DELETE/MERGE on registered views to the correct tier and, on a SELECT DuckDB will run, the spellings DuckDB lacks (`date_bin`, `::jsonb`, the JSON builders, the JSON operators); `planner_hook` folds bound parameters into such a read; `ProcessUtility_hook` handles DDL; the hook lazily ATTACHes the Iceberg catalog on the first query touching a tiered view. | PostgreSQL |
 | Lakekeeper | Provides the Iceberg REST catalog as a single Rust binary. | Apache 2.0 |
 | S3-compatible store or Azure ADLS Gen2 | Stores the cold data; any S3-compatible store works, including SeaweedFS, MinIO, AWS S3, and GCS, and Azure ADLS Gen2 works through `set_storage_secret_azure`. | Varies |
 | Archiver (tiered mode) | Moves rows from hot to cold; this Go binary is a thin SQL orchestrator that cron invokes. | PostgreSQL |
@@ -258,13 +258,19 @@ in the statement the view is named (a CTE, a sub-select, a set-operation
 branch): `date_bin`, the `::jsonb` cast, `jsonb_array_length` and the JSON
 builders (`jsonb_build_object`, `jsonb_agg` and their `json_` twins) are
 rewritten into spellings both engines accept (see
-[Supported Column Types](../supported_types.md)). A
-`planner_hook` folds bound parameters into such a read before pg_duckdb plans
-it when a parameter sits where DuckDB cannot type a placeholder (a direct
-argument of a pg_duckdb function, any argument of a table function); the plan
-cache's generic-plan build, which has no values, gets a PostgreSQL plan priced
-above any custom plan, so under the plan cache's cost-based selection the read
-is planned from its values on every execution.
+[Supported Column Types](../supported_types.md)). The hook also rewrites the
+JSON operators DuckDB lacks (`#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`) into
+DuckDB's JSON functions. A `planner_hook` folds bound parameters into such a
+read before pg_duckdb plans it when a parameter sits where DuckDB cannot type a
+placeholder (a direct argument of a pg_duckdb function, any argument of a table
+function); the plan cache's generic-plan build, which has no values, gets a
+PostgreSQL plan priced above any custom plan, so under the plan cache's
+cost-based selection the read is planned from its values on every execution.
+When an operand of those JSON operators holds a bound parameter, the
+parse-analyze hook leaves the rewrite to the `planner_hook`. The parse-analyze
+hook keeps `#>` and `#>>`, and turns the other operators into
+`coldfront._jsonb_*` stand-ins that PostgreSQL can parse. The `planner_hook`
+then folds the read's bound parameters and rewrites those operators.
 `plan_cache_mode = force_generic_plan` bypasses that selection and picks the
 value-less generic plan, which fails with `only works with DuckDB execution`
 (see [Supported Column Types](../supported_types.md)).
@@ -545,11 +551,12 @@ The cross-cutting limitations are:
   that references a DuckDB read (all-or-nothing plan takeover), so the cold
   branch cannot produce a PG `jsonb` directly. The view generator casts `jsonb`
   columns to DuckDB-safe `json` on both sides: hot emits `"col"::json`, cold
-  emits `r['col']::json`, the UNION unifies on `json`. Standard JSON access
-  (`->>`, `->`, `#>`) works without any caller-side cast; jsonb-only operators
-  (`?`, `@>`, containment, de-dup) need an explicit `data::jsonb` in the
-  caller's query. Storage in Iceberg remains VARCHAR (Iceberg has no JSON type
-  either). The cold sink sends the incoming value as its text form.
+  emits `r['col']::json`, the UNION unifies on `json`. The `->>` and `->`
+  operators reach DuckDB unchanged. ColdFront's read rewrite translates the
+  JSON operators that DuckDB lacks. For the operator list and the `data::jsonb`
+  cast rule, see [Supported Column Types](../supported_types.md). Storage in
+  Iceberg remains VARCHAR (Iceberg has no JSON type either). The cold sink
+  sends the incoming value as its text form.
 
 - Lakekeeper remote signing may not work with all S3-compatible stores; the
   workaround is `ACCESS_DELEGATION_MODE NONE` with a direct DuckDB S3 secret.

docs/caveats.md

@@ -10,9 +10,13 @@ tables:
 
 - `jsonb` reads come back as `json`. The `->`/`->>` operators work, and
   ColdFront translates the `::jsonb` cast, `jsonb_array_length`,
-  `jsonb_build_object` / `jsonb_agg` and `date_bin`; other jsonb operators and
-  functions are unsupported on cold or cross-tier reads. A hot-only read of the
-  view runs in PostgreSQL with full jsonb (see
+  `jsonb_build_object` / `jsonb_agg` and `date_bin`. ColdFront also rewrites
+  the `#>`, `#>>`, `?`, `?|`, `?&`, `@>` and `<@` operators and the functions
+  behind them. A path or key for these operators must be a constant or a bound
+  parameter. The `@>` and `<@` operators never match an integer against the
+  same value written as a decimal. Other jsonb operators and functions are
+  unsupported on cold or cross-tier reads. A hot-only read of the view runs in
+  PostgreSQL with full jsonb (see
   [Supported Column Types](supported_types.md)).
 - Cross-tier isolation differs from PG-native isolation. The cold tier keeps
   one Iceberg snapshot for the whole transaction, even at READ COMMITTED, so a

docs/changelog.md

@@ -355,6 +355,18 @@ this project adheres to
   it, since DuckDB has `<=>` and `<->` as operators of its own and no `<#>`.
   The read now spells it as the function DuckDB has under the same name,
   `list_negative_inner_product`; `<=>` and `<->` are unchanged.
+- The JSON operators `#>`, `#>>`, `?`, `?|`, `?&`, `@>` and `<@` failed
+  whenever DuckDB ran a tiered or decoupled read. DuckDB has no JSON form of
+  these operators, so `#>` failed with "syntax error at or near #". The read
+  rewrite turned the documented `data::jsonb` cast into `json`, so `?` failed
+  with "operator does not exist: json ? text". The read rewrite now turns each
+  operator, and the function behind each operator, into DuckDB's JSON
+  functions. A path or key must be a constant or a bound parameter. ColdFront
+  plans a read from its bound values on every execution when a parameter is an
+  operand of these operators. The rewritten reads return PostgreSQL's answers
+  with two exceptions. DuckDB writes a returned container or number in its own
+  form. The `@>` and `<@` operators never match an integer against the same
+  value written as a decimal.
 - Once a `DROP COLUMN` returned a tiered table to an earlier schema, the next
   column change failed in the Iceberg mirror with "Attempted to add schema
   with id N, but this already exists in the table!" and rolled back the

docs/supported_types.md

@@ -102,12 +102,31 @@ also lacks) and `date_bin` on read: the builders become the `concat` /
 `to_json` / `array_agg` form both engines evaluate identically (the result
 is JSON-equal to jsonb's rendering, in compact form and in argument order),
 and `date_bin` becomes DuckDB's `time_bucket`, which takes the same
-arguments and agrees on every fixed-width bucket. The jsonb-only operators
-(`?`, `@>`, `<@`, `#>`, `#>>`) and most jsonb functions (`jsonb_typeof`,
-`jsonb_extract_path`, `jsonb_extract_path_text`, `jsonb_set`, `jsonb_path_*`,
-`jsonb_each`, `jsonb_object_keys`) are not supported on tiered or decoupled
-data: DuckDB either lacks them or its same-named function differs in
-signature or result. Reach into the document with `->`/`->>`.
+arguments and agrees on every fixed-width bucket.
+
+ColdFront also rewrites the JSON operators that DuckDB lacks into DuckDB's JSON
+functions, which return PostgreSQL's answers. The rewrite covers:
+
+- the `#>`, `#>>`, `?`, `?|`, `?&`, `@>` and `<@` operators.
+- the `json_extract_path` and `jsonb_extract_path` functions and their `_text`
+  forms.
+- the `jsonb_exists`, `jsonb_exists_any` and `jsonb_exists_all` functions.
+- the `jsonb_contains` and `jsonb_contained` functions.
+
+A path or key for these operators and functions must be a constant or a bound
+parameter, not a column. A path can hold at most eight negative, signed or
+zero-padded index elements, such as `-1`, `+1` or `01`. PostgreSQL has no
+`json` form of `?`, `?|`, `?&`, `@>` and `<@`, only a `jsonb` one. Cast the
+view's `json` column to `jsonb` first, as in `data::jsonb ? 'k'`. Two kinds of
+result differ from PostgreSQL's. DuckDB writes a returned container or number
+in its own form, such as `{"b":"c"}`, or `1.5` for `1.50`. The `->>` operator
+returns values in the same form. The `@>` and `<@` operators never match an
+integer against the same value written as a decimal. For example, a stored
+`{"p": 10.0}` does not contain `{"p": 10}`, because DuckDB compares JSON
+numbers by kind. ColdFront does not support most other jsonb functions on
+tiered or decoupled data, such as `jsonb_typeof`, `jsonb_set`, `jsonb_path_*`,
+`jsonb_each` and `jsonb_object_keys`. DuckDB either lacks these functions, or
+its function of the same name differs in signature or result.
 
 These limits apply only to reads that go through a tiered or iceberg-only view,
 which pg_duckdb plans entirely in DuckDB. Ordinary (non-tiered) PostgreSQL
@@ -120,9 +139,11 @@ their context (`ts > $1`); one that is a direct argument of a DuckDB function
 with several overloads (`time_bucket`'s origin, so `date_bin`'s) or any
 argument of a table function (`generate_series`) it cannot, so ColdFront plans
 such a read from the bound values on every execution instead of caching a
-generic plan. Under `plan_cache_mode = force_generic_plan` those reads cannot
-run: the forced generic plan has no values and fails with
-`only works with DuckDB execution`.
+generic plan. The rewrite of `#>`, `#>>`, `?`, `?|`, `?&`, `@>` and `<@` also
+needs the value of any parameter in an operand. ColdFront plans such a read
+from the bound values on every execution as well. Under
+`plan_cache_mode = force_generic_plan` those reads cannot run: the forced
+generic plan has no values and fails with `only works with DuckDB execution`.
 
 A hot-only read is the exception. When a `SELECT` reads a tiered view directly
 (no join, CTE, or sub-query) and its `WHERE` provably restricts to the hot

extension/coldfront/Makefile

@@ -29,7 +29,7 @@ REGRESS     = load_order update_unregistered_view update_heap_table \
               type_map hot_guard adopt_type_map adopt_read_only adopt_iceberg_table \
               cold_write_json_agg duckdb_temp_dir \
               vector_centroids vector_type_map vector_cold_render array_cold_render vector_param_render vector_ops vector_probe vector_status vector_multicolumn \
-              duckdb_keyword_name partition_clause failed_txn_block tiered_insert_single_pass tiered_view_no_trigger copy_tiered_view param_in_subquery merge_on_view delete_without_where update_from_without_alias settings_registered ensure_replicated read_inner_product \
+              duckdb_keyword_name partition_clause failed_txn_block tiered_insert_single_pass tiered_view_no_trigger copy_tiered_view param_in_subquery merge_on_view delete_without_where update_from_without_alias settings_registered ensure_replicated read_inner_product read_json_operators \
               async_write_before_claim tiered_insert_stream cold_render_canonical tiered_insert_isolation clustered_append_generation \
               whitespace_in_value
 REGRESS_OPTS = --inputdir=test --outputdir=test

extension/coldfront/coldfront--1.0.sql

@@ -2671,6 +2671,22 @@ BEGIN
 END;
 $$;
 
+-- Stand-ins for the jsonb operators ?, ?|, ?& and @> on a read DuckDB runs when an
+-- operand holds a bound parameter. The read rewrite turns each operator into
+-- DuckDB's JSON functions and needs the operand's value to do so, which only the
+-- planner has; until then the stand-in keeps the rewritten read valid for
+-- PostgreSQL, which has these operators on jsonb alone where the read has json,
+-- and the planner hook rewrites it once the value is bound. Each body gives the
+-- operator's own answer.
+CREATE OR REPLACE FUNCTION coldfront._jsonb_exists(json, text)
+RETURNS boolean LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT $1::jsonb ? $2 $$;
+CREATE OR REPLACE FUNCTION coldfront._jsonb_exists_any(json, text[])
+RETURNS boolean LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT $1::jsonb ?| $2 $$;
+CREATE OR REPLACE FUNCTION coldfront._jsonb_exists_all(json, text[])
+RETURNS boolean LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT $1::jsonb ?& $2 $$;
+CREATE OR REPLACE FUNCTION coldfront._jsonb_contains(json, json)
+RETURNS boolean LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT $1::jsonb @> $2::jsonb $$;
+
 -- ============================================================================
 -- Canonical text. A value that leaves the caller's session as text (a cold
 -- write's rendered parameter, a streamed row's text column, a cold VARCHAR