diff --git a/tools/covering/README.md b/tools/covering/README.md new file mode 100644 index 00000000..324563ac --- /dev/null +++ b/tools/covering/README.md @@ -0,0 +1,52 @@ +# Covering-column generator (TemporalParquet) + +MobilityDuck's projection of the MEOS catalog `temporalCovering` descriptor: the +DuckDB macros that materialise the [TemporalParquet][tp] **covering columns** of a +temporal value, so a table prunes at the Parquet row-group and Iceberg manifest +level. The covering schema is generated from the single catalog source of truth, +not hand-written per type. + +[tp]: https://github.com/MobilityDB/MobilityLakehouse + +## Files + +| File | Role | +| --- | --- | +| `temporal-covering.json` | the descriptor, vendored from MEOS-API (the source of truth) | +| `codegen_covering.py` | the generator: descriptor → `covering.sql` | +| `covering.sql` | generated macros — **do not edit by hand** | +| `test_covering.sql` | loads the macros and checks they materialise the columns | + +## Use + +```sql +.read tools/covering/covering.sql + +-- materialise the covering columns next to the lossless value +COPY ( + SELECT mmsi, asBinary(traj) AS traj, c.* + FROM (SELECT mmsi, traj, covering_spatial(traj) AS c FROM trajectories) +) TO 'shard.parquet' (FORMAT PARQUET); +``` + +`covering_spatial` covers the spatial temporal types (STBOX: `xmin..zmax`, +`tmin/tmax`, `srid`); `covering_number` covers the numeric ones (TBOX: +`vmin/vmax`, `tmin/tmax`); `covering_timeOnly` covers the time-only ones +(`tbool`, `ttext`: `tmin/tmax`, no spatial box). The type-to-macro mapping is +listed at the foot of `covering.sql`. `zmin/zmax` are `NULL` for 2D values. + +## Regenerate + +```bash +python3 tools/covering/codegen_covering.py +``` + +## Status + +The generator is pin-independent (it reads the descriptor JSON), and the emitted +macros use box accessors already present in the extension, so `covering.sql` runs +on the current MobilityDuck build today. To land this as the covering surface: + +1. vendor `temporal-covering.json` from the merged MEOS-API descriptor, +2. wire `covering.sql` into the extension's load-time SQL, +3. re-verify against the pinned `libmeos`. diff --git a/tools/covering/codegen_covering.py b/tools/covering/codegen_covering.py new file mode 100644 index 00000000..68b6e971 --- /dev/null +++ b/tools/covering/codegen_covering.py @@ -0,0 +1,78 @@ +"""Generate covering.sql from the MEOS-API temporal-covering descriptor. + +MobilityDuck's projection of the catalog `temporalCovering` block: per covering +class, a DuckDB macro that returns the TemporalParquet covering columns of a +temporal value as a STRUCT. The covering schema is therefore generated from the +single catalog source of truth, not hand-written. + + python3 tools/covering/codegen_covering.py # -> tools/covering/covering.sql + +Pure dict -> text; pin-independent to run. The emitted macros use box accessors +already present in the extension (`stbox`/`tbox`/`Xmin`/.../`SRID`). +""" + +import json +from pathlib import Path + +HERE = Path(__file__).parent +DESCRIPTOR = HERE / "temporal-covering.json" +OUT = HERE / "covering.sql" + +# MEOS C symbol -> MobilityDuck SQL function (the binding's box-accessor map). +# The only binding-specific knowledge; everything else comes from the descriptor. +DUCK_FN = { + "tspatial_to_stbox": "stbox", "tnumber_to_tbox": "tbox", + "stbox_xmin": "Xmin", "stbox_xmax": "Xmax", + "stbox_ymin": "Ymin", "stbox_ymax": "Ymax", + "stbox_zmin": "Zmin", "stbox_zmax": "Zmax", + "stbox_tmin": "Tmin", "stbox_tmax": "Tmax", + "tbox_xmin": "Xmin", "tbox_xmax": "Xmax", + "tbox_tmin": "Tmin", "tbox_tmax": "Tmax", + "tspatial_srid": "SRID", + "temporal_start_timestamptz": "startTimestamp", + "temporal_end_timestamptz": "endTimestamp", +} + + +def _column_expr(column: dict, box_from: str) -> str: + fn = DUCK_FN[column["accessor"]] + if column["source"] == "value": + return f"{fn}(t)" + return f"{fn}({DUCK_FN[box_from]}(t))" + + +def build_sql(descriptor: dict) -> str: + classes = descriptor["classes"] + out = [ + "-- Generated by tools/covering/codegen_covering.py from the MEOS-API", + "-- temporal-covering descriptor. Do not edit by hand; regenerate.", + "--", + "-- Each macro returns a temporal value's TemporalParquet covering columns", + "-- as a STRUCT. Materialise them with:", + "-- SELECT base.*, c.* FROM (SELECT base, covering_(value) AS c FROM ...)", + "", + ] + by_type = {} + for class_name, spec in classes.items(): + box = spec.get("box") + box_from = box["from"] if box else None + fields = ", ".join( + f"'{c['name']}': {_column_expr(c, box_from)}" for c in spec["columns"]) + out.append(f"CREATE OR REPLACE MACRO covering_{class_name}(t) AS {{ {fields} }};") + for t in spec["types"]: + by_type[t] = class_name + out.append("") + out.append("-- temporal type -> covering macro:") + for t in sorted(by_type): + out.append(f"-- {t}: covering_{by_type[t]}") + return "\n".join(out) + "\n" + + +def main() -> None: + sql = build_sql(json.loads(DESCRIPTOR.read_text())) + OUT.write_text(sql) + print(f"wrote {OUT}") + + +if __name__ == "__main__": + main() diff --git a/tools/covering/covering.sql b/tools/covering/covering.sql new file mode 100644 index 00000000..f1a3e029 --- /dev/null +++ b/tools/covering/covering.sql @@ -0,0 +1,25 @@ +-- Generated by tools/covering/codegen_covering.py from the MEOS-API +-- temporal-covering descriptor. Do not edit by hand; regenerate. +-- +-- Each macro returns a temporal value's TemporalParquet covering columns +-- as a STRUCT. Materialise them with: +-- SELECT base.*, c.* FROM (SELECT base, covering_(value) AS c FROM ...) + +CREATE OR REPLACE MACRO covering_spatial(t) AS { 'xmin': Xmin(stbox(t)), 'xmax': Xmax(stbox(t)), 'ymin': Ymin(stbox(t)), 'ymax': Ymax(stbox(t)), 'zmin': Zmin(stbox(t)), 'zmax': Zmax(stbox(t)), 'tmin': Tmin(stbox(t)), 'tmax': Tmax(stbox(t)), 'srid': SRID(t) }; +CREATE OR REPLACE MACRO covering_number(t) AS { 'vmin': Xmin(tbox(t)), 'vmax': Xmax(tbox(t)), 'tmin': Tmin(tbox(t)), 'tmax': Tmax(tbox(t)) }; +CREATE OR REPLACE MACRO covering_timeOnly(t) AS { 'tmin': startTimestamp(t), 'tmax': endTimestamp(t) }; + +-- temporal type -> covering macro: +-- tbigint: covering_number +-- tbool: covering_timeOnly +-- tcbuffer: covering_spatial +-- tfloat: covering_number +-- tgeogpoint: covering_spatial +-- tgeography: covering_spatial +-- tgeometry: covering_spatial +-- tgeompoint: covering_spatial +-- tint: covering_number +-- tnpoint: covering_spatial +-- tpose: covering_spatial +-- trgeometry: covering_spatial +-- ttext: covering_timeOnly diff --git a/tools/covering/temporal-covering.json b/tools/covering/temporal-covering.json new file mode 100644 index 00000000..111282f7 --- /dev/null +++ b/tools/covering/temporal-covering.json @@ -0,0 +1,74 @@ +{ + "_comment": "Temporal-covering descriptor — the single codegen source of truth for projecting a MEOS temporal column into Parquet/Iceberg covering columns (GeoParquet 1.1 `covering.bbox`). Every binding/engine generates the IDENTICAL covering schema from this mapping, so a temporal table prunes the same way on every platform (Iceberg manifest pruning + Parquet row-group min/max) with no spatial-aware engine. Curated canonical data keyed by temporal-type FAMILY (a `class`), not per type — adding a type is one entry in its class. The canonical MEOS-WKB value column is unchanged and lossless; the covering columns are denormalised derivations of the value's bounding box. RFC #870 (TemporalParquet) / #913 (Temporal Data Lake).", + "provenance": { + "rfc": "MobilityDB RFC #870 (TemporalParquet) + #913 (Temporal Data Lake)", + "discussion": "MobilityDB#861 (edge-to-cloud SQL portability: one query, three platforms)", + "geoParquet": "GeoParquet 1.1 covering.bbox (geoparquet.org/releases/v1.1.0)", + "benchmark": "MVB v3 — the scalar AND-chain on materialised covering columns prunes row groups identically to the spatial-aware path and ~10x faster, with no DuckDB spatial extension" + }, + "version": "1.0.0", + "valueCodec": { + "asHexWkb": "temporal_as_hexwkb", + "fromHexWkb": "temporal_from_hexwkb", + "note": "The canonical MEOS-WKB stays the lossless value column (BLOB); covering columns are denormalised and never the source of truth." + }, + "metadataKeys": { + "temporal": "temporal", + "geo": "geo", + "covering": "bbox" + }, + "classes": { + "spatial": { + "doc": "Spatial temporal types — STBOX covering (x/y[/z] extent + time extent + SRID).", + "box": {"type": "STBOX", "from": "tspatial_to_stbox"}, + "srid": "tspatial_srid", + "types": ["tgeompoint", "tgeogpoint", "tgeometry", "tgeography", "tcbuffer", "tnpoint", "tpose", "trgeometry"], + "columns": [ + {"name": "xmin", "sqlType": "double", "accessor": "stbox_xmin", "source": "box"}, + {"name": "xmax", "sqlType": "double", "accessor": "stbox_xmax", "source": "box"}, + {"name": "ymin", "sqlType": "double", "accessor": "stbox_ymin", "source": "box"}, + {"name": "ymax", "sqlType": "double", "accessor": "stbox_ymax", "source": "box"}, + {"name": "zmin", "sqlType": "double", "accessor": "stbox_zmin", "source": "box", "when": "hasZ"}, + {"name": "zmax", "sqlType": "double", "accessor": "stbox_zmax", "source": "box", "when": "hasZ"}, + {"name": "tmin", "sqlType": "timestamptz", "accessor": "stbox_tmin", "source": "box"}, + {"name": "tmax", "sqlType": "timestamptz", "accessor": "stbox_tmax", "source": "box"}, + {"name": "srid", "sqlType": "int", "accessor": "tspatial_srid", "source": "value"} + ] + }, + "number": { + "doc": "Numeric temporal types — TBOX covering (value range + time extent).", + "box": {"type": "TBOX", "from": "tnumber_to_tbox"}, + "srid": null, + "types": ["tint", "tfloat", "tbigint"], + "columns": [ + {"name": "vmin", "sqlType": "double", "accessor": "tbox_xmin", "source": "box"}, + {"name": "vmax", "sqlType": "double", "accessor": "tbox_xmax", "source": "box"}, + {"name": "tmin", "sqlType": "timestamptz", "accessor": "tbox_tmin", "source": "box"}, + {"name": "tmax", "sqlType": "timestamptz", "accessor": "tbox_tmax", "source": "box"} + ] + }, + "timeOnly": { + "doc": "Time-only temporal types — no spatial box; time extent only.", + "box": null, + "srid": null, + "types": ["tbool", "ttext"], + "columns": [ + {"name": "tmin", "sqlType": "timestamptz", "accessor": "temporal_start_timestamptz", "source": "value"}, + {"name": "tmax", "sqlType": "timestamptz", "accessor": "temporal_end_timestamptz", "source": "value"} + ] + } + }, + "deferred": { + "pointcloudCellIndex": { + "types": ["tpcpoint", "tpcpatch", "th3index", "tquadbin"], + "reason": "STBOX covering via a type-specific box path (e.g. tpcbox_to_stbox); fold into the `spatial` class once the catalog confirms a uniform temporal->STBOX converter for these families." + } + }, + "notes": [ + "The covering columns are a denormalisation of the value's bounding box; the canonical MEOS-WKB BLOB remains the lossless source of truth.", + "Materialising the covering columns as primitive Parquet columns gives Iceberg manifest-level file pruning and Parquet row-group min/max pruning, with no spatial-aware engine.", + "zmin/zmax are emitted only for 3D values (`when: hasZ`); 2D values omit them or store null.", + "`source: box` accessors take the box returned by `class.box.from(value)`; `source: value` accessors take the temporal value directly.", + "This descriptor is type-agnostic per class exactly as `portable-aliases.json` is type-agnostic per operator family — codegen consumes it identically across every binding." + ] +} diff --git a/tools/covering/test_covering.sql b/tools/covering/test_covering.sql new file mode 100644 index 00000000..0d583006 --- /dev/null +++ b/tools/covering/test_covering.sql @@ -0,0 +1,24 @@ +-- Loads the generated covering macros and checks they materialise the covering +-- columns. Run with a MobilityDuck-enabled DuckDB: +-- duckdb < tools/covering/test_covering.sql +.read tools/covering/covering.sql + +-- spatial: a tgeogpoint trajectory -> xmin..zmax, tmin/tmax, srid +WITH t AS ( + SELECT 7 AS mmsi, + tgeogpointSeq(list(TGEOGPOINT(ST_Point(lon, lat), ts) ORDER BY ts)) AS traj + FROM (VALUES (TIMESTAMPTZ '2026-02-26 08:00:00+00', 4.40, 51.20), + (TIMESTAMPTZ '2026-02-26 10:00:00+00', 4.95, 51.48)) v(ts, lon, lat) +) +SELECT mmsi, c.xmin, c.xmax, c.ymin, c.ymax, c.zmin, c.tmin, c.tmax, c.srid +FROM (SELECT mmsi, covering_spatial(traj) AS c FROM t); + +-- number: a tfloat -> vmin/vmax, tmin/tmax +WITH t AS (SELECT tfloat '[1.5@2026-02-26 08:00:00+00, 9.2@2026-02-26 10:00:00+00]' AS v) +SELECT c.vmin, c.vmax, c.tmin, c.tmax +FROM (SELECT covering_number(v) AS c FROM t); + +-- time-only: a tbool -> tmin/tmax (no spatial box) +WITH t AS (SELECT tbool '[true@2026-02-26 08:00:00+00, false@2026-02-26 10:00:00+00]' AS v) +SELECT c.tmin, c.tmax +FROM (SELECT covering_timeOnly(v) AS c FROM t);