From a3fb6259a23f8231d616c4d5b380580ae15e0162 Mon Sep 17 00:00:00 2001 From: jiangjianyuan Date: Mon, 26 Oct 2026 10:12:18 +0800 Subject: [PATCH 1/2] sink: add native Databend sink and self-contained E2E Consume native change events through bounded memory and quota-limited local spool into retained raw rows, target MERGE and compaction. Preserve key identity, type mapping, DDL epochs and callback completion ordering. Register Databend/lake schemes, explicit conflict keys and metrics with limited existing-sink integration. Stop on ambiguous mutation outcomes and document controlled single-capture target-exclusivity requirements. Include design and acceptance reports, focused unit/race tests, real backend response-loss tests and an owned local Databend/TiUP/TiCDC runner. No collector credit protocol, automatic HA service or full snapshot initialization is introduced. --- Makefile | 7 +- ...databend-native-sink-design-review-v0.2.md | 1236 +++++++++++++++++ ...databend-native-sink-design-review-v0.3.md | 447 ++++++ .../databend-native-sink-design-review.md | 395 ++++++ .../design/databend-native-sink-e2e-report.md | 131 ++ ...atabend-native-sink-parity-fault-report.md | 198 +++ docs/design/databend-native-sink-plan.md | 278 ++++ .../databend-native-sink-review-prompt.md | 74 + ...tabend-native-sink-scope-review-comment.md | 94 ++ downstreamadapter/sink/databend/client.go | 218 +++ .../sink/databend/client_test.go | 110 ++ downstreamadapter/sink/databend/completion.go | 94 ++ .../sink/databend/completion_test.go | 88 ++ downstreamadapter/sink/databend/config.go | 156 +++ .../sink/databend/config_test.go | 67 + .../sink/databend/integration_test.go | 94 ++ downstreamadapter/sink/databend/lane.go | 337 +++++ .../sink/databend/live_fault_test.go | 405 ++++++ downstreamadapter/sink/databend/metrics.go | 46 + downstreamadapter/sink/databend/native.go | 263 ++++ .../sink/databend/native_test.go | 282 ++++ downstreamadapter/sink/databend/row.go | 245 ++++ downstreamadapter/sink/databend/row_test.go | 140 ++ .../sink/databend/runtime_test.go | 497 +++++++ downstreamadapter/sink/databend/schema.go | 707 ++++++++++ .../sink/databend/schema_test.go | 207 +++ downstreamadapter/sink/databend/sink.go | 278 ++++ downstreamadapter/sink/databend/sql.go | 126 ++ downstreamadapter/sink/databend/sql_test.go | 92 ++ downstreamadapter/sink/databend/writer.go | 160 +++ .../sink/databend/writer_test.go | 176 +++ downstreamadapter/sink/metrics/databend.go | 29 + downstreamadapter/sink/sink.go | 5 + downstreamadapter/sink/sink_test.go | 18 + pkg/common/event/table_schema_store.go | 2 +- pkg/common/event/table_schema_store_test.go | 9 + pkg/common/types.go | 1 + pkg/config/sink_protocol.go | 6 + pkg/errors/databend.go | 27 + pkg/errors/helper.go | 2 + pkg/metrics/downstream_type.go | 2 + pkg/metrics/downstream_type_test.go | 3 + pkg/metrics/init.go | 1 + pkg/sink/spool/spool.go | 6 + pkg/sink/spool/spool_test.go | 6 + .../databend_native/.gitignore | 1 + .../databend_native/README.md | 301 ++++ .../databend_native/baseline.py | 96 ++ .../baseline_parity_test.go.txt | 120 ++ .../databend_native/extended.py | 299 ++++ .../databend_native/local.py | 483 +++++++ .../databend_native/options.py | 176 +++ .../integration_tests/databend_native/run.py | 769 ++++++++++ .../integration_tests/databend_native/run.sh | 10 + .../databend_native/test_local.py | 160 +++ 55 files changed, 10178 insertions(+), 2 deletions(-) create mode 100644 docs/design/databend-native-sink-design-review-v0.2.md create mode 100644 docs/design/databend-native-sink-design-review-v0.3.md create mode 100644 docs/design/databend-native-sink-design-review.md create mode 100644 docs/design/databend-native-sink-e2e-report.md create mode 100644 docs/design/databend-native-sink-parity-fault-report.md create mode 100644 docs/design/databend-native-sink-plan.md create mode 100644 docs/design/databend-native-sink-review-prompt.md create mode 100644 docs/design/databend-native-sink-scope-review-comment.md create mode 100644 downstreamadapter/sink/databend/client.go create mode 100644 downstreamadapter/sink/databend/client_test.go create mode 100644 downstreamadapter/sink/databend/completion.go create mode 100644 downstreamadapter/sink/databend/completion_test.go create mode 100644 downstreamadapter/sink/databend/config.go create mode 100644 downstreamadapter/sink/databend/config_test.go create mode 100644 downstreamadapter/sink/databend/integration_test.go create mode 100644 downstreamadapter/sink/databend/lane.go create mode 100644 downstreamadapter/sink/databend/live_fault_test.go create mode 100644 downstreamadapter/sink/databend/metrics.go create mode 100644 downstreamadapter/sink/databend/native.go create mode 100644 downstreamadapter/sink/databend/native_test.go create mode 100644 downstreamadapter/sink/databend/row.go create mode 100644 downstreamadapter/sink/databend/row_test.go create mode 100644 downstreamadapter/sink/databend/runtime_test.go create mode 100644 downstreamadapter/sink/databend/schema.go create mode 100644 downstreamadapter/sink/databend/schema_test.go create mode 100644 downstreamadapter/sink/databend/sink.go create mode 100644 downstreamadapter/sink/databend/sql.go create mode 100644 downstreamadapter/sink/databend/sql_test.go create mode 100644 downstreamadapter/sink/databend/writer.go create mode 100644 downstreamadapter/sink/databend/writer_test.go create mode 100644 downstreamadapter/sink/metrics/databend.go create mode 100644 pkg/errors/databend.go create mode 100644 tests/integration_tests/databend_native/.gitignore create mode 100644 tests/integration_tests/databend_native/README.md create mode 100644 tests/integration_tests/databend_native/baseline.py create mode 100644 tests/integration_tests/databend_native/baseline_parity_test.go.txt create mode 100644 tests/integration_tests/databend_native/extended.py create mode 100755 tests/integration_tests/databend_native/local.py create mode 100644 tests/integration_tests/databend_native/options.py create mode 100755 tests/integration_tests/databend_native/run.py create mode 100755 tests/integration_tests/databend_native/run.sh create mode 100644 tests/integration_tests/databend_native/test_local.py diff --git a/Makefile b/Makefile index 215d263f8f..e4f5b8d24d 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ generate-protobuf generate_mock \ cdc kafka_consumer storage_consumer pulsar_consumer filter_helper kafka_auth_server \ prepare_test_binaries \ - unit_test_in_verify_ci integration_test_build integration_test_build_fast integration_test_mysql integration_test_kafka integration_test_storage integration_test_pulsar \ + unit_test_in_verify_ci integration_test_build integration_test_build_fast integration_test_mysql integration_test_kafka integration_test_storage integration_test_pulsar integration_test_databend \ generate-next-gen-grafana check-next-gen-grafana @@ -255,6 +255,11 @@ check_failpoint_ctl: tools/bin/failpoint-ctl integration_test: integration_test_mysql +# Self-contained local source/capture/Databend; does not require unrelated Kafka, +# Pulsar, MinIO, diff-inspector or downstream TiDB test binaries. +integration_test_databend: + python3 tests/integration_tests/databend_native/local.py + integration_test_mysql: check_third_party_binary tests/integration_tests/run.sh mysql "$(CASE)" "$(START_AT)" diff --git a/docs/design/databend-native-sink-design-review-v0.2.md b/docs/design/databend-native-sink-design-review-v0.2.md new file mode 100644 index 0000000000..30e3077abc --- /dev/null +++ b/docs/design/databend-native-sink-design-review-v0.2.md @@ -0,0 +1,1236 @@ +# RFC：TiCDC 原生 Databend Lake Warehouse Sink 详细设计与评审报告 + +| 项目 | 内容 | +|---|---| +| 文档版本 | v0.2，2026-10-24，吸收两份独立评审后的协议修订版 | +| 决策状态 | Proposed；存在生产阻塞项，不是 Ready for Production | +| 实现状态 | 仅设计文档；未新增 sink、未提交、未部署 | +| 用户已确定要求 | 原生 TiCDC sink；迁移 bend-cdc 核心能力；批量写 staging/MERGE；第一版必须有内存 + 限额本地磁盘 spool + 背压 | +| 分析对象 | Databend SQL warehouse；不泛指 Iceberg/Delta/Hudi 文件格式 | +| 主动评审目标 | 找出丢数据、状态回退、错误确认、DDL 重放、双写、内存/磁盘失控及不可恢复场景 | + +> 阅读提醒：本文将“源码事实”“拟定设计”“待验证假设”分开描述。章节完整不代表所有协议已证明。第 21 节列出阻塞项；任何 agent 都不应把它们自动视为已解决。 + +## 0. 评审导航与版本固定 + +### 0.1 两份源码基线 + +| 对象 | 本地仓库 | 固定 commit | +|---|---|---| +| bend-cdc | `/Users/jiangjianyuan/work/git/pingcap/webapi`，子目录 bend-cdc | `b6481621021e82aee659ad88e8e6c00f6dd11906` | +| 新架构 TiCDC | `/Users/jiangjianyuan/work/git/pingcap/ticdc` | `2bf6dacd0aec9ac2fad464600e47341f9aa951f2` | + +TiCDC remote 是 `git@github.com:sdojjy/tiflow.git`；被分析的新架构分支名为 `feat/create-paused-changefeed`。这是 fork 分支，不是上游最新主干，也没有证据证明它就是线上部署版本。 + +**当前 checkout 已切换**：本报告编写时，本地 TiCDC 是旧架构 `master` / `03ed2432bd7782198742d3aa20e07edcc47097e6`。因此新架构代码一律通过 `git show` 读取固定 commit；没有切分支，没有覆盖其他工作。不能在当前 HEAD 下搜索不到新接口，就认定报告引用错误；也不能拿旧架构接口直接实现本文。 + +复核方法（只读，无需 checkout）： + +```bash +git -C /Users/jiangjianyuan/work/git/pingcap/ticdc show \ + 2bf6dacd0aec9ac2fad464600e47341f9aa951f2:downstreamadapter/sink/sink.go +git -C /Users/jiangjianyuan/work/git/pingcap/ticdc show \ + 2bf6dacd0aec9ac2fad464600e47341f9aa951f2:pkg/sink/spool/spool.go +git -C /Users/jiangjianyuan/work/git/pingcap/webapi show \ + b6481621021e82aee659ad88e8e6c00f6dd11906:bend-cdc/pkg/sink/databend/databend_cdc.go +``` + +### 0.2 推荐阅读顺序 + +1. 目标与非目标（§1–2）。 +2. 安全不变量、事件与表状态机（§4–5）。 +3. 内存/磁盘预算和攒批（§6–7）。 +4. MERGE/恢复/DDL/所有权（§8–12）。 +5. 类型、配置、运行保障（§13–17）。 +6. 测试、上线、阻塞项、评审问题（§18–23）。 + +评审结论应给出具体执行时序和反例，不只判断“总体可行”。文末 TiCDC 源码索引用 T 编号，bend-cdc 用 C 编号，阻塞项用 B 编号，三者不混用。TiCDC 链接固定 commit；bend-cdc 本机链接须配合本节固定 commit 复核。 + +### 0.3 v0.2 评审处置与变更摘要 + +本轮只修订设计，不把新增协议描述当作实现/实验成功。两份评审中有价值的反例均进入验收；对无证据的事实断言和不充分修复，作如下限定： + +| 评审主题 | 处置 | 本版对应内容 | +|---|---|---| +| PK不变、二级UK更新也会拆成同TSO DELETE/INSERT | 接受；是普通UPDATE的支持契约问题，不只是PK变键 | §11.4，固定raw-change模式；B05 | +| DDL不能套用DML事务fence；KillQuery返回nil不是终态 | 接受 | §9.4、§12.2分mutation矩阵；B01/B03 | +| 增量分片提前确认/整事件回收死锁 | 接受 | §5.2、§7.4；新增B17 | +| 入口共享线程与共享chunk绕过预算 | 接受，需要跨层接入，不是只改sink | §6.7；新增B16 | +| raw/tombstone在维护成功时仍随历史键增长 | 接受 | §9.6；新增B18 | +| no-op MERGE必然不消费stream，是规范要求 | 不接受其普遍性断言；存在版本相关已知修复，必须测实际SQL/版本 | §9.7；B02 | +| enable-table-across-nodes=false等于禁用HA | 不接受；它限制拆表分布，不阻止整表迁移/接管 | §12.5；T15 | +| 固定sleep、进程panic或取消成功能保证旧SQL终止 | 不接受；只能在可信终态/真正fence后放行 | §12.2、§12.5 | +| Segment pinning、stream失效、bootstrap竞态完全遗漏 | 原版已识别但不可验收，细化并分别跟踪 | §6.8、§9.8、§10.6；B13–B15 | +| 必须用事务DAG归约/唯一约束journal/标记行推进stream | 不预设实现成立；先验证源净事件、目标能力与原子性 | §10–12；不盲选SQL tie-break/无界事务缓存 | + +新阻塞项编号保留评审建议的B13–B15，另外增加B16入口信用、B17完成封口、B18远端容量。既有B01–B12不重编号；“设计已明确”与“实现/实验已关闭”分别记录。 + +## 1. 背景、目标与成功标准 + +### 1.1 当前链路及代价 + +```text +TiDB → TiCDC 原生事件 → Canal JSON 编码 → S3/OSS 文件 + → bend-cdc LIST/GET/SQS → Canal JSON 解码 + → 元数据/类型处理 → Databend raw → stream → MERGE → target +``` + +中间链路有编码/解码、对象请求、文件扫描、网络流量及独立消费进程成本。它也提供较长的中间缓冲、审计与回放窗口；去掉对象存储不等于只删成本而不损失能力。 + +### 1.2 目标链路 + +```text +TiDB → TiCDC 原生 DMLEvent / DDLEvent + → 事件适配/元数据/有界 admission + → 内存热缓冲 → 必要时溢写限额本地 spool + → 同表同 schema epoch 攒批 → Databend raw INSERT + → append-only stream → MERGE → 目标表可见 + → 释放覆盖的 spool + 完成 flush callback → checkpoint 可推进 +``` + +目标是：保留核心同步正确性，降低增量中间链路成本；不是完全消除 JSON、磁盘或 Databend MERGE。第一版 raw_data 仍可用 JSON 作为受控行载体，但不再使用 Canal JSON envelope 传输。 + +### 1.3 成功标准 + +- 最终状态与源在同一已完成逻辑水位上一致；包括 DELETE、不发生旧数据复活。 +- source-to-target 按 at-least-once 传输，重复与重启后幂等收敛。 +- 新增可确认进度必须有目标应用成功的证据；正常路径要求MERGE成功，合法幂等no-op要求同等级的已支配/消费证据（§9.7），绝不以仅raw提交代替。 +- 支持权威建表、复合主键、批量 staging、核心类型映射和明确白名单 DDL。 +- 进程内存、spool 磁盘、节点总预算有界；满载后背压而不是无界积压。 +- 达到双方约定的吞吐、target-applied lag 和恢复时间；数值待压测/产品确认，不预设达标。 +- 固定相同数据量、下游和延迟 SLO 后，总成本实测低于旧链；不预先承诺节省百分比。 + +## 2. 范围、非目标与第一版支持矩阵 + +### 2.1 本设计包含 + +原生增量 sink、连接与鉴权、schema bootstrap、类型转换、内存/磁盘 spool、批处理、raw/stream/MERGE、checkpoint 回调、DDL 屏障与恢复、所有权/接管、配额、监控、灰度和回退。 + +### 2.2 非目标 + +- 不把 TiCDC 改成 MySQL/Postgres/MongoDB 通用捕获器。 +- 不把新 sink 当成自动全量导出器；全量初始化继续独立执行。 +- 不保证跨表或跨物理片段的源事务原子可见，也不保证读取者获得源数据库级一致快照。 +- 不承诺复现所有 TiDB/MySQL 约束、索引、collation、默认表达式和生成列计算行为。 +- 不支持外部应用同时改目标表/内部 raw/stream/journal。 +- 不以本地 spool 提供节点永久丢失后的恢复能力；它不是 WAL。 +- 不支持无限期下游停机、无限 raw 历史或无限磁盘预算。 + +### 2.3 第一版边界 + +| 能力 | 第一版目标 | 约束 | +|---|---|---| +| 非分区表，非空单列/复合 PK | 支持 | PK 列保留于完整 after image，明确 before image 契约 | +| 无 PK/nullable UK/force-replicate | 拒绝 | 不自动换用隐式 handle，也不静默 append-only | +| 分区表/table-across-nodes | 拒绝 | 未解决 logical table owner 汇聚前不开放 | +| 同逻辑目标表一个 owner | 必须 | 正常调度与异常接管都需成立 | +| PK不变、仅二级UK值更新 | 必须支持 | 固定raw-change事件模式，禁止继承默认拆分模式；见§11.4 | +| 主键值更新 | 验证门槛，未验收前不承诺支持 | 必须证明入口净变化、旧键删除与重放；不能假设所有变键都以UPDATE暴露而承诺可检测拒绝 | +| 主键定义改变 | 拒绝 | 改 PK 列集合/类型可能重定义 row identity | +| 历史空表自动建表 | 必须提供路径 | start-ts schema bootstrap 或全量初始化预建 | +| 列 DDL | 白名单 | ADD/DROP/RENAME 非键列，有限 MODIFY | +| TRUNCATE | 核心对齐项 | table incarnation 与中断恢复验收前不能上线 | +| DROP/RENAME TABLE/SCHEMA、复杂分区 DDL | 初期阻塞 | 不以 warning 代替正确处理 | +| 二级索引 DDL | 有条件 no-op | 已证明不改变行身份/投影后才确认 | +| 不支持的同步选项 | 创建时拒绝 | 如未经设计的 active-active、BDR、syncpoint，不能悄悄忽略 | + +“第一版目标”不等于实现完成；若某核心项仍未验证，只能发布带明确限制的 PoC，不能对外声称 bend-cdc 核心能力全量对齐。 + +## 3. 代码现状与迁移策略 + +### 3.1 bend-cdc 实际入口 + +事实：TiDB 路径是 `TiDBRunner → TiDBCDCManager → DatabendMergeSink → DatabendSink/CDCManager`（C01–C03）。不是简单 `DatabendSink.WriteBatch` 的固定 id INSERT/DELETE 实现。 + +| 已有职责 | 迁移策略 | +|---|---| +| 文件 discovery、SQS、Canal 解码、S3 key checkpoint | 删除这层职责，使用 TiCDC 原生事件与上游进度 | +| 每表 PK 推导、目标路由、epoch 管理 | 保留语义，从 TableInfo/DDL 获取权威数据 | +| raw 参数化 INSERT、stream、MERGE | 迁移核心算法，不照搬 goroutine/循环生命周期 | +| 乱序抑制、最新水位、DELETE tombstone | 必须保留 | +| 类型转换、binary/ENUM/SET/JSON 测试 | 保留测试语义，重写原生 datum 表示 | +| schema 白名单和 TRUNCATE | 保留并补充 TiCDC 原生恢复协议 | +| 每表连接池/后台循环、全局 Background context | 不照搬；改为受限共享资源与可取消生命周期 | +| Dumpling/CSV/COPY INTO | 独立初始化路径，接口共享 mapper，任务生命周期不并入 sink | + +事实：raw 是普通持久化表，附带 append-only stream；不是临时内存表。严格顺序模式查询 retained raw history；compaction 保留每个键最新版本，删除也保留墓碑（C03）。因此 raw 容量可能随**历史键集合**增长，而不只是当前目标行数。 + +### 3.2 TiCDC 现有接入点 + +T01 的 Sink 接口支持 AddDMLEvent、FlushDMLBeforeBlock、WriteBlockEvent、SetWriteGate、Run、Close。T02 提供 CommitTs、TableInfo、before/after 和 enqueue/flush 分离回调。 + +不能仅改 factory：需要 scheme/config、SinkType、TableSchemaStore、dispatcher、恢复、metrics 和测试等配套。冒充 MysqlSinkType 会触发 `*mysql.Sink` 强转（T07），不可采用。 + +### 3.3 复用现有 S3 spool 的边界 + +固定新架构基线已有 spool（T03–T05），具备内存热数据、segment 溢写、磁盘配额等待、释放和回调拆分。拟复用/提取通用能力，不直接复用 Canal 编码器或 S3 payload builder。 + +已发现必须审查的差异： + +1. `TryEnqueue` 对单 entry 大于磁盘配额有“内存接受后立即刷”的例外，不能当严格内存上限。 +2. `prepareWorkDir` 启动清空所属目录，证明这不是重启恢复 WAL。 +3. `acceptEntryLocked` 在溢写时先序列化 blob，存在额外瞬时内存；必须计入预算。 +4. S3 `buildPayload` 使用 bytes.Buffer 拼完整上传批次；新 sink 不能照搬后声称磁盘回读完全流式。 +5. 基线 S3 路径有 unlimited channel 等结构；不应复制到要求硬预算的新 sink。 +6. 现有 spool 预算不是现成的节点所有 changefeed 总预算，更不是全部磁盘消费者的全局预算。 + +## 4. 设计原则与安全不变量 + +| 编号 | 必须始终成立的约束 | +|---|---| +| I1 | 未有目标应用成功或合法幂等no-op完成证据的事件不得触发flush callback；仅raw提交、结果行/计数不构成证据 | +| I2 | 同事件重放不能使目标版本回退、重复插入业务行或复活已删除键 | +| I3 | 同逻辑目标表只有一个有效 merge/DDL owner；锁不能只存在单进程 | +| I4 | DDL fence 前的 DML 已应用；fence 后 DML 不使用旧投影；旧 DML 重放不能污染新 schema | +| I5 | collector交付/共享chunk、spool、队列、编码、回读、元数据预算有界；磁盘有单任务和节点总预算；远端raw有停写阈值 | +| I6 | 正在等待配额的生产者不能阻止负责 MERGE/回收的消费者前进 | +| I7 | 未物理释放的 segment 空间不能从磁盘预算中提前扣掉 | +| I8 | 失租/停止后的异步工作不能继续执行新的下游变更；在途旧请求必须被接管协议覆盖 | +| I9 | 最新键水位与 DELETE tombstone 不能被普通 TTL 清除 | +| I10 | schema bootstrap/version 与“某 DDL 前数据已应用”的 durable fence 不能混为一谈 | +| I11 | metrics/result parsing 失败不能被简单当成业务未提交；提交不确定也不能被简单当成功 | +| I12 | 所有“不支持”功能可诊断地拒绝/阻塞，不用静默降级代替正确性 | + +## 5. 组件划分与状态机 + +### 5.1 组件 + +```text +TiCDC dispatcher + │ DMLEvent/DDL + ▼ +DatabendSink adapter + ├─ capability/config validation + ├─ schema/route registry + ├─ budgeted admission + typed row normalizer + ├─ hot memory + disk spool + transaction completion tracker + ├─ per-table lane scheduler + │ └─ raw INSERT → MERGE → covered-batch completion + ├─ DDL barrier/recovery coordinator + └─ target client + write gate + owner fencing adapter + +Capture-wide BudgetManager / free-space guard + └─ 所有 Databend changefeed 共享，若与 S3/redo 共盘还需联合规划 +``` + +候选代码目录： + +- `/Users/jiangjianyuan/work/git/pingcap/ticdc/downstreamadapter/sink/databend/`：原生事件、callback、lane、barrier、生命周期。 +- `/Users/jiangjianyuan/work/git/pingcap/ticdc/pkg/sink/databend/`：配置、客户端、mapper、SQL、journal、恢复。 +- `/Users/jiangjianyuan/work/git/pingcap/ticdc/pkg/sink/spool/`：仅必要的通用预算与存储扩展，避免破坏 S3 行为。 +- `/Users/jiangjianyuan/work/git/pingcap/ticdc/tests/integration_tests/databend_sink/`：真实链路验证。 + +以上是新架构上的拟定目录，不表示当前旧架构 HEAD 已有这些代码。 + +### 5.2 事件/分片状态机 + +事件完成与分片生命周期必须分开，不再把“事件ACK后才释放全部entry”当顺序要求： + +```text +事件tracker：OPEN → SEALED → COMPLETED + └────────────→ FAILED / CANCELLED（未完成不确认） +分片：REGISTERED → STAGED → RAW_COMMITTED → APPLYING → APPLIED → RECLAIMED + └→ COMMIT_UNKNOWN → RECONCILING +APPLIED分片可独立回收；其源事件仍可处于OPEN或SEALED +``` + +APPLIED表示目标已包含该分片，或按§9.7证明它被已应用版本正确支配；不是只看到raw中有更高版本。回收失败不撤销APPLIED、不重复MERGE，但空间继续计费。 + +tracker协议（拟实现，非现有DMLEvent保证）： + +1. 单一编码producer持有OPEN状态；每个分片必须先原子登记、增加registered/pending，再发布给消费者。发布失败也不能把它伪装成已应用，需重试该分片或置FAILED。 +2. 只有编码遍历完整个源事件、没有尚未登记的行，才可Seal；Seal与失败/完成检查在同一状态同步域内。若先前分片均已完成但producer还OPEN，仍不得确认。 +3. 确认条件为 `SEALED && pending == 0 && !failed && CAS(completed,false,true)`；Seal与最后一个Apply竞争时由CAS保证只触发一次。零分片也必须Seal，过滤语义通过后才完成。 +4. 每个分片APPLIED转换也必须一次性；仅递减pending一次，并独立释放payload。只保留未完成分片的小描述符和有界tracker，不永久保存所有已回收分片的位图；每事件/每lane限制未完成窗口。 +5. DetachPostCallbacks不捕获DMLEvent，但其flush闭包自身没有once保护，只有enqueue有CAS（T02）；由tracker提供flush的一次性。失败/取消后迟到完成只能清理资源，不能把未完成事件改成成功。 +6. 一个大事件可边编码边应用/回收，累计编码量可以大于spool额度；并发驻留量、源chunk与元数据仍受§6约束。不能靠一次性复制整事件换取封口。 + +反例验收：暂停F2登记，让F1完成，此时flush次数必须为0；随后登记F2、Seal、应用F2后恰为1。再测试大于可用spool的事件、重复完成、Seal/Apply并发、发布失败与取消。这里的“事件”是TiCDC完成单位，不承诺跨表事务原子性。 + +### 5.3 表状态机 + +```text +DISCOVERED → BOOTSTRAPPING → ACTIVE +ACTIVE → DRAINING_BEFORE_DDL → DDL_PENDING → DDL_APPLIED → ACTIVE +任意状态 → FENCED / STOPPING / FAILED +重启：RECOVERING → 验证 owner/schema/journal/未完成 raw → ACTIVE +``` + +只有 ACTIVE 接受正常新 epoch DML；旧版本重放走受控验证/跳过流程，不以普通写入路径解释。 + +v0.2补充暂停/恢复转移，避免把所有阻塞都混成retry： + +| 状态 | 可继续的工作 | 恢复ACTIVE的必要条件 | +|---|---|---| +| LOCAL_RESOURCE_BLOCKED | 已接管分片的编码窗口/应用/回收与控制，受保留池约束 | 物理配额/信用回到低水位，未丢唤醒且无fatal IO | +| REMOTE_CAPACITY_BLOCKED | 已raw提交coverage的应用、安全维护与故障协调 | 扩额或安全清理后的新鲜测量+增长预留通过；旧chunk不能无界排队 | +| COMMIT_UNKNOWN | 同lane仅执行权威终态查询/协调 | 服务端提交/回滚证据及coverage核验；不能仅等到timeout | +| HANDOFF_BLOCKED / FENCED | 终态核验、释放自有资源；无新业务mutation | 新有效owner取得mutation排他，完整恢复后走RECOVERING | +| RECOVERY_REQUIRED | 按证据执行授权恢复，正常admission关闭 | journal/ledger/raw/stream/binding一致且unknown均已处置 | +| REINITIALIZATION_REQUIRED | 隔离旧目标、显式新快照初始化，不在旧lane继续写 | 新基线/新内部对象校验与显式切换；不是直接改回ACTIVE | + +多个阻塞原因并存时保存原因集合；释放一个配额不能清掉COMMIT_UNKNOWN或DDL fence。RESOURCE_BLOCKED是本地资源阻塞的通称，对外状态使用LOCAL_RESOURCE_BLOCKED区分远端容量。进程失败/取消仍遵守§15，不因解除某一局部阻塞自动恢复写入。 + +## 6. 内存、磁盘 spool 与预算 + +### 6.1 为什么第一版必须有 spool + +这是已确定需求。热数据留内存，下游抖动或待 MERGE 数据增加时溢写磁盘；磁盘不是无限缓存。防止写爆依靠**硬预算 + 可用空间保护 + 背压**，不是“用了磁盘就安全”。 + +### 6.2 预算对象 + +定义： + +- `M_hot`：spool 内存 payload。 +- `M_encode`：规范化、序列化和参数构造临时内存。 +- `M_read`：磁盘回读和 SQL 批次缓冲。 +- `M_meta`：entry、callback、表状态和事务 tracker。 +- `M_in`：尚未转交/释放的源 chunk 和入口队列。 +- `D_live`：segment 实际占用，包括已逻辑释放但尚未回收部分。 +- `D_reserved`：在途写预留，包括头部、对齐、允许的分配开销。 + +要求 admission 预留预算后才接受自有数据；不能只统计 M_hot 然后无限持有 M_in/M_meta。进程 RSS 还包括 TiCDC 其他组件与 Go/runtime 开销，应单独观测，不能保证等于 sink 配额。 + +单任务 `D_live + D_reserved ≤ D_feed`，所有任务共享 `ΣD ≤ D_capture`。每个批次先占有预算，再读回/编码；并发 worker 数乘以最大批次不能超过 M_read 总池。 + +### 6.3 磁盘 admission 算法 + +```text +1. 估计/增量计算 entry 的编码及落盘字节，申请内存/磁盘预算。 +2. 在 capture 共享锁下预留预算；不能持锁做网络 IO。 +3. 留内存或 append segment；失败回滚尚未使用的预留。 +4. 对实际文件分配/长度进行结算，超出预留视为实现错误或停止 admission。 +5. 额度不足：通知压力调度；等待仅发生在有信用约束的异步任务，不阻塞共享dynstream处理线程（§6.7）。 +6. 删除/截断成功后才释放实际磁盘额度并唤醒等待者。 +``` + +文件系统 guard 要考虑：statfs 可用空间、配置的保留量、未落盘预留、segment 分配开销及 inode。不同实例共享预算时必须原子预留；其他进程写同一文件系统无法由本组件完全控制，ENOSPC 仍需容错，不能承诺绝不发生。 + +建议 dedicated spool volume 或明确与 eventstore/redo/log 的共盘预算。仅限制 Databend spool 不会限制 TiCDC 其他组件的磁盘增长。 + +### 6.4 队列、callback 与超大事务 + +- 所有数据队列按**字节与元素数**同时限额；每个小事件也占 M_meta，避免零小 payload 绕过限制。 +- admission 前不能无界 detach/复制所有 chunk；增量编码和所有权转移需有明确时机。 +- callback 不得捕获整块原始 DMLEvent/chunk；使用独立 completion tracker（参考 T02），但要证明所有数据引用已解除。 +- 大事务按行/字节拆分；原子性不扩大为整个源事务，多片 MERGE 期间目标可能看到部分事务，这是公开限制。 +- 单行大于支持上限时终止该表/changefeed并保留错误，不采用任意内存直通。 +- 最大并行事务 tracker 与等待确认 entry 数另有限制，防止“数据落磁盘了，callback 仍把内存占满”。 + +### 6.5 高低水位和死锁规避 + +达到高水位时暂停新 admission，低水位后恢复，避免频繁抖动。具体水位按有效内存+磁盘预算定义，不盲用 S3 的比例。 + +MERGE、清理、lease/DDL 控制必须有独立可运行资源。配额等待者不得占住唯一 worker、表锁或整个 executor。额度不足可强制提前 MERGE，忽略通常的 merge-trigger-rows/interval；否则“额度满但没达到 MERGE 阈值”会永久等待。 + +### 6.6 文件生命周期与重启 + +- 目录按 sink-kind、capture/进程实例、changefeed 实例隔离；启动只能清理已确认非活跃的自身旧目录。 +- orphan 目录即使不用于恢复，也必须计入磁盘扫描与回收流程，不能因进程换 ID 每次留下 10 GiB。 +- MERGE 后 entry 逻辑释放不保证整个 segment 可删除，必须处理长寿命 entry 造成空间钉住；优先缩短 segment、密度控制或安全复制存活 entry。 +- 本地文件不要求承担 crash durability；重启仍从 checkpoint 重放。是否 fsync、何时 fsync要按临时 spool 语义确定，不能给出 WAL 级承诺。 +- checksum/长度校验发现运行中 spool 损坏：停止、不确认，从 checkpoint 恢复；不跳过坏记录继续。 + +### 6.7 collector→dispatcher→sink入口信用协议 + +**已确认接入事实**：BasicDispatcher同步调用AddDMLEvent，dynstream线程同步等待Handle返回；collector在调用HandleEvents之前已执行BatchDMLEvent.AssembleRows，多个DMLEvent可共享chunk（T12–T14）。因此原接口本身不能证明本设计的端到端有界性。 + +拟新增的能力协议如下，名称是概念接口，不表示当前代码已有： + +```text +ReceiveEnvelope(deliveryID, path, sessionEpoch, wireBytes, rowCount, decodedBound) + → collector TryReserveDelivery(bytes, elements, decodeExpansion) + 无信用：block该path并保留最多一个已计费delivery；返回共享线程 + 有信用：受限executor解码/AssembleRows，附SharedChunkLease + → dispatcher一次登记该delivery的全部tableProgress blocker + → sink TryTransfer(deliveryID, lease, events) + 接管：转移有界入口队列的所有权，快速返回；异步增量编码 + 暂停：delivery仍归上层，block该path；异步信用通知后重投同delivery + → 行数据复制到有预算的内部格式后逐步解除chunk引用 + → 最后一个使用者释放SharedChunkLease；归还decoded预算 +``` + +必须实现的细则： + +- 在线路收包/排队阶段就限制最大消息字节、元素数和同时在途delivery，不能等AssembleRows后才算账。发送端按事件边界/行数/字节切batch，协议协商硬上限；解码膨胀按保守上界预留并增量校验，超限拒绝/重建会话，不继续分配。 +- 传输编码大小不等于chunk内存。SharedChunkLease按实际分配容量保守计费**一次**，覆盖所有DMLEvent视图；每个事件视图和callback再计M_meta。不通过“这个事件只用了一行”只计那一行。 +- 原始chunk生命周期由lease所有者管理，sink不能擅自把共享Rows清空而破坏同batch其他视图。编码worker复制受限行窗口后归还引用；最后一个视图释放前全chunk仍计费。 +- 信用是额度凭证，不是“承诺之后归还”的无界债务。collector/dynstream已有内存控制若可复用，必须证明同一buffer单一拥有/跨层转移，无遗漏、无双计；接入前做分配链路审计。 +- TryTransfer未接管时不Detach回调、不发PostEnqueue，也不重复注册tableProgress。用稳定deliveryID和sessionEpoch记状态：RECEIVED/PROGRESS_REGISTERED/TRANSFERRED；重投仅重试转移。所有事件先加blocker再允许任何一个完成，沿用基线防checkpoint跳跃约束。 +- AddDMLEvent旧签名不能表达拒收/稍后重试。新增可选AdmissionSink capability和dispatcher/collector分支（或等价显式接口）；旧sink走原路径。不能仅改Databend.Add内部实现后宣称协议已接通。 +- 每path最多一个已交付等待batch，跨path有字节/数量总上限；阻塞的是该path，不是共享handler线程。只有在sink入口lease和元数据已接管后才可PostEnqueue；它仍不表示应用成功。 +- data信用与控制事件/DDL/lease通知分池，并为B表保留最小公平信用。只有A达到每path上限、全局仍有保留额度时，保证B可推进；全局预算耗尽时允许整个feed背压，但MERGE/回收/控制必须可运行。不承诺在全节点资源耗尽时无限保障B的新DML。 +- 释放信用时先完成可重入的状态更新，再通知可重试path，避免丢唤醒；session失效的旧delivery只释放，不能重投新owner。 + +`max-delivery-bytes`包括已解码chunk上界而非仅wire长度，`max-delivery-events`限制小事件元数据。若现有eventservice不能拆分单个过大原生事件，必须增加有序分片/完成标记协议或明确在接收前拒绝该输入；不能因为源事务大而无界AssembleRows。`max-row-bytes`不能代替这些上限。 + +本节扩大了原方案范围：B16未实现并通过共享线程公平性与真实内存实验前，不能宣称128/256MiB示例是端到端内存硬界。大源事务可支持为多个有界delivery，不要求所有源事务进一个chunk。 + +### 6.8 Segment pinning与压力排空协议 + +保留物理字节计费，不以逻辑Release数估算空间。拟给segment维护有界的存活引用→lane反向索引；压力时： + +1. BudgetManager以pressureEpoch合并通知，唤醒所有持有受压segment引用的lane，而不只唤醒最新申请者。 +2. lane跳过通常行数/时间阈值，刷出当前可应用分片；OPEN大事件的已应用片也可回收。DDL/COMMIT_UNKNOWN阻塞的lane记录原因，不能为释放空间越过barrier。 +3. 独立M_read/控制池执行排空；数据admission不能占用这部分保留量。按最老entry和公平轮转调度，避免热点饿死冷表。 +4. 首版优先验收“强制排空+物理回收”，不把segment搬迁当现成解法。若仍有长寿命pin且有必要搬迁，需预留maintenance disk/memory，先复制并校验、原子切换引用、等待读者退出，再删旧segment；复制期间两份都计费。不能在配额已满时临时多写一份。 +5. 不能排空/搬迁时进入RESOURCE_BLOCKED并告警，冻结新输入；通过恢复故障或人工扩额解除，不删除存活数据。可诊断的下游阻塞不等于无条件保证活性。 + +验收需覆盖热表A与冷表B、同segment混排、额度满但未达MERGE阈值、坏segment/删除失败、搬迁额度不足，以及大事件OPEN期间空间循环。原版已有时间触发，故不能把“冷表永远等5万行”作为既定事实；测试的是压力传播和保留资源是否真正有效。 + +## 7. 攒批、调度和确认语义 + +### 7.1 两层批处理 + +第一层按**目标表 + incarnation + schema version**攒 raw INSERT；任一条件触发：行数、编码字节、最长等待、DDL barrier、配额压力、优雅停止。 + +第二层合并若干 raw 批次再 MERGE；行数/时间/压力/barrier 均可触发。计时从最老未完成记录/批次入队起算，不能每次新事件到来都重置导致小表永不刷出。 + +上次示例 5000 行、8 MiB、1s raw flush，50000 行或 5s MERGE，只是压测起点，不是默认 SLO。真实延迟还包括排队、网络、SQL、重试、调度和 checkpoint 广播。 + +### 7.2 第一版采用保守的表级串行 lane + +拟定决策：同一逻辑目标表在一个 merge cycle 内，先提交选中的 raw 批次，再停止该表新的 raw INSERT，完成 MERGE及必要状态确认后，开启下一 cycle。新的源事件仍可进入受配额控制的 spool。 + +这样避免“MERGE 固定 stream 快照后，另一 raw writer 插入更高版本，被全历史过滤看到但并未一起应用”的复杂竞态。不同表有限并行。该决策以吞吐换证明简单，后续若并发 raw 与 MERGE，必须另写 snapshot/coverage 协议，不能直接去掉互斥。 + +多个 DMLEvent 可以共用一个 cycle；一个大事件也可跨 cycle，但回调等所有片段 APPLIED。数据 replay 事件可以被更新版本覆盖，不等于必须逐个中间状态呈现在目标。 + +### 7.3 覆盖集合与回调 + +每个cycle固定本地 `coverage = {entry/fragment IDs}`，其raw INSERT已明确成功，且不混入尚未应用DDL的schema。MERGE终态与消费覆盖证明满足§9后才将分片APPLIED；事件是否可回调还须§5.2的SEALED条件。 + +本地 generation/entry ID 是覆盖记账工具，不是持久化恢复身份，也不能当作 TiDB row version。进程重启后重新从 source checkpoint 构造完成关系。 + +`PostEnqueue` 只表示 sink 接管数据且满足背压策略，不能用于推进 durable checkpoint。`PostFlush` 才表示 target applied。沿用 detach 回调时要审查每个 hook 对动态流唤醒、流控与内存释放的影响。 + +### 7.4 释放与回收顺序 + +下游应用/消费已确认 → 分片一次性APPLIED → 归还其payload/entry并触发物理回收 → 更新tracker pending；满足SEALED完成条件后独立执行事件callback。上述状态更新需有同步与可重入保证；不要求磁盘unlink成功才回调，但未释放物理空间仍计费。 + +关键区别：F1已应用时即使事件仍OPEN、F2未登记或未应用，也可回收F1。tracker只保留已完成计数/有界未完成集合，不捕获F1数据；从而事件总大小可超过spool可用额度而不循环等待整事件ACK。 + +callback 不能重复执行；回收失败不撤销 APPLIED、不重复业务 SQL，继续保留磁盘计费。callback 后进程马上崩溃而上游 checkpoint 未持久化时，重放仍应安全。 + +### 7.5 对 TiCDC 接口的具体映射 + +| 接口 | 拟定职责 | 禁止事项 | +|---|---|---| +| AdmissionSink能力 / AddDMLEvent接入 | 先通过§6.7信用协议；原Add只接收已拥有入口lease的交付，快速转入异步编码 | 在共享处理线程等磁盘；未接管却发PostEnqueue；重复登记progress | +| FlushDMLBeforeBlock | 向受影响 lane 插入有序 flush marker，强制 marker 前 coverage 完成 | 拿着 admission 锁等待消费者；把本地排空当全局 DDL 完成 | +| WriteBlockEvent | 由协调后的指定执行者执行 DDL/recovery，成功才 PostFlush | 所有 capture 重复执行同一个 TRUNCATE | +| AddCheckpointTs | 接收已计算进度用于观测或另行设计的维护，不承担必要的首次 MERGE 触发 | 等 checkpoint 回调才 MERGE，导致进度闭环死锁 | +| SetWriteGate | 将许可检查接入每个实际 mutation 边界 | 只在 sink 构造时检查一次 | +| SetTableSchemaStore | 注册 schema 名称/范围依赖；补齐专属 SinkType 分支 | 假设名称清单就是可用的历史完整 schema | +| Run / Close | 生命周期、受限 worker、取消、正常 drain/失租区分 | detached Background worker 或无限等待 | + +必要关系是 `独立计时/行数/压力触发 MERGE → PostFlush → TiCDC 计算 checkpoint → AddCheckpointTs`，不能反过来依赖 AddCheckpointTs 推动第一步。全局 checkpoint 是已确认覆盖的聚合结果，不是单表 `MAX(commit_ts)`。 + +cycle 的控制伪代码（不表示现有接口已有该实现）： + +```text +entries = lane.selectBoundedEntries(schema, memoryReadBudget) +coverage = freeze(entries) +for batch in splitByRowsBytesParams(entries): + assertCurrentOwnerAndWriteGate() + rawInsertAndClassifyCommit(batch) # UNKNOWN: 停留协调，不越过 +freezeFurtherRawWritesForThisTable() +assertCurrentOwnerAndWriteGate() +result = mergeAndClassifyCommit(schema, incarnation, coverage) +if result in {APPLIED_CONFIRMED, NOOP_CONFIRMED} and coverageAndConsumptionProved(result): + markApplied(coverage) + releaseEntriesAndSchedulePhysicalReclaim() + tryCompleteSealedEventsOnce() +else: + keepCoverageUnackedAndReconcileOrFail() +``` + +原始输入、batch 和 coverage 的引用必须在各预算内；伪代码的 entries 不能实现为一次把磁盘积压全部读回内存。 + +## 8. 持久化数据模型与事件身份 + +### 8.1 隔离域 + +`source identity + changefeed instance + logical table ID + table incarnation + target mapping` 共同定义同步对象。复用 changefeed 名称或 DROP 后同名 CREATE 不能复用旧状态。 + +表名只是显示与路由名，不是唯一历史身份。内部对象名使用稳定 ID/hash 加可读后缀，避免用户表名长度/转义、同名表和 `_cdc_raw` 撞名。初始化必须验证内部对象 metadata 指向同一源与任务，不能见到 IF NOT EXISTS 就接管。 + +### 8.2 行身份与版本 + +- `row_key`：按 PK 列 ID、类型、长度前缀和值构造确定性编码；保留完整 canonical key。hash 仅用于索引/命名，不能以碰撞概率为零代替相等判断。 +- `commit_ts`：事务 TSO，使用无损整数表示，不能转 float64；选择 Databend UINT64 或经范围验证的 DECIMAL(20,0)。 +- `event_id`：上述隔离域、schema version、canonical key、commit_ts、净动作的确定性摘要/编码。 +- `payload_hash`：同 event_id 对应规范化 payload 相同；冲突时 fail-closed，不能通过到达时间选一条。 +- `add_time`：观测/清理辅助字段，绝不定义业务先后。 + +**commit_ts 不是每行唯一 ID**。同一事务不同键共享 TSO。相同键同一 TSO 的多条输入，必须通过已验证的源净变化契约归约；无法区分时不能用 dispatcher Seq、slice index、进程时钟创造伪顺序。 + +### 8.3 概念 raw schema（不是可直接执行的完整 DDL） + +```text +RawRecord { + source_id, changefeed_instance, logical_table_id, incarnation, + source_schema_version, + row_key, event_id, payload_hash, + commit_ts, + action: UPSERT | DELETE, + payload: exact JSON / typed normalized row, + ingest_time +} +``` + +第一版可以每目标表一套 raw/stream，但必须全任务共享 pool/budget；不要给每张表单独起一套无限后台 goroutine。 + +### 8.4 元数据/journal + +需要区分至少三类状态： + +1. `SchemaDescriptor`：列 ID、名称、类型、精度、PK、源 schema version、目标 fingerprint。 +2. `DDLJournal`：DDL ID/commitTs、before/after fingerprint、操作内容、pending/applied、incarnation。 +3. `OwnerRecord/Binding`：任务实例、表身份、target 映射、所有权代际/能力状态。 + +schema version 表示“如何解释行”，applied DDL fence 表示“这一边界前已处理到什么程度”；bootstrap 加载已有 schema 不自动证明旧 DML 已应用（I10）。本地 writeGen/mergedGen 和数据库每键水位同样不能互相替代。 + +### 8.5 applied witness与“更高raw版本已应用”的证明 + +retained raw本身是ingested watermark，不天然是applied watermark。若SQL过滤旧candidate依赖更高版本，必须能证明该更高版本已应用；否则I1不成立。 + +拟定首选证据为`AppliedCycle(operation_id, incarnation, schema_fence, coverage_digest, consumed_boundary)`，与target MERGE及stream消费**同一已验证事务提交**。若P0证实该组合可原子提交，恢复时用该证据关联operation ledger，而不是把本机mergedGen写到数据库后声称原子。APPLIED标记不存所有payload，长期保留/汇总与journal同受远端预算控制。 + +如果无法提供上述原子证据，不允许单独“MERGE成功后补写marker”冒充无窗口协议。可另行验证在单owner冻结的同schema/incarnation范围，对受影响key从retained raw重算最高有效净动作并在本cycle重新应用，避免把未证实higher raw仅用作过滤条件；跨schema历史不自动这样重投。该替代需要明确SQL快照、coverage和no-op消费证明，未验收就保持RECOVERY_REQUIRED，不静默降级。 + +因此持久化applied witness/等价重应用算法是B02/B03的一部分；新raw schema的event_id唯一性、目标一行内容或MAX(TSO)都不能替代它。 + +## 9. raw INSERT、stream、MERGE 与 compaction + +### 9.1 raw INSERT + +使用参数绑定，不拼接业务值；显式列清单和全限定表名，避免 USE/session 对连接池的隐式依赖。控制 SQL/请求字节和参数数；大 batch 在字节预算内拆分。 + +重试必须使用同一逻辑 event_id 与 payload。INSERT 成功响应丢失可能带来物理重复 raw 行；必须先设计 MERGE去重，不能假设数据库替我们提供唯一约束。 + +### 9.2 stream 的用途和能力前提 + +raw 的每次业务变化都 INSERT 一条带 action 的记录，因此 stream 使用 append-only；业务 DELETE 是 raw 的 INSERT tombstone，不依赖 stream 捕获 raw DELETE。raw 历史清理不应成为新业务删除。 + +公开文档确认有 append-only stream，且 CREATE STREAM 页面标注 Enterprise 能力；部署许可/可用性需预检，不能假定所有 Databend 版本/发行版都有。[CREATE STREAM](https://docs.databend.com/sql/sql-commands/ddl/stream/create-stream) + +公开指南描述 stream 消费与提交绑定、并发消费者有冲突限制；**这不等于已经证明本文 MERGE/事务/no-op 的全部语义**。[Stream workflow](https://docs.databend.com/guides/load-data/continuous-data-pipelines/stream) + +第一版默认要求 stream 能力。现有 bend-cdc 建 stream 失败时回退全 raw 查询（C03），新 sink 不应悄悄照搬：性能和覆盖语义改变，必须显式配置、单独验收，默认 fail-closed。 + +### 9.3 MERGE 算法草图 + +在单 owner、表 lane 已冻结 raw INSERT、schema 一致的前提下： + +1. 确定本 cycle coverage，验证每个 fragment 已 raw committed。 +2. 从该 stream 待消费 delta 选 candidate，并参考同 incarnation、允许 schema fence 下的 retained raw history。 +3. 按canonical key选最高commit_ts的有效净动作；必须先满足§11.4源事件模式/净变化契约。不能把默认UK拆分产生的合法同版本DELETE/UPSERT误判为源数据冲突。契约满足后才对剩余同版本异payload fail-closed，完全相同replay合并。 +4. 过滤低于 retained key watermark 的旧 replay；不能因 newer 仅存在于尚未应用的另一个未协调 writer 的 raw 就确认旧事件。 +5. 使用显式列投影转换目标值，按 typed PK 相等匹配：匹配 UPSERT 更新，匹配 DELETE 删除，不匹配 UPSERT 插入，不匹配 DELETE不插入。 +6. 正常写入和零物理修改分别按§9.4/§9.7取得终态、coverage与消费证据后标记APPLIED；不是要求目标行数必须大于0。缺证据进入COMMIT_UNKNOWN/停写，不发callback。 + +MERGE 的增删改语法由官方文档支持，但这里的去重/覆盖/fence 算法是拟定设计，需真实 SQL 验证。[MERGE](https://docs.databend.com/sql/sql-commands/dml/dml-merge) + +不能直接把本文草图当生产 SQL：特别要验证 raw self-reference 与 stream 读取是否共享正确快照、无匹配行或只含被抑制旧事件时 stream 是否前进、查询投影失败后是否完全回滚。C03 的既有 SQL可作起点，不是证明。 + +### 9.4 提交结果判定 + +#### 9.4.1 adapter返回的证据结构 + +拟定TargetClient返回下面的逻辑结构，而不是`executed bool`： + +```text +ExecutionEvidence { + operation_id, sql_digest, owner_generation, table_incarnation, + schema_fence, coverage_digest, mode: AUTOCOMMIT | EXPLICIT_TXN, + attempt_id, request_id, server_query_id?, server_transaction_id?, + terminal: SUCCEEDED | FAILED | ROLLED_BACK | UNKNOWN, + commit: COMMITTED | NOT_COMMITTED | UNKNOWN, + stream_before?, stream_after?, consumption_proof?, + error_class, server_error_code?, evidence_source, observation_counts?, + observed_at, server_version, driver_version +} +``` + +operation_id属于一次固定SQL/coverage；attempt_id区分物理发送。query_id不等于幂等键，重试标签也不自动覆盖DDL。request/attempt必须在发请求前登记；收到server ID立即补记，不能仅保存在即将关闭的Rows里。SQL digest不含业务原文；错误与证据日志脱敏。 + +结果分为`APPLIED_CONFIRMED`、`NOOP_CONFIRMED`、`NOT_APPLIED_CONFIRMED`、`COMMIT_UNKNOWN`。`OBSERVATION_FAILED`是附加观测状态，不覆盖前三种提交判定。显式事务中单条MERGE的SUCCEEDED仍非COMMITTED，必须另取COMMIT终态，且同连接/会话正确绑定。 + +#### 9.4.2 首选取得路径与错误分类 + +首版拟使用受控HTTP/driver adapter：保留原始响应状态/错误与query ID，持续读取/轮询到服务端成功终态；自动提交需证明该终态与提交绑定，显式事务还需COMMIT成功。只经database/sql拿到Rows不能提供全部证据；若v0.9.4不暴露必需信息，应增加窄接口或单独客户端，不能从私有字段猜测。具体endpoint、认证、终态字段、节点路由、保留期需在P0按实际server版本落实，未落实就禁止生产/自动接管。 + +| 观察结果 | 分类与动作 | +|---|---| +| 权威服务端确认语法/权限失败且无提交 | NOT_APPLIED_CONFIRMED，停止或按错误策略重试；不callback | +| 已收到结果页，但最终轮询报执行错误 | 不是统计失败；依据服务端提交/回滚语义判断，不能证明则UNKNOWN | +| 网络断开/超时/首响应丢失 | UNKNOWN，停止同lane后续mutation，查询已有query/request关联状态 | +| Rows值解码/类型转换错误 | 与提交证据分离；有独立COMMITTED证据才保留成功，否则UNKNOWN | +| 服务端成功终态已确认，仅操作计数解析失败 | APPLIED或NOOP保持，统计unavailable，禁止为补统计重做SQL | +| Kill/Close返回nil、查询列表中暂时不存在 | 不是提交/终止证明，保持UNKNOWN | +| 显式事务中语句成功、COMMIT响应丢失 | UNKNOWN，按transaction/query关联恢复；不能按语句成功确认 | + +已核实D01：v0.9.4 `waitForData`在有缓冲行时可退出，不强制ReadFinished；KillQuery丢弃请求重试错误并返回nil。C07则在QueryContext返回Rows后立即设executed。由此只能得出“现有标记不足以作为证明”，**不能声称已复现MERGE先返回数据再提交失败**。 + +#### 9.4.3 不确定结果、重启与找不到query + +1. 进入UNKNOWN即冻结该lane及可能冲突的DDL/compaction，不确认coverage。保留本地payload直至明确结果或受控停止；不得无限重发。 +2. 在旧owner仍有效时，用有界协调worker查询服务端权威终态；证明已提交则按原coverage继续，证明未提交/已回滚且旧请求不再运行才可重试。 +3. 发请求前把在途操作描述写入目标durable operation ledger并确认（或由P0证明等价外部持久化查询目录）。ledger状态写入本身也有提交不确定性：未明确登记成功不发送业务SQL；更新丢响应按相同operation_id读回核验。它只存小元数据，不存全部源payload，也不构成分布式锁。 +4. 重启先取得§12排他性，再扫描未终结ledger、关联query/transaction终态。若数据库不支持客户端指定/关联request ID，首响应丢失时仍可能无法定位SQL，此时自动恢复不可用，进入RECOVERY_REQUIRED；不能假装ledger消除了这个窗口。 +5. query记录过期、查不到、权限不足或只查到“当前不在运行”均不等于NOT_COMMITTED。可通过已证明原子提交的应用标记/消费元数据等替代证据协调；**目标中查到一行业务值或MAX(TSO)不足以证明整个coverage**。 +6. 无可信证据则fail-closed，运维确认所有旧查询终结并完成目标核验后恢复或重新初始化。超时只转人工状态，不自动转成功。 + +operation ledger的持久化/并发控制与ID查询路径尚未实测，B03必须以接口字段、实际请求响应、故障实验和重启证据关闭，不能以“已审driver”关闭。 + +### 9.5 compaction + +第一版同表 compaction 与 raw INSERT/MERGE 串行。只在已明确成功的 cycle 之后删除被支配历史；不能删除唯一最新水位/墓碑，也不能删除恢复 pending DDL 所需状态。 + +最新事件的物理重复行要有明确去重策略：仅删除严格更旧版本不能清除相同 event_id 的重复，长期重试会膨胀；用 ROW_NUMBER 删除相同行也不能误删全部。是否支持稳定物理 row ID、重写去重表或安全重建 stream，列为待验证项 B06。 + +compaction 失败独立记账和告警；APPLIED 不回退，但持续失败超过 backlog/空间阈值应停止 admission。raw 保留全 payload 的最新行成本高，后续可分离小 watermark/tombstone 表与短期原始 payload；第一版不借此增加跨表原子性依赖。 + +### 9.6 远端raw/watermark容量与成功运行下的停写边界 + +持续新PK INSERT→DELETE可让target接近空而raw每键保留一个墓碑；MERGE和compaction全部成功仍增长。因此新增独立于本地spool的远端预算，覆盖raw、历史watermark/tombstone、stream依赖保留快照、内部journal/ledger及物理重复。 + +拟定控制： + +- 每feed配置remote-max-bytes、remote-max-history-keys、检查周期和软/硬水位；部署必须给出有限值，不以0代表默认无限。 +- capacity observer使用真实版本支持的表统计/存储计量；明确active逻辑字节、retained snapshot与物理计费字节差异，不把COUNT(*)当物理占用。历史键数使用保守上界，不能低估删除键。 +- observer结果过旧或查询失败超过限期后停止新raw admission。新cycle先预留“最大可能增长”额度，再raw INSERT；并发表共享feed预算，禁止每表分别看一遍旧统计后各自超额写入。 +- 测量延迟、SQL失败残留、压缩/快照放大意味着应用阈值不是数据库物理硬quota。设置安全余量，预算覆盖已预留在途写；若要求绝对计费上限，还需warehouse/provider硬配额能力，未证实时不承诺。 +- 软水位告警并提高安全compaction优先级；到admission阈值进入REMOTE_CAPACITY_BLOCKED，停止新raw INSERT，允许已提交raw的MERGE、必要DDL恢复和安全清理。DDL自身要保留维护余量，不能借“维护”无限扩张。 +- 恢复仅有三种：显式扩额并重新测量；有证明地消除物理重复/被支配历史；从新一致快照S2重建独立目标与内部状态，校验后切换。重建需要额外预算与旧feed冻结/隔离，禁止原地删最新墓碑。 + +验收：固定target有效行数、持续新键churn，记录历史键、物理存储/计费增长、扫描量与成本；compaction均成功时达到预算仍必须可诊断停写，停止后新raw写入次数为0（已预留请求除外且计入余量）。B18跟踪停写控制，B06跟踪去重和保留算法。 + +### 9.7 no-op、已支配重放与stream消费 + +零目标行修改不等于失败，也不等于已经完成。确认分为： + +| 情况 | 完成所需证据 | +|---|---| +| 正常MERGE改写业务行 | 服务端COMMITTED终态 + 本cycle覆盖/stream消费契约成立 | +| 原coverage全部因更高版本而被抑制 | 更高版本已应用证明（不能仅raw存在）+本次终态/消费证明，返回NOOP_CONFIRMED | +| DELETE匹配不到目标或完全重复写 | 相同版本净动作/墓碑一致 +终态/消费证明，不要求操作计数>0 | +| 空stream但本地coverage非空 | 默认恢复异常；需证明coverage已由历史消费覆盖，否则不能确认 | +| 无待确认coverage的空cycle | 直接结束调度，不制造源事件callback | + +本版选择保守规则：所有已写入raw的coverage，在当前或可证明历史消费覆盖后才完成；不因为“raw中有更大TSO”提前确认。单owner恢复先解决遗留raw与未终结查询，再接入新cycle；不能用未应用的新raw支配旧事件。 + +**版本事实**：Databend已合并[#20671](https://github.com/databendlabs/databend/pull/20671)，修复某些no-op DML跳过目标提交时丢失stream offset更新的问题，并说明planner-level no_effect短路不在修复范围。这不是“所有版本空MERGE都不消费”的规范，也不证明部署已含补丁。 + +P0必须取得server build与补丁覆盖证据，分别测实际SQL的数据依赖过滤为空、常量空集/优化短路、仅旧重放、DELETE不命中、重复事件、autocommit与显式COMMIT/ROLLBACK。cursor证据采用真实版本可核验的metadata或等价前后delta实验；不能只看counts=0或假定一定有可查offset API。 + +若实际组合不能可靠消费，首版拒绝该server/SQL组合或修复服务端再验收。显式WITH CONSUME/标记表只能另行证明与MERGE、owner、coverage同一提交边界后采用；禁止在MERGE前独立消费，禁止向业务target塞伪业务行。没有消费证明时不callback，但必须告警/进入恢复态，不能静默无限循环重刷。 + +### 9.8 stream过期/丢失后的恢复顺序 + +检测stale/缺失/依赖snapshot丢失立即冻结lane，不静默新建空stream。先排除旧owner/未知SQL；核验目标incarnation、DDL journal、最新墓碑与尚未应用raw的完整性。若raw仍完整，需通过显式full-raw回放覆盖算法重建stream起点并验收，且不混旧schema；“CREATE新stream”本身不会补已存在raw。 + +缺任何恢复证据就进入REINITIALIZATION_REQUIRED，从新一致快照重建；从上游checkpoint重放不是充分条件，因为上次已确认但被目标外部破坏的数据可能不会重发。重建路径未实现前，首版只支持诊断停写+人工重新初始化，不承诺自动修复。B14专门验收，不能因已有fail-closed就标成自动恢复完成。 + +## 10. DDL、建表与 schema 演进 + +### 10.1 初始 schema 与历史空表 + +已有表不会因为新建 changefeed 而重新执行历史 CREATE TABLE。首条 DML 带 TableInfo 可懒建非空表，但永不变更的历史空表必须由 start-ts schema snapshot bootstrap 或独立初始化任务预建。 + +bootstrap 要与同一时点 schema 匹配：不能查询“当前最新 schema”再解释较早 DML。TableSchemaStore 中名称清单不等于完整列定义。具体元数据读取 API 接口与调度顺序需实现前落定（B04）。 + +建目标表使用权威列顺序、精度、NULL/default 规则；建后 DESCRIBE/fingerprint 校验。目标已存在时不盲目接管：结构、主键唯一性与初始化水位均需验证。未经初始化的既有目标可能本身有重复 PK，MERGE 不能自动修复这一前置问题。 + +### 10.2 正常 DDL 协议 + +```text +收到 DDL(D) + → 禁止受影响表的新 epoch 进入 raw 写路径 + → 本地 flush marker，强制所有旧 epoch entry MERGE完成 + → maintainer 完成受影响 dispatcher 的全局协调 + → 验证当前 owner 与 before fingerprint + → durable journal(PENDING, D, operation, before/after schema) + → 执行翻译后的 Databend DDL / 重置 + → 校验 after fingerprint + → journal(APPLIED) + → DDL PostFlush + → 放行新 schema DML +``` + +FlushDMLBeforeBlock 只证明它负责的本地队列完成，不能冒充全局屏障。DB/All 范围 DDL 必须枚举所有影响表；TableSchemaStore 需求分支需新增（T06）。NotSync/过滤 DDL 的语义必须显式接入。 + +### 10.3 DDL 白名单 + +| 操作 | 处理 | +|---|---| +| CREATE DATABASE/TABLE | 翻译并校验；不原样下推 TiDB DDL | +| ADD 非键列 | 字面量默认值/NULL 语义明确才支持；对已有行回填需与源一致 | +| DROP/RENAME 非键列 | 旧 epoch排空，更新 fingerprint 和投影；防止 drop 后同名 re-add 混列 | +| MODIFY | 沿用有限安全白名单；不因目标已碰巧同类型就跳过未验证变更 | +| 默认值/NULL 属性 | 区分未声明、DEFAULT NULL、空字符串和表达式；不支持表达式则拒绝 | +| 二级索引 | 证明无行投影/身份变化后 no-op，仍更新必要元数据 | +| PK 定义更改/复杂多操作 | 拒绝/阻塞 | +| RENAME/DROP TABLE/SCHEMA | 未实现前明确阻塞，不能仅 warning继续 | + +复用 C04/C05 白名单时要保留适用限制：已有 VARCHAR→TEXT、TINYINT→INT、INT→BIGINT、部分 INT→文本，并不等价于支持任意类型改写。 + +### 10.4 journal结构、恢复优先级与普通DDL转移 + +DDL identity使用`source/任务实例/逻辑表血缘/源DDL ID或可证明唯一的等价身份`，包含commit_ts、old/new physical table ID与DDL内容摘要。只用同名表或全局MAX(ddlTs)不可区分重建/多表DDL。journal记录before/after列ID fingerprint、drain fence证据、phase、owner generation、operation/query关联和新代际是否已开放。 + +不假设Databend支持journal上强制唯一约束；单有效owner下同DDL身份单调推进，读回重复内容必须一致，多owner排他由§12提供。journal写入也经§9.4分类，不明确持久化就不得越过该步骤；冲突/倒退/未知身份fail-closed。 + +恢复优先级固定为： + +```text +所有新mutation保持关闭 + → 取得owner排他并处置旧在途SQL + → 核验source/target绑定、journal/ledger完整性与schema snapshot + → 恢复PENDING DDL/TRUNCATE（先于任何旧DML普通解码/写入） + → 安装已证实的applied-DDL fence与incarnation映射 + → 对旧事件执行身份校验后的skip/recovery规则 + → 协调遗留raw/stream与未知cycle + → 完成当前schema bootstrap核验，开放正常增量 +``` + +| durable phase / 当前证据 | 允许动作 | 禁止动作 | +|---|---|---| +| 无journal且收到新DDL | drain并持久化PREPARED，成功后发送DDL | 先DDL后补journal | +| PREPARED +无未知SQL +可信before fingerprint | 按同DDL身份执行白名单DDL，记录执行attempt | 在旧DDL可能仍运行时重发 | +| EXECUTION_UNKNOWN | 查询服务端终态；只有after结构可唯一证明本DDL完成时才收敛 | 只见列不存在/目标同类型就猜成功 | +| 可信after fingerprint +执行终态/等价完成证据 | 写VERIFIED，再写APPLIED与drain fence，均确认持久化 | journal未确认即放行新schema DML | +| APPLIED +相同DDL重放 | 验证目标仍属于该代际，跳过业务DDL，只完成重放事件 | 重做ALTER/TRUNCATE | +| 旧schema DML重放 | 只有身份匹配且drain fence证明该前缀已应用/由TRUNCATE合法清除时跳过 | 仅凭当前schema更新时间丢DML | +| 元数据/目标不一致或无法判定中间状态 | RECOVERY_REQUIRED，维持写关闭 | 用IF EXISTS掩盖冲突继续 | + +fingerprint不是万能提交证明：TRUNCATE前后schema相同；有外部DDL时相同结构也可能来自别的操作。目标禁止外部写是前置条件，仍须记录mutation身份。多操作DDL首版不支持，不能拆成多句后只验最终schema而忽略中途失败。 + +TiCDC基线MySQL专属恢复查询/skip标记需抽窄RecoveryCapability或Databend专属分支，覆盖普通dispatcher和merge task入口（T07）。恢复返回的是明确skip范围、代际与证据，不是笼统`lastDDLTS`。该API/持久化实现仍由B04/B09阻塞，表格只固定允许的状态转移。 + +### 10.5 TRUNCATE逐步恢复算法 + +TRUNCATE改变源物理表ID/代际；目标名可能不变。新代际不得在APPLIED记录持久化前接收任何DML。优先新建带代际后缀的raw/stream、隔离旧对象，旧对象物理清理后移，不先删唯一恢复证据。 + +| phase | 正常动作 | 崩溃/响应丢失后的恢复 | +|---|---|---| +| R0 DRAINED | 全局旧DML排空；登记DDL身份、old/new incarnation、reset目标与drain证明 | 无可信drain证明则重新协调，不直接清空 | +| R1 PREPARED | durable记录新代际CLOSED；旧SQL已终结后发送目标TRUNCATE，登记operation | 先协调unknown旧TRUNCATE。只有证明新代际从未开放且无新数据写入、目标独占时，才可在旧SQL终结后重复reset；否则停写 | +| R2 TARGET_RESET | 终态确认后记目标reset完成；创建新代际raw/stream并验证binding | 不从“表为空”推导旧query已终结；新对象存在须核验identity，不能见名接管 | +| R3 NEW_BOUND | 持久化新binding/schema/fence，旧raw标retired而非可消费 | 新代际仍CLOSED；补完缺失journal后读回一致性 | +| R4 APPLIED | 所有前置步骤完成且持久化APPLIED，再允许DDL回调/开放新代际 | 相同DDL只skip；即使checkpoint还在TRUNCATE之前也不得再清target | +| R5 ACTIVE | 新代际可写；旧raw/stream仅由受fence维护清理 | APPLIED/绑定冲突或丢失时停写核验，绝不通过再TRUNCATE“修复” | + +安全依据是“新代际写入发生在可信APPLIED之后”，不是TRUNCATE天然幂等。R1的可重做条件必须由journal、排他所有权与未开放协议共同证明，不能只依赖本机布尔值。若目标/journal被外部删除或查询无法确认，算法选择阻塞而不是猜测。 + +验证每个R0–R5之前/之后崩溃、步骤提交后响应丢失、journal更新不确定、旧owner晚到TRUNCATE。重点：R5写入@510后从@490重放@500 TRUNCATE，@510不得被再次清空。即使@510理论上可能重放，也不把这当允许重复破坏的理由。 + +### 10.6 历史空表bootstrap与DDL交叉 + +初始化取一致schema snapshot S并建立bootstrap epoch，表清单与列定义必须属于同一个S；枚举和分页期间收到的DDL按源顺序缓存于有界队列/上游保留，不能最新schema补齐旧清单。snapshot完成并持久化binding前，相关表不开放DML写入。 + +若DDL(D>S)先到而某空表尚未建好：该表先以S建表/核验，再走正常D的drain/journal；若采用直接建D后schema的优化，必须证明跳过区间无未应用数据，首版不默认采用。schema读取失败/历史版本不可用就停止bootstrap，不退回查询最新DESCRIBE。 + +测试无历史CREATE重放的空表、分页时ADD/DROP列、同名列drop/re-add、DDL后首批DML、bootstrap中断重启。B15验证时序，B04验证获取API与dispatcher恢复接入，两者不能互相代替。 + +## 11. 主键、净变化与乱序处理 + +### 11.1 普通更新和删除 + +INSERT/UPDATE 归一成完整 after-image UPSERT；DELETE 使用旧 PK 与 tombstone。完整 image 要由源契约保证，不得将缺失字段自动填 NULL。 + +同键较大 commit_ts 胜出；DELETE 也参加比较。收到 `UPSERT@100, DELETE@200, delayed UPSERT@150` 后，目标必须仍不存在，raw 保留至少 DELETE@200 的版本水位。 + +### 11.2 主键值更新 + +必须处理旧键消失和新键存在。源入口可能将变键 UPDATE 拆成 DELETE/INSERT，也可能暴露 before/after；两种都要真实测试。 + +同事务多行交换键/复用键需要按最终 KV净状态归约，不能简单规定“最后到达赢”，也不能未证明就让 DELETE 总比 INSERT 新。若源不能提供确定净结果，第一版必须拒绝该能力而不是静默留脏行（B05）。 + +### 11.3 collation 与身份 + +canonical key 的字节相等、源 PK 约束相等与 Databend SQL JOIN 相等可能不同。测试大小写、Unicode、尾空格、binary/text 键、DECIMAL 规范化和 NULL。 + +不能只在 raw 用区分大小写的字符串 key、target 用另一种 collation，然后宣称相等语义一致。第一版可限制到已验证 PK 类型/collation 集合；拒绝规则必须在创建/发现新表时可执行。 + +### 11.4 原生事件模式：二级UK、真实变键与净变化 + +源码事实（T11）：outputRawChangeEvent=false时，IsUKChanged对索引列/unique标志列比较，更新二级唯一键也可能SplitUpdate；INSERT可被缓存到DELETE之后甚至跨分片输出。T12的初始化只为CloudStorage/Kafka读取该开关，新SinkType若不加分支就保持false。 + +**本版决策：Databend固定要求outputRawChangeEvent=true，作为内部事件契约，不开放随意切回false的调优项。** 必须贯通manager→SharedInfo→dispatcher注册/重注册→eventservice→scanner；bootstrap、迁移、重试/重建session、兼容版本都校验实际生效。无法协商/确认则拒绝启动，不以sink端猜DELETE优先级补救。排除会改写/过滤DML两半的未经设计选项。 + +| 输入/场景 | 规范化规则与限制 | +|---|---| +| PK相同，二级UK从a→b | 保留完整before/after UPDATE，输出同PK一个UPSERT@C；不制造DELETE@C | +| 同PK普通列UPDATE | 完整after UPSERT；不完整image拒绝，不用NULL补齐 | +| UPDATE明确携带oldPK!=newPK | 只有已验证该源表示和事务净契约后输出旧键DELETE+新键UPSERT；不以到达顺序解决同TSO冲突 | +| clustered PK/record handle变更以独立KV DELETE/INSERT出现 | raw=true未必合并成UPDATE；必须验证提交时按record key净结果及跨分片契约，不能假定旧/new可配对 | +| 同事务反复修改/复用同键 | 先证明源提交事件已是确定净结果；若还需归约，必须有稳定事务结束边界/有界或可溢写归约状态，不能把DMLEvent结束等同完整事务结束 | +| 同key/TSO的相反净动作或不同payload在契约后仍存在 | 停止并保留诊断，不用ROW_NUMBER任意选一条，也不规定DELETE/UPSERT一律胜出 | + +因此，raw=true解决的是**避免人为UK拆分**，不是对真实PK变化、同事务多键替换和所有冲突归约的万能证明。第一版在真实TiDB事件与重启测试通过前不宣称支持变键；如果输入无法区分“变键”与普通DELETE/INSERT，就不能承诺可检测地禁用PK UPDATE，必须收紧可保证的源使用范围或补源端契约/能力后再发布。 + +测试用例先验证源SQL合法性。不能直接把可能违反即时唯一约束的单句PK互换当作可复现负载；选择TiDB实际允许的中间键/多语句事务与最终KV事件，保存事件dump。必须覆盖PK不变仅UK更新、raw开关true/false对照、两半强制跨事件/cycle/重启、同事务多行相同TSO、真实handle变更。不能为了证明确定性而在sink构造虚假的行序号。 + +此决策应纳入scheme创建时校验及DDL新增唯一索引后的持续校验，B05的关闭条件同时覆盖普通UPDATE与可选变键，不再缩写成“PK更新测试通过”。 + +## 12. 所有权、失租和故障接管 + +### 12.1 为什么进程 mutex 不够 + +每个 TiCDC capture 都可能建立 sink，同表/同 stream 不能有两个独立 merge owner。即使表跨节点调度关闭，正常迁移、旧节点网络分区及未结束 SQL 仍可能重叠。 + +T08 的 write gate 检查新请求是否可以发出，但请求通过检查后失租，服务端可能仍在运行。`ctx.Cancel()` 或客户端断连不等于服务端已回滚。 + +### 12.2 所有权协议要求 + +正常 handoff：旧 owner停 admission → drain/停止提交 → 确认其在途 mutation 的终态 → 释放 owner → 新 owner取得新 generation → 恢复 journal/raw → 放行。 + +异常接管：必须有目标侧原子 fencing/冲突机制，或可证实旧 SQL 已终止的机制，再开放新 owner。旧 DDL/compaction/stream 消费尤其危险；仅依靠幂等 row event_id 不能保护 schema 和 truncate。 + +候选A仅适用于可证明同事务原子冲突的mutation，不能泛化到DDL。官方说明DDL会提交当前多语句事务，所以先检查owner再DDL不提供该保护。[BEGIN/DDL限制](https://docs.databend.com/sql/sql-commands/ddl/transaction/begin) + +| mutation类别 | 首版排他要求 | A候选是否可直接覆盖 | +|---|---|---| +| raw INSERT /目标MERGE | 单owner lane；接管前旧query终态确认；候选事务fence需验证隔离/写冲突及提交原子性 | 仅待P0证明，普通SELECT owner再写不够 | +| journal/operation ledger/binding DML | 与owner代际关联，单调、读回一致；旧owner写元数据也须被排除 | 同上，元数据表不自动是锁 | +| CREATE/ALTER/TRUNCATE/DROP | 控制面串行授权 +所有相关旧SQL已终结 +本DDL终态与journal恢复 | 否，不把DDL塞BEGIN当fence | +| stream创建/替换/重建 | 与DDL同级排他；先冻结消费和raw写，验证未丢coverage | 否 | +| raw compaction/退役对象清理 | 独立登记mutation；只处理已证实retired/被支配对象，排除旧owner维护SQL | 若为事务DML需另测，不能默认 | + +候选B要求服务端终态确认，而非“调用取消成功”。D01的KillQuery丢弃错误，返回nil可能根本未取消；即使取消请求被接收，也要确认旧query不再能提交。未知request ID、服务端查询记录过期、节点不可达、取消接口受阻都保持HANDOFF_BLOCKED。 + +正常handoff与异常接管都使用同一mutation registry，涵盖业务与维护SQL。需要证明查询枚举/登记完整，不能只查当前客户端连接或某一个query ID。旧owner已退出只防止未来发送，不能证明已发送SQL终止。 + +**不同mutation的保护均未经过实际server验证，B01仍为生产阻塞项。** 外部锁可序列化新授权，不能撤销已发送SQL；先读owner表再MERGE也不原子。首版可先做受控非HA PoC，但必须满足§12.5，不把候选描述标成已落地。 + +### 12.3 mutation 边界 + +raw INSERT、MERGE、CREATE/ALTER/TRUNCATE、journal写入、stream 创建/重建、compaction和内部对象清理都必须检查 write gate/owner。验证连接不应随便创建数据库等副作用；需要区分只读预检与用户授权初始化。 + +### 12.4 多任务和映射冲突 + +同库同表的绑定注册必须跨 changefeed 生效，不是每 capture 一个 map。规则变更、大小写折叠、前后缀、同名源库映射都可能撞目标。未能实现可靠排他时要求运维静态独占并在重启核验 metadata；这仍不是完整生产自动化方案。 + +### 12.5 未闭合fencing时的PoC运行边界 + +`scheduler.enable-table-across-nodes=false`只禁止一表拆span跨节点，并不禁止整表调度、故障接管或维护迁移（T15）。因此它是必要规模限制，不是“禁用HA”的实现。 + +受控PoC必须增加Databend恢复准入：任何新capture/session owner在取得授权后仍保持写gate关闭，完成registry扫描/旧query终态核验/DDL恢复后才开放。未实现跨节点准入时只能在隔离单capture实验环境运行，禁用自动替换/重启写入并要求人工恢复；不能只配置一个false就接入生产调度。 + +失租立即阻止本机新mutation并取消worker；进程fail-stop可作纵深措施，不构成服务端fence。禁止用固定sleep>query timeout或驱动Kill nil作为安全接管证据。无法证明query超时上界、取消终态与枚举完整性时保持fail-closed。 + +真实故障验收：阻断取消/终态接口并延迟旧TRUNCATE，新owner必须始终拒绝写；恢复可达并取得可信终态后，才完成恢复并放行。控制面锁、客户端write gate、目标query终态三者角色分别记录，不混为一个“lease有效”布尔值。 + +## 13. 类型映射与 schema/value 统一 + +### 13.1 统一 mapper + +一套 mapper 同时服务 schema bootstrap、DDL、raw 编码和 MERGE projection；输入是 FieldType/flags/Flen/Decimal/Elems/列 ID，不是样本值。沿用 C03/C06 的测试案例，避免各路径字符串映射漂移。 + +| 源类型 | 目标原则 | 必测边界 | +|---|---|---| +| signed integer | 保留精确宽度或无损提升 | 最小/最大值，负值 | +| unsigned integer | 使用对应 UINT，BIGINT→UINT64 | 大于 2^63−1，最大 2^64−1 | +| DECIMAL(p,s) | 精确保留 p/s，不经 float64 | DECIMAL(65,0)、最大 scale、负数/前导零 | +| BIT(n) | UINT64 | BIT(64) 全 1，NULL | +| FLOAT/DOUBLE | 明确非有限值策略 | NaN/Inf 支持差异、精度 | +| BINARY/BLOB | 原始 bytes→HEX或已验证二进制参数 | 0x00–0xFF、空 bytes、NULL | +| ENUM/SET | typed value/Elems→约定字符串表示 | 空值、非法 ordinal、声明顺序、逗号/转义 | +| JSON | 精确 JSON 文本或 typed value，不经通用 float64 map | 大整数、SQL NULL vs JSON null、数组/嵌套 | +| DATE/DATETIME/TIMESTAMP | 明确时区/精度/范围；DATETIME不隐式平移 | 零日期、DST、微秒、超范围 | +| TIME/YEAR | TIME 字符串，YEAR 小整数 | 负 TIME、超过 24h、零 YEAR | +| TEXT/CHAR/VARCHAR | 字符集解码与目标字符语义明确 | 引号、反斜杠、控制字符、Unicode | +| VIRTUAL/STORED generated | VIRTUAL 不写；STORED写物化值 | 初建表、DDL后、仅生成列 | +| vector/geometry/其他 | 未定义前拒绝或显式安全映射 | 不静默 infer VARCHAR | + +原生 binary 不能套 Canal Latin-1 还原；原生 ENUM label 不能再当 Canal ordinal 解码。JSON 验证不应 unmarshal 成 float64 导致精度损失。 + +### 13.2 raw payload 与 projection + +第一版可用规范化 JSON：decimal/大整数字符串或精确数字文本必须约定一种表示；binary采用明确 HEX marker/列类型；SQL NULL 使用单独 null语义。不要让同一字段有时 raw bytes、有时 base64、有时 Latin-1 字符串。 + +所有值转换失败必须停止对应应用，不以 TRY_CAST 静默填 NULL；可选容错模式不属于第一版。缺少字段与字段值 NULL必须可区分，必要时带列 presence，避免不完整行覆盖。 + +### 13.3 默认值与目标侧行为 + +源值优先于目标 default；CDC INSERT已有完整数据时不重新计算 CURRENT_TIMESTAMP。ADD COLUMN 的历史回填只在明确兼容时执行；普通建表 default 的翻译不能成为源时间语义漂移来源。 + +## 14. 配置接口草案 + +**全部 Databend 新字段均未实现，以下不是当前可运行配置。** 数字是测试起点，不是已承诺默认值。 + +连接标识：`databend://HOST:PORT/DB_NAME?sslmode=require`。DB_USER/TOKEN应由受保护配置或运行时 credential provider注入；具体接口待安全设计，避免把真实 token 写进日志、提交记录或 CLI 参数。Bend-cdc 已有 runtime AccessTokenLoader，可借鉴但要接入 TiCDC生命周期。 + +```toml +[filter] +rules = ["DB_NAME.*"] + +[scheduler] +enable-table-across-nodes = false + +[sink.databend-config] +auto-create-table = true +# 镜像模式固定处理DELETE；不开放allow-delete=false +# outputRawChangeEvent内部强制true，不提供切换为false的用户项 + +batch-max-rows = 5000 +batch-max-bytes = 8388608 # 8 MiB，编码后数据预算；请求开销另计 +flush-interval = "1s" +merge-trigger-rows = 50000 +merge-interval = "5s" +merge-concurrency = 4 # 跨表，同表串行 + +spool-memory-quota = 67108864 # 64 MiB 热数据，不等于整个 sink RSS +spool-disk-quota = 10737418240 # 10 GiB，含维护预留，单 changefeed +spool-maintenance-reserve = 67108864 # 64 MiB，包含在磁盘总额内，不可用于普通admission +max-inflight-bytes = 268435456 # 256 MiB，含入口/热数据/编码/读回/元数据及控制预留 +max-row-bytes = 8388608 # 规范化后的单行含key/元数据；大事件应拆分 +max-delivery-bytes = 33554432 # 32 MiB，解码后chunk容量预算；wire另有协议上限 +max-delivery-events = 1024 # 元数据数量限制，不代表允许一次交付1024个最大事件 + +remote-max-bytes = 1099511627776 # 1 TiB示例admission预算，不是已证明的物理计费硬quota +remote-max-history-keys = 100000000 # 1亿历史键示例；按实际行宽/成本重新设定 +remote-check-interval = "30s" +remote-observation-max-age = "2m" # 过期后停新raw，不能永久盲写 +``` + +节点级拟定（server配置；字段名尚未实现/定稿）： + +```toml +[sink-spool] +base-dir = "/PATH_TO_DEDICATED_SPOOL" +total-disk-quota = 107374182400 # 100 GiB，capture 所有相关任务共享 +min-free-disk-bytes = 21474836480 # 至少保留 20 GiB，容量按节点实际调整 +``` + +配置校验：非负/上下界、单行与batch预算、内存池及并发可行性、磁盘总额/余量、未知字段拒绝、不支持功能拒绝。max-inflight必须包含编码和读回池，不能让四个 worker各额外分配无限 batch。 + +256MiB只是便于预算拆分的示例，不是已验证RSS保证。例如热64、入口32、编码32、读回64、元数据16、控制48MiB，总和256；真实分池根据最大row/delivery/request放大、并发数校验。任何一个8MiB行若连同参数/协议开销不能装进预留，必须提前拒绝或调小有效行上限，不能临时超额分配。sync.Pool只是复用机制，不是预算；超大buffer不能无限留在pool。 + +远端软水位与admission安全阈值由上述预算扣除观测误差、增长预留和维护余量推导，生产配置需给出推导结果；示例没有承诺1TiB成本封顶。每个pool和阈值的计费定义/保守放大系数应在启动诊断中展示，但不泄漏业务数据。 + +v0.2决定镜像模式固定处理DELETE，不再开放`allow-delete=false`；旧计划/客户端传此不支持项时显式拒绝，不静默忽略。删除版本和墓碑按统一规则保留。非镜像append模式不属于第一版。 + +更改配额下降到当前用量以下：停止新 admission、继续消费者排空，不直接删文件。改变目标映射、PK策略、spool根目录、类型映射等不默认支持在线生效，需停止和显式迁移。 + +## 15. 错误、生命周期与安全 + +### 15.1 错误分类 + +| 错误 | 行为 | +|---|---| +| 临时网络/限流 | 有限指数退避 + jitter，保持原 event_id，不确认 | +| 类型/权限/语法/不支持 DDL | 永久或人工介入状态，保留首因，停止对应 feed | +| COMMIT_UNKNOWN | 等待/查询终态/协调，不能盲目与新 owner 并发重试 | +| token过期 | 受控刷新并重试；刷新失败停止，不打印 token | +| spool配额满 | 背压并主动调度 MERGE，不是丢数据错误 | +| 入口信用耗尽 | block所属path，快速返回共享线程；等待信用重投，不重复progress登记 | +| 远端预算达到/观测过期 | REMOTE_CAPACITY_BLOCKED，停新raw；保留排空与安全维护路径 | +| stream失效/应用证据缺失 | RECOVERY_REQUIRED或REINITIALIZATION_REQUIRED，不静默重建空stream | +| ENOSPC/文件损坏/IO失败 | 停 admission，明确告警；不能假装写入成功 | +| compaction/回收失败 | 业务成功不回退，空间继续计费；达到运维阈值停止新输入 | +| metrics解析失败 | 业务终态独立；观测标不可用，不重复执行已提交业务 | + +只保留一层主要 retry预算，避免内外两层循环把错误隐藏几十分钟。所有重试可取消，受单操作 timeout和feed级最大恢复策略约束。 + +### 15.2 正常关闭 vs 失租 + +正常关闭：停止 admission，按设定期限尝试 drain/MERGE，成功的确认，超时未完成不确认，再取消并关闭；不能无限 Background flush。 + +失租/强制终止：不执行“为了收尾再写一次”的新 mutation。停止工作，收集在途状态并交给接管协议。取消、等待和pool关闭需幂等；不得边清 spool边还有 worker读写。 + +### 15.3 安全与权限 + +TLS默认要求验证证书；最小权限覆盖目标 DML、必要建表/ALTER、内部元数据/stream，不要求仓库管理员。只读 Verify和有副作用初始化分离。标识符全量转义，业务值参数绑定；不同任务内部对象权限隔离。 + +本地 spool含业务明文可能敏感：受限目录权限、独占卷/宿主盘加密、保留与清理策略；不能因为“临时文件”就忽略数据保护。清理仅限实例拥有目录，不跟随不可信符号链接。 + +## 16. 可观测性、容量与成本模型 + +### 16.1 指标分层 + +- input accepted/bytes、spool memory/physical disk/reserved/metadata、排队最老年龄。 +- raw committed rows/bytes、raw待应用批次、MERGE周期耗时和终态。 +- target applied progress vs source resolved/checkpoint，分别展示；MAX(commitTs)不等于覆盖水位。 +- quota/free-space waiting、ENOSPC、segment reclaim failure/orphan bytes、每feed公平性。 +- DDL pending duration、owner generation/fenced、commit unknown query数、token刷新失败。 +- target mutation counts和source event counts分开，失败读取计数时 unavailable，不编造 0。 +- delivery credit/共享chunk已分配字节、blocked path、重投/重复转移、progress登记次数。 +- OPEN/SEALED tracker数、pending分片、Seal等待年龄、duplicate completion、已应用未回收bytes。 +- remote active/retained/reserved bytes、历史键上界、观测年龄、远端预算停写原因。 +- no-op confirmed/消费证据缺失、ledger未终结operation与服务端终态查询失败。 + +标签避免无限高基数；事件原始值、PK、token不写常规日志。异常样本需脱敏、本机保存、访问受控。 + +### 16.2 容量估算 + +若平均编码输入速率为 R bytes/s、有效本地可用配额为 D，粗略缓冲时间 `T ≈ D / R`；编码放大、元数据、未回收segment与已有积压会缩短窗口。多feed还受共享预算和公平分配约束。 + +spool只是有限吸收抖动。长时间停机后必须在上游 GC/日志保留范围内恢复；容量不够要背压，不能承诺不停机/零延迟。内部 eventstore等其他磁盘也要计入运维容量。 + +### 16.3 成本对照 + +固定输入变更率/行宽/更新热点/表数/目标可见延迟后比较： + +```text +旧成本 = TiCDC编码 + S3请求/容量/网络 + bend-cdc消费计算 + + Databend raw/MERGE/compaction + 运维资源 +新成本 = TiCDC原生适配/spool IO/资源增量 + + Databend raw/MERGE/compaction + 网络 + 运维资源 +``` + +必须记录 TiCDC CPU、RSS、磁盘吞吐/写放大、spool空间、Databend credit/query cost、MERGE次数、raw扫读量、小表对象数。不能仅用少一个 deployment证明总成本下降。 + +## 17. 全量初始化与 start-ts 接续 + +sink只做增量。独立初始化协议： + +1. 选择可重现一致快照 S，固定 schema S。 +2. 在保留窗口允许范围内启动/准备从 S接续的changefeed，确保增量不丢；可停留在受控bootstrap状态，不能让内存无限积压。 +3. Dumpling导出、COPY INTO目标，使用同一mapper；全量失败可按任务批次恢复。 +4. 校验全量数据/目标结构/元数据绑定，登记 baseline S。 +5. 放行约定边界之后的增量；确认 TiCDC实际 start-ts 是排除还是包含边界，不能靠口头假设。 + +初始化期间既要容忍 DDL，也要明确是否禁止 DDL或按快照版本隔离；不能查询最新 schema处理旧快照。全量导入不能在增量写完后覆盖回旧状态。必要的全量S3文件与增量链路去S3不矛盾。 + +## 18. 恢复矩阵与反例 + +| 中断点 | 允许的残留 | 预期恢复 | 不允许做什么 | +|---|---|---|---| +| 内存/磁盘 spool 后，raw前 | 本地临时文件 | 从checkpoint重放 | 把spool落盘当确认 | +| raw成功、响应丢失 | 相同event_id重复raw | 协调后幂等重试 | 新增本机时间当业务版本 | +| raw成功、MERGE前 | raw积压/stream增量 | 回放源并重新应用 | 只依赖丢失的pendingEvents内存map | +| MERGE明确失败 | target未推进 | 重试/人工修复 | 触发PostFlush | +| MERGE/COMMIT结果不明 | SQL可能仍执行 | COMMIT_UNKNOWN协调 | 新owner立即并发DDL/清理 | +| MERGE成功、callback前 | target已更新 | 重放不回退 | 清除最新墓碑 | +| callback后checkpoint保存前 | 重复source事件 | 幂等收敛 | 把重复当新写入时间更晚版本 | +| MERGE成功、compaction失败 | raw大、spool可能已释放 | 保留APPLIED，重试维护/限制新输入 | 重复业务操作修复统计 | +| entry释放、segment删除失败 | 真实磁盘仍占用 | 继续计费、重试回收/告警 | 配额假释放继续写满 | +| DDL pending后 | journal存在、目标旧schema | fingerprint确定后执行 | 跳过检查直接applied | +| ALTER成功、journal未applied | 目标新schema | reconcile并隔离旧DML | 旧行按新列名写入 | +| applied TRUNCATE后重启 | 新代际数据可能已存在 | 跳过同DDL，恢复正确代际 | 再次清空目标 | +| capture永久丢失 | 本地spool丢失 | 新owner在fence后从checkpoint重放 | 依赖本地磁盘保证数据可恢复 | +| stream过期/内部对象被删 | 恢复证据缺失 | 停止并进入显式重建协议 | 静默新建空stream丢旧raw | +| F1已应用、F2未登记 | OPEN tracker、F1可已回收 | 不回调；重启从checkpoint重放整个源事件 | 只看registered==applied就确认 | +| delivery已登记progress但未转入sink | 上层仍有lease/blocker | 同session按deliveryID重投转移，或取消恢复 | 再加一遍progress、丢chunk计费 | +| no-op成功但消费未知 | target未改/stream可能未推进 | 核验版本和消费证据，未证明就阻塞 | counts=0当失败循环重刷或当成功直接确认 | +| 远端持续新键churn | target小、raw墓碑持续增大 | 到远端预算停新raw，显式扩额/重建 | 以本地spool已排空宣称容量安全 | + +关键反例： + +- DELETE后延迟INSERT复活：检验I2/I9。 +- 高于MERGE快照的新raw抑制旧candidate却未应用：检验§7串行lane和coverage。 +- 生产者拿着表锁等磁盘，MERGE也要同锁：检验I6。 +- 同一segment一个长事务钉住空间，其余全释放：检验I7和物理回收。 +- 失租旧TRUNCATE晚于新owner写入提交：检验I3/I8，幂等DML不能解决。 +- 初始schema updateTS被误当已应用DDL fence导致漏首批数据：检验I10。 + +## 19. 测试与验收计划 + +### 19.1 分层验证 + +| 层级 | 验证内容 | 证据要求 | +|---|---|---| +| 单测/property | mapper、key规范化、状态机、预算、回调exactly-once-in-process | 固定seed与随机序列，包含错误路径 | +| mock客户端 | commit unknown、cancel、权限、retry、回收失败 | 精确检查未完成事件不确认 | +| 真实Databend | stream/MERGE、no-op、事务、session、原子恢复 | 固定server/driver版本、SQL记录与最终表状态 | +| TiDB/TiCDC E2E | 原生before/after、DDL、快照边界、回放、故障调度 | 固定版本与事件时序，checksum/count/schema | +| 并发/资源 | race、内存、磁盘、并发feed、满盘恢复 | 实际RSS/allocated bytes、死锁与泄漏检查 | +| 灰度/成本 | 影子目标、同SLO、费用与可见延迟 | 不用本地单测替代线上或计费证据 | + +### 19.2 必测清单 + +1. 空批、低流量、热点更新、大表、小表海量、混合表大小、整库空表。 +2. PK/复合PK、变键、多行同事务、换键、同TSO净变化；不支持表在发现时拒绝。 +3. 所有§13边界值、字段缺失vsNULL、case/collation、生成列与DDL后mapper刷新。 +4. insert/update/delete/reinsert、乱序、重复、同event_id异payload冲突。 +5. 每个§18中断点；网络断开、超时、服务端SQL继续运行、节点失租。 +6. PENDING/APPLIED DDL、无后续DML、drop/re-add同名列、TRUNCATE多步中断。 +7. 多feed共享配额、热/冷表公平性、segment pin、孤儿清理、ENOSPC/inode耗尽。 +8. 配额达到但merge-trigger未达到、等待者被取消、长事务超过可用spool、单行超限。 +9. stream没新数据、只有重复或被抑制旧数据、no-op MERGE、SELECT是否消费、回滚后cursor。 +10. 正常Close、失租Close、快速重复start/stop、token刷新、所有权切换、目标被外部改写。 +11. 同名不同库映射、目标schema已存在但不兼容、内部对象属于别的任务。 +12. 快照S前后写入与DDL、源GC保留不足、stream失效和恢复失败。 + +### 19.2.1 v0.2新增的可执行验收矩阵 + +下面每项均未执行；它们是实现/P0必须交付的测试，不是本文已通过的证据。 + +| ID | 注入点与负载 | 必须断言 | 关闭项 | +|---|---|---|---| +| V01 | 固定PK仅改二级UK，分别raw=true/false，切分cycle并在两半间重启 | 正式模式只输出正确净变化；最终仅新值，无版本冲突/迟到DELETE破坏；迁移重注册仍raw=true | B05 | +| V02 | F1先APPLIED，暂停F2登记；并发Seal/Apply/重复完成 | OPEN时callback=0，全集完成后=1；已APPLIED分片可提前回收 | B17 | +| V03 | 累计事件大于spool，可用空间小于事件总量，多个有界delivery | 编码/应用/回收循环前进；所有分片未完成前不确认；入口chunk/元数据不越预算 | B08/B16/B17 | +| V04 | A/B共用dynstream线程；A用尽path信用，预留B信用且下游健康 | A handler快速返回；B DML/DDL在测试设定的有界期限推进；重投progress/lease各一次 | B16 | +| V05 | 多DMLEvent共享大chunk，wire小但解码膨胀；断session/取消 | chunk容量只计一次且释放恰一次，无丢计/双计；超限在分配前拒绝；RSS剖析含临时内存 | B08/B16 | +| V06 | 热/冷表共享segment；配额满，冷表未达普通阈值；unlink/搬迁失败 | 受压引用lane均被唤醒；保留池可排空；不假释放物理quota，无无限busy loop | B07/B13 | +| V07 | 首结果页后注入服务端失败/传输断开；提交后响应丢失；计数解析失败 | 前两种无独立证据不ACK；已提交统计失败不重做；UNKNOWN可跨重启协调或明确停写 | B03 | +| V08 | 阻断Kill接口，延迟旧TRUNCATE/compaction，新owner请求接管 | Kill nil不放行，固定sleep不放行；取得可信终态后才能恢复写 | B01 | +| V09 | 实际no-op SQL各形态，旧/含修复build对照；事务提交/回滚 | 目标可0修改但消费证据与callback正确；无planner短路漏消费；重试标签不偷消费下一批 | B02 | +| V10 | R0–R5每步前后崩溃；R5写新数据后回放旧TRUNCATE | 新数据不被再清空；unknown query先协调；journal缺失/冲突不猜测 | B04/B09 | +| V11 | stream依赖snapshot过期/内部对象删除 | 可诊断停写；恢复不跳过遗留raw；不足证据进入显式重初始化 | B10/B14 | +| V12 | schema S分页时并发空表ADD/DROP/re-add，bootstrap中断 | 每表按正确历史schema接DDL；无首批漏写/错列 | B04/B15 | +| V13 | target恒定小、持续新键churn，compaction始终成功 | 远端预算触发停新raw；统计过期也停写；不删最新墓碑；多表在途增长预留一致 | B06/B18 | + +V07的“服务端先出数据后失败”可先用mock验证客户端分类，但真实MERGE是否可出现该时序须另记录，不把mock当服务端复现。V04对B的期限应由固定压测配置定义；不在全局资源耗尽时作无法兑现的公平性承诺。 + +### 19.3 属性/模型测试建议 + +构建内存reference model：按源净事件顺序维护 PK→值/墓碑，随机插入重复、乱序、cancel、重启和DDL边界，比较“完成全部已确认覆盖后”的目标最终状态。 + +断言：确认集合不超出已证明APPLIED；每个回调最多一次；预算无负数/超额预留；无未计费文件；取消后无新mutation；旧DDL/旧epoch不污染新代际。 + +### 19.4 发布门槛 + +必须关闭所有会造成数据丢失、错误确认、双owner、不可恢复DDL、资源无界的阻塞项。性能不足可以限制规模后发布PoC；正确性缺口不能只通过缩短flush interval掩盖。 + +## 20. 上线、回退与交付拆分 + +### 20.1 阶段 + +| 阶段 | 交付物 | 退出标准 | +|---|---|---| +| P0语义验证 | 原生事件契约、tracker、入口信用、终态/fence四份协议；server/driver/SQL能力实验 | V01/V02/V04/V07/V08/V09设计/原型证据；无法证明fence只能限制隔离单capture实验，不能开放生产HA | +| P1纵切PoC | 原生scheme/config、跨层信用接入、SEALED tracker、spool、单表PK批写/MERGE、远端预算 | 受限单owner故障重放与资源有界，包含UK UPDATE与大事件；不能跳过端到端入口预算 | +| P2核心对齐 | 复合PK/类型/建表/空表、DDLjournal/恢复/TRUNCATE | 支持矩阵所有已承诺项完成E2E | +| P3灰度 | 影子目标、切换/回退、容量与成本结果 | 正确性、可见延迟、恢复和费用均验收 | +| P4可选扩展 | 分区、跨节点汇聚、typed staging、高并行raw | 单独RFC与不变量证明 | + +此前“PoC约1–2周”的估算未包含完整共享配额/owner/DDL/真实故障覆盖，不能当本详细范围的排期承诺。先做P0再按已关闭风险重估。 + +### 20.2 灰度切换 + +旧链继续写旧目标，新sink写独立影子目标，不能共用raw/stream/journal。固定初始化S和对照水位，验证目标checksum/count/schema与业务查询，连续观察后切读，再停旧链。 + +影子对照必须明确水位：任意时刻比较两个异步目标的行数可能自然不同，不直接当错误。新sink不保证全changefeed在同一时刻的snapshot read，需要在受控drain/已对齐边界比较。 + +### 20.3 回退 + +保留旧链数据与checkpoint覆盖窗口；先让旧链追平到验证边界再切读。不把新sink内部raw/journal当旧bend-cdc文件游标，不直接复用内部表。 + +升级涉及raw schema、key编码、schema journal或spool格式需版本化。临时spool可丢弃重放不等于目标journal可丢；回退二进制必须理解已有持久化元数据，否则阻止启动。 + +## 21. 已知不足、待验证假设与生产阻塞项 + +| ID | 风险/未闭合问题 | 当前拟定措施 | 关闭所需证据 | +|---|---|---|---| +| B01 | 各mutation的fencing/旧在途SQL与新owner重叠未证明 | §12分DML/DDL/stream/维护排他；取消返回值不是证据 | V08及完整registry、正常迁移/异常接管；非HA准入不能只配across-nodes=false | +| B02 | stream+MERGE提交/快照/no-op/cursor/applied witness未实测 | 固定build及补丁、§8.5/§9.7覆盖和消费证明 | V09、实际SQL与版本矩阵、atomic witness或等价算法，含planner短路 | +| B03 | QueryContext/Rows/COMMIT不确定结果判定与重启定位 | §9.4证据结构、错误分类、durable operation ledger；UNKNOWN停写 | V07、真实endpoint/字段/保留期、首响应丢失/查不到query及ledger自身不确定结果 | +| B04 | start-ts完整schema bootstrap/空表与恢复API接入未定稿 | 权威snapshot与窄recovery接口 | 既有空表、schema变更并发、重启E2E | +| B05 | 普通二级UK更新拆分与PK/换键净事件契约 | §11.4固定raw=true贯通注册；真实handle变更另验 | V01、迁移重注册、源合法事务dump与checksum；不能只测试PK UPDATE | +| B06 | raw相同event_id物理重复与墓碑长期容量 | 逻辑去重+安全compaction，禁止TTL删最新 | 等值重复清理SQL、stream不丢数据、增长基准 | +| B07 | node级预算与其他磁盘消费者共盘 | 全feed共享budget+free guard，建议独占卷 | 多feed/多组件满载测试、物理字节计费 | +| B08 | 源chunk/序列化/blob/readback/callback瞬时内存 | 分阶段预算，禁止无界channel和oversized旁路 | 内存剖析、极端行/事务压力与race测试 | +| B09 | DDL journal不是现成原生恢复协议，TRUNCATE多步 | before/after fingerprint、generation、skip/recovery接入 | 所有DDL中断点/旧DML重放测试 | +| B10 | STREAM企业能力、许可、保留窗口/失效 | 明确capability校验，默认不静默fallback | 实际warehouse版本/权限/许可与失效演练 | +| B11 | 跨表原子性不提供，目标读取可能见部分事务 | 产品明确语义，影子比对在受控边界 | 使用方接受与文档/测试一致 | +| B12 | 性能与成本尚无数据；保守串行lane可能限制热点吞吐 | P3同SLO benchmark，先保正确性 | 固定负载和费用证据 | +| B13 | 跨lane segment pin与压力/回收活性 | §6.8反向引用、压力广播、保留排空池；搬迁需双份预算 | V06，热/冷表、删除失败、资源不足不假释放；不以原版有定时器就认为已解决 | +| B14 | stream失效后的安全恢复/重新初始化 | §9.8冻结、证据核验；不够证据只允许显式重初始化 | V11，full-raw恢复若未实现不得宣称自动修复 | +| B15 | 历史空表bootstrap与DDL/schema竞态 | §10.6同一S快照、bootstrap epoch、顺序DDL | V12，分页/同名列重建/中断，不能以最新schema兜底 | +| B16 | collector→dynstream→sink入口信用与共享chunk责任 | §6.7新增capability、deliveryID、progress只登记一次、跨层lease | V03/V04/V05、协议batch上限、兼容旧sink、真实分配/RSS与控制事件推进 | +| B17 | 动态分片全集封口与独立回收 | §5.2 OPEN/SEALED/COMPLETED、先登记后发布、once完成 | V02/V03及race/失败/取消；flush闭包自身不是once | +| B18 | 成功维护下远端历史键增长/观测过期 | §9.6有限远端预算、增长预留、停写/扩额/重建流程 | V13，active/retained计量与费用，不能借删除最新墓碑降量 | + +本轮B01–B18均未因“写了设计”关闭。B01–B10、B13–B18若影响受支持场景的正确性/资源边界，必须在该场景发布前用实现和实验关闭；不能以缩短flush interval或新增风险编号代替。B11需明确语义接受，B12需按固定SLO验收。首版范围缩减必须可执行，不能写一个无法在源事件上识别的“禁用PK更新”开关冒充保护。 + +## 22. 替代方案与取舍 + +| 方案 | 优点 | 代价/拒绝原因 | +|---|---|---| +| 保留S3+bend-cdc | 变动小，有对象存储缓冲/回放 | 中间编码/消费成本仍在，可作为fallback | +| TiCDC纯内存直写 | 实现较简单 | 已被用户否决；无法满足第一版磁盘spool要求 | +| 原生sink + 本地spool + raw/stream/MERGE | 复用核心语义、去中间链、有限吸收抖动 | owner/DDL/提交/预算仍需正式实现；本文选择 | +| raw落盘立即确认，后台异步MERGE | 易释放本地资源、吞吐可能更高 | 需要独立durable任务恢复、applied水位和接管；第一版不选 | +| 每条直接UPSERT/DELETE目标 | 链路短 | 小SQL多，成本高；旧事件防回退和删除墓碑仍需解决 | +| 每batch临时表然后MERGE并删除全部staging | 局部逻辑简单 | 丢失跨批次watermark/tombstone，迟到重放可能回退 | +| 多capture直接共享同一stream | 表面并发高 | ownership/覆盖与DDL竞态未解决，不选 | +| 无stream，全raw扫描MERGE | 不依赖stream许可 | 扫描成本和coverage策略不同，需独立验收，非静默fallback | +| typed staging/独立watermark表 | 可能减少JSON与raw历史成本 | schema/事务复杂度高，放后续独立优化 | + +## 23. 发给 reviewer 的具体问题和输出约定 + +请优先寻找以下反例，每条提供触发条件、事件序列、违反的不变量、影响和最小修正： + +1. MERGE后才确认的coverage定义，在重放、空stream和旧事件被抑制时是否充分？ +2. 单表raw/MERGE串行是否仍存在进程间/接管竞态？什么条件才能真正排除？ +3. 本地磁盘配额计费能否被segment pin、回收失败、孤儿文件、其他capture绕过？ +4. 超大事务、callback tracker和chunk引用会不会让spool有界而RSS仍无界？ +5. 配额不足时是否有消费者等待生产者、DDL等待callback、callback等待DDL的循环？ +6. retained raw历史与目标schema变更是否会混用epoch？初始bootstrap是否可能错误跳过DML？ +7. PK变键/同事务换键是否存在无法靠commitTs消歧的合法输入？ +8. 有无在不提供目标侧fence时仍安全自动HA的证明？如果没有，应缩减什么承诺？ +9. DDL结果不确定、TRUNCATE部分完成和旧owner晚到SQL如何收敛？ +10. stream无业务行更新时是否一定消费？许可/失效/保留期会不会让恢复停摆？ +11. 同event_id重复物理记录的compaction是否可能删光水位，或无限累积？ +12. 固定镜像DELETE语义后，已有目标数据、collation、full-image缺失等前置条件是否仍不可执行或存在漏洞？ +13. 此设计是否真的完整覆盖bend-cdc“建表、批写、MERGE、类型改写、DDL”核心能力？遗漏请逐项列出。 +14. 配置数量/共享模块改动是否过度设计？能否在不破坏不变量下缩小首版？ + +输出建议：`结论(接受/修改后接受/拒绝) → P0/P1/P2发现 → 每项证据/时序/修复 → 发布阻塞 → 可后移优化`。不要以测试没跑作为全部结论；需要指出该测试要证明哪个语义。也不要替本文补全缺失假设后默默给通过。 + +## 24. 证据、实际验证与可复核性 + +### 24.1 已执行与未执行 + +- 本会话前序已执行 bend-cdc：`go test ./pkg/sink/databend ./pkg/source/tidb -count=1`，两包通过；这是已有代码的本地测试结果，不是新sink验证。 +- 未配置真实Databend测试连接，相关集成测试按条件跳过。未跑真实stream/事务、TiDB/TiCDC新sink E2E、故障注入、benchmark或部署。 +- 本次新增/修改Markdown并检查格式、代码证据引用；未实现新配置或SQL，未构建TiCDC，未创建分支/提交/PR。 +- 编写时读取的是固定Git对象与当前bend-cdc源码；公开文档仅用于能力与限制背景，不能代替部署版本/真实运行证据。 + +### 24.2 新架构TiCDC固定源码索引 + +以下链接基于已用git对象读取的commit；远程仓库可见性受reviewer权限影响，无法打开时使用§0的git show。引用的固定路径是Git对象路径，不要求当前旧架构checkout存在对应文件。 + +- **T01**：[Sink接口与factory](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/sink.go)。 +- **T02**：[DMLEvent、原生行与回调](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/dml_event.go)。 +- **T03**：[Spool、TryEnqueue、oversized与目录生命周期](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/sink/spool/spool.go)。 +- **T04**：[S3 buffer manager与完整payload构造](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/cloudstorage/buffer_manager.go)。 +- **T05**：[S3 writer、写入后回调与release](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/cloudstorage/writer.go)。 +- **T06**：[TableSchemaStore的sink-type需求](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/table_schema_store.go)。 +- **T07**：[dispatcher manager的MySQL恢复分支](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go)；[merge task恢复分支](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/task.go)。 +- **T08**：[capture写许可gate](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/writelease/write_gate.go)。 +- **T09**：[DDL原生事件](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/ddl_event.go)；[dispatcher屏障](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatcher/basic_dispatcher.go)。 +- **T10**：[S3 writers创建spool及上游队列](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/cloudstorage/dml_writers.go)。 +- **T11**：[UPDATE的UK拆分，L970–1014](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/eventservice/event_scanner.go#L970-L1014)；[IsUKChanged，L219–259](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/codec.go#L219-L259)。 +- **T12**：[raw-change初始化，L320–350](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go#L320-L350)；[dispatcher同步交付与progress先登记，L295–335](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatcher/basic_dispatcher.go#L295-L335)；[session注册传播，L590–655](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/eventcollector/dispatcher_session.go#L590-L655)。 +- **T13**：[dynstream同步Handle与path blocking，L340–356](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/utils/dynstream/stream.go#L340-L356)。 +- **T14**：[collector提前AssembleRows并交付，L360–395](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/eventcollector/dispatcher_stat.go#L360-L395)。 +- **T15**：[across-nodes配置定义，L34–38](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/config/scheduler_config.go#L34-L38)；[控制初始span拆分，L201–208](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/maintainer/span/span_controller.go#L201-L208)。 + +### 24.3 bend-cdc源码索引 + +下列本机链接在本报告编写时的bend-cdc commit上有效；异地reviewer应按§0的bend-cdc基线读取同路径的Git对象，不能以未来HEAD替代。C编号只表示源码，不是§21的B编号阻塞项。 + +- **C01**：[TiDBRunner](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/service/tidb_runner.go)。 +- **C02**：[DatabendMergeSink、PK/schema/转换](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go)。 +- **C03**：[CDCManager、raw/stream/MERGE/compaction](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go)。 +- **C04**：[DDL翻译](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/schema_ddl.go)；[schema元数据与建表](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/schema_meta.go)。 +- **C05**：[列变更白名单](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/schema_change.go)；[epoch](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/schema_epoch.go)；[TRUNCATE](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/truncate.go)。 +- **C06**：[类型与连接](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend.go)；[TiDB值转换](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/tidb_values.go)。 +- **C07**：[MERGE结果判定](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/merge_result.go)。 +- **C08**：[applied watermark](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/applied_watermark.go)；[Dumpling初始化](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/dumpling_executor.go)。 + +### 24.4 官方文档证据边界 + +1. [MERGE语法](https://docs.databend.com/sql/sql-commands/dml/dml-merge)：支持source/subquery驱动的增删改；不证明源端幂等算法。 +2. [CREATE STREAM](https://docs.databend.com/sql/sql-commands/ddl/stream/create-stream)：append-only选项、Enterprise标记；需实际部署许可验证。 +3. [Stream工作流](https://docs.databend.com/guides/load-data/continuous-data-pipelines/stream)：提交消费与并发限制的公开说明；MERGE/no-op组合仍需实验。 +4. [Transaction](https://docs.databend.com/sql/sql-commands/ddl/transaction)：事务控制入口；不据此假设所有DDL/owner校验支持所需隔离与原子性。 +5. [BEGIN中DDL限制](https://docs.databend.com/sql/sql-commands/ddl/transaction/begin)：DDL提交当前多语句事务；不能泛化DML事务fence到DDL。 +6. [no-op stream offset修复#20671](https://github.com/databendlabs/databend/pull/20671)：已合并版本相关修复，明确存在未覆盖的planner短路；不证明目标部署已包含，也不是所有空MERGE的规范定义。 + +### 24.5 驱动证据与本轮验证边界 + +- **D01**：bend-cdc go.mod引用databend-go v0.9.4；本机读取[waitForData，rows.go L27–50](/Users/jiangjianyuan/go/pkg/mod/github.com/datafuselabs/databend-go@v0.9.4/rows.go#L27-L50)、[KillQuery，client.go L746–767](/Users/jiangjianyuan/go/pkg/mod/github.com/datafuselabs/databend-go@v0.9.4/client.go#L746-L767)。异地reviewer需获取相同版本，不把本机路径当远程可用。 +- 本次读取文件SHA256：rows.go `065c09c10a3a7b334da795949de1b717a5fd0014000d55be80e5fbd4fb5da6b9`；client.go `b3d9b42925ae4d584e84afbc244bfa97096eb5591eda257ecd2747894a35aecf`。这是文件内容指纹，不代表已运行驱动集成实验。 +- v0.2只增加协议/状态转移/验收条件并更新评审入口；V01–V13均未实际执行。Markdown结构、TOML语法、文件/Git对象引用检查与新sink正确性验证严格分开。 +- 不把独立评审的P0/P1标签当复现证据；例如no-op规范、PK互换SQL合法性、Kill接口能力、永久丢数据时序，都需按本文限制重新核验。没有创建新review agent、没有切分支、没有修改业务Go代码。 + +--- + +**最终建议**：方向可行，进入独立评审和P0验证；不要直接宣布可生产实施。优先要求reviewer击穿owner fencing、提交结果判定、DDL恢复和资源硬预算这四条关键路径，而不是只讨论字段命名或批大小。 diff --git a/docs/design/databend-native-sink-design-review-v0.3.md b/docs/design/databend-native-sink-design-review-v0.3.md new file mode 100644 index 0000000000..6e3768308d --- /dev/null +++ b/docs/design/databend-native-sink-design-review-v0.3.md @@ -0,0 +1,447 @@ +# RFC v0.3：TiCDC 原生 Databend Sink——直接 raw 批次与最小接入 + +| 项目 | 结论 | +|---|---| +| 版本 / 日期 | v0.3 / 2026-10-24 | +| 状态 | Proposed；唯一候选主路径，待实现及实验，不是生产就绪 | +| 目标 | 将 bend-cdc 的建表、类型转换、批写 staging、MERGE、必要 DDL/恢复迁入原生 sink，去掉增量 Canal JSON→S3→消费者链 | +| 改动约束 | 核心位于 Databend sink；外部仅必要注册、raw-change、schema/recovery、失败生命周期接入 | +| 首版运行范围 | 隔离单 capture、静态独占目标、人工核验旧 SQL 后恢复的受控 PoC；不承诺自动 HA | +| 全量工具范围 | 不迁入 Dumpling/CSV/COPY INTO，不实现或编排全量初始化；只处理 TiCDC 交付的增量事件及必要表元数据 | +| 缓冲 | 有界入口/内存 + 限额本地 spool + 现有 PostEnqueue 背压 | +| 应用主路径 | manifest → raw → 本批主键 K → retained history H → 每键净动作 S → MERGE → flush callback | +| 不采用 | stream cursor、消费 hint/witness、普通 DML durable applied ledger、新 delivery/信用协议、progress 迁移、跨 capture 强一致预算服务 | +| 实现状态 | 未实现新 sink，未执行真实 Databend/新 sink 故障实验 | + +**本文替代 v0.2 作为本轮唯一实施候选。** [v0.2 历史设计](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review-v0.2.md)只供风险分析，不再构成并行实施要求；其中 B01–B18/V01–V13 保留为历史编号。当前仅跟踪本文 Q1–Q7、A01–A12，不把删掉旧协议当作其风险已通过实验。 + +## 0. 固定基线与本轮裁决 + +| 对象 | 仓库 | 固定 commit | +|---|---|---| +| 新架构 TiCDC | `/Users/jiangjianyuan/work/git/pingcap/ticdc` | `2bf6dacd0aec9ac2fad464600e47341f9aa951f2` | +| bend-cdc | `/Users/jiangjianyuan/work/git/pingcap/webapi` 下 `bend-cdc` | `b6481621021e82aee659ad88e8e6c00f6dd11906` | + +TiCDC origin 为 `git@github.com:sdojjy/tiflow.git`，上述是 fork 新架构基线，不代表上游/生产最新版本。编写时 checkout 是旧架构 master / `03ed2432bd7782198742d3aa20e07edcc47097e6`；只通过 Git 对象读取新架构，没有切分支。 + +```bash +git -C /Users/jiangjianyuan/work/git/pingcap/ticdc show \ + 2bf6dacd0aec9ac2fad464600e47341f9aa951f2:downstreamadapter/sink/sink.go +``` + +本轮采纳三项裁决： + +1. 采用“manifest 明确成员 + K/H/S + 直接 MERGE”，不再把 stream 消费范围当确认边界；普通 DML 不默认要求 durable applied ledger。 +2. 复用现有 dispatcher/progress/callback 生命周期；transport reset 不丢 sink 已接管数据。接管失败必须让旧生命周期停止，不能只 reset session 接着跑。 +3. 自动 HA 和严格 feed 全局远端额度移出受控 PoC 承诺；保留旧 SQL 排他检查、容量监控和可诊断停止，不以缩小范围掩盖数据错误。 + +## 1. 流程、职责与核心约束 + +```text +TiCDC dispatcher（现有 progress 登记） + → AddDMLEvent 非阻塞接管到有界入口 + → 异步按权威 schema/主键转换原生行 + → 内存热缓冲，必要时溢写限额 spool + → 同表/同 incarnation/同 raw_generation 攒批并封口 manifest + → 批量 INSERT raw，核验所有 manifest 成员到齐 + → 独占该表 lane：冻结其他 raw/DDL/compaction + → K = manifest 主键；H = K 对应 retained raw；S = 每键最高版本净动作 + → MERGE target USING S，取得提交成功终态 + → 本批分片 APPLIED，独立释放本地数据 + → 源事件 SEALED 且全部分片完成，flush callback 一次 + → TiCDC 原有机制推进 checkpoint +``` + +明确约束： + +- raw 是普通持久化 staging 表，不是 TEMP，也不是本地 spool。raw INSERT 成功不等于目标应用完成。 +- commit_ts 是事务 TSO，不是每行唯一 ID。版本不使用到达时间、批次序号或进程时钟。 +- 同逻辑目标表首版一条串行 lane；跨表有限 worker；没有外部应用改 target/raw/journal。 +- 不承诺跨表/跨分片事务原子可见。不实现、调用或编排全量导出/导入，不自动扫描源表历史行；目标基线由使用方负责，不构成sink工作包。 +- 不确认未应用的数据、不伪造 flush 清进度；所有“不支持”操作显式失败。 +- 限制 sink 自持内存与磁盘，不宣称硬限制全 TiCDC RSS；源 chunk 在进入 sink 前的分配不是本组件能消除的。 + +### 1.1 核心功能迁移 + +| bend-cdc 职责 | 原生 sink 处理 | +|---|---| +| Canal JSON 解码、文件 discovery、SQS、S3 游标 | 不迁移，改吃原生事件与现有 checkpoint | +| 主键、库表路由、建表、类型转换 | 按 TableInfo/列 ID/FieldType 适配，复用既有语义与测试案例 | +| raw、stream、MERGE | 保留 raw/MERGE；stream 不作为首版必要依赖 | +| 乱序、重试、最新版本和删除墓碑 | 必须保留，使用 K/H/S 实际应用最高有效版本 | +| DDL 白名单、pending/applied、TRUNCATE | 保留必要持久化恢复，接入现有 DDL 屏障 | +| Dumpling/CSV/COPY INTO | 不迁移、不引入依赖，也不开发独立全量初始化模块或共享全量mapper接口 | + +## 2. 接入边界与首版支持范围 + +| 修改位置 | 允许范围 | +|---|---| +| Databend sink 新包 | adapter、buffer/spool、manifest、tracker、raw/MERGE SQL、mapper、DDL journal、错误与指标 | +| scheme/type/config/factory/metrics | 新 sink 正常注册与校验，不伪装 MySQL SinkType | +| dispatcher manager/session 注册 | Databend 强制 raw-change=true 的小分支，贯通初建/重注册 | +| TableSchemaStore / recovery | 必要类型分支及专属恢复结果；沿用既有 dispatcher 创建/DDL 协调 | +| manager fatal 错误生命周期 | 先验证既有关闭路径；确有缺口才加有限失败分支，不引入通用交付协议 | +| collector/dynstream/TableProgress/调度核心 | 不改 delivery/信用/重投/progress 迁移协议；如确实不能接入，先给最小阻塞证据 | + +首版支持：非分区表、非空单列/复合 PK、完整行镜像、普通增删改与二级 UK 值更新、权威建表、有限非键列 DDL、通过恢复测试的 TRUNCATE。 + +首版不承诺：无 PK/nullable UK/force-replicate、分区或一表跨节点、多任务撞同一目标、复杂 DDL/PK 定义变更、active-active/BDR/syncpoint、外部写入、全自动 HA。也不要求历史空表自动发现/预建；没有增量事件的历史表可以暂不出现在目标，不能默认为会重放历史CREATE。 + +PK/handle 值变化及同事务复用键必须按真实源事件验证；不能假设全部变键都能识别为 UPDATE。未通过测试时不宣称支持，也不拿一个实际上无法识别所有变键的开关冒充拒绝保护。 + +## 3. 原生事件、类型转换与内部数据 + +### 3.1 原生事件模式 + +固定基线在 raw-change=false 时会因 UK 变化拆 UPDATE，PK 不变也可能产生同 key/TSO 的 DELETE/INSERT。Databend 内部固定 true，不开放用户切回 false；校验 manager→SharedInfo→注册/重注册→eventservice 实际生效。未知旧协议拒绝，不靠 SQL 任意选动作补救。 + +- INSERT / PK 不变 UPDATE：完整 after-image UPSERT。 +- DELETE：旧 PK + tombstone。 +- 同 key/TSO 完全相同净事件：重放，逻辑去重。 +- 同 key/TSO 异动作/异 payload：先检查源净变化契约，未解释的冲突在 MERGE 前失败；不能 ROW_NUMBER 任意决胜，也不能规定 DELETE 永远胜出。 +- raw=true 不等于真实 record-handle 变化已归约。测试先确认源 SQL 合法，再保存实际事件与最终键集合。 + +### 3.2 类型与 schema + +一个 mapper 同时服务建表、raw 编码、MERGE projection、DDL。输入是列 ID/FieldType/flags/精度/Elems，不按样本值猜类型。 + +| 类别 | 要求 | +|---|---| +| signed/unsigned、BIT | 无损整数，验证 UINT64/BIT64 边界，commit_ts 不转浮点 | +| DECIMAL | 明确目标精度能力，精确文本/参数，不经 float64；不支持范围拒绝 | +| binary/blob | 原生 bytes 用固定 HEX/已验证参数；不用 Canal Latin-1 还原 | +| ENUM/SET | 按原生 datum/Elems 转换，不重复做 Canal ordinal 解码 | +| JSON | 保留大整数，区分 SQL NULL/JSON null/字段缺失 | +| DATE/DATETIME/TIMESTAMP/TIME/YEAR | 时区/微秒/零值/负 TIME/超范围策略明确，不隐式修正 | +| 字符串/PK collation | 源键相等、canonical key 与 target JOIN 语义须一致，限制到已验证集合 | +| generated | VIRTUAL 不写，STORED 使用物化值；不把目标生成逻辑当源语义替代 | + +转换失败停止，不 TRY_CAST 静默写 NULL。ADD COLUMN 历史回填/default 必须与源相容；复杂表达式未定义前拒绝。 + +### 3.3 最小内部记录 + +```text +RawRow { + source_binding, logical_table, incarnation, raw_generation, + source_schema_version, row_key, commit_ts, + action: UPSERT | DELETE, + event_id, payload_hash, payload, + batch_id +} +``` + +row_key 使用确定类型/长度编码，保留完整键；hash 不代替相等性。event_id 基于源身份、代际、规范键、TSO、净动作等稳定信息，不依赖重启后重分批。batch_id 只标识一次逻辑 raw 批次，同批重试复用；重启可重新分批，业务幂等依赖事件身份与版本，不要求重建原 batch_id。 + +## 4. 有界接管、正常背压与失败生命周期 + +### 4.1 正常接管 + +```text +dispatcher 按现状登记该组事件的 progress + → AddDMLEvent 尝试非阻塞接管 + → 成功：sink 保留事件与回调的责任，有界入口中的引用计费 + → worker 增量编码/spool，不把整个事务一次复制 + → 根据水位/驻留引用与剩余额度，延迟或触发原有 PostEnqueue + → raw/MERGE 后按 tracker 触发 PostFlush +``` + +PostEnqueue 释放的是现有表路径的背压，不是 durable 确认。正常磁盘压力优先通过 spool、高低水位、强制小批 MERGE、延迟 PostEnqueue 吸收；不能只因普通水位升高就报 fatal。 + +入口按保守字节与元素数限制，并预留一个现有交付组可能带来的有限突发空间。共享 chunk 按其被 sink 引用的实际容量保守计费；可多计不能漏计。Detach 回调后逐步解除源数据引用；必须证明后台任务没通过闭包继续捕获整个 chunk。 + +现有交付组超过可接纳范围、单行超支持上限、磁盘/IO 严重错误属于明确异常策略：失败停机而非无限分配或无限等待。现有源 batch 没有足够硬上限时，不承诺永不触发该策略；用压测记录支持规模，不为此默认改 collector。 + +### 4.2 transport reset 与 owner/lifetime 分离 + +- 同 manager/dispatcher 的 transport session reset 不取消或丢弃 sink 已接管的数据，不因 collector epoch 改变清空入口/spool。 +- 重放可能与已接管事件重复，按源身份收敛；每个现有 progress 登记仍由自己的真实完成回调完成，不能因为逻辑事件重复就丢其 callback。 +- 未接管发生致命失败后，不在同 dispatcher 中尝试自定义 delivery 重绑/清 blocker；结束旧生命周期，重建后从安全 checkpoint 重放。 +- 真正 owner/lifetime 结束时，停止该旧 sink 新写入；这不证明已经发出的远端 SQL 已终止，见§9。 + +### 4.3 接管失败不是只设 IsNormal=false + +必须以首次 fatal 原因驱动单调状态： + +```text +RUNNING → FAILED（不可在本生命周期内复活） + → 立即关闭本地 admission 和 mutation gate + → Run 返回非context.Canceled的首因；上报控制面 + → 旧 manager/dispatcher 最终关闭，未完成事件不flush + → 核验旧远端 SQL；新生命周期从安全 checkpoint 重建 +``` + +固定基线中 Run 错误经 handleError/collectErrors 上报并重发，**并不立即销毁 dispatcher**。因此： + +1. sink 自己立即拒绝后续 Add/DDL/新 SQL；所有 worker 检查相同 failed gate。后续调用不能继续排队；失败回调只记首因，不忙循环。 +2. 不为清理旧 blocker 伪造 PostFlush。旧生命周期保留未完成 blocker 是安全停机状态；不能把它当健康任务继续 session reset。 +3. 验证控制面收到错误后关闭旧 manager/dispatcher，恢复时创建新对象。若既有路径不保证关闭，增加 Databend fatal 专属小接入，复用已有关闭/LocalFence能力；不能在运行 goroutine 里等待自身退出，不能取消上报上下文后把首因吞成 Canceled。 +4. 关闭/上报链未实测前，受控 PoC 需要人工停掉旧进程并确认 manager 不再运行，不承诺自动恢复。不得只调用一个 Close/LocalFence 就宣称整个控制面路径已验证。 + +## 5. spool、批处理与事件完成 + +### 5.1 缓冲策略 + +内存热数据→限额 spool segment；spool 是临时缓冲，不是恢复 WAL。预算包含入口引用、编码临时值、payload、回读批次、tracker/callback 元数据。超大行无内存直通例外，sync.Pool 不等于配额。 + +磁盘按实际占用/预留计费，未删成功的 segment 继续占额；启动只清理确认不活跃的自身目录。共享本地 BudgetManager 若可复用则复用，不能因多个 feed 各有10GiB就忘记节点空间；不把预算扩成分布式服务。 + +压力触发唤醒所有钉住受压 segment 的表，不只热点申请者；独立读回/应用资源保证消费者能排空。首版先验证强制排空和物理回收，不默认新增复杂搬迁系统；不能排空则告警/停止新接管。ENOSPC 仍可能来自共盘其他组件，要明确处理，不承诺绝不发生。 + +### 5.2 表内串行与攒批 + +每表/代际/raw_generation 一条 lane，按行数、编码字节、最长等待、DDL marker、压力触发。计时从最老待处理数据起算。不同表有限并发，单表 raw/核验/MERGE/DDL/compaction 不交错。 + +raw batch 可包含多个事件分片；大事件可跨多批。通过有界窗口边编码边应用，不能把累计事件总量作为必须同时驻留的大小。 + +### 5.3 SEALED tracker 和独立回收 + +```text +事件：OPEN → SEALED → COMPLETED（或 FAILED） +分片：REGISTERED → STAGED → RAW_VERIFIED → MERGING → APPLIED → RELEASED +``` + +- 分片先登记、增加 pending，再发布给消费者；生产者遍历完整事件后才 Seal。 +- 只有 `SEALED && pending==0 && !failed && CAS(completed)` 才调用 flush 一次。 +- APPLIED 分片立即可释放 payload/entry、回收空间，不等整事件完成；只保留有界未完成描述符和小 tracker。 +- Detached flush 闭包自身没有 once 保护，由 tracker 保证；PostEnqueue 与 flush 不混用。 +- 停止后未完成分片不确认；Close 可回收临时文件，恢复靠源 checkpoint 重放，不能靠文件删除当作已完成。 + +## 6. manifest、K/H/S 与直接 MERGE + +### 6.1 本批 manifest + +每批在 sink 内有只读 manifest，含：batch_id、binding/incarnation/raw_generation/schema、分片到成员映射，以及每个成员的稳定 event_id、完整 row_key、commit_ts、净动作、payload_hash。 + +manifest 是有界 batch 元数据，可随 spool 暂存；不要求普通 DML durable applied ledger。不能只保存预期行数或摘要：需要能精确判断成员存在、内容一致以及属于哪个事件分片。源恢复可重建新批次/manifest。 + +### 6.2 raw 完整性核验 + +1. 参数化批量写 raw，可拆多个有限 SQL；失败重试同批成员，不改变业务版本。 +2. 读取/核验该 batch 的 raw 逻辑成员集合，核对完整事件身份和内容；重复物理行按相同身份去重，相同身份异内容为错误。 +3. 缺任何 manifest 成员则继续补写/核验，不 MERGE、不确认；额外未知成员说明 batch 绑定错误，停止。 +4. 核验结果若分页需读取完整，不能 LIMIT一页、COUNT相等或比较一个无成员能力的digest就当全集证明。核验本身也在有界批次/内存预算内。 + +### 6.3 集合定义与应用算法 + +```text +M = 已验证的本批 manifest 成员 +K = DISTINCT row_key FROM M +H = 当前 binding/incarnation/raw_generation 合法范围内、row_key∈K 的 retained raw +S = H 中每键最高 commit_ts 的确定净动作（含 DELETE tombstone） +``` + +顺序：冻结该表其他 raw/DDL/compaction → 验证 M 全到齐 → 验证 H 的同 key/TSO 冲突 → 形成 S → MERGE target USING S → 成功终态 → 仅完成 M 对应分片。 + +**不是直接 MERGE 本批旧行，也不是看到 higher raw 就跳过旧行。** 例如 M 有 K@150 而 H 有 K@200：本次实际将 K@200 的净动作写入目标，即使@200之前只是raw写入尚未应用，也能支配@150。不能仅过滤掉@150后声称它已完成。 + +概念 SQL 结构（不是已验证可执行 SQL）： + +```sql +-- 在实际实现中使用转义标识符和绑定参数；先核验 manifest/同版本冲突。 +WITH batch_keys AS ( + SELECT DISTINCT row_key FROM RAW_TABLE + WHERE batch_id = BATCH_ID AND raw_generation = RAW_GENERATION +), latest AS ( + SELECT r.* FROM RAW_TABLE r JOIN batch_keys k ON r.row_key = k.row_key + WHERE r.raw_generation = RAW_GENERATION + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY r.row_key ORDER BY r.commit_ts DESC + ) = 1 +) +MERGE INTO TARGET_TABLE t USING latest s ON TYPED_PK_MATCH +WHEN MATCHED AND s.action = 'DELETE' THEN DELETE +WHEN MATCHED AND s.action = 'UPSERT' THEN UPDATE SET EXPLICIT_COLUMNS +WHEN NOT MATCHED AND s.action = 'UPSERT' THEN INSERT EXPLICIT_COLUMNS; +``` + +ROW_NUMBER只在同版本净动作/内容已验证一致后用于消除物理重复，不能拿它解决冲突。真实SQL必须包含完整binding/代际约束、精确payload投影/NULL规则，不能跨schema误读。 + +### 6.4 覆盖条件 + +本批完成需同时证明： + +- M完整进入raw；K准确对应M，没有未解释缺失/额外成员。 +- 对每个M成员，S包含同key、版本不低于它的合法净动作；其更高版本被**实际纳入本次MERGE**。 +- SQL无额外LIMIT/漏JOIN/过滤，目标PK匹配/投影覆盖完整S；同表冻结期间核验到MERGE使用一致有效数据，快照/查询计划行为需真实验证。 +- MERGE服务端提交终态明确成功；零业务行修改也可以完成，如删除不存在的键，但不能仅凭返回计数判断成功。 + +S可能顺便应用H中其他批次的更高事件，但只完成M对应callback；其他批次仍走自己的核验/幂等应用，不猜其进度。若只覆盖部分分片，未覆盖部分保持pending，不一次确认整事件。 + +没有stream，stream hint/cursor/no-op消费witness不参与正确性；但MERGE源集合与终态证明仍不可省略。若无法证明上述条件，停止，不加一张写有“计划coverage”的表假装解决。 + +## 7. 提交、重试与恢复：不引入普通 DML applied ledger + +| 情况 | 行为 | +|---|---| +| 部分 raw INSERT 成功 | 不确认，补写缺少成员；同事件物理重复允许但逻辑去重 | +| 明确未提交/回滚 | 在旧请求不再运行且所有权有效后重试 | +| MERGE成功、callback之前崩溃 | 从checkpoint重放，重建manifest并按K/H/S重应用，不要求恢复旧batch_id | +| MERGE成功、响应丢失 | COMMIT_UNKNOWN，不确认、不发下一批；先核验旧请求终态 | +| 业务提交已确认，仅操作统计解析失败 | 保留成功，统计unavailable，不为补统计重做SQL | +| 查询记录缺失、断连、取消返回nil | 不能推导未提交/已终止，进入人工恢复 | + +最小 TargetClient 需能保留当前操作/query ID、读取完整结果和服务端执行终态，并区分执行错误、传输错误、值解码错误与统计解析失败。QueryContext返回Rows不是通用提交证明，驱动Kill/Close成功也不是服务端终止证明。 + +不要求把每次普通DML attempt写入durable applied ledger。代价是：重启丢query关联、旧查询查不到时，**不承诺自动恢复**；人工确认所有旧mutation已终结且目标独占，再从安全checkpoint幂等重应用。进程退出/固定sleep不能替代确认。业务幂等不意味着可以允许旧SQL晚于新批次提交。 + +DDL仍有必要durable journal，因为重复TRUNCATE会破坏新代际；不能将“普通DML无需ledger”扩展为“所有持久化恢复元数据都不要”。 + +## 8. 建表、DDL、schema fence 与 TRUNCATE + +### 8.1 建表与历史数据范围 + +首次DML按权威TableInfo建表/核验，接收到CREATE TABLE等受支持DDL时也按事件建表；不查询最新schema解释旧事件。目标已存在须验证结构、任务绑定与PK唯一性，不盲目IF NOT EXISTS接管。没有DML/DDL的历史空表不要求出现在目标，不做全库发现/预建，也不借此引入全量工具。 + +同一raw_generation只接受可由同一projection解释的数据。已完成会改变行投影的DDL后，优先切换新的raw_generation；旧raw隔离，不把旧payload直接CAST成新schema。此前目标行的回填/默认值由已验证DDL处理，后续完整after-image重新建立新generation的raw记录。 + +旧schema事件的跳过依据是durable applied-DDL fence，必须证明该边界前数据已排空或被TRUNCATE合法清除；不是当前schema的更新时间。只有对源顺序前缀/fence契约成立时才可安全退役旧raw墓碑，否则停止而非清理。 + +### 8.2 普通 DDL + +```text +现有 FlushDMLBeforeBlock / 全局 DDL 协调 + → 强制旧分片全部MERGE完成 + → sink核验DDL身份/owner/当前schema + → durable journal PREPARED（含before/after fingerprint及drain fence） + → 执行白名单DDL，取得终态并校验 + → 切换/验证projection和必要raw_generation + → journal APPLIED持久化 + → DDL PostFlush，放行新schema +``` + +支持CREATE、有限非键ADD/DROP/RENAME/MODIFY；默认值/NULL/生成列须明确。复杂多操作、PK定义更改、DROP/RENAME表等未定义项停止。二级索引DDL只有不改变行身份/投影时才可no-op，raw-change契约持续有效。 + +### 8.3 DDL/TRUNCATE最小恢复顺序 + +先关闭新写→人工/受控确认旧SQL终结和目标独占→核验binding/journal→恢复pending DDL/reset→安装已应用fence→协调旧DML重放→开放当前代际。 + +| durable状态 | 恢复行为 | +|---|---| +| PREPARED，尚未发送或明确未提交 | 核验before条件，按同DDL身份执行；未知旧SQL先协调 | +| 执行结果不明 | 查终态/校验可唯一证明的after状态；不能只见列不存在就猜成功 | +| APPLIED但上游checkpoint尚未保存 | 相同DDL只跳过，不重复破坏性操作；旧DML按已证fence处理 | +| metadata冲突/丢失 | 停止、人工核验或重新初始化，不用重新TRUNCATE“修复” | + +TRUNCATE记录源DDL身份、旧/新表血缘和代际，不只用表名/全局MAX(TSO)： + +```text +DRAINED → PREPARED(new incarnation CLOSED) + → TARGET_RESET已确认 + → 新raw_generation/binding校验完成 + → APPLIED持久化 + → 才可开放新代际DML +``` + +PREPARED中重做reset仅在旧SQL明确终结、目标独占且新代际从未开放/无新写入证据充分时允许；APPLIED之后绝不重做。journal写入本身不确定就停在该步骤，不越过。不存在假定的强制唯一约束/跨DDL事务万能锁。 + +重点验收：TRUNCATE@500完成，新代际@510写入后从@490重放，@510不得被再次清空。此顺序属于sink必要恢复，不因此重写调度框架。 + +## 9. 受控 PoC 的排他与生命周期边界 + +首版只在隔离单capture、静态独占目标环境验证；禁止正常运行时跨owner漂移，自动替换/重启后也不能未经核验直接开放写入。`enable-table-across-nodes=false`仅限制拆表分布，不等于禁止整表HA，不能将其当唯一保护。 + +失租/fatal立即关本地gate，禁止新raw/MERGE/DDL/compaction；已发送的远端SQL仍需查终态。旧MERGE晚到也可能覆盖新值，旧TRUNCATE晚到可清空新代际，去掉stream并不解决这些问题。 + +若目标版本/接口不能可靠定位并终结旧SQL，保持停机，人工核验或重初始化；不发布自动HA。自动HA若需要目标侧fence或更多控制面工作，单独RFC，不夹进新增sink的首版合入。 + +## 10. 清理、容量与成本 + +- raw保留每键最新版本及删除墓碑，删除严格被支配历史；最新物理重复需安全去重策略，不因为SQL成功就认为空间不再增长。 +- raw compaction与同表写入/MERGE串行；不能删manifest仍需要的成员/恢复证据。未处理完的批次受本地预算约束，不能清raw后才发现还需要核验成员。 +- 持续新键INSERT→DELETE会让target很小但raw增长，必须监控raw/墓碑/retained快照、历史键及扫描成本，达到阈值停新raw；旧SQL/清理在途也计入告警判断。 +- 首版是**监控型停写阈值**，有测量延迟、物理放大和多capture超调风险；不承诺全局原子feed硬额度。受控PoC不引入跨capture预算服务；将来需要严格额度时单独评估静态切分或已有外部配额。 +- 不删最新墓碑解决超额。恢复方式是显式扩额、安全去重或新一致快照重初始化；物理文件/快照释放可能延迟，不把逻辑删除量当已回收字节。 +- 无stream不等于便宜：K关联H可能扫很多raw。固定事件率、行宽、历史键数、热点、小表数量、可见延迟SLO，实测Databend扫描/credit、TiCDC CPU/RSS/spool IO，再与旧链比较。 + +## 11. 配置草案与增量接入边界 + +所有Databend新字段均未实现；以下是可读的压测起点，不是可运行配置。认证通过受保护配置/运行时凭据注入，不把真实TOKEN写入URI/日志。 + +连接标识：`databend://HOST:PORT/DB_NAME?sslmode=require`。 + +```toml +[filter] +rules = ["DB_NAME.*"] + +[scheduler] +enable-table-across-nodes = false # 不是禁用HA；PoC还需隔离部署和恢复准入 + +[sink.databend-config] +auto-create-table = true +batch-max-rows = 5000 +batch-max-bytes = 8388608 +flush-interval = "1s" +worker-count = 4 # 跨表，同表串行 +entry-queue-max-events = 1024 +sink-memory-quota = 268435456 # 含入口引用/编码/回读/metadata，不等于进程RSS +spool-memory-quota = 67108864 # 包含在sink-memory-quota内 +spool-disk-quota = 10737418240 +raw-observe-interval = "30s" +raw-stop-threshold-bytes = 1099511627776 # 示例告警/停写阈值，非硬计费quota +``` + +镜像模式固定处理DELETE，内部固定raw-change=true；不开放allow-delete=false、stream消费hint、DML ledger模式、自动HA、分布式预算开关。单行/交付规模、SQL请求开销和内存分池必须校验兼容，不能让queue count乘以巨大chunk绕过字节预算。配置减少到用量以下时停新输入继续安全排空，不直接删文件。 + +sink从TiCDC交付的增量事件开始处理，遵守已配置start-ts与现有checkpoint，不负责搬运该边界之前的存量行。用户若要求目标成为全量镜像，需要在本功能之外准备可衔接的目标基线；本设计不指定工具、不实现或编排该流程，也不强制所有增量用例先做全量导入。没有基线且历史行一直不变时，这些行不会自动出现在目标，这是增量范围边界而非自动补数能力。源保留窗口不足仍应失败，不承诺无限期背压恢复。 + +## 12. 具体验收：先实现最小路径，不继续扩通用协议 + +以下均为待执行测试，未因为本文写出而通过。 + +| ID | 场景 | 必须证明 | +|---|---|---| +| A01 | PK固定仅UK更新；raw=true初建/重注册；合法真实PK变化对照 | 普通UPDATE无人工DELETE/INSERT冲突；未验证变键不虚报支持 | +| A02 | 一批多个raw子写，部分成功、重复物理行、缺成员/同身份异payload | manifest逐成员核验，缺失不MERGE、不提前回调；跨batch重复收敛 | +| A03 | UPSERT@100→DELETE@200→迟到UPSERT@150；higher raw已入但未应用 | K/H/S实际应用最高净动作，删除不复活，不因仅raw存在就确认 | +| A04 | 实际MERGE SQL、不同schema/代际、查询投影失败、零目标修改 | 完整S应用与终态成立；无stream hint依赖；不跨schema解释旧payload | +| A05 | F1完成时F2未登记；事件总量>spool；Seal/Apply/取消竞争 | OPEN不确认、分片可独立回收、全集完成恰回调一次、驻留有界 | +| A06 | 正常磁盘压力、冷表pin、删除失败、入口无法接管 | 常规压力由背压/排空吸收；异常非阻塞fatal，无假释放/无限队列 | +| A07 | 已接管后transport reset；Add接管失败；Run错误上报/控制面关闭 | reset不丢数据；fatal后无新接管/SQL；旧blocker不伪flush；新manager从安全checkpoint恢复而非旧session复活 | +| A08 | MERGE提交后响应丢失、callback前崩溃、统计解析失败、旧query查不到 | 无未知结果确认；终态排他后幂等重应用；统计失败不重做；查不到时人工停机 | +| A09 | DDL各journal中断、schema generation切换、旧DML/DDL重放 | 正确fence/投影，APPLIED前不开新schema；不靠最新版schema猜跳过 | +| A10 | TRUNCATE每步骤崩溃、新数据写入后回放旧TRUNCATE、晚到旧SQL | 新代际不再被清空；旧SQL终态无法证明则不放行 | +| A11 | target小但历史键churn；同SLO旧链/无stream SQL成本对照 | 成功清理下仍能告警停写；量化延迟/超调，不声称严格quota；扫描/费用实测 | +| A12 | start-ts/checkpoint接入、首条DML/CREATE建表、类型边界/复合键/collation | 只验增量交付边界和数据转换，不要求全量工具或历史空表预建；既有bend-cdc增量核心案例逐项对照 | + +### 12.1 当前仅保留七个落地门槛 + +| ID | 待关闭事项 | 处理边界 | +|---|---|---| +| Q1 | manifest成员核验、K/H/S SQL、同版本净事件与重放 | sink内实现+A01–A04，不能用新ledger替代集合证明 | +| Q2 | 非阻塞接管、正常背压与fatal生命周期闭环 | sink为主+A06/A07；仅证明确需时加manager小接入 | +| Q3 | SEALED tracker、spool字节预算和回收活性 | sink内实现+A05/A06，不能承诺框架全RSS硬界 | +| Q4 | 服务端终态与旧SQL排他 | sink客户端+A08/A10；未证明则人工恢复PoC，不扩自动HA承诺 | +| Q5 | DDL journal、schema fence、TRUNCATE与恢复分支 | sink为主、schema/recovery小接入+A09/A10 | +| Q6 | raw保留/去重、监控停写、扫描与机器总成本 | sink内+A11，严格跨capture额度不在首版 | +| Q7 | 事件元数据建表、类型矩阵、真实源契约和兼容接入 | 增量mapper/建表及小注册分支+A01/A12；不包含全量工作包 | + +Q1–Q7均未以实现/实验关闭。功能或规模缩减必须可执行；若最小路径在既有框架不能正确停止/恢复，先提出精确小补丁，不继续扩写通用信用/progress/调度系统。 + +## 13. 交付顺序、灰度和回退 + +1. P0：固定server/driver，验证K/H/S SQL、raw完整核验、提交终态、原生事件及现有fatal关闭调用链；不创建生产资源。 +2. P1：受控单capture、PK表DML纵切，完成buffer/spool、manifest、tracker、raw/MERGE、错误停机；覆盖A01–A08。 +3. P2：事件驱动建表/类型、有限DDL与TRUNCATE恢复，对齐bend-cdc已支持的增量核心行为，完成A09–A12；不引入全量导出/导入工具。 +4. P3：独立影子目标、对齐快照/水位比对，测容量与成本后决定是否继续生产化；自动HA另议，不因PoC通过默认开放。 + +旧链写旧目标，新sink写独立影子目标，不共享raw/journal；受控drain/对齐边界比较，不能任意时刻比两个异步目标行数。切读前保留旧链回放窗口，回退先追平旧链再切读。内部raw/journal格式版本化，不直接复用bend-cdc文件游标或不兼容旧schema。 + +## 14. 源码证据与验证状态 + +以下是固定源码证据，不代表新sink已经运行。路径基于Git对象，无需当前checkout存在同文件。 + +- [Sink接口/factory](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/sink.go)。 +- [raw-change初始化与Run错误传递](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go)；[session注册/重注册传播](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/eventcollector/dispatcher_session.go)。 +- [UK拆分](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/eventservice/event_scanner.go#L970-L1014)；[IsUKChanged](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/codec.go#L219-L259)。 +- [同步Add与progress先登记](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatcher/basic_dispatcher.go#L293-L335)；[progress按flush移除](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatcher/table_progress.go#L97-L125)。 +- [Run错误进入handleError，L393–404](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go#L393-L404)；[错误队列/持续上报，L699–749](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go#L699-L749)；[TryClose/LocalFence/stopWritePath，L1057–1173](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go#L1057-L1173)。 +- [flush/enqueue及Detach闭包](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/dml_event.go#L660-L711)；[spool实现](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/sink/spool/spool.go)。 +- [TableSchemaStore类型分支](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/table_schema_store.go)；[dispatcher恢复专属分支](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/task.go)。 +- bend-cdc：[实际MERGE适配入口](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go)、[raw/MERGE/compaction](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go)、[DDL epoch](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/schema_epoch.go)、[TRUNCATE](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/truncate.go)、[现有MERGE结果标记](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/merge_result.go)。 + +验证状态： + +- 前序已运行bend-cdc既有两包本地测试通过，真实Databend连接未配置、相关集成用例跳过；不能当作新sink证据。 +- 本轮只统一文档和评审入口，保留v0.2历史快照；未改业务Go代码、未切分支、未构建/部署、未创建PR。 +- A01–A12、K/H/S真实SQL、错误控制面关闭链、终态/恢复、扫描和费用实验均未执行。 +- 文档结构/TOML/Git对象链接检查与上述运行验证分开报告。去掉旧协议是范围裁决，不是正确性已被证明。 diff --git a/docs/design/databend-native-sink-design-review.md b/docs/design/databend-native-sink-design-review.md new file mode 100644 index 0000000000..95780135e9 --- /dev/null +++ b/docs/design/databend-native-sink-design-review.md @@ -0,0 +1,395 @@ +# 定稿 v1.1：TiCDC 原生 Databend Sink——按 bend-cdc 既有语义迁移 + +| 项目 | 当前裁决 | +|---|---| +| 版本 / 日期 | v1.1 / 2026-10-25；v1.0定稿后按用户要求增加conflict-key及指定DDL跳过 | +| 状态 | 设计已接受，首版实现及受控真实链路验收通过；不表示生产HA/成本已验收 | +| 实现分支 | `feat/databend-native-sink`，从固定新架构commit `2bf6dacd0aec9ac2fad464600e47341f9aa951f2` 创建 | +| 正确性基线 | 对齐固定版本 bend-cdc 的 TiDB 增量路径，不为迁移另造更强的通用同步系统 | +| 实施主线 | 迁移现有 Databend raw/MERGE、mapper、schema epoch 核心；新增原生事件、spool、TiCDC回调适配 | +| 输入与来源默认值 | 与固定版本CloudStorage默认配置保持拆分模式 raw-change=false；首版固定raw source，stream仅保留后续验收入口 | +| 不新增 | batch manifest持久化/逐成员读回、K/H/S新SQL算法、event_id/payload_hash体系、AppliedCycle/DML ledger、raw_generation协议、collector信用/progress迁移、分布式锁/预算服务 | +| 缓冲 | 保留有界内存 + 限额本地spool + 现有PostEnqueue背压 | +| 全量范围 | 不支持/编排Dumpling、CSV、COPY INTO，不实现历史空表发现/预建 | +| 首版边界 | 沿用单逻辑目标表单writer、目标静态独占前提；故障恢复先验证，不承诺新建自动HA体系 | +| 已验证 | 本地/race/lint/build及真实TiUP→CDC→staging Lake：9组PASS、32列、磁盘spool/排空重启；见§8.2–8.3，未验收生产HA/成本 | + +**本文是唯一当前定稿设计。** [v0.3](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review-v0.3.md)和[v0.2](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review-v0.2.md)仅保留历史分析；其中manifest、K/H/S、raw generation等不是并行实施要求。设计评审不再以新增通用协议扩大范围；实现中若发现不可规避的范围变更，单独说明并复审。 + +## 0. 固定源码与原则 + +- TiCDC：`/Users/jiangjianyuan/work/git/pingcap/ticdc`，commit `2bf6dacd0aec9ac2fad464600e47341f9aa951f2`，origin `git@github.com:sdojjy/tiflow.git`。 +- bend-cdc：`/Users/jiangjianyuan/work/git/pingcap/webapi` 下 `bend-cdc`，commit `b6481621021e82aee659ad88e8e6c00f6dd11906`。 +- 设计期间曾在旧架构master / `03ed2432bd7782198742d3aa20e07edcc47097e6` 下按Git对象核对；用户批准实现后已创建并切到上述新架构基线的实现分支，没有在旧master开发。 + +原则：**替换输入和运行时外壳，不先重设计已经存在的目标应用算法。** 新增约束必须归入三类之一：既有行为迁移、移除S3后必须补的接入、可延期的增强；不能只因理论上可加强就进入首版。 + +“同样消费TiCDC”不意味着故障语义完全相同：原链有S3持久缓冲、文件/epoch顺序与独立消费进程；直接sink必须把必要的顺序/完成语义接到现有dispatcher/callback。也不能把旧代码注释/假设当成全部合法输入都已正确的证明。 + +本轮只补六个局部缺口：拆分输入兼容、规范化一次、raw source默认值、内层UNKNOWN截断、spool无成功回调清理、落库精度排序。没有重新引入manifest/ledger/witness或collector/progress协议。默认值来自固定源码，不代表已核实任何线上changefeed配置；旧链若显式启用raw=true，其PK更新行为需另列既有缺口。 + +## 1. 实际代码基线：已有的保证、假设与限制 + +| 主题 | bend-cdc实际代码 | 迁移处理 | +|---|---|---| +| 核心输入 | CanalJSONMessage转common.Event，再经normalizeCDCEvent交CDCManager | 原生DMLEvent经NativeNormalizer进入规范行入口，不二次做Canal解码 | +| 版本 | 优先 `_tidb.commitTs` → TiDBCommitTS → event_pos；无扩展时兼容退回毫秒TS | 直接用原生CommitTs，复用排序规则；不继承“TSO每行唯一”的错误注释 | +| 主键 | 完整PK集合canonical化、复合conflictIdentity与MERGE JOIN | 直接从TableInfo获取相同键语义，保留复合键测试 | +| raw记录 | `id, table_name, raw_data, add_time, event_pos, action` | 优先保持形状，不为迁移增加manifest/event_id表或新一套版本身份 | +| UPDATE/DELETE | UPDATE使用After，DELETE使用Before；默认拆分模式提供PK/UK变更的DELETE/INSERT | 保持默认拆分和输出顺序；不把raw=true变键UPDATE直接交After-only核心 | +| 顺序与重放 | 严格模式按(event_pos,add_time)选候选，并查询retained raw抑制旧值 | 先迁移既有SQL和清理语义；同TSO合法净动作须做源事件对照，不用到达时间掩盖新增差异 | +| 清理 | 删除被更高版本支配历史，保留每键最新watermark/tombstone | 保留，不引入TTL删最新墓碑；不承诺历史键集合有界 | +| stream | 创建append-only stream；失败时用raw作为同一个MERGE builder的source | 首版显式固定raw；stream代码可保留，但未验收不创建/消费，不用静默fallback当完成证明 | +| 完成进度 | raw成功后writeGen递增；mergeTable捕获startGen，成功后mergedGen推进；Flush排空 | 复用批次/代数级完成，映射TiCDC callback，不默认读回每行 | +| DDL顺序 | 文件schema版本边界先Flush，ApplySchema保存pending/applied epoch | 使用现有TiCDC屏障代替文件队列，迁移epoch/reconcile | +| TRUNCATE | Flush→pending epoch→drop stream/raw→truncate target→applied | 迁移现有reset顺序，测试重启重放，不增加多套代际状态机 | +| 不支持DDL | 某些DDL只warning并推进epoch，并没有执行目标DDL | 不把warning跳过算“已支持”；只对齐已正确投影的能力，不承诺任意DDL | + +代码事实不等于运行证明。例如writeGen/mergedGen假定MERGE覆盖了已记账raw批次；QueryContext返回Rows被现有代码视作executed；这些前提接入callback前需要局部验证，不能靠新增一套架构掩盖，也不能不经验证就宣布安全。 + +## 2. 最小实现：三个主体，而非新框架 + +```text +现有TiCDC dispatcher / TableProgress / DDL协调 + → NativeAdapter + TableInfo、CommitTs、before/after → 等价内部事件/原生类型转换 + → Buffer + Spool + TableWorker + 有界接管、按表攒批、内存/本地磁盘、有限并发、callback记账 + → 迁移的DatabendCore + 建表、参数化raw INSERT、已有MERGE/compaction、schema epoch/TRUNCATE +``` + +对每张表，第一版由一个worker串行控制： + +```text +取一个有界批次 + → 全部raw子INSERT成功 + → 调用等价的FlushTable（复用现有Flush/mergeTable逻辑） + → 确认本轮排空成功 + → 完成覆盖的分片、释放spool + → 源事件全部完成时flush callback一次 +``` + +这样无需在单表内并发raw写入与MERGE，也无需manifest逐成员核验。普通INSERT成功按完整SQL执行结果记账；如果分多条SQL，全部成功才使该批具备完成资格。失败/结果未知不确认，局部重试或停止后从checkpoint重放；不把“写了一部分raw”当作一批成功。 + +实现可保留writeGen/mergedGen或等价本地批次序号，属于进程内callback记账，不是持久化应用日志。移植核心算法，不照搬每表连接池、Background context、无界goroutine/重复后台循环，也不让TiCDC依赖整个webapi服务模块。 + +## 3. raw/stream/MERGE如何复用，哪些不能盲抄 + +### 3.1 不再重设计MERGE来源：首版固定raw + +保留现有 `buildMergeSQL(target, source, raw, schema)`、复合键JOIN、列投影和compaction；首版调用显式传 `source=raw`，初始化不创建stream，也不在后台自动切换为stream。复用现有raw分支，不是新K/H/S算法。 + +- 首版使用现有full-raw路径：单writer停止新写，完整MERGE当前epoch的raw，再确认该批。无stream cursor、消费hint或witness依赖。 +- stream作为后续增量读取优化保留，不以“删除stream代码”作为简化目标。启用时必须证明当前server/每个连接的有效设置下Flush覆盖所记批次，且验证no-op消费；如果只处理一部分，不能按startGen确认全部，也不能循环现有Flush冒充排空，因为pending可能已经被清零。 +- 不强制新增batch_id/manifest、K/H/S或读回所有raw成员；也不因为不用stream就保证更省钱。full-raw可能扫描更多，性能决定是否需要启用已有stream优化。 + +验收门槛很具体：两次raw INSERT、两个不同键，Flush必须将两键应用后才完成两个callback；加入no-op、旧事件重放。stream消费限额是后续启用测试，不能“与旧代码一样”就提前确认；不新增消费witness或逐成员读回。 + +### 3.2 同TSO和类型：先比较等价输入 + +**本版撤销强制raw-change=true。** 固定CloudStorageConfig未设置该开关时返回false，manager对新增SinkType也默认false；首版沿用拆分模式，不需要新增“强制true”分支。启动/重注册测试确认传播值为false，用户不能把它当调优项切成true。该模式下PK/UK更新由既有scanner拆分；adapter/worker必须保留其DELETE/延后INSERT输出顺序，不能重新逐行交错或仅按TSO排序。 + +例如非聚簇PK从1变2，兼容路径应收到并应用DELETE(1)、INSERT(2)；不能将带oldPK!=newPK的raw UPDATE直接送After-only核心。若意外收到这种UPDATE，首版显式拒绝/停写，不静默仅写After。将来需要raw=true时，须补oldPK删除和同事务顺序适配后再开放，不因此新建事务DAG。 + +source事务TSO不是每行唯一；同key/TSO合法相反动作可能依赖上述既有源顺序。分片/重启后同样要通过最终键集合测试，不用本机时钟创造源业务版本。 + +**规范化恰一次**：迁移核心提供两个边界清晰的入口，汇合后共享raw构造/INSERT/MERGE： + +```text +Canal原始事件 → normalizeCDCEvent（原ordinal/Latin-1等规则） ┐ +Native datum → NativeNormalizer（按FieldType取label/bytes） ┴→ NormalizedRow → raw构造/INSERT +``` + +NormalizedRow携带规范PK/列值/动作和CommitTs；Native路径不再进入normalizeCDCEvent。Source仍是TiDB，不通过伪装其他Source绕过解码；旧getTiDBSequence仍依赖它，或在规范行接口中显式传递同一CommitTs。 + +比较的是规范化后的键、payload、动作、TSO，而非native datum与Canal原始字符串。数字label `ENUM('2','x')`：Canal ordinal `"1"`和native label `"2"`均只能最终得到`"2"`，不能二次解码成`"x"`。还需SET高位、全字节二进制、精确decimal、大整数JSON、SQL NULL/JSON null、时间精度；raw JSON序列化不再次做类型解码。 + +### 3.2.1 add_time按落库精度单调，而非仅Go时间单调 + +既有核心按1ns递增，但raw参数只输出6位小数，可能落成相同时间。依赖同key/TSO顺序的兼容路径需在raw构造中局部修正：先将candidate量化到实际存储/参数精度（当前格式为微秒），若不大于该表lastStored，则取lastStored+1个存储精度单位，再格式化。不要先做纳秒比较再截断。 + +单writer按源顺序调用该分配器；测试必须检查最终SQL参数及真实raw时间/目标状态，而不只检查内存Time.After。add_time仍是既有同TSO决胜辅助，不成为新的业务版本体系，也不解决任意源冲突/失租并发；同payload纯重复不作为该错误反例。 + +### 3.3 完成结果与旧缺口 + +**必须在迁移的客户端/核心最内层分类，不能只在外层worker判断UNKNOWN。** 旧executeMergeInto在QueryContext返回Rows后设executed，后续Rows.Err也携带true；旧mergeTable会把executed错误改成成功，其他临时错误则可能已在内部重发。原分支不能透明照搬。 + +迁移后使用最小提交状态 `COMMITTED / NOT_COMMITTED / UNKNOWN`，统计解析错误单独携带；raw写、MERGE和显式COMMIT同样适用。每次结果进入核心后，**先判断UNKNOWN，再判断观测失败或可重试错误**： + +- COMMITTED且仅统计失败：业务成功、统计unavailable，不为补统计重做。 +- 明确NOT_COMMITTED且可重试：才进入既有有界重试；不能把所有网络错误都归入该类。 +- UNKNOWN：立即退出内层循环，上层不得按transient再重试；不推进mergedGen/pending，不callback，不进下一批/DDL。先核验旧SQL终态，取得排他后用retained raw幂等重应用；无法证明就停写、人工恢复。 + +mock检查错误分支与物理请求次数；真实实验检查提交响应丢失后的恢复。mock构造的“首页后失败”不是声称真实MERGE已复现。只改Databend客户端/核心，不引入DML ledger、AppliedCycle、查询注册中心或分布式fence。 + +## 4. 哪些仍是直接sink必须新增的适配 + +### 4.1 MERGE后才确认TiCDC + +bend-cdc已有MergeProgressReporter/applied watermark区分“文件处理/raw写入”与“目标已MERGE”。原链的文件游标、S3持久数据和lake applied进度不是同一件事。 + +新sink用现有PostFlush表达目标应用完成，不能在raw INSERT成功时就推进TiCDC checkpoint。否则去掉S3后，可能没有一个可靠消费者负责把已确认raw继续应用。这不是追求比bend-cdc更强的exactly-once，而是把已有目标完成语义接到不同的上游接口。 + +同一事件的回调由本地completion对象持有；分片先登记后发布，编码完成Seal，`SEALED && pending==0`时once回调。已应用分片可独立回收，避免大事件等整事件确认才释放空间。只保留有界计数/未完成描述符，不增持久化tracker。 + +### 4.2 保留spool和现有流控，不改collector + +- 有界入口、内存热数据、限额磁盘、字节/元素数记账；单行超上限明确失败，不无限内存直通。 +- 正常压力延迟PostEnqueue、强制小批Flush；磁盘物理未回收继续计费，压力唤醒钉住segment的表。 +- 已接管事件不因transport session reset丢弃。失败不伪造PostFlush，不清迁旧TableProgress。 +- AddDMLEvent不无限等待共享线程；真正无法接管/IO致命错误关闭sink本地admission和写gate，Run报首因；旧manager/dispatcher需最终结束，不只reset旧session继续跑。 +- 先验证现有关闭链，确有缺口仅补Databend fatal小分支。上游chunk已在sink前分配，不承诺全进程RSS硬上限；不为此改collector/dynstream信用协议。 + +**spool包装有两个不可透明继承的行为**：固定spool的Discard会先跑entry postFlush再Release；TryEnqueue对超过disk quota的entry有oversized内存接受分支。native包装必须隔离： + +1. 源事件成功callback只由SEALED tracker持有；spool message/entry的postFlush列表保持为空，也不能间接挂上会递减tracker成功计数的回调。取消/fatal/Close仅走无成功回调的资源Release；不调用语义不匹配的Discard来确认未应用数据。PostEnqueue只用于背压，不转成成功确认。 +2. enqueue前按包含metadata/编码开销的保守字节上界拆分entry；单行仍超限就拒绝，不进入oversized旁路。若缺少一致大小估算接口，仅补spool小型计量/包装，不改collector；“接受后发现oversized再失败”不能算预防超额。 +3. 正常MERGE成功时由worker显式完成tracker并释放分片；重入/迟到Release不得再次完成源事件。分别在raw前、raw后MERGE前注入fatal/Close，源flush次数必须为0；普通压力仍可提前Flush恢复，而非频繁fatal。 + +## 5. DDL：迁移现有epoch，而不是新建DDL协调系统 + +沿用 `ApplySchema`、`SchemaChanges`、`SchemaEpoch/SaveSchemaEpoch`、reconcile和TruncateCDCTable已有语义： + +```text +TiCDC现有屏障排空旧DML + → 读取目标epoch；已应用的同DDL跳过 + → Flush该表 + → 保存pending epoch + → 翻译/执行已支持DDL或TRUNCATE reset + → 保存applied epoch + → DDL PostFlush，再放新DML +``` + +固定TiCDC的SchemaFile.Build明确使用event.FinishedTs填TableVersion，因而迁移字段映射直接固定： + +| 字段 | native取值/用途 | +|---|---| +| raw event_pos | DMLEvent.CommitTs | +| 实际DDL epoch | DDLEvent.FinishedTs | +| TableInfoVersion/更新时间 | 解释列元数据，不自动成为applied fence | + +还必须迁移旧DatabendMergeSink.writeBatch的外层保护，不只复制CDCManager：写前读epoch；CommitTs<=epoch的旧事件仅在其排空/DDL恢复前提已成立时按旧规则跳过；pending时新数据禁止写入，先reconcile到applied。启动加载schema只读/核验，不能凭空SaveSchemaEpoch标成已处理历史。epoch==pending也不等于DDL已经执行，实际DDL仍按原恢复分支处理。 + +TRUNCATE继续清理旧raw/stream并清目标，applied前不得写新代际。测试执行后checkpoint未保存时重放，不重复清空新数据。若旧epoch记录不足以识别特定中断，只补必要字段/检查，不默认增加before/after全量快照、独立raw_generation和多份journal。 + +普通增量自动建表仍使用权威事件元数据；类型映射/有限列DDL按bend-cdc现有支持范围。warning跳过但未执行的DDL保留明确限制，不能当镜像一致性保证。复杂/无证明操作可停止，不为追平“任意DDL”扩大首版。 + +## 6. 外部改动边界与非目标 + +| 范围 | 当前处理 | +|---|---| +| 新Databend包/测试 | 主要工作量：提取迁移核心、native adapter、buffer/spool、worker/callback | +| scheme/type/config/factory/metrics | 常规sink注册小改动 | +| raw-change注册/重注册 | 首版沿用默认false；复用并测试现有传播，不新增强制true分支 | +| TableSchemaStore/recovery | 增加必要类型/恢复分支；不伪装MySQL，不改进度算法 | +| manager fatal关闭 | 仅验证不足时的有限小补丁 | +| collector/dynstream/TableProgress/调度核心 | 不做通用协议重构 | + +不实现Dumpling/CSV/COPY INTO、全量编排、历史空表发现/预建；不实现新的全库schema snapshot服务；不要求跨表事务原子性、分布式预算、目标分布式锁或通用自动HA系统。 + +原有单目标writer、无外部写入的前提必须在TiCDC部署中成立，不能因为来源都是TiCDC就自动假定。across-nodes=false只禁止拆表分布；旧owner在途SQL不是客户端cancel就结束。首版在受控单capture/目标独占下做迁移与恢复实验，不为新增sink顺带重做HA。无法确认旧SQL结束时停写，这不是已交付生产HA。 + +远端raw增长按现有compaction和监控告警处理，可加本地阈值停新raw；不叫严格跨capture quota，不新增容量账本。历史PK新增后删除仍会保留墓碑，这项成本按旧链同样负载比较，不因它存在就要求另建watermark存储系统。 + +## 7. 迁移验收：证明不退化，另测接入差异 + +以相同输入和相同能力前提运行旧Canal→bend-cdc核心与新native→迁移核心，比较最终目标状态、错误与DDL处理。不是只看“两个都返回nil”，也不把旧实现的已知错误当金标准。 + +| ID | 迁移测试 | 核查重点 | +|---|---|---| +| T1 | 两路径规范化结果，数字label ENUM、SET高位、全字节binary、decimal/JSON/NULL/时间 | 规范化恰一次；Source/CommitTs不改丢；键、payload、动作、TSO等价 | +| T2 | 默认拆分、非聚簇PK单行变键、UK更新、合法同TSO键复用、同微秒时间、跨分片/重启 | 旧键消失新键唯一；保留源DELETE/INSERT顺序；检查SQL时间参数及raw落库精度，墓碑阻止旧值复活 | +| T3 | 首版raw source、低流量/no-op；后续stream强制部分消费 | 目标真正覆盖记账工作才确认；stream不得只凭pending/mergedGen自证排空，未消费B不得回调B | +| T4 | raw子SQL失败/UNKNOWN、MERGE首响应/提交响应丢失、callback前重启 | 最内层UNKNOWN前置截断，检查请求次数不重发；不提前确认，终态排他后重放收敛 | +| T5 | FinishedTs映射、外层epoch过滤、pending阻新DML、TRUNCATE中断/重放 | bootstrap不写applied fence；旧schema不污染新结构，旧TRUNCATE不清新数据 | +| T6 | OPEN/分片回收、spool超限、raw前/raw后MERGE前fatal/Close、reset、压力 | 源flush不得由Discard/清理触发，未应用为0；无oversized旁路，正常压力不变异常，旧生命周期结束 | +| T7 | 同吞吐/行宽/历史键/热点/延迟SLO比较旧链和新sink | 记录扫描及full-raw重复更新保留键造成的目标重写、CPU/RSS/IO/credit与总费用；不预先承诺比例 | + +T1–T7是完整迁移验收门槛：当前已有其中的本地单测/HTTP模拟覆盖，真实TiDB→Databend、原生/Canal差分及成本实验仍待执行。完成它们所需的最小Databend局部修正是范围内工作；把它们替换成新调度/交付/事务协议不是默认答案。 + +## 8. 本轮已执行与未执行 + +源码静态核对覆盖：DatabendMergeSink输入/PK/配置、CDCManager raw/Flush/mergeTable/SQL/compaction、MergeProgressReporter/applied watermark、ApplySchema/schema epoch/TRUNCATE、现有相关单测与TiCDC回调接入。 + +设计阶段v0.4在bend-cdc执行（v0.4.1评审时未重复运行，也尚未实现新sink）： + +```text +go test ./pkg/sink/databend ./pkg/source/tidb -count=1 +ok databendcloud/bend-cdc/pkg/sink/databend 20.922s +ok databendcloud/bend-cdc/pkg/source/tidb 8.784s +``` + +未配置BENDCDC_TEST_DSN/TIDB_DDL_INTEGRATION_DSN，相应真实warehouse测试跳过。这证明既有本地单测/模拟测试通过，不证明stream全量消费、driver终态或新sink恢复已实测。虽然source包包含全量工具单测，运行它们不意味着迁移那些工具。 + +v0.4.1新增静态核对：CloudStorage raw-change默认false/manager默认值、normalizeCDCEvent的Canal专属解码/getTiDBSequence、spool Discard/oversized分支、raw时间格式与单调分配、SchemaFile.Build的FinishedTs映射、writeBatch外层epoch过滤。在v0.4.1评审时，六项修正仅在设计中明确，未标记为代码已修复；定稿后的首批实现状态见下文。 + +上述记录属于设计阶段。用户批准后已创建实现分支，现已完成sink运行时接入与主程序本地构建，并在2026-10-25通过受控真实TiDB/Lake链路（§8.3）；未部署共享CDC、未做在途真实故障或成本实验，未提交或推送。历史版本不作当前规范。 + +### 8.1 首批实现记录（2026-10-24，历史进度） + +首批代码集中在 `/Users/jiangjianyuan/work/git/pingcap/ticdc/downstreamadapter/sink/databend/`：规范行入口、复合键身份、参数化raw/既有MERGE/compaction SQL、微秒时间分配、提交状态前置截断、SEALED completion及对应单测。错误码位于 `/Users/jiangjianyuan/work/git/pingcap/ticdc/pkg/errors/databend.go`。 + +当时仅为核心实现切片，不是已可创建changefeed的完整sink（后续接入见§8.2）：真实HTTP/driver终态客户端、DMLEvent datum适配、配置/factory注册、spool worker、自动建表/DDL epoch恢复尚待后续接入。当前writer通过执行函数边界要求明确提交结果，不能将测试注入的结果当真实服务端终态协议已完成。完整T1–T7仍待后续集成验收。 + +本批5个生产文件、4个测试文件（15个测试函数），没有修改collector、TableProgress、dynstream或scheduler；未注册不可用的scheme/factory。验证记录： + +```text +GOMAXPROCS=4 go test -p 2 ./downstreamadapter/sink/databend ./pkg/errors -count=1 +PASS（两个包） +GOMAXPROCS=4 go test -p 2 -race ./downstreamadapter/sink/databend ./pkg/errors -count=1 +PASS（两个包） +GOMAXPROCS=4 go vet -p 2 ./downstreamadapter/sink/databend +PASS +``` + +gci/gofumports、当前文档本地链接/代码围栏/空白检查通过。单测覆盖规范值保持、实际raw参数的微秒排序、SQL构造、子INSERT/MERGE未知结果不重发、已提交后统计失败不重试，以及OPEN/SEALED/失败/并发重入回调。SQL仅检查构造和调用顺序，未在真实Databend执行；尚无完整sink运行入口，因此未运行sink集成、真实故障与性能测试，也未执行全仓测试/主程序构建。 + +### 8.2 运行时接入记录（2026-10-25） + +实现分支仍为 `feat/databend-native-sink`，基线未变；没有提交、推送或更改webapi。 + +- 原生接入：`native.go`直接读取TableInfo/typed chunk；ENUM/SET取label、binary保留字节、decimal/大整数/JSON保持精确文本，SQL NULL与JSON null分开。TIMESTAMP按TiCDC配置的源时区转UTC；DATETIME保留wall clock，HTTP会话固定UTC。PK变键的未拆分UPDATE显式拒绝。 +- 运行时：`sink.go`/`lane.go`固定数量lane按逻辑目标表hash，保持单表串行；源事件池按字节/元素有界，Add不阻塞共享handler。正常spool压力提前排空/延迟PostEnqueue，容量异常停止整个旧生命周期；不操作collector/progress条目。 +- 攒批：支持行数/编码字节/定时/DDL屏障/配额压力触发；大事件分片边应用边回收，封口前不flush源事件。复用共享spool；新增的通用接口仅有`AccountingBytes`，用于接纳前阻止oversized旁路。源成功回调始终不放入spool消息。 +- 客户端：直接使用Databend HTTP Query API的`params`与终态，绕开驱动内层POST重试。仅匹配query ID的Succeeded确认mutation；Running首页含结果不算成功。UNKNOWN带请求ID报错、停止lane/整个sink，不自动重发、不根据取消调用返回值宣布旧SQL结束。统计不可用不撤销已确认提交。URI继续请求限制同origin，禁止重定向和错误中泄漏SQL/参数/凭据。 +- SQL：参数化raw INSERT→首版固定full-raw MERGE→已有strict compaction；排序时间按微秒递增，并在重启时读取retained raw的MAX(add_time)，避免本机时钟回退导致同TSO决胜顺序倒退。源列名`action`不再与内部动作列冲突；真实验收后投影改为`get(raw_data, 字面键)`，避免引号列名被错误解释为JSONPath。 +- schema：按权威类型创建目标/raw；复用每表schema epoch，补充一份有界的post-DDL定义/翻译步骤供pending恢复。它不是DML ledger、manifest或新generation协议。pending恢复先于新DML，applied epoch按FinishedTs过滤旧输入；bootstrap不保存applied。TRUNCATE先清raw/target再applied，applied重放不重复清新数据。 +- 接入点：scheme/type/factory、TableSchemaStore类型分支、错误分类和metrics；没有改collector、dynstream、TableProgress、调度或manager算法。Run/DDL错误使用既有unretryable路径结束旧生命周期；真实UNKNOWN故障下的控制面关闭/人工恢复仍待验收。 +- 本地验证包含：原生精确类型、原始UPDATE拒绝、实际raw时间参数、OPEN/SEALED、并发/重入回调、部分raw结果未知、MERGE未知、低流量/transport epoch变化、配额压力、多表排空、Close/接纳竞争、pending列DDL与TRUNCATE恢复、applied重放、远端监控阈值、factory注册。 + +HTTP fixture只模拟协议与journal持久化，不执行SQL，不能证明真实Databend提交原子性。真实入口为`TestDatabendNativeLiveIntegration`，仅在显式配置`DATABEND_NATIVE_TEST_URI`时运行，创建/删除独立随机测试database；运行时接入阶段曾跳过，随后获授权连接staging Lake通过（§8.3）。没有启动或修改生产资源。 + +本轮验证命令与最终结果见§10.4；全量`pkg/common/event`测试曾在TiDB测试依赖中出现`duplicate metrics collector registration attempted` panic。未改无关依赖；缩小到本次改动相关的TableSchemaStore测试后通过。全仓测试未运行。 + +### 8.3 真实 TiUP→CDC→staging Lake 验收（2026-10-25） + +获用户授权，使用其staging Lake和本地TiUP v8.5.1，当前分支构建新架构CDC；只操作随机独立库。修复后从新库完整重跑，9组全部PASS：DML/PK/UK、复合键、特殊标识符、32列类型和值、有限列DDL、TRUNCATE、批处理/spool、受控进程重启、DROP TABLE。 + +- 实测发现并修复DESCRIBE unsigned类型别名不兼容、含双引号列名的JSONPath投影；补回归测试并重跑完整链路。增加`lake://` HTTPS别名以支持用户实际URI。 +- 1200行事务使用64 KiB磁盘spool、极小内存热集，实际41次磁盘读回；证明本负载能边应用边回收，不等于性能/RSS硬上限证明。 +- 排空/暂停并终止旧CDC，停机期间积压DML/DDL，同data-dir重启后从原checkpoint恢复并逐行比对；不是在途UNKNOWN或自动HA实验。 +- 真实核心测试PASS（13.36s）：旧UPSERT不复活删除键、applied过滤和重复TRUNCATE。6包race、sink lint、主程序构建通过。 +- 源/目标测试库、feed按具体名字确认已清理，本次本地CDC/TiUP已停止；未提交、推送或部署共享集群。 + +详见[真实验收报告](./databend-native-sink-e2e-report.md)及[可重复测试说明](../../tests/integration_tests/databend_native/README.md)。真实UNKNOWN故障、跨capture/HA、长期墓碑增长及成本/SLO未验收，T1–T7不据此整体关闭。 + +### 8.4 bend-cdc 对照、DDL矩阵与真实响应丢失（2026-10-25） + +用户要求补齐的测试已执行，详见[补充验收报告](./databend-native-sink-parity-fault-report.md)。这里纠正之前过宽的范围表述：TiCDC的`GetPrimaryKeyColumnNames`包含非空唯一键fallback；无显式PK但有NOT NULL UNIQUE的原生DML/变键已实测通过，不需要新增framework协议。 + +固定bend-cdc真实core对照确认：RENAME TABLE和DROP DATABASE不执行目标操作，只跳过/推进epoch，不能算已支持；真正无可用PK/非空UK时，旧版可用显式conflictKey，native尚未迁移；旧core能扁平投影分区元数据及普通DML，native仍拒绝分区dispatcher。多列组合DDL旧版拒绝，native支持的ADD/DROP/RENAME组合已实测。 + +本轮还复现并局部修复INT→VARCHAR(200)被过度拒绝：只开放完整容纳有符号INT32的VARCHAR(n>=11)，不开放同字符类型未知宽度/长度缩小。原failed feed更换本地二进制后从原checkpoint恢复成功；生产改动仅在sink/schema.go。 + +真实Lake故障注入覆盖部分raw已提交/第二条未发、MERGE成功但成功响应被丢弃，以及CREATE/ALTER/TRUNCATE/DROP共19个关键提交边界。客户端未取得证据时无成功callback、旧sink停止；在测试方已独立取得匹配终态的前提下重建manager，恢复journal及重放验证通过。该测试不伪造SQL执行，但不是每个OS指令点SIGKILL、服务端执行中崩溃或自动HA证明。具体最终矩阵与清理证据以补充报告为准。 + +### 8.5 显式conflict-key与跳过DDL（2026-10-25） + +按用户最新要求：增加URI `conflict-key`，对应bend-cdc持久化配置`conflict_key`/运行时`ConflictKey`，不是新建SQL主键。与旧版相同，单列override可以替代无键/单列源键，但不允许覆盖复合键；按可见列元数据正名，NULL身份或未拆分的变键UPDATE拒绝。操作者保证业务键唯一、非空、稳定；真正无索引键的源表还需启用既有force-replicate。DDL journal保存实际选取的键，DML和DDL共用这一配置，不改通用框架。 + +RENAME TABLE/RENAME TABLES与DROP DATABASE在旧DML排空后warning跳过、正常回调，不更改目标DDL或伪造applied epoch。历史目标保留；重命名后的DML按新表名进入新投影，不会搬迁历史数据。旧DML结果未知仍必须停止，不能靠跳过后续DDL绕过。 + +真实TiUP→CDC→staging Lake补测：无键CRUD+ADD COLUMN及单PK override、重命名后进度/后续DML、DROP DATABASE后进度/保留目标/新库表DML，3组均通过。详细证据见[补充报告§8](./databend-native-sink-parity-fault-report.md)。分区保护未放开：lane只约束单进程，跨capture同一逻辑表排他未证明；不是Lake无法存储分区数据。 + +### 8.6 本地 Databend 自动后台 E2E（2026-10-26） + +新增 `make integration_test_databend`，由 [本地 runner](/Users/jiangjianyuan/work/git/pingcap/ticdc/tests/integration_tests/databend_native/local.py) 自动管理独立 Docker Databend（meta/query、fs存储）、TiUP PD/TiKV/TiDB及本分支TiCDC，复用已有真实行值、DDL和故障断言。无需外部Lake凭据、S3或Dumpling。仅修改测试、Make入口及文档，没有修改TiCDC框架或新增生产运行时协议。 + +macOS arm64 + Docker Desktop + TiDB v8.5.1 + Databend `v1.2.925-patch-13` 固定多架构digest实跑：9组全部PASS，包含27个Python场景及3个真实后端Go测试；DDL响应丢失覆盖19个提交边界。磁盘spool观测41次实际读回，排空后的进程重启及后续DML/DDL恢复通过。7个不依赖服务的runner测试通过。各suite对象清理及runner进程/容器清理PASS；不会停止共享Docker daemon。Linux/amd64 CI尚未运行，不能把镜像多架构等同于跨平台实测。 + +旧 `v1.2.852-nightly` 镜像参数绑定预检失败，不列为支持版本，不引入字符串插值降级。新入口还修复了测试固定重启端口、owner早于coordinator就绪、TIME_WAIT导致连续重跑失败、macOS路径别名导致清理误判的问题。受控本地fs后端测试不替代云网关、S3存储、成本/容量或自动HA验收。使用方法及测试边界见 [E2E说明](/Users/jiangjianyuan/work/git/pingcap/ticdc/tests/integration_tests/databend_native/README.md)。 + +## 9. 可复核源码 + +bend-cdc链接按固定commit的本地源码核对；远端reviewer按相同Git对象路径读取。 + +- [Canal事件转换与现有目标配置，L376–559](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go#L376-L559)。 +- [PK/schema与ApplySchema，L93–163](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go#L93-L163)。 +- [raw写入与代数记账，L640–840](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L640-L840)；[MergeProgress，L399–416](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L399-L416)。 +- [Flush排空，L503–578](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L503-L578)；[MERGE/代数推进/清理，L1230–1416](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L1230-L1416)。 +- [现有MERGE SQL，L1628–1705](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L1628-L1705)；[stream创建及fallback，L889–942](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L889-L942)。 +- [applied watermark与MergeProgress接口](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/applied_watermark.go)；[schema epoch](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/schema_epoch.go)；[TRUNCATE](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/truncate.go)。 +- [schema队列的flush/警告范围](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/schema_queue.go)；[Flush错误传播测试](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/flush_permanent_test.go)。 +- [TiCDC Sink接口](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/sink.go)；[原生DML回调](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/dml_event.go#L660-L711)。 +- [raw-change与sink Run/error/close接入](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go)；[原生DDL字段](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/ddl_event.go)。 +- [CloudStorage raw-change默认值，L774–784](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/config/sink.go#L774-L784)；[scanner拆分与延后INSERT，L970–1014](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/eventservice/event_scanner.go#L970-L1014)。 +- [规范化入口，L14–39](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/tidb_values.go#L14-L39)；[外层epoch保护，L215–235](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go#L215-L235)。 +- [spool oversized接纳，L503–510](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/sink/spool/spool.go#L503-L510)；[Discard执行postFlush，L734–746](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/sink/spool/spool.go#L734-L746)。 +- [raw微秒格式化，L995–1003](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L995-L1003)；[纳秒单调时间，L1755–1763](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L1755-L1763)；[SchemaFile使用FinishedTs，L208–215](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/cloudstorage/schema_file.go#L208-L215)。 + + +## 10. 首版使用方式与实际支持边界 + +### 10.1 配置入口 + +所有Databend专属参数放在sink URI，不引入一套通用框架配置。`lake://`与`databend://`均为同一原生HTTPS sink（已通过真实Lake验收），不是MySQL协议。HTTPS： + +```text +databend://DB_USER:TOKEN@HOST/DB_NAME?exclusive=true&warehouse=WAREHOUSE&workers=4&batch-rows=2048&batch-bytes=1048576&flush-interval=1s&spool-bytes=10737418240&memory-ratio=0.02&source-bytes=268435456&source-events=128&max-entries=4096&query-timeout=2m +``` + +`DB_USER`/`TOKEN`必须URL编码；不应把实际密码粘贴到review或日志。`databend+http://DB_USER:TOKEN@HOST:PORT?...`仅用于明确允许明文的隔离环境。留空database路径时使用源schema名；指定database时同名源表不能汇入同一目标表，不实现任意表路由。 + +| 参数 | 默认 | 意义 | +|---|---|---| +| exclusive | 无，必须为true | 操作者确认目标静态独占；不是锁或自动HA证明 | +| conflict-key | 未指定 | 可选feed级单列override，替代无/单列源键；不能覆盖复合键，必须唯一、非NULL且稳定，不能在原目标上随意更换 | +| workers | 4，最大64 | 固定lane数；表hash到lane，单表串行 | +| batch-rows / batch-bytes | 2048 / 1 MiB | 攒批/分片阈值；单行编码超限立即停止，不无界直通 | +| flush-interval | 1s | 低流量定时排空 | +| spool-bytes | 10 GiB | 共享本地磁盘预算；已释放但物理未回收仍由spool计费 | +| memory-ratio | 0.02 | spool内存热数据额度占磁盘预算比例；独立于source池 | +| source-bytes / source-events | 256 MiB / 128 | 整个sink已接纳原生事件池；共享chunk保守按每个event完整计费 | +| max-entries | 4096 | 每lane待应用分片/缓存表元数据上限，达到分片上限主动排空 | +| spool-dir | TiCDC data-dir下 | spool根目录；按capture/feed隔离，不是恢复日志 | +| query-timeout | 2m | 单SQL及分页总超时；到期不代表服务端已停止 | +| warehouse | 未指定 | HTTP warehouse头 | +| remote-bytes-limit | 未开启 | 基于system.tables中目标database raw压缩字节测量的停写阈值；不是跨capture硬额度 | + +源chunk在进入sink之前已经分配，source池、编码、读回、HTTP和元数据分别有界不等于全进程RSS严格上限。batch/spool参数必须联合满足最小额度，未知/重复参数拒绝。普通压力用背压和提前排空吸收；事件池无法接管属于明确fatal容量异常，不伪造成功回调。 + +### 10.2 已实现的首版能力 + +- 具备TiCDC可选冲突键（显式PRIMARY KEY或全列NOT NULL的UNIQUE键），或显式配置conflict-key的普通非分区表；INSERT/UPDATE/DELETE，PK/UK变化沿用raw-change=false的scanner拆分顺序。 +- 增量自动建表，复合键与保留墓碑，参数化批量raw INSERT、full-raw MERGE、strict历史清理。 +- 主要TiDB标量/文本/binary/ENUM/SET/bit/decimal/JSON/date/time/datetime/timestamp映射，生成列只投影CDC可见的STORED值，不生成目标表达式。 +- CREATE DATABASE、CREATE TABLE/RECOVER、DROP TABLE、TRUNCATE；有限ADD/DROP/RENAME/CHANGE/MODIFY列，保守类型widening、字面默认值与nullability；二级索引DDL不创建Databend索引但推进已应用schema fence。 +- 实际列DDL遇到缺失目标时，可依据该事件定义创建空投影；不扫描或预建没有事件的历史空表。 +- pending schema步骤重入恢复、applied DDL重放跳过、旧DML fence过滤；无成功回调的fatal/Close清理。 + +### 10.3 必须保留的限制 + +- 受控单capture、目标静态独占；旧SQL终态无法确认时人工排除后才能resume。`exclusive=true`是运行前提声明，不自动检测/获得下游租约。跨capture自动故障接管没有验收，不承诺生产HA。 +- 首版拒绝分区dispatcher、跨节点拆表、syncpoint、active-active、redo、low-latency模式、列选择、任意路由和table级事务原子性。分区调度可能使同逻辑目标出现多writer，因此在证明隔离前不静默接纳。 +- 拒绝删除/重命名冲突键、修改复合冲突键、不安全type conversion、除INT32→VARCHAR(n>=11)之外的参数化字符MODIFY/CHANGE及表达式默认值DDL；复杂分区等不做隐式有损翻译。按用户最新裁决，表重命名（含多表）及DROP DATABASE只warning后跳过，目标对象不变、进度继续；不宣称镜像执行了这些DDL。其他unsupported操作仍fail-closed。 +- 不支持Dumpling/CSV/COPY INTO或全量编排；首次创建空目标不补历史存量。用户必须已独立准备一致的起始状态与start-ts。 +- 不创建stream、DML ledger、manifest/witness或新owner协议。retained raw最新墓碑不TTL删除；不同历史PK持续churn仍会增长。 +- HTTP端点必须实际支持Query API参数绑定与终态。Verify以绑定参数SELECT做预检；不会自动降级到字符串插值。本轮staging端点的参数绑定、终态、支持DDL/类型与排空重启已实测；不同端点/版本仍需预检，不推广到所有版本/许可或在途故障恢复。 +- 当前journal是新sink专属的`epoch/applied/plan`形状，不自动接管正在运行的bend-cdc目标或其旧两列journal。迁移必须停旧链并独立验证起点；不绕过DDL恢复证据。 + +### 10.4 验证与发布状态 + +代码已接入factory并可构建；用户随后提供staging Lake，真实核心及TiUP全链路通过（§8.3），不再仅有HTTP fixture。用户随后要求提交并推送当前实现分支，提交/远端状态以该分支Git记录为准；下文实验记录描述测试当时的dirty工作区。未建PR、未部署共享CDC。成本/SLO与在途故障/自动HA未验收，T1–T7不能标为全部关闭。 + + +2026-10-25 本地最终验证记录（最后一批代码）： + +```text +GOMAXPROCS=4 go test -p 2 -race ./downstreamadapter/sink/databend ./downstreamadapter/sink ./pkg/sink/spool ./pkg/errors ./pkg/config ./downstreamadapter/sink/metrics -count=1 -timeout=180s +PASS：6个包 +GOMAXPROCS=4 go test -p 2 -race ./pkg/common/event -run 'TestTableSchemaStore|TestNeedTableNameStore' -count=1 +PASS +GOMAXPROCS=4 go test -p 2 ./pkg/metrics -run TestDownstreamTypeFromSinkURI -count=1 +PASS +GOMAXPROCS=4 tools/bin/golangci-lint run --timeout 5m ./downstreamadapter/sink/databend/... ./downstreamadapter/sink/metrics/... +0 issues +GOMAXPROCS=4 go vet -p 2 ./downstreamadapter/sink/databend ./downstreamadapter/sink ./pkg/sink/spool ./pkg/common/event ./pkg/errors ./downstreamadapter/sink/metrics +PASS +GOMAXPROCS=4 go test -p 2 ./downstreamadapter/sink/databend -run TestDatabendNativeLiveIntegration -v -count=1 +PASS：显式配置用户授权staging URI，13.36s；独立随机库自动清理（URI不写命令/报告） +GOMAXPROCS=4 go build -p 2 -o /tmp/ticdc-databend-native ./cmd/cdc +PASS，临时产物不进入仓库；version子命令可执行，未注入发布版本元数据 +``` + +新增Databend包为11个生产文件、11个测试文件、54个测试函数（包含3个显式opt-in live测试）；另有错误定义和sink metrics文件。factory/类型/schema store/配置scheme/错误分类/metrics/spool计量等现有文件仅有限接入改动。HTTP fixture仍仅是进程内协议测试；真实SQL/TiDB链路另见§8.3，生产故障恢复不据此宣称通过。 diff --git a/docs/design/databend-native-sink-e2e-report.md b/docs/design/databend-native-sink-e2e-report.md new file mode 100644 index 0000000000..a8c00dd116 --- /dev/null +++ b/docs/design/databend-native-sink-e2e-report.md @@ -0,0 +1,131 @@ +# TiCDC 原生 Lake Sink：本地 TiUP → staging Lake 实测报告 + +日期：2026-10-25（Asia/Shanghai）。状态：**本轮受控端到端验收通过；未提交、未推送、未部署到共享 CDC 集群。** + +后续追加的bend-cdc对照、DDL默认值/nullability矩阵和真实响应丢失实验见[补充报告](./databend-native-sink-parity-fault-report.md)。下述9组是首轮记录，不替代后续实验；“无PK不支持”应精确为“无可用PK/非空唯一键且依赖显式conflictKey的表”。 + +## 1. 结论与证据边界 + +使用本地 TiUP TiDB/PD/TiKV、当前分支新架构 CDC，以及用户授权的真实 staging Lake，修复兼容性问题后，**从全新随机库重新执行的 9 组测试全部通过**。不是仅运行 mock，也没有使用 S3/Canal JSON/bend-cdc 代写。 + +- DML、有限列 DDL、TRUNCATE、DROP TABLE、32 个目标列的类型/值比对通过。 +- 1,200 行事务超过 64 KiB 磁盘 spool 配额，仍能边应用边回收；实际发生 41 次 spool 磁盘读回。 +- 排空、暂停、终止旧 CDC 后，制造源端 DML/DDL 积压，重启同一 data-dir 并从原 checkpoint 恢复，最终数据一致。 +- 独立真实 Lake 核心测试验证旧 UPSERT 不复活已删除键、applied fence 与重复 TRUNCATE。 +- **不代表**任意 DDL/列类型均已支持、在途 UNKNOWN 请求故障接管已通过、多 capture HA 已通过，或成本/吞吐已达到生产要求。 + +当前设计与限制见 [定稿设计 §10](./databend-native-sink-design-review.md)。可重复运行的测试见 [runner](../../tests/integration_tests/databend_native/run.py) 和 [操作说明](../../tests/integration_tests/databend_native/README.md)。 + +## 2. 实际环境与版本 + +| 组件 | 本次使用 | +|---|---| +| 源 TiDB / PD / TiKV | TiUP playground v8.5.1，各 1 个；TiDB `SELECT VERSION()` 返回 `8.0.11-TiDB-v8.5.1` | +| CDC 源码 | 分支 `feat/databend-native-sink`；基线 `2bf6dacd0aec9ac2fad464600e47341f9aa951f2` 加当前未提交实现 | +| CDC 运行 | 当前工作区构建，`--newarch --tz UTC`；单 capture，独占本次目标 | +| CDC 版本标签 | 本地注入 `v8.5.1` / `2bf6dacd-dirty-e2e`，仅用于源版本兼容检查，**不是官方 v8.5.1 发布包** | +| Lake | `tnww1vflk.gw.aliyun-ap-northeast-1.default.staging.lake.tidbcloud.com:443`，warehouse `default`，HTTPS Query API | +| Lake 实际版本 | `Lake Query v1.2.931-nightly-0ca7e6af79(rust-1.94.0-nightly-2026-08-02T22:24:51.279542089Z)` | +| 测试库 | `native_e2e_20261024_171016_bbf3e64f`（源、目标相同的独立随机库，测试后删除） | +| Changefeed | `native-e2e-20261024-171016-bbf3e64f`（测试后删除） | + +用户原 URI 的 database 被替换为本次随机库；没有修改 `default` 内现有表。凭据仅从仓库外 0600 文件读取，不记录到本文或测试代码。 + +本地 TiUP SQL/PD/TiKV/CDC API 分别为 `127.0.0.1:14000/12379/30160/18300`。本机空间不足触发 TiDB 快速 ADD INDEX 的磁盘保护，故仅对本次独立 TiUP 关闭 `tidb_ddl_enable_fast_reorg` 和 `tidb_enable_dist_task` 后运行索引 DDL。这是**源端测试环境调整**，不属于 sink 修复，不能据此更改共享源配置。 + +## 3. 最终全新运行：9 / 9 PASS + +耗时是该组 wall time，含远端 SQL 和等待收敛，不是吞吐基准。 + +| 测试组 | 具体断言 | 结果 / 秒 | +|---|---|---| +| DML | INSERT/UPDATE/DELETE；固定 PK 改二级 UK；非聚簇 PK 变键；同事务 DELETE+重插后最终键集合一致 | PASS / 16.571 | +| 复合键 | 包含分隔符/转义字符的复合主键不碰撞，源目标逐键一致 | PASS / 10.622 | +| 特殊标识符 | 表/列包含双引号；源列 `action`、`__ticdc_operation` 不覆盖内部动作列 | PASS / 10.246 | +| 类型/值 | 32 个投影列，初始、NULL、极值及更新后比对；VIRTUAL 不投影 | PASS / 12.801 | +| 列 DDL | ADD NOT NULL 字面默认值及旧行回填、RENAME、INT→BIGINT 后写大整数、DROP、TEXT NOT NULL 隐式空默认；二级 ADD/DROP INDEX | PASS / 42.352 | +| TRUNCATE | 旧行清空、旧 raw 代际清理后仅保留新 DML | PASS / 18.195 | +| 批处理/spool | 单事务 1,200 行×256 字符；随后 UPDATE 一行、DELETE 100 行；分别比对 1,200/1,100 行；低流量尾批排空 | PASS / 40.592 | +| 进程重启 | checkpoint 排空后暂停并终止旧 CDC；停机期间 UPDATE/INSERT/ADD COLUMN；同 data-dir 重启/恢复，完整比对积压结果 | PASS / 22.769 | +| DROP TABLE | 源 DROP 后目标表及对应 raw 表均不存在 | PASS / 12.387 | + +### 类型覆盖与比较方法 + +| 源类型/边界 | 真实目标观察/比较 | +|---|---| +| TINY/SMALL/MEDIUM/INT/BIGINT，含 UNSIGNED | 覆盖有符号正负边界、`BIGINT` 最小值、`UINT64` 最大值；精确十进制文本比较，无 float64 中转 | +| DECIMAL(65,30) | 大正数、大负数、30 位小数和最小小数位更新；精确比较 | +| FLOAT/DOUBLE | 源端以足够精度读回后按浮点容差比对，不能用 MySQL CLI 默认短显示当精确值 | +| CHAR/TEXT/BOOLEAN | Unicode、emoji、引号、反斜线、换行、NULL；BOOLEAN 保留 TiDB TINYINT 表示 | +| BLOB/BINARY/VARBINARY | 0–255 全字节、定长补零、变长及更新；canonical hex 精确比较 | +| BIT(64) | 全 64 位值映射到 unsigned BIGINT，精确比较 | +| ENUM/SET | 数字 label `ENUM('2','x')` 不二次解码；64 项 SET 的 bit63 及多 label 更新 | +| JSON | 超过有符号范围的大整数、数组/布尔；显式 `is_null_value` 区分 JSON null 与 SQL NULL | +| DATETIME(6)/TIMESTAMP(6)/DATE | 微秒与日期保持；本轮源会话/CDC 为 UTC，不据此宣称任意时区组合已实测 | +| TIME(6)/YEAR | 负且超过 24h 的 TIME 精确文本、YEAR 数值 | +| 生成列 | STORED 实际值同步，含 BIGINT 最大值；VIRTUAL 列不在目标 DESCRIBE 中 | + +按主键 canonical 化后比较，不依赖两端数值/VARCHAR 的不同排序规则。最终 Lake `DESCRIBE types_test` 为 **32 列**；此数字是测试表覆盖列数，不是“所有 MySQL 类型的数量”。 + +### spool 与恢复的实际证据 + +测试配置：`workers=1`、`batch-rows=64`、`batch-bytes=32768`、`flush-interval=200ms`、`spool-bytes=65536`、`memory-ratio=0.001`。真实 `ticdc_databend_spool_load_bytes_count` 累计 **41**，不是只根据阈值推测使用了磁盘。该测试证明本负载可在小配额下推进，不构成全进程 RSS/物理磁盘硬封顶或性能报告。 + +重启时旧 PID `41562` 已终止，新 PID `41917`；正数源事务读 TSO 为 `469988010726522887`，恢复后 checkpoint 为 `469988011722670087`。未覆盖 checkpoint，未跳过积压。**这是受控排空重启，不是在 MERGE/TRUNCATE 结果未知时直接启动新 owner。** + +## 4. 本轮发现并修复的真实兼容问题 + +### 4.1 DESCRIBE 的 unsigned 类型别名 + +Lake 返回 `TINYINT/SMALLINT/INT/BIGINT UNSIGNED`，原校验只认内部 UINT8/16/32/64，导致合法目标被安全拒绝。`canonicalType` 统一别名及 Nullable 包装，不放宽 Decimal 精度检查；增加回归测试。真实极值类型测试已重跑通过。 + +### 4.2 双引号列名不是 JSONPath + +原 SQL 的 `s.raw_data:"a""b"` 虽经过 SQL 标识符转义，仍触发真实 Lake `1006 InvalidKeyPath`。改为字面对象键 `get(s.raw_data, 'a"b')`，统一 SQL 字面量转义，并测试点号、引号、反斜线。没有替换 raw/MERGE 主算法，也没有引入 stream/manifest。 + +先核验旧错误请求为终态后，用 checkpoint 和 retained raw 恢复该失败案例，目标正确；随后使用修复后的完整代码，从新库完整重跑，9 组均通过。不是把多个失败运行的局部结果拼成一次全绿。 + +### 4.3 接入用户实际 URI + +增加 `lake://` 作为同一 HTTPS native sink 的 scheme 别名，接通 factory/config/downstream metrics；`exclusive=true` 等首版限制保持不变。用户名/密码继续按 URI URL 编码，不降级明文。 + +测试脚本本身还修正了 reserved identifier、源 FLOAT 读出精度、hex 大小写、JSON null 观察、`/tmp` realpath 和 coordinator 启动等待等环境/fixture 问题，未将其计为数据复制缺陷。 + +## 5. 独立真实核心与本地回归 + +真实 Lake 核心测试 `TestDatabendNativeLiveIntegration`：**PASS，13.36s**（package 15.127s）。另外创建/清理独立随机库，验证 DELETE 后旧 UPSERT 不复活、applied epoch 过滤旧输入、相同 FinishedTs 的 TRUNCATE 重放不清空新数据。它不是 TiUP 源端完整链路的替代证明。 + +最终代码验证： + +```sh +GOMAXPROCS=4 go test -p 2 -race ./downstreamadapter/sink/databend ./downstreamadapter/sink ./pkg/sink/spool ./pkg/errors ./pkg/config ./downstreamadapter/sink/metrics -count=1 -timeout=180s +# PASS，6 个包 +GOMAXPROCS=4 tools/bin/golangci-lint run --timeout 5m ./downstreamadapter/sink/databend/... ./downstreamadapter/sink/metrics/... +# 0 issues +``` + +当前 Databend 包包含 45 个测试函数（含 1 个 opt-in 真实 Lake 测试）。历史本地回归还通过 TableSchemaStore 针对性测试、scheme metrics 测试、go vet 和主程序构建。未运行全仓测试；之前全量 common/event 曾因既有 TiDB 测试依赖重复注册 metrics panic，未改无关依赖，针对相关测试通过。 + +## 6. 清理与证据位置 + +- 最终 run 状态和 cleanup 均为 PASS。 +- 对 4 次运行的**具体自有库名/feed ID**查询确认：Lake 测试库、源测试库、changefeed 均无残留；不是只看 DELETE 返回成功。 +- 本轮启动的 CDC 与 TiUP 已停止；本地 14000/12379/30160/18300 端口均已关闭。 +- 凭据不入 Git；临时 credential 文件在完成清理核验后删除。日志/result 保留在本机私有目录用于复核，不保证跨机器可访问。 + +证据根目录： + +```text +/var/folders/lq/3n41d_w97w55hrx2mzmd42w80000gn/T/ticdc-native-e2e-0t2q351n +``` + +其中 `acceptance-4/result.json` / `acceptance-4.log` 为最终全新运行；`live-core-test-final.log` 为真实核心测试；`cleanup-verification.json` 为逐名字清理核验；`acceptance-3/quoted-recovery.json` 为特殊列名修复后恢复案例。最终测试 runner SHA-256：`81c753f3f38e471f7c6bb17a83d342215ea6352fb4147fbfca2c7056a7f25226`。 + +## 7. 尚未验收，不应扩大结论 + +- 在途 HTTP 断连/首页后失败、服务端仍执行的 UNKNOWN SQL、旧 owner 与新 owner 并发、进程任意中断点恢复：本地 fixture 覆盖部分分支，**本轮没有真实故障注入证明**。 +- 多 capture、自动 HA、跨表事务原子性、分区表、无可用PK/非空唯一键、复杂/不安全 DDL、任意路由、非 UTC 时区矩阵不在本轮验收范围。 +- 长期 retained raw 墓碑增长、多表高并发、成本/扫描量/SLO、长期满盘/IO 故障未做 benchmark/soak。 +- 从既有 bend-cdc journal/目标在线无缝接管、历史全量初始化不在范围；不能把首次自动建空表当存量已同步。 + +因此本轮可以确认“首版支持范围内，真实 TiDB→当前 CDC→该 staging Lake 的功能纵切通过”，不能把 T1–T7 全部风险或生产就绪标为关闭。 diff --git a/docs/design/databend-native-sink-parity-fault-report.md b/docs/design/databend-native-sink-parity-fault-report.md new file mode 100644 index 0000000000..73ac537b89 --- /dev/null +++ b/docs/design/databend-native-sink-parity-fault-report.md @@ -0,0 +1,198 @@ +# bend-cdc 能力对照与 native sink DDL / 故障补充验收 + +日期：2026-10-25。结论：**已完成真实 Lake 能力对照、11 组真实源端矩阵、2 类 DML 故障及 19 个 DDL 提交边界恢复测试；发现并修复一个局部迁移退化。仍有明确未迁移能力，不宣称全部替代 bend-cdc。** + +**后续更新**：下述对照矩阵是上一轮状态；按用户新要求，`conflict-key`现已实现，RENAME/DROP DATABASE改为跳过不阻塞。最新实现/真实测试见§8；分区保护仍保留。 + +## 1. 先纠正能力判断 + +固定基线为 bend-cdc `b6481621021e82aee659ad88e8e6c00f6dd11906`，不是另一个 HEAD。旧库保持干净，通过 Go overlay 添加临时测试，不改其源码。Native 为 TiCDC `2bf6dacd0aec9ac2fad464600e47341f9aa951f2` 加当前分支未提交实现；不改 collector/dispatcher/progress/调度框架。 + +| 场景 | bend-cdc 固定版实际证据 | native 当前结果 | 判定 | +|---|---|---|---| +| 分区表普通 DML | 真实 core 接受分区 CREATE 元数据，创建扁平目标，Canal 行 INSERT/Flush 后值存在 | 真实源 CREATE 分区表触发 unretryable，未建新目标 | **仍有迁移缺口**；旧 core 验证不等于已测完整 S3 分区文件/分区 DDL | +| 无显式 PK，但有 NOT NULL UNIQUE | native 与旧路径可使用 TiCDC 提供的隐式冲突键，不等于真正无键 | 真实 TiDB→CDC→Lake 的 INSERT、唯一键变更、DELETE 全通过，未开 force_replicate | **已支持**；之前笼统说“无 PK 都不支持”不准确 | +| 无可用 PK/非空唯一键 | 无 pkNames 且无 override 时拒绝；设置 `conflictKey=id` 后真实 ApplySchema、WriteBatch、Flush 通过 | force_replicate 后实际 INSERT，native 拒绝且 checkpoint 停止 | **仍缺显式 conflictKey override**；旧版依赖操作者保证所选键有效，不是任意无键表都安全 | +| RENAME TABLE | 真实旧 ApplySchema 返回成功，但旧表仍在、新表不存在；代码按 unsupported 跳过并推进 epoch | 拒绝，旧目标保留 | **旧版没有实际重命名支持**，不应把 nil error 当已支持 | +| DROP DATABASE | 旧 SchemaChanges 判 unsupported；给旧 ApplySchema 带表元数据探针后仍保留目标库/表 | 真实源 DROP DATABASE 后 native 拒绝，目标库表保留 | **旧版没有实际 DROP DATABASE 投影**；旧探针不是完整 S3 DROP DATABASE 测试 | +| 默认值/nullability | 真实旧 MODIFY 默认值、NOT NULL→NULL 通过 | 真实源多种组合通过，见§2 | 对应支持场景已补证据 | +| INT→VARCHAR(200) | 真实旧 core 通过，原值转文本保留 | 修复前真实 failed；局部修复后通过且原 feed 重放恢复 | **已修复迁移退化** | +| 多操作列 DDL | 旧 parser 对两列 ADD 等组合明确报 multi-operation 错误 | ADD+DROP+RENAME、CHANGE 完整定义等受测组合通过 | 不属于旧版全支持、新版遗漏；仍不承诺任意组合 | +| 缩窄/表达式默认值 | 旧版有明确限制，不是任意 DDL 都执行 | 本轮真实 source 探针验证 fail-closed | 拒绝测试通过≠功能支持 | + +非空唯一键的源码依据是 [GetPrimaryKey 隐式键选择](../../pkg/common/table_info_helper.go#L593-L637)，native 复用该 helper，**不是本轮新增框架机制**。旧 unsupported 分类及处理见 [SchemaChanges](../../../webapi/bend-cdc/pkg/source/tidb/schema_ddl.go) 和 [ApplySchema](../../../webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go)。 + +## 2. 修复后全新库：11 / 11 PASS + +真实环境：本地 TiUP TiDB/PD/TiKV v8.5.1、当前分支新架构单 capture CDC、用户授权 staging Lake。TiUP 仅用本轮独立实例；源为 UTC。CDC 注入的 `v8.5.1` 是本地兼容标签，不是官方 CDC 发布包；最终 git_hash 标签 `2bf6dacd-dirty-extended-e2e`。 + +最终随机库 `native_e2e_20261025_012759_8da770af`，结果及 cleanup 均 PASS。下面时间包括网络与收敛等待，不是性能指标。 + +| 测试 | 秒 / 结果 | +|---|---| +| 无显式 PK 的 NOT NULL UNIQUE 表，包含键变更/删除 | 11.240 / PASS | +| INT→VARCHAR(200)、INT 最小值保留、后续 `'0012'` 文本及默认值 | 16.797 / PASS | +| 同类型 MODIFY 字面默认值，已有行不被回填改写 | 8.122 / PASS | +| NOT NULL→NULL，显式 DEFAULT NULL | 7.830 / PASS | +| 清除源 NULL 后 NULL→NOT NULL，后续默认值写入 | 9.606 / PASS | +| MODIFY 完整定义省略 NOT NULL，使目标恢复可空 | 9.225 / PASS | +| MODIFY 省略 DEFAULT，删除旧默认值 | 8.022 / PASS | +| TINYINT→INT→BIGINT，VARCHAR→TEXT | 12.184 / PASS | +| ADD Decimal/文本/NULL 字面默认值，旧行回填及转义字符串 | 16.237 / PASS | +| CHANGE 重命名及完整默认值/nullability 定义 | 8.363 / PASS | +| 同一 DDL 的 ADD+DROP+RENAME，随后 DML | 9.161 / PASS | + +不只比行数：按键比较实际值，并用真实目标 DESCRIBE 检查默认值和可空性。`VARCHAR(200)` 只存在于源定义，目标按既有 mapper 使用不定长 VARCHAR。 + +另有 **6 个独立真实拒绝探针**通过：分区表、真正无键表（显式 force_replicate 并写行）、表重命名、DROP DATABASE、INT→TINYINT 缩窄、CURRENT_TIMESTAMP 表达式默认值。断言为稳定 failed、checkpoint 不再推进、旧目标值未变/新目标未创建；不能将这些 PASS 计为支持了相应操作。 + +初始无键探针若不 force_replicate 会在上游被 eligibility 过滤，feed 可保持 normal，不能当 sink 拒绝证据。初始误把 NOT NULL UNIQUE 当“不支持无 PK”也导致测试预期失败；读取真实目标、追踪 helper 后已改为正向测试。保留这些原始结果，未把失败记录改成通过。 + +## 3. 两类真实 DML 故障：PASS + +[实现与复现入口](../../downstreamadapter/sink/databend/live_fault_test.go)。SQL 都由真实 Lake 执行，不使用 HTTP fixture 伪造数据库成功。 + +### 3.1 部分 raw INSERT 成功 + +四行批次拆为两条 INSERT：第一条真实提交两行，第二条在发送前注入传输失败。 + +- 观察真实 raw=2、target=0，applyResult 未应用。 +- writer 未重发未知请求、未继续 MERGE（匹配 raw POST 总数为2，第二条未送服务器）。 +- 测试已知旧第一条成功、第二条未发，替换所有本地 manager/writer 状态，再完整重放四行。 +- 最终真实 target=4、strict compaction 后 raw=4。 + +这是“部分子 SQL 成功”，不是声称同一条 INSERT 内部半提交。 + +### 3.2 MERGE 已提交，但成功响应丢失 + +委托真实 HTTP transport 执行，独立取得**相同 query_id 的 Succeeded**后，在 sink 看到该响应之前丢弃它。 + +- 真实 target 已有两行，但旧 sink 因 UNKNOWN 终止，源 flush callback=0。 +- MERGE POST 只有1次，无内层/外层暗中重发。 +- 测试方已独立取得旧 query 的终态且关闭旧 sink 后，重建 manager/writer、重放同一输入。 +- 最终 target=2，compaction 后 raw=2，没有重复目标行。 + +原任意 UNKNOWN 请求在实际运维中仍必须取得终态及目标排他后才能恢复；测试夹具掌握的证据不等于新建了自动终态发现/HA 服务。 + +## 4. DDL 真实提交边界：19 / 19 PASS + +每个断点均先在真实服务器提交该语句，再丢弃客户端响应；经实际 `Sink.WriteBlockEvent` 路径观察错误、Run 停止、DDL callback=0。随后在旧请求终态已知的前提下，用新 schemaManager 只读取真实 Lake 的 journal/表状态恢复，不继承旧内存状态。 + +| DDL | 丢失响应的关键提交边界 | 数量 | +|---|---|---| +| CREATE | pending journal、目标建表、raw建表、applied journal | 4 | +| ALTER | pending、ADD、RENAME、MODIFY、raw确保存在、applied | 6 | +| TRUNCATE | pending、DROP raw、TRUNCATE target、重建raw、applied | 5 | +| DROP | pending、DROP target、DROP raw、applied | 4 | + +每例均验证恢复至 epoch=150/applied=true；非 DROP 场景再写 ts=160 的新数据并核对完整值，重放相同 DDL 不清新数据，ts=120 旧 DML 被 fence 跳过；DROP 后 target/raw 均不存在。ALTER 同时核对旧行 `label='old',extra=NULL` 和新行 `label='new',extra=7`，不只核对行数。 + +最终命令带 `-race`，2个父测试及21个故障子场景通过，package **199.112s**。19条匹配 query_id 的真实终态证据保留在 `fault-summary.json` 和原始日志。 + +**边界**：这是客户端真实响应丢失、旧 sink 生命周期终止和本地 manager 状态重建，不是服务端执行中崩溃、OS 任意指令点 SIGKILL、跨 capture 故障接管。没有声称“所有 DDL 的所有可能中断点”已覆盖，也没有对旧 S3/bend-cdc 整条链做同样故障注入。 + +## 5. 本轮修复与真实升级恢复 + +生产行为仅修改 [schema.go](../../downstreamadapter/sink/databend/schema.go) 的参数化字符转换判定：此前将所有参数化 VARCHAR MODIFY/CHANGE 一律拒绝，误挡住 bend-cdc 的 INT→VARCHAR(200)。现在仅放行 **INT32→VARCHAR(n>=11)**（足够完整容纳包括符号的 INT32 十进制域）；同字符类型未知原宽度、VARCHAR(10)、CHAR(200)、BIGINT→VARCHAR 等仍按保守规则拒绝。增加边界单测，不扩展通用 casting/框架协议。 + +真实复现先保留 failed feed，再停止旧 CDC、换本轮构建、同 data-dir 重启，从**原 checkpoint**手工 resume，不覆盖时间戳： + +- 原 PID `67290` → 新 PID `68998`。 +- checkpoint `469995752722530332` → `469995795137953793`。 +- 原行 `-2147483648` 转文本保留；积压新行 `'0012'` 不丢前导零。 +- 原错误 DDL 重放恢复成功，独立记录 `upgrade-recovery.json`；随后另起新库完整跑§2矩阵。 + +## 6. 验证、清理和可重复运行 + +新增/扩展测试入口： + +- [extended.py](../../tests/integration_tests/databend_native/extended.py)：源端矩阵与拒绝探针。 +- [baseline.py](../../tests/integration_tests/databend_native/baseline.py) / [固定版 Go overlay](../../tests/integration_tests/databend_native/baseline_parity_test.go.txt)：旧 core 对照。 +- [live_fault_test.go](../../downstreamadapter/sink/databend/live_fault_test.go)：真实 Query API 故障注入。 +- [README](../../tests/integration_tests/databend_native/README.md)：私有凭据、启动与命令。 + +固定 bend-cdc 的7个真实对照子测试及原有 IndexWarning/TRUNCATE 集成测试通过（package **38.659s**）；选定的原有分类/默认值/冲突键单测也通过。旧 checkout 未修改。 + +最终修改后，6包 race、sink lint（0 issues）、go vet、当前分支主程序构建通过。Databend包现49个测试函数，其中3个 opt-in live 父测试。全仓测试、生产部署、成本/长期负载没有执行。 + +本轮按**具体自有名字**确认16个源命名空间、26个远端命名空间（含baseline/fault随机库）均无残留，对应changefeed均已删除；没有操作原 `default` 中的已有表。自有本地CDC/TiUP停止，临时凭据清除；没有提交、推送或建PR。 + +私有证据根目录（仅本机可访问）： + +```text +/var/folders/lq/3n41d_w97w55hrx2mzmd42w80000gn/T/ticdc-native-extended-il8v40pb +``` + +关键记录：`ddl-matrix-fixed-final/result.json`、`baseline-fixed-final/bend-live-parity.log`、`live-fault-final.log`、`fault-summary.json`、`upgrade-recovery.json`、`cleanup-verification.json`。首轮9组DML/32列/spool/排空重启仍见[首轮报告](./databend-native-sink-e2e-report.md)，与本轮不同运行分开记账。 + +## 7. 当前仍未闭合的事项 + +1. **分区普通 DML**与**显式 conflictKey override**仍是具体迁移差异，不能写成已覆盖；本轮没有用跳过错误或删除安全检查冒充支持。 +2. 任意复杂/不安全DDL、所有类型/时区组合不是11组测试能够穷举；RENAME和DROP DATABASE也不能因旧版“成功返回”就宣称正确支持。 +3. 服务端仍在执行时的 UNKNOWN 终态取得、自动HA、长期资源与性能成本继续保留未验收状态。 + +所以本轮关闭的是具体功能/恢复测试缺口及一个局部转换退化，不是宣称 bend-cdc 全功能、全部故障或生产就绪已验收。 + +## 8. 最新用户要求：conflict-key 配置、RENAME/DROP DATABASE 跳过 + +本节取代§1/§7中相应的“尚未迁移/拒绝”状态；不是改写上一轮实验记录。分区保护仍未开放。 + +### 8.1 配置及语义 + +bend-cdc 在 `DatabendConfig` 中保存单列 `conflict_key`，TiDB wrapper 的 `ConflictKey` 将其交给 raw identity / MERGE 作为业务键；**没有给源表自动添加 PRIMARY KEY 或唯一索引**。旧 `resolveConflictKeys` 允许覆盖无键或单列源键，拒绝覆盖复合源键。 + +新 sink 已按该规则增加 URI 参数：在原 URI 查询串追加： + +```text +&exclusive=true&conflict-key=id +``` + +`exclusive=true`只应出现一次，已有时只追加`conflict-key`。真正没有可用源索引键的表，还需既有TiCDC选项： + +```toml +force-replicate = true +``` + +API对应 `replica_config.force_replicate=true`。无需改TiCDC通用配置结构。 + +规则： + +- 未配置时仍自动选择显式 PK / NOT NULL UNIQUE；配置后是feed级**单列override**，列名按可见元数据大小写归一。 +- 不能覆盖复合源键；缺失/不可见列、NULL身份拒绝。此配置适合独立feed，不能随意把复合键表混入同一override任务。 +- 业务方保证值唯一、非NULL且稳定。重复业务键会按同键更新处理，不会自动获得源端唯一约束。 +- 任意未索引逻辑键不一定触发TiCDC拆分；未拆分UPDATE改变该键时native拒绝，不静默留下旧行。本轮没有新造通用变键协议。 +- DML与DDL journal均使用相同的实际选取键；不要在原target/raw上直接更换键，须独立重新初始化。 + +生产改动集中于 sink 包的 `config.go`、`native.go`、`lane.go`、`schema.go`；不改collector、dispatcher、progress或调度算法。 + +### 8.2 跳过语义 + +`RENAME TABLE`（含多表类型）和 `DROP DATABASE` 在旧DML成功排空后 warning 并正常完成DDL回调；不发目标DDL，不创建虚假的applied epoch。不再因这些指定操作停任务；其他不安全DDL仍拒绝。 + +重要影响：目标库/旧表/旧数据保留；**重命名不会搬迁历史行**。后续DML使用新表名，按需创建新投影。DROP DATABASE不删除目标库；源库重建后的新表仍能同步。同名库表重建不会自动清理旧目标数据，需要空目标时应独立重新初始化。这是用户要求的跳过策略，不是完整DDL镜像。 + +如果前序DML的MERGE结果未知，屏障仍会失败并停止；不能以“跳过后续DDL”绕过旧写入的安全问题。 + +### 8.3 真实验收：3组PASS + +使用本地TiUP v8.5.1、当前新架构CDC及同一授权staging Lake；CDC本地标签`2bf6dacd-dirty-options-e2e`，不是官方发布包。入口为[options.py](../../tests/integration_tests/databend_native/options.py)。 + +| 场景 | 结果 | +|---|---| +| 无任何PK/UK的表：force-replicate + `conflict-key=ID`，INSERT/UPDATE/DELETE/ADD COLUMN，journal键为`id`；另测覆盖不同的单列源PK | PASS / 32.927s | +| 真实RENAME后checkpoint跨过DDL、旧目标数据保留；对新表名继续UPDATE/INSERT，新投影值正确且旧表不变 | PASS / 16.873s | +| 真实DROP DATABASE后checkpoint继续、目标库和数据保留；重建源库、新建表的DML继续成功 | PASS / 16.143s | + +单测/race同时验证：配置格式、大小写、空值、NULL身份、复合键拒绝、未拆分变键拒绝、配置贯穿DML/journal、跳过三种DDL无目标SQL/无虚构epoch、回调顺序、后续DML推进，以及旧UNKNOWN写不能被跳过掩盖。4包race、lint（0 issues）、go vet、主程序构建通过；全仓测试/HA/成本未跑。 + +私有证据根目录： + +```text +/var/folders/lq/3n41d_w97w55hrx2mzmd42w80000gn/T/ticdc-native-options-gdj9c83o +``` + +确切路径由本机`/tmp/ticdc-native-options-current`指针记录；`manual-key-1/result.json`、`skip-ddl-1/result.json`和`cdc.log`分别保存结果及warning。两个run的status/cleanup均PASS；最终按具体名字核验源/目标库及feed清理，自有CDC/TiUP停止，临时凭据移除。未提交、推送或部署共享CDC。 + +### 8.4 为什么仍拒绝分区dispatcher + +限制来自首版主动保护，不是Lake不能写分区数据。一个capture内，同一逻辑目标已经hash到同一lane串行；但不同物理分区可由不同capture承担，多个进程可能同时MERGE同一target。`enable_table_across_nodes=false`并不等于所有物理分区都在同一capture，`exclusive=true`也不是跨capture锁。当前仍拒绝分区输入，避免把“单进程串行”误当跨进程排他；这轮没有顺手删除该保护或引入框架锁协议。 diff --git a/docs/design/databend-native-sink-plan.md b/docs/design/databend-native-sink-plan.md new file mode 100644 index 0000000000..60dc756710 --- /dev/null +++ b/docs/design/databend-native-sink-plan.md @@ -0,0 +1,278 @@ +# TiCDC 原生 Databend / Lake Warehouse Sink：代码分析与迁移计划 + +日期：2026-10-24。状态：历史分析摘要；当前设计已评审定稿，首版实现及受控真实链路验收通过，未提交、未部署共享CDC。 + +当前规范见 [定稿设计 v1.1](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review.md)，配套 [实现评审入口](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-review-prompt.md)。本文件保留为初始分析历史摘要，不是当前实施规范；唯一主路径与范围以v1.0为准。实现分支为 `feat/databend-native-sink`；v0.2/v0.3已另存历史版本，不同时实施。 + +v0.4.1局部修正：首版raw-change=false保持默认拆分和源顺序，首版固定raw source；Native只规范化一次；UNKNOWN在核心内层截断；spool清理不挂成功回调且不进oversized旁路；add_time按落库精度单调；DDL epoch明确取FinishedTs并迁移外层过滤。上述为已接受的范围；首批核心、运行时接入与待验收状态见定稿§8.1–8.2/§10，没有扩大框架范围。 + +最新裁决：对齐bend-cdc既有正确性，迁移raw/stream或raw source/MERGE、mapper和schema epoch核心，只新增native adapter、spool与TiCDC callback适配。删除manifest/K-H-S新算法、DML ledger与新raw_generation，不因review无限增设框架协议。保留MERGE后确认、SEALED/分片独立回收和必要失败关闭；stream全批覆盖、终态、原生事件与旧路径差异要局部验证。当前有T1–T6的部分本地覆盖及[9组真实TiUP/Lake功能验收](./databend-native-sink-e2e-report.md)，在途故障/HA与成本仍未验收，T1–T7不整体关闭；无全量工具工作包。 + +需求修订（2026-10-24）：第一版必须采用新架构 S3 sink 类似的 **内存缓冲 + 有配额本地磁盘 spool + 背压**，不再采用仅内存攒批的初版建议。磁盘 spool 是临时缓冲，不是恢复日志；确认点仍是 MERGE 成功。 + +## 1. 结论与边界 + +可行，建议新增独立的 Databend sink，直接消费 TiCDC 的原生 DML/DDL 事件，迁移 bend-cdc 的 **schema/type mapping + raw staging + stream + MERGE + ordering/tombstone + schema epoch**，不搬运它的对象存储消费者。 + +本文的 lake 指 bend-cdc 当前实际连接的 Databend SQL warehouse，不是泛化的 Iceberg/Delta/Hudi 文件 sink。如果目标接口不同，需要另做 writer/事务语义设计。 + +现状： + +```text +TiDB → TiCDC → Canal JSON 编码 → S3/OSS + → bend-cdc LIST/GET/SQS → Canal JSON 解码 + → schema/类型转换 → Databend raw table → stream → MERGE → target +``` + +建议： + +```text +TiDB → TiCDC DMLEvent / DDLEvent + → Databend sink [原生类型适配 + schema barrier + 有界批处理] + → 有界内存 / 限额本地磁盘 spool（超限背压） + → Databend 持久化 raw table → append-only stream → MERGE → target + ↓ + 成功后触发 PostFlush +``` + +去掉的是增量链路中的 Canal JSON envelope、文件生产/扫描/下载、独立 bend-cdc consumer，以及相应 CPU、内存和网络开销。**不去掉** Databend 自身存储、raw 写入、MERGE、必要的重放水位，也不承诺完全没有 JSON：第一版仍可用 raw_data JSON 承载行内容。 + +代码分析基线：webapi `b6481621021e82aee659ad88e8e6c00f6dd11906`，ticdc `2bf6dacd0aec9ac2fad464600e47341f9aa951f2`，调查开始时两边 working tree 干净。均为本地源码证据，不代表生产版本。 + +## 2. 现有实现真正要迁移的内容 + +### 2.1 路径与职责 + +| 模块 | 已确认行为 | 迁移处理 | +|---|---|---| +| `service/tidb_runner.go` | TiDB 专用 runner，明确绕过通用 channel pipeline，保存对象游标 | 不迁 runner；让 TiCDC 管生命周期/checkpoint | +| `source/tidb/canal_json.go` | 读取 pkNames/mysqlType/_tidb.commitTs，文件分类与 schema version | 不迁解码；从 DMLEvent/TableInfo 取信息 | +| `source/tidb/manager.go`、`schema_queue.go` | 表级队列、存储进度、schema 对象排在新 epoch DML 前 | 改为 TiCDC dispatcher + sink barrier | +| `source/tidb/databend_merge_sink.go` | 每目标表一个 DatabendSink；PK 推导、库表映射、schema epoch、事件转换 | 迁移语义，重写入口及资源管理 | +| `sink/databend/databend.go` | 连接、建表、DESCRIBE cache、类型推导、加列 | 迁移；用权威 TableInfo 替代首行猜类型 | +| `sink/databend/databend_cdc.go` | raw 批量参数化 INSERT、append-only stream、MERGE、去重、重试、清理 | 核心迁移；重写调度/确认/所有权 | +| `sink/databend/schema_change.go`、`schema_epoch.go`、`truncate.go` | 列变更白名单，pending/applied epoch，清空 target/raw/stream | 迁移并接入 TiCDC DDL 恢复流程 | +| `sink/databend/tidb_values.go`、`enum_set.go` | Canal 字符串格式的 binary/ENUM/SET/JSON 还原 | 迁移语义和测试，不照搬 Canal 专用表示 | +| `source/tidb/applied_watermark.go` | 区分对象消费进度与 MERGE 可见进度 | 保留“可见才确认”的契约，不迁对象轮询算法 | +| `source/tidb/dumpling*.go` | 全量 schema、CSV、COPY INTO、快照元数据 | 独立保留，不能用增量 sink 自动替代 | + +**不要选错入口**：`source/tidb/databend_sink.go` 是简单 INSERT/DELETE 路径（DELETE 固定取 id），实际 TiDB runner 创建的是 `DatabendMergeSink`。只迁前者会丢失现有核心能力。 + +### 2.2 “临时表”实际上是持久化状态 + +当前建的是普通 `_cdc_raw`，不是 TEMP TABLE。包含 `id/table_name/raw_data/add_time/event_pos/action`；另外创建 `
_cdc_stream`，使用 `APPEND_ONLY = true`。 + +`buildMergeSQL` 使用 ROW_NUMBER 按冲突键选最新事件；严格顺序模式还查 retained raw history，拒绝被更大 event_pos 覆盖的旧事件。DELETE 同样保留最新 raw 记录作为墓碑。`buildRawCompactionSQL` 删除旧版本而保留每键最新水位，避免旧事件重放后复活已删除数据。 + +所以迁移目标应是 **持久化 staging + 安全 compaction**，不能在 MERGE 后直接清空全部 raw，不能只按 24h TTL 删除最新墓碑。保留每键水位意味着空间通常随历史键集合增长，删除过的键也会占空间；它不是固定大小缓存。 + +## 3. TiCDC 已有接入能力，以及不能漏掉的改动 + +`downstreamadapter/sink/sink.go` 已有统一接口： + +- `AddDMLEvent` 接受含 TableInfo、CommitTs、Row/PreRow 的原生事件。 +- `FlushDMLBeforeBlock` 是 DDL/syncpoint 前的 flush/barrier 钩子。 +- `WriteBlockEvent` 成功后需要调用事件 `PostFlush`。 +- `Run/Close/IsNormal` 管理生命周期及错误。 +- `SetWriteGate` 要求在真实下游写边界再次检查写许可。 +- `AddCheckpointTs` 接收已经计算出的 changefeed checkpoint，不是写入完成确认接口，不能反过来用它触发必须先完成的 MERGE，形成进度依赖环。 + +建议新目录： + +```text +downstreamadapter/sink/databend/ # 事件适配、队列、barrier、callback、生命周期 +pkg/sink/databend/ # 客户端、配置、schema、类型、raw、merge、恢复 +tests/integration_tests/databend_sink/ # 真实 TiDB/TiCDC/Databend 故障与正确性测试 +``` + +配置建议 `databend://...` + 独立 `DatabendConfig`；不是 MySQL scheme，也不是 Canal JSON protocol。下面是设计项，当前均未实现： + +1. sink `New/Verify` 分支、scheme 常量、独立 `DatabendSinkType`。 +2. `pkg/config/sink.go`、changefeed config 的默认值、校验、克隆/序列化/更新、`RmUnusedFields`，避免新增字段被清掉。 +3. `pkg/common/event/table_schema_store.go` 新 sink 的需求分支；现有默认分支会 panic。DB/All 级 DDL 所需表集合要明确维护。 +4. dispatcher/dispatchermanager 中 sink-type 分支、恢复、routing、调度兼容性；不能只改 factory。 +5. 不能返回 `MysqlSinkType` 偷用流程：manager 和 task 中会将 sink 强转成 `*mysql.Sink` 调 `GetTableRecoveryInfo`。 +6. 将必要恢复能力抽成窄接口，或新增 Databend 专属恢复分支；不借此重构所有 sink。 +7. URI/DSN/token 脱敏、TLS、token 刷新、权限预检、metrics、示例配置、mock/集成测试。 + +生产实现不建议直接 import 整个 bend-cdc module：会耦合多源捕获、应用配置和日志体系，还会影响 TiDB parser/驱动/Arrow 依赖选择。先做有来源记录的最小代码迁移，保留适用版权与 NOTICE；只有确有两边持续共享需求时再抽公共库。 + +## 4. 第一版必须确定的正确性契约 + +### 4.1 完成确认：MERGE 成功后再推进 checkpoint + +当前 bend-cdc `Write()` 返回只代表 raw INSERT 成功；MERGE 是异步的。TiCDC 不能把它直接当作“目标已写完”。 + +第一版建议： + +1. 原生事件转换成自有、不可变记录，按目标表/epoch 进入内存/磁盘 spool，保存事务 flush callback；内存与磁盘预算均显式受限，不能无限接收。 +2. 数据转交后的 enqueue 与最终 flush 分开；可参考 storage sink `DetachPostCallbacks` 的做法，不能提前释放仍被 worker 引用的 chunk。 +3. 达到行数、字节数或 flush interval 任一上限就批写 raw；低流量表也必须被定时 flush。 +4. 同一目标表只运行一个 MERGE；明确它覆盖了哪些已提交 raw 批次。 +5. MERGE 确认提交后，释放其覆盖的 spool entry，并只完成已覆盖事件的 callback，每个事务 callback 恰好一次；拆批事务必须等全部片段完成。不能因本地落盘或 raw INSERT 成功提前调用 PostFlush。 +6. MERGE 已提交但 callback/上游 checkpoint 未保存：允许重放，依靠稳定身份和版本去重收敛。 +7. MERGE 失败：不确认；临时错误有限重试，永久错误由 `Run` 返回并使 changefeed 可见地失败。 + +目标保证是 **at-least-once 传输 + 幂等最终状态**，不是未经证明的 exactly-once，也不默认保证跨表事务原子可见。 + +raw 落盘即确认也可以设计，但那需要独立持久化的待 MERGE 任务、重启扫描、接管协议及另一套 applied watermark；不建议第一版为此引入额外复杂度。bend-cdc 的 pendingEvents/writeGen/mergedGen 是内存状态，不能直接充当接管后的恢复日志。 + +### 4.2 事件身份与顺序 + +- 直接使用 `DMLEvent.CommitTs`，不用毫秒时间或本机 add_time 定义业务顺序。 +- **commit TSO 是事务版本，不是每行唯一 ID**。同一事务可以更新多行，现有 Canal 代码“每行唯一”的注释不能成为新实现假设。 +- 用源集群/命名空间、changefeed、逻辑表身份/建表代际、schema epoch、规范化复合键定义隔离域；事件版本另保存 commitTs，必要的同事务子操作顺序必须来自稳定的归约规则，不能使用重启后改变的 dispatcher seq/本机时钟。 +- 同一键、同一版本重放必须内容一致；不一致要检测并报错，不能通过新的 add_time 随机覆盖。 +- PK 值变更要处理 before/after 身份：旧键 tombstone + 新键 upsert。现有 CDCManager 的 UPDATE 只使用 After，不能直接宣称覆盖主键值更新。还要验证 TiCDC 入口是否已拆成 DELETE/INSERT，覆盖同事务换键/交换键的净结果。 +- 第一版仅接受已验证的非空单列/复合主键，不把 TiCDC handle key 一概当作用户主键；无 PK、nullable unique key、force-replicate 表拒绝或单独设计 append-only 模式。 +- 字符串键需测试大小写、collation、尾空格与 Unicode；复合键编码避免分隔符歧义。目标库表映射必须检测多源碰撞。 + +### 4.3 多节点与接管 + +bend-cdc 是单任务管理目标表，map/mutex 只保护进程内；TiCDC 每节点都有 sink，不能原样复制后同时消费同一个 stream、MERGE 同一目标表。 + +第一版限制为：**每个逻辑目标表一个明确 owner**，禁用 table-across-nodes，拒绝多 changefeed 写同一目标表，并验证调度/迁移时确实满足限制。分区表可能存在多个 physical-table dispatcher 汇入同一个逻辑表，禁用 table-across-nodes 不自动解决这个问题；未实现 owner 汇聚前第一版拒绝分区表。 + +接管必须包括停旧 writer、取消/等待其在途 SQL、fence、恢复 pending DDL、识别未完成 raw、继续 MERGE；验证 SQL 结果不确定时的重放。`SetWriteGate` 要覆盖 raw INSERT、MERGE、建表/ALTER、epoch 写、cleanup，不只是 AddDMLEvent 入口。capture lease 只是必要条件，还要证明调度转移期间旧请求不会晚于新 owner 落库；不能声称进程 mutex 或入口 gate 就提供分布式 fencing。 + +更大规模阶段再选择：逻辑表单 owner 汇聚多个 dispatcher，或分 writer staging + 单 merge owner。不能让每个节点自行操作共享 stream cursor。 + +### 4.4 DDL 与恢复 + +继续采用：旧 epoch DML 全部 MERGE → durable pending DDL journal → ALTER/TRUNCATE → applied journal → DDL PostFlush → 新 epoch DML。 + +需要复用 TiCDC 的 block 协调：本地 `FlushDMLBeforeBlock` 完成本节点相关队列屏障，配合 maintainer 对受影响 dispatcher 的协调，不把一个节点的 flush 当成整个 changefeed 的全局屏障。 + +仅保存 applied epoch 不够：还应保存可恢复的 DDL 标识/类型、目标身份、schema fingerprint/操作内容与 pending 状态。重启从 TiCDC checkpoint 较早位置重放时，必须识别已执行 DDL、跳过已覆盖的旧 schema DML，并禁止旧 DML 用新表结构解释。当前只有 MySQL-class 有显式 `GetTableRecoveryInfo`，这是新 sink 的正式开发项，不是自动继承能力。 + +支持策略： + +| DDL | 第一版建议 | +|---|---| +| 建库/建表 | 支持，依据权威 TableInfo，保留列顺序/精度/NULL 属性 | +| ADD/DROP/RENAME COLUMN | 非键列、明确默认值语义下支持，保留 epoch 恢复 | +| MODIFY COLUMN | 沿用明确的安全转换白名单，不做任意 CAST | +| 二级索引 DDL | 明确 no-op，并推进必要 schema 状态 | +| TRUNCATE | 支持前需完成 table ID 代际、raw/stream 重置和中断恢复测试 | +| PK 定义变更、复杂多操作 ALTER | 第一版拒绝 | +| RENAME/DROP TABLE/SCHEMA、分区 DDL | 明确阻塞/拒绝，第二阶段扩展，不静默忽略 | + +现有 bend-cdc 对某些不支持 DDL 会告警并推进 epoch，新 sink 不应无条件照搬；只有已证明不影响数据投影/身份的操作才可跳过。DDLEvent 的 `NotSync` 与过滤语义也要遵守。 + +**既有空表建表是独立问题**：新 changefeed 不保证为所有历史表重放 CREATE TABLE。首个 DML 的 TableInfo 可用于懒建表，但空表永远没有首行；需从 start-ts 对应 schema snapshot 获取元数据做 bootstrap，或由全量初始化任务预建表。不能把 schema store 里的名称清单当成完整列定义。 + +## 5. 类型改写：语义保留，表示重写 + +以 TableInfo/ColumnInfo 的 FieldType、flags、Flen/Decimal、Elems 和源时区为权威；同一套 mapper 服务建表、DML 编码及 DDL，消除 bend-cdc 中多个字符串映射函数的漂移。 + +| 源类型 | 目标与编码原则 | +|---|---| +| signed/unsigned integer | 精确宽度映射；BIGINT UNSIGNED → UINT64，禁止 float64 中转 | +| DECIMAL(p,s) | 精确保留 p/s；不沿用缺 metadata 时 DECIMAL(76,30) 的妥协，更不能转 DOUBLE | +| BIT(n) | UINT64，验证 BIT(64) 最大值 | +| BINARY/VARBINARY/BLOB | 目标 binary；直接从 datum 原始 bytes 转 HEX/二进制参数，禁止套 Canal 的 Latin-1 字符串还原 | +| ENUM/SET | 目标字符串，依据 typed datum/Elems 还原；不能把原生 label 再当 Canal 数字解析 | +| JSON | VARIANT/JSON；保留数值文本精度，区分 SQL NULL 与 JSON null；只解析一次 | +| DATE/DATETIME/TIMESTAMP | 明确时间范围、时区、精度和零日期策略；DATETIME 不应意外做时区平移 | +| TIME/YEAR | 延续 TIME 字符串、YEAR 小整数语义，验证负 TIME 和超 24h 值 | +| CHAR/VARCHAR/TEXT | 转 string；测试字符集、大小写、引号/反斜杠/控制字符 | +| generated column | 沿用 TiCDC 可见列策略：VIRTUAL 不写，STORED 作为物化值写入，不复制 TiDB 生成表达式到 Databend | +| 其他/不支持类型 | 明确报错或显式配置映射，不静默推成字符串 | + +首版可继续使用 parameterized INSERT 写 raw_data，再用受控 projection 转目标类型。后续评估 typed staging 或驱动批量上传路径；不为追求“完全零 JSON”阻塞正确性迁移。 + +官方 [MERGE 文档](https://docs.databend.com/sql/sql-commands/dml/dml-merge) 确认支持基于 source/subquery 的 INSERT/UPDATE/DELETE。目标 warehouse 的 stream、事务、DDL、session sticky state、结果不确定恢复与性能，仍须用实际版本验证；公开语法支持不能替代真实兼容性测试。 + +## 6. 性能与资源设计 + +1. 不沿用“每张表一个连接池 + 一组后台循环”：建议每 changefeed/目标 endpoint 共享有界连接池和 worker pool，表状态按需维护，设置空闲回收。 +2. 同时配置 max batch rows、max bytes、flush interval、max in-flight bytes 和 merge concurrency。按目标表串行、多表有限并行，避免大量小表各自频繁小 MERGE。 +3. backpressure 要贯通到 TiCDC dispatcher；不能无限保存未确认事件，也不能用无限 goroutine 批触发 MERGE。 +4. raw INSERT、MERGE、compaction 分别计时/计费；cleanup 失败不能把已成功业务写入误报为未提交并导致无界重试。compaction 不能删除尚未安全应用的更新或最新墓碑。 +5. 所有后台任务使用 sink 生命周期和超时；不把现有 context.Background() merge 循环直接带入 TiCDC，避免 Close/失租后继续写。 +6. 监控区分 accepted、raw durable、target applied、checkpoint，以及 raw backlog/bytes、MERGE SQL 时间、重试、schema 阻塞、写许可阻塞。吞吐不能只统计进入队列的行数。 + +### 6.1 第一版必需：有界内存 + 限额磁盘 spool + +正常在内存中攒批；达到内存预算后，新增记录溢写本地 segment 文件。按批大小、等待时间或 DDL barrier 刷到 Databend raw，再执行 MERGE。第一版将 spool entry 保留至对应 MERGE 确认提交后释放，使未完成工作受同一套预算约束。 + +采用基线 `2bf6dacd0` 的 `spool.TryEnqueue`、segment 管理和 flush marker 思路，但不能直接照搬全部默认策略：该实现有“单 entry 超过磁盘配额时在内存接受并立即刷出”的特殊路径。新 sink 要对超大记录/事务单独定义限制，不能声称复用后内存天然存在严格硬上限。 + +必须具备的空间与背压约束： + +- **单 changefeed 配额 + capture 共享总预算**：多张表、多 writer 共享所属配额；所有 changefeed 还受节点总预算约束，不能每表/每 worker 各自分配一份上限后将磁盘累加写满。 +- **磁盘实际占用计费**：包括 segment 文件头、未回收空间和在途写入预留；删除/截断失败时继续计费，不能只扣逻辑消息大小就宣称空间已释放。 +- **最低剩余空间保护**：写入前同时检查 spool 配额及文件系统可用空间，为日志、其他 TiCDC 数据及其他进程留余量；多个 spool 实例原子预留共享预算。其他进程仍可能抢占空间，因此 ENOSPC/IO 错误必须可诊断、停止接收且不提前确认。 +- **达到限制立即背压**：先调度已有 raw flush/MERGE，释放空间后再接收；不删除未完成数据腾空间，不无限重试落盘，也不通过无界内存队列绕过配额。 +- **防止调度死锁**：MERGE、回收、DDL barrier 等控制路径不与等待配额的生产者互相占用唯一 worker/锁；下游不可用时允许进度停住，恢复后应能自动排空积压。 +- **读回同样有界**：spool 回读、编码缓冲和 SQL batch 都有预算，不能一次将所有磁盘数据加载回内存。源事件/chunk 的生命周期及队列元数据也必须计入内存设计，不能把 spool payload 配额当成进程 RSS 上限。 +- **超大事务/单行**：事务可以按预算拆分持久化/写入，回调等所有片段 MERGE 后完成；单行超过支持的最大大小时明确报错，不无限加内存或磁盘预算。 +- **重启恢复**：spool 不作为 WAL。重启/节点迁移仍由已保存 checkpoint 重放，依赖 raw/MERGE 幂等；清理 spool 只能操作当前实例拥有的目录，不能删除其他活跃实例数据。 +- **监控与验收**：增加 spool 内存/实际磁盘字节、共享预算使用率、可用空间、配额等待、segment 回收失败；覆盖多 changefeed、磁盘满、回收失败、超大行、慢 MERGE、失租、重启和背压解除测试。 + +配置接口建议（待实现，不是当前可运行配置；容量仅为示例，需按节点磁盘和 changefeed 数量定额）： + +```toml +[sink.databend-config] +spool-memory-quota = 67108864 # 64 MiB，单 changefeed 热数据预算 +spool-disk-quota = 10737418240 # 10 GiB，单 changefeed 磁盘上限 +``` + +节点级还必须有跨 changefeed 的 spool 总预算和最低剩余空间阈值；这两项属于 capture/server 级配置，不应复制成每表配置。具体字段名及与 TiCDC 现有 quota 的整合方式在实现前确定。上述预算不限制 Databend 端 raw 的历史水位/墓碑容量，后者仍按安全 compaction 单独管理。 + +成本比较固定同样输入/目标延迟/SLO：原链 TiCDC 编码成本 + 存储请求与容量 + consumer LIST/GET/解码/调度成本，对比新链新增的 TiCDC writer 成本；另单独计算 Databend raw/MERGE/compaction 与网络费用。直接 sink 会失去对象存储的长时间缓冲，warehouse 故障将更快反馈到 TiCDC lag 和上游保留窗口，需约定最长可恢复停机时间。 + +不提前承诺节省百分比；如果 Databend MERGE 计算是主成本，链路缩短不一定等比例降低总费用。 + +## 7. 全量初始化、上线与回退 + +增量 sink 不负责导出现存 TiDB 全量。保留 Dumpling + COPY INTO 作为独立初始化任务：选一致快照 S → 从 S 建 changefeed 并确保增量可保留 → 按 S 的 schema/数据导入 → 校验 → 放行大于 S 的增量。实施前验证 start-ts 边界和 GC/日志保留；全量期间不能直接让增量覆盖后又被旧快照写回。 + +切换建议使用 **独立影子目标库表**：旧链继续写原目标，新 sink 写影子目标；对齐快照/TSO 后核对 count/hash/关键业务查询及 schema，再切读并停旧链。禁止两个链路同时消费同一 stream、共用 raw 或写同一目标表。 + +回退需预留旧链原始数据和消费进度，在连续覆盖窗口内追平到已验证水位再切读，不能只替换二进制。新 sink 目标的 target/raw/stream/epoch/owner 状态按一组管理；不自动复用旧 bend-cdc 的内部表，也不自动转换其文件游标。 + +## 8. 分阶段工作包与验收 + +| 阶段 | 工作与产物 | 退出标准 | +|---|---|---| +| P0：语义验证 | 锁定 warehouse/驱动版本；验证 raw/stream/MERGE、重复批次、提交结果不确定、sticky session、fencing/接管、DDL 中断恢复 | 核心恢复模型明确；无法证明时先不扩展功能 | +| P1：增量纵切 PoC | 独立 scheme/type/config；原生事件适配；PK/复合 PK；内存/磁盘 spool；节点共享配额与背压；raw/MERGE；MERGE 后 ack；单 owner | insert/update/delete、重放、低流量 flush、超大事务、磁盘满/回收失败/多 changefeed 配额、cancel/Close、失租不丢数据 | +| P2：核心能力对齐 | 权威建表/空表 bootstrap；类型矩阵；列 DDL/epoch 恢复；TRUNCATE；库表映射；错误与监控 | 达到现有已支持核心行为，补齐 PK 值更新及故障案例；不支持项可诊断地拒绝 | +| P3：灰度与优化 | 快照接续、影子流量、全链故障注入、成本基准、多表并发；再评估分区/跨节点 owner 汇聚及 typed staging | 正确性与目标 SLO 达标，成本实测下降，可演练回退 | + +粗估：一名熟悉两端代码的工程师，受限 PoC 约 1–2 周；DDL/恢复/完整类型和灰度再预留数周。该估算不是交付承诺，跨节点/fencing 和真实 warehouse 兼容性可能成为关键路径，应在 P0 后重新排期。 + +必须覆盖的验收用例： + +- 复合键、NULL/极值、大整数与 DECIMAL、二进制全字节、ENUM/SET、JSON 大数、时区和生成列。 +- 同毫秒不同事务；同事务多行；同键旧版本延迟；DELETE 后旧 INSERT 重放；PK 值更新和换键。 +- raw INSERT 前后、MERGE 提交前后、callback 前后强杀；响应丢失；重启/调度接管；重复数据不复活/不回退。 +- pending/applied DDL 各中断点，rename/drop/re-add column、TRUNCATE 代际、不带后续 DML 的 DDL、历史空表。 +- 多节点失租/恢复、旧节点在途请求、目标撞名、分区/跨节点不支持时的创建拒绝。 +- raw cleanup 与 MERGE 并发、墓碑保留、warehouse 长时间不可用、超限背压、token 刷新、权限变化。 +- 内存溢写磁盘、节点总预算、最低剩余空间、ENOSPC、segment 回收失败、超大 entry 限制、配额等待时 MERGE 仍能推进并释放空间。 +- 快照 S 接续与影子切换，不把 source checkpoint lag 当作 target applied lag。 + +## 9. 本次实际验证与证据索引 + +已执行：`go test ./pkg/sink/databend ./pkg/source/tidb -count=1`（在 bend-cdc module），两包通过。未设置 BENDCDC_TEST_DSN/TIDB_DDL_INTEGRATION_DSN，真实 Databend 集成测试按现有条件跳过，因此这些结果只验证现有本地单元/模拟测试，不证明原生 sink 已可运行。 + +未改生产代码，未跑新 sink E2E、故障注入、成本 benchmark 或部署。TiCDC 尚未新增 sink，且本机默认 Go 1.25.5 与该 repo go.mod 要求 1.26.8 不同；本次不构建 TiCDC。 + +主要源码证据（绝对路径便于本机复核）： + +- [实际 TiDB runner，L66–138](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/service/tidb_runner.go#L66-L138) +- [schema epoch 与写入适配，L93–260](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go#L93-L260) +- [复合主键与有序模式，L305–474](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go#L305-L474) +- [raw 写入与异步调度，L642–855](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L642-L855) +- [持久化 raw/stream 与参数化写入，L888–1016](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L888-L1016) +- [墓碑 compaction，L1188–1203](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L1188-L1203) +- [严格顺序 MERGE，L1625–1704](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L1625-L1704) +- [现有列 DDL 白名单，L80–188](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/schema_ddl.go#L80-L188) +- [TiCDC sink 接口和工厂，L36–105](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/sink.go#L36-L105) +- [原生 DML 元数据，L375–430](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/dml_event.go#L375-L430) +- [DML flush/enqueue callback，L660–705](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/dml_event.go#L660-L705) +- [TableSchemaStore sink-type 分支，L65–109](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/table_schema_store.go#L65-L109) +- [MySQL 专属恢复分支，L551–570](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go#L551-L570) +- [storage sink callback 解耦参考，L51–68](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/cloudstorage/task.go#L51-L68) diff --git a/docs/design/databend-native-sink-review-prompt.md b/docs/design/databend-native-sink-review-prompt.md new file mode 100644 index 0000000000..67dab2a07e --- /dev/null +++ b/docs/design/databend-native-sink-review-prompt.md @@ -0,0 +1,74 @@ +# 定稿后的实现评审入口：按 bend-cdc 既有语义迁移 + +用户已确认设计评审通过，[定稿v1.1](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review.md)按v0.4.1范围进入 `feat/databend-native-sink` 分支实现。后续优先评审实际代码与该定稿的偏差，不重新扩展通用协议；实现状态见设计§8.1–8.3及§10；[真实验收报告](./databend-native-sink-e2e-report.md)记录本地TiUP→当前CDC→staging Lake的9组PASS、32列及排空重启。单测与受控链路通过不等于在途故障/自动HA验收。 + +最新补充：[bend-cdc能力对照与真实故障验收](./databend-native-sink-parity-fault-report.md)。重点看已验证的迁移差异，不把旧版“warning跳过”算已支持；非空唯一键fallback已验证，参数化字符widening局部修复见源码。 + +最新用户裁决已实现：URI `conflict-key`、指定RENAME/DROP DATABASE warning跳过；真实3组通过，见补充报告§8。此前对应“仍拒绝/未迁移”的结论是历史状态；分区保护未开放。 + +## 基线与范围 + +- 新架构TiCDC：`/Users/jiangjianyuan/work/git/pingcap/ticdc`，固定commit `2bf6dacd0aec9ac2fad464600e47341f9aa951f2`。 +- bend-cdc：`/Users/jiangjianyuan/work/git/pingcap/webapi` 下 `bend-cdc`，固定commit `b6481621021e82aee659ad88e8e6c00f6dd11906`。 +- 设计基线按固定Git对象核验；实现位于上述新分支，以实际分支/工作区代码为准，不切分支或覆盖开发中的改动。 +- 用户要求：只对齐bend-cdc当前TiDB增量路径已正确支持的语义，核心改动集中sink，不为迁移重做TiCDC框架。 +- [v0.3](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review-v0.3.md)、[v0.2](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review-v0.2.md)以及旧review comment都是历史讨论；manifest/K-H-S/AppliedCycle/新raw_generation不是当前要求。 + +## 唯一主线 + +```text +原生事件adapter（不经Canal编解码） + → 有界内存/限额spool与按表worker + → 迁移现有Databend raw写入、Flush/MERGE、compaction + → 目标完成后回调；SEALED分片计数独立回收 +``` + +复用现有raw字段、复合PK、TSO排序、墓碑、stream/raw source、schema epoch和TRUNCATE顺序，不默认增加per-row manifest读回、event_id/payload_hash、DML ledger、分布式锁/预算、collector信用/progress迁移。 + +普通DML无需逐成员回读，但多条raw SQL必须全部成功且Flush覆盖对应工作后才确认。**首版显式固定raw source，不自动创建/消费stream**；stream代码可保留，启用前验证batch limit/no-op与代数记账覆盖，不能循环pending已清零的旧Flush冒充排空。full-raw扫描和大量保留键重复更新成本仍要测。 + +## 本轮六项明确决策 + +1. 首版沿用固定CloudStorage默认拆分模式`raw-change=false`，撤销强制true的接入；保留scanner的DELETE/延后INSERT顺序。非聚簇PK变键要真实验证；若意外收到oldPK!=newPK的原始UPDATE则拒绝，不直接交After-only核心。默认源码不等于实际旧任务配置，后者须另核验。 +2. Canal原始值入口与NativeNormalizer规范行入口分离，汇合后共享raw写入/MERGE；规范化恰一次，不伪造Source绕过解码/丢失TSO。数字label ENUM('2','x')必须测试。 +3. 默认raw source解决首版部分stream消费前提；writeGen/mergedGen仍不是独立覆盖证明，不因“沿用”就跳过T3。 +4. 在客户端/核心**内层重试之前**传播COMMITTED/NOT_COMMITTED/UNKNOWN；UNKNOWN不得被executed分支吞成成功或transient分支重发，外层也不能再重试。 +5. spool成功回调列表为空，源callback只在SEALED tracker；取消/Discard/Release不能间接完成源事件。enqueue前拆分/拒绝oversized，不接受后再声称有界。 +6. add_time先按真实存储精度量化再保证单调；检验SQL参数/落库，而非只测Go内存递增；不把它提升为任意源冲突的业务版本。 + +schema映射固定：raw event_pos=CommitTs，实际DDL epoch=FinishedTs，TableInfoVersion只解释schema。迁移外层writeBatch的旧事件过滤与pending阻新数据；bootstrap不得凭空保存applied epoch。 + +不做Dumpling/CSV/COPY INTO、全量编排或历史空表预建。单writer/无外部写的运行前提必须成立；不因来源相同就假设旧SQL、原生session/callback与S3文件恢复完全相同。 + +## 当前实现与证据边界 + +运行时代码集中在 `/Users/jiangjianyuan/work/git/pingcap/ticdc/downstreamadapter/sink/databend/`。当前已经有native mapper、HTTP终态/参数绑定、按表lane、共享spool与回调、schema epoch恢复，以及factory接入；不是只有接口草图。 + +先读定稿§8.2–8.3/§10：受控PoC明确拒绝分区dispatcher/复杂DDL，未验证自动HA；新journal附带必要的post-DDL定义与步骤，不自动接管旧bend-cdc目标。local/race/lint/build通过不代表真实Databend SQL和完整控制面恢复通过。请特别核对源码已有的scope限制、边界输入与SQL实际语义，避免把HTTP fixture当成数据库实现。 + +## 请按“是否退化”评审，不默认增加理想化要求 + +每个发现请标注： + +1. **迁移退化**：bend-cdc已正确处理，而当前native/spool/callback适配会破坏；必须修。 +2. **接入差异**：去掉S3或使用TiCDC原生生命周期后新增的必要责任；给最小sink/接入补丁。 +3. **既有缺口**：旧代码也存在假设或潜在bug；明确证据及是否在支持场景下会造成错误，不能盲抄，也不能默认为迁移要重建整个系统。 +4. **增强需求**：比基线更强的HA/跨表原子性/全局硬预算等，默认后移，不作为新增sink首版合入要求。 + +优先检查： + +- native与Canal转换后的内部行差分：复合键、二进制、ENUM/SET、decimal/JSON/时间、同TSO合法变化、UK更新。 +- 现有WriteToRawTable/Flush/mergeTable/writeGen/mergedGen是否足以绑定完成的callback；stream部分消费时是否可能错误确认，最小修正是什么。 +- raw子SQL部分成功、MERGE提交未知、callback前重启后重放；不新增ledger能否在受控排他前提下收敛。 +- 现有schema epoch的数值含义与原生FinishedTs/表版本是否正确映射，pending/applied与TRUNCATE重放是否会退化；不能用新raw_generation替代必要源码审查。 +- spool正常压力/封口/回收、transport reset不丢已接管数据、fatal结束旧生命周期；既有框架接入若不够，给确切最小修改，不恢复整套delivery协议。 +- 现有warning跳过的DDL不算完整支持；已知问题不能因“等价迁移”被宣称正确，支持范围之外不无限扩展。 +- 对相同SLO/负载比较raw source与stream、旧链与新sink的扫描和总成本；不预设任一方案更便宜。 + +## 输出 + +总体：接受 / 修改后接受 / 拒绝。 + +每项：证据commit/定位、基线与新方案差异、最小触发时序、影响、最小修复、验证断言；标注上述四种类型以及“sink内 / 小接入 / 超出范围”。按正文T1–T7提出具体验收，不把旧本地测试通过当新sink通过。 + +本轮只读评审；不要修改文件/代码、切分支、创建任务或执行生产操作。 diff --git a/docs/design/databend-native-sink-scope-review-comment.md b/docs/design/databend-native-sink-scope-review-comment.md new file mode 100644 index 0000000000..54f68e0150 --- /dev/null +++ b/docs/design/databend-native-sink-scope-review-comment.md @@ -0,0 +1,94 @@ +# Review comment：请先评审“sink 内聚、最小框架改动”的收敛方案 + +> 状态更新：设计已接受，当前规范为[定稿v1.0](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review.md)，以bend-cdc既有正确性为基线。以下是历史review comment，其中manifest/K-H-S候选及raw-change=true已由当前基线迁移决策替代，不是第二套实施要求；继续评审请使用[当前任务说明](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-review-prompt.md)。方向裁决不代表实现/实验通过。 + +## 结论与评审约束 + +v0.2 对数据正确性的讨论有价值，但新增 collector delivery/信用协议、progress 迁移和跨 capture 账本，会把“新增一个 Databend sink”扩大成 TiCDC 框架改造，不利于合入。本轮请优先评审下面的收敛方案，**不要默认通过增加框架协议来修复每个反例**。 + +这是针对 v0.2 的架构调整提议，不是已实现或已验证的 v0.3。原报告尚未整体替换为本文；有冲突处作为本轮待裁决事项，不能把两套协议同时当实施要求。 + +固定源码基线： + +- TiCDC：`2bf6dacd0aec9ac2fad464600e47341f9aa951f2`，仓库 `/Users/jiangjianyuan/work/git/pingcap/ticdc`。 +- bend-cdc：`b6481621021e82aee659ad88e8e6c00f6dd11906`，仓库 `/Users/jiangjianyuan/work/git/pingcap/webapi` 下的 `bend-cdc`。 +- 当前 checkout 不代表上述新架构版本，请用固定 Git 对象复核，不切分支。 + +## 1. 建议保留的工作流程 + +```text +现有 dispatcher 交付原生 DML/DDL + → sink 内取 TableInfo、主键、commit_ts、before/after + → 统一类型转换与内部行记录 + → 有界内存缓冲 + 限额本地 spool + → 同表/同 schema 攒批，不同表有限并行 + → 批量 INSERT 持久化 raw staging（带 batch_id / 事件身份） + → MERGE INTO 目标表 + → 分片应用成功后独立释放本地资源 + → 事件 SEALED 且全部分片完成后，现有 flush callback 恰执行一次 + → TiCDC 原有机制推进 checkpoint +``` + +- 保留内存+磁盘 spool，不改回纯内存；raw 写成功不能确认源事件。 +- 同表 raw INSERT、MERGE、DDL、compaction 首版串行，避免先引入高并发覆盖协议。 +- 保留 `OPEN → SEALED → COMPLETED` 和已应用分片独立回收，不为等事件全集确认而钉住全部 spool。 +- Databend 内部固定 `raw-change=true`，只增加必要的初始化/注册接入；普通二级 UK 更新必须正确。真实 PK/handle 变更仍需源事件实验，不把这个开关当万能解决方案。 + +## 2. 首版考虑不以 stream 作为应用/确认依据 + +建议评审一个更小的替代：**按明确 batch_id 从 raw 读取本批数据进行 MERGE，不依赖 stream 的消费 cursor 判断完成。** Stream 不作为首版必要依赖，后续若成本实测证明有必要，再单独引入。 + +这样可以移除本轮“stream hint 只消费了 F1，却给 F1/F2 写了 witness”的故障路径；也不再需要为 stream 构造跨层消费证明系统。但必须注意： + +1. batch_id 是下游写入批次身份，不是源业务版本；本次逻辑批次的重试复用身份，重启重新分批仍需按事件身份/commit_ts 幂等收敛。 +2. 每个批次写 raw 可拆多个 SQL；全部待应用记录明确写入成功后才执行 MERGE。不能用 INSERT/MERGE 返回的操作行数充当事件成员覆盖证明。 +3. 不能简单把本批旧值直接覆盖目标。候选思路是：取本批涉及主键，结合同代际、合法 schema 范围的 raw 历史，重算各键最高有效版本并实际应用，包含 DELETE 墓碑。 +4. 不能只因 raw 有更高版本就跳过并确认旧事件；更高版本也可能尚未应用。若采用上述重算，应真的将最高版本纳入当前 MERGE，而非仅用它过滤旧行。 +5. 相同 key/TSO 的冲突、跨 schema 历史、同键乱序/重放、未完成批次、compaction 都必须有明确行为；SQL 与快照一致性仍待验证。 +6. 若某批只部分可应用，未覆盖分片不能确认。`batch_id + MERGE 成功`不是自动正确性证明，必须说明 SQL 来源集合完整、无意外过滤/限制，以及对应 callback 的成员集合。 + +请比较这个直接批次方案与 stream 方案的正确性、SQL复杂度和扫描成本。**去掉 stream 可能增加 raw 扫描，不能预先认定一定更便宜；本提议也不视为关闭所有覆盖风险。** + +## 3. 不改 collector/dynstream/TableProgress 的入口处理 + +删除 v0.2 新增的 delivery 重投、transport epoch 重绑、progress blocker 迁移等定制协议，优先沿用现有 AddDMLEvent / PostEnqueue / PostFlush 生命周期。 + +- 已被 sink 接管的数据由 sink 生命周期管理，不因 collector 的 transport session reset 直接丢弃。源重放由现有框架处理，sink 负责幂等；不能伪造 PostFlush 清 blocker。 +- sink 的入口队列、编码窗口、spool 和元数据必须有界。PostEnqueue 按真实接管/现有表路径背压语义触发，不提前虚报。 +- 不允许在 AddDMLEvent 中无限等待磁盘并占住共享处理线程。无法在有界入口内接管时，首版可通过既有 sink 错误路径停止任务，不无限排队、静默丢弃或自行搬迁 progress；需要复核现有错误路径确实会完成必要的生命周期清理。 +- 上述取舍不承诺“只靠 sink 就能硬限制全 TiCDC RSS”，也不承诺资源耗尽时其他表完全不受影响。原生大 chunk 已在 sink 前分配的问题要明确限制/测量，不靠文档声称消失。 + +请给出“不新增 delivery 协议”的最小调用时序，核查 session reset、接管失败、任务停止/重启时是否仍会遗留 blocker 或漏事件。如果原框架存在无法规避的问题，先给具体反例和最小补丁位置，不能直接恢复整套信用协议。 + +## 4. DDL、恢复及容量控制的简化边界 + +- 建表、类型映射、raw/MERGE、白名单 DDL、DDL journal 与 TRUNCATE 恢复保留在 sink。使用已有 DDL 屏障，不重做全局调度；历史空表可由独立初始化预建，不为此新增框架 schema 服务。 +- 保留 TRUNCATE 的持久化身份/代际和“APPLIED 前不开新代际”的恢复顺序；不能为了减少代码去掉必要的破坏性操作幂等保护。 +- 提交未知时不确认，不继续同表下一批；先核验旧 SQL 终态。驱动取消返回 nil、进程退出、固定等待时间均不是终态证明。 +- 正常迁移/异常接管仍需排除旧在途 SQL；across-nodes=false 不等于禁用自动 HA。若只能依赖大规模框架变更才可生产安全，首版应明确缩为隔离、受控、需人工恢复的 PoC，不虚报自动 HA。 +- 远端 raw/tombstone 增长仍需监控、阈值停写和扩容/重初始化流程。但首版不为此默认新增跨 capture 强一致预算服务；严格 feed 总预算应另评估静态配额切分/已有外部配额。无法保证时，明确是监控型停止阈值而非全局原子硬额度，不能对外承诺严格封顶。 + +## 5. 允许与不允许的修改范围 + +| 范围 | 本轮要求 | +|---|---| +| 新 Databend sink 包 | 事件适配、类型/SQL、buffer/spool、batch worker、tracker、DDL、恢复、错误与指标 | +| scheme/type/config/factory/metrics | 必要注册与参数校验 | +| raw-change模式、schema store、recovery接入 | 允许有限分支/小接口，需列出确切触点及原因，不能伪装成MySQL sink | +| collector/dynstream/TableProgress/调度核心 | 不做通用协议改造；如发现确实阻塞，先提交最小必要变更论证 | + +“主要改在 sink”不等于强求外部零改动；但新增 sink 不应变成 TiCDC 全链路流控/调度重构。 + +## 6. 请 reviewer 给出明确裁决 + +请重点回答： + +1. 不新增 delivery/信用协议，复用原有接口能否做到不丢数据、正常完成、资源失败可诊断？列出最小必要的外部改动。 +2. 直接 batch_id + raw MERGE 能否比 stream/witness 方案明显简单？请给出候选 SQL/成员映射及会击穿它的最小反例，不只说“需验证”。 +3. 旧事件重放、不同 batch 重复、最新墓碑、同TSO合法更新、跨schema恢复是否仍正确？ +4. raw扫描/历史保留成本是否会抵消去掉中间链路的收益，最小实验怎么设计？ +5. 哪些风险必须在sink中解决，哪些需要小接入补丁，哪些只能通过明确缩减首版范围处理？ + +输出：**接受 / 修改后接受 / 拒绝**；每个发现写证据、触发时序、影响、最小修正、验证方法，并标注“sink内修改 / 小接入修改 / 超出本轮范围”。不要为了给通过而补隐含前提，也不要只为追求理想化全局保障无限扩大框架范围。 + +本轮仅评审，未实现该收敛方案，未运行真实 Databend/新 sink 故障实验。不要修改代码、切分支、创建任务或执行生产操作。 diff --git a/downstreamadapter/sink/databend/client.go b/downstreamadapter/sink/databend/client.go new file mode 100644 index 0000000000..909ed51027 --- /dev/null +++ b/downstreamadapter/sink/databend/client.go @@ -0,0 +1,218 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "strings" + "sync/atomic" + "time" + + "github.com/google/uuid" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/writelease" +) + +const maxResponseBytes = 8 << 20 + +type queryPage struct { + ID string `json:"id"` + NodeID string `json:"node_id"` + State string `json:"state"` + Error *struct { + Code int `json:"code"` + } `json:"error"` + Data json.RawMessage `json:"data"` + NextURI string `json:"next_uri"` + FinalURI string `json:"final_uri"` +} + +// httpClient intentionally has no POST retry, redirect or driver-level retry. +// Every statement is autocommit; DDL is never placed inside a DML transaction. +type httpClient struct { + opts options + transport *http.Client + gate atomic.Pointer[writelease.Gate] +} + +func newHTTPClient(o options) *httpClient { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConnsPerHost = o.workers + 2 + return &httpClient{opts: o, transport: &http.Client{Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}} +} + +func (c *httpClient) mutate(ctx context.Context, sql string, args []any) (mutationResult, error) { + _, state, err := c.query(ctx, sql, args, false) + return state, err +} + +func (c *httpClient) selectRows(ctx context.Context, sql string, args []any) ([][]*string, error) { + rows, state, err := c.query(ctx, sql, args, true) + if err != nil { + return nil, err + } + if state.state != committed { + return nil, errors.ErrDatabendExecution.GenWithStackByArgs() + } + return rows, nil +} + +func (c *httpClient) query(ctx context.Context, sql string, args []any, readData bool) ([][]*string, mutationResult, error) { + ctx, cancel := context.WithTimeout(ctx, c.opts.queryTimeout) + defer cancel() + if err := writelease.WaitForWrite(ctx, c.gate.Load()); err != nil { + return nil, mutationResult{state: notCommitted}, errors.WrapError(errors.ErrDatabendExecution, err) + } + id := uuid.NewString() + payload := struct { + SQL string `json:"sql"` + Params []any `json:"params,omitempty"` + Session any `json:"session"` + Pagination any `json:"pagination"` + }{sql, args, map[string]any{"database": "default", "settings": map[string]string{"timezone": "UTC"}}, map[string]int{"wait_time_secs": 1, "max_rows_per_page": 1024, "max_rows_in_buffer": 1024}} + body, err := json.Marshal(payload) + if err != nil { + return nil, mutationResult{state: notCommitted}, errors.WrapError(errors.ErrDatabendExecution, err) + } + method, path := http.MethodPost, "/v1/query" + node, route := "", "" + var rows [][]*string + var total int + state := mutationResult{state: commitUnknown} + for { + page, h, err := c.request(ctx, method, path, body, id, node, route) + if err != nil { + return nil, state, err + } + // A pre-start SQL rejection has no server query ID. All started + // queries, especially successful mutations, must echo our identity. + preStartFailure := method == http.MethodPost && page.State == "Failed" && page.ID == "" + if page.ID != id && !preStartFailure { + return nil, mutationResult{state: commitUnknown}, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "query identity mismatch") + } + if page.NodeID != "" { + node = page.NodeID + } + if v := h.Get("X-DATABEND-ROUTE-HINT"); v != "" { + route = v + } + if page.State == "Failed" { + code := 0 + if page.Error != nil { + code = page.Error.Code + } + return nil, mutationResult{state: notCommitted}, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "server terminal failure code "+jsonInt(code)) + } + if page.Error != nil || (page.State != "Succeeded" && page.State != "Starting" && page.State != "Running") { + return nil, mutationResult{state: commitUnknown}, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "missing or inconsistent execution state") + } + if page.State == "Succeeded" { + state.state = committed + if !readData { + c.finalize(ctx, page.FinalURI, id, node, route) + return nil, state, nil + } + } + if readData && len(page.Data) > 0 { + var data [][]*string + if err := json.Unmarshal(page.Data, &data); err != nil { + return nil, state, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "invalid result data") + } + // Administrative reads are intentionally bounded, independently of paging. + total += len(page.Data) + if total > maxResponseBytes { + return nil, state, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "result exceeds metadata budget") + } + rows = append(rows, data...) + } + if page.NextURI == "" || strings.HasSuffix(page.NextURI, "/final") { + if state.state != committed { + return nil, state, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "no terminal execution evidence") + } + c.finalize(ctx, page.FinalURI, id, node, route) + return rows, state, nil + } + method, path, body = http.MethodGet, page.NextURI, nil + select { + case <-ctx.Done(): + return nil, state, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "query polling interrupted; verify server termination before resuming") + case <-time.After(10 * time.Millisecond): + } + } +} + +func jsonInt(n int) string { b, _ := json.Marshal(n); return string(b) } + +func (c *httpClient) request(ctx context.Context, method, path string, body []byte, id, node, route string) (queryPage, http.Header, error) { + // Never forward credentials to an absolute or network-path URI supplied by a server. + u, err := url.Parse(path) + if err != nil || u.IsAbs() || u.Host != "" || !strings.HasPrefix(path, "/v1/query") { + return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "invalid query continuation URI") + } + req, err := http.NewRequestWithContext(ctx, method, c.opts.endpoint+path, bytes.NewReader(body)) // #nosec G704 -- endpoint is operator configured; continuation above is restricted to the same origin. + if err != nil { + return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "invalid request") + } + req.SetBasicAuth(c.opts.username, c.opts.password) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-DATABEND-QUERY-ID", id) + if c.opts.warehouse != "" { + req.Header.Set("X-DATABEND-WAREHOUSE", c.opts.warehouse) + } + if node != "" { + req.Header.Set("X-DATABEND-STICKY-NODE", node) + req.Header.Set("X-DATABEND-NODE-ID", node) + } + if route != "" { + req.Header.Set("X-DATABEND-ROUTE-HINT", route) + } + if method == http.MethodPost { + if err := writelease.WaitForWrite(ctx, c.gate.Load()); err != nil { + return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "write admission closed before request") + } + } + resp, err := c.transport.Do(req) // #nosec G704 -- intentional configured sink endpoint, same origin only and redirects disabled. + if err != nil { + return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "transport failed; verify server termination before resuming") + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "HTTP status "+jsonInt(resp.StatusCode)) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) + if err != nil || len(data) > maxResponseBytes { + return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "truncated or oversized response") + } + var page queryPage + if err := json.Unmarshal(data, &page); err != nil { + return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "invalid execution response") + } + return page, resp.Header, nil +} + +func (c *httpClient) finalize(ctx context.Context, path, id, node, route string) { + if path == "" { + return + } + // Releasing result buffers is observation/cleanup only, never a new mutation. + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + _, _, _ = c.request(ctx, http.MethodGet, path, nil, id, node, route) +} diff --git a/downstreamadapter/sink/databend/client_test.go b/downstreamadapter/sink/databend/client_test.go new file mode 100644 index 0000000000..3a35f4ed83 --- /dev/null +++ b/downstreamadapter/sink/databend/client_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/writelease" + "github.com/stretchr/testify/require" +) + +func TestHTTPClientRequiresTerminalEvidence(t *testing.T) { + for _, tc := range []struct { + name string + second any + want commitState + }{ + {"result before failure", map[string]any{"state": "Failed", "error": map[string]any{"code": 1001}}, notCommitted}, + {"result before disconnect", nil, commitUnknown}, + {"committed statistics malformed", map[string]any{"state": "Succeeded", "data": "not result rows", "stats": "not statistics"}, committed}, + } { + t.Run(tc.name, func(t *testing.T) { + var posts, gets atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := r.Header.Get("X-DATABEND-QUERY-ID") + if r.Method == "POST" { + posts.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{"id": id, "node_id": "node", "state": "Running", "data": [][]string{{"1"}}, "next_uri": "/v1/query/next"}) + return + } + gets.Add(1) + if tc.second == nil { + w.WriteHeader(503) + return + } + response := tc.second.(map[string]any) + response["id"] = id + _ = json.NewEncoder(w).Encode(response) + })) + defer server.Close() + c := newHTTPClient(options{endpoint: server.URL, queryTimeout: time.Second, workers: 1, username: "root"}) + defer c.transport.CloseIdleConnections() + result, err := c.mutate(context.Background(), "MERGE INTO TARGET", nil) + require.Equal(t, tc.want, result.state) + if tc.want == committed { + require.NoError(t, err) + } else { + require.Error(t, err) + } + require.EqualValues(t, 1, posts.Load()) + require.EqualValues(t, 1, gets.Load()) + }) + } +} + +func TestHTTPClientProtectsCredentialsAndDoesNotRetryPost(t *testing.T) { + var leaked atomic.Int64 + foreign := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { leaked.Add(1) })) + defer foreign.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"id": r.Header.Get("X-DATABEND-QUERY-ID"), "state": "Running", "next_uri": foreign.URL + "/v1/query/steal"}) + })) + defer server.Close() + c := newHTTPClient(options{endpoint: server.URL, username: "user", password: "SECRET", queryTimeout: time.Second}) + result, err := c.mutate(context.Background(), "INSERT SECRET_PAYLOAD", nil) + require.Error(t, err) + require.Equal(t, commitUnknown, result.state) + require.Zero(t, leaked.Load()) + require.NotContains(t, err.Error(), "SECRET") + gate := writelease.NewGate() + c.gate.Store(gate) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result, err = c.mutate(ctx, "DELETE TARGET", nil) + require.Error(t, err) + require.Equal(t, notCommitted, result.state) +} + +func TestHTTPClientRejectsWrongOrMissingSuccessfulIdentity(t *testing.T) { + for _, id := range []string{"", "another-query"} { + t.Run(id, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"id": id, "state": "Succeeded", "data": [][]string{{"1"}}}) + })) + defer server.Close() + c := newHTTPClient(options{endpoint: server.URL, queryTimeout: time.Second}) + result, err := c.mutate(context.Background(), "INSERT TARGET", nil) + require.Error(t, err) + require.Equal(t, commitUnknown, result.state) + }) + } +} diff --git a/downstreamadapter/sink/databend/completion.go b/downstreamadapter/sink/databend/completion.go new file mode 100644 index 0000000000..7c3f70721c --- /dev/null +++ b/downstreamadapter/sink/databend/completion.go @@ -0,0 +1,94 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "sync" + "sync/atomic" +) + +// eventCompletion owns the source flush callback independently of spool entries. +// Spool release/discard/close must never call a fragment's applied method. +type eventCompletion struct { + mu sync.Mutex + pending int + sealed bool + failed bool + completed bool + callback func() +} + +type fragmentCompletion struct { + done atomic.Bool + parent *eventCompletion +} + +func newEventCompletion(callback func()) *eventCompletion { + return &eventCompletion{callback: callback} +} + +// register must happen before publishing a fragment to a worker. +func (c *eventCompletion) register() (*fragmentCompletion, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.sealed || c.failed || c.completed { + return nil, false + } + c.pending++ + return &fragmentCompletion{parent: c}, true +} + +func (c *eventCompletion) seal() { + c.mu.Lock() + c.sealed = true + callback := c.finishLocked() + c.mu.Unlock() + if callback != nil { + callback() + } +} + +func (c *eventCompletion) fail() { + c.mu.Lock() + c.failed = true + c.callback = nil + c.mu.Unlock() +} + +func (c *eventCompletion) finishLocked() func() { + if !c.sealed || c.failed || c.completed || c.pending != 0 { + return nil + } + c.completed = true + callback := c.callback + c.callback = nil + return callback +} + +// applied is called only with confirmed target application. Its caller may +// release the fragment payload immediately, even while the event is OPEN. +func (f *fragmentCompletion) applied() { + if !f.done.CompareAndSwap(false, true) { + return + } + c := f.parent + c.mu.Lock() + c.pending-- + callback := c.finishLocked() + c.mu.Unlock() + if callback != nil { + callback() + } +} diff --git a/downstreamadapter/sink/databend/completion_test.go b/downstreamadapter/sink/databend/completion_test.go new file mode 100644 index 0000000000..f159cebd1d --- /dev/null +++ b/downstreamadapter/sink/databend/completion_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCompletionRequiresSealAndAllFragments(t *testing.T) { + var calls atomic.Int64 + c := newEventCompletion(func() { calls.Add(1) }) + first, ok := c.register() + require.True(t, ok) + first.applied() + first.applied() + require.Zero(t, calls.Load(), "registered==applied while OPEN is not complete") + second, ok := c.register() + require.True(t, ok) + c.seal() + require.Zero(t, calls.Load()) + second.applied() + c.seal() + second.applied() + require.EqualValues(t, 1, calls.Load()) + _, ok = c.register() + require.False(t, ok) +} + +func TestCompletionFailureAndEmptyEvent(t *testing.T) { + var calls atomic.Int64 + c := newEventCompletion(func() { calls.Add(1) }) + f, ok := c.register() + require.True(t, ok) + c.fail() // Resource cleanup must not manufacture an application result. + c.seal() + f.applied() // Even a late worker result cannot revive a failed event. + require.Zero(t, calls.Load()) + _, ok = c.register() + require.False(t, ok) + empty := newEventCompletion(func() { calls.Add(1) }) + empty.seal() + empty.seal() + require.EqualValues(t, 1, calls.Load()) +} + +func TestCompletionConcurrentSealAndReentrantCallback(t *testing.T) { + for range 50 { + var calls atomic.Int64 + var last *fragmentCompletion + c := newEventCompletion(func() { + calls.Add(1) + last.applied() // Source callbacks may re-enter after completion. + }) + fragments := make([]*fragmentCompletion, 16) + for i := range fragments { + f, ok := c.register() + require.True(t, ok) + fragments[i] = f + } + last = fragments[len(fragments)-1] + var wg sync.WaitGroup + for _, f := range fragments { + wg.Go(func() { + f.applied() + f.applied() + }) + } + wg.Go(c.seal) + wg.Wait() + require.EqualValues(t, 1, calls.Load()) + } +} diff --git a/downstreamadapter/sink/databend/config.go b/downstreamadapter/sink/databend/config.go new file mode 100644 index 0000000000..20f2319db7 --- /dev/null +++ b/downstreamadapter/sink/databend/config.go @@ -0,0 +1,156 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "net/url" + "strconv" + "strings" + "time" + + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/util" +) + +// All quotas are local to this sink instance. Exclusive is an operator assertion, +// not a distributed lock or permission to automatically fail over an old query. +type options struct { + endpoint, database, username, password, warehouse, spoolDir string + workers, batchRows, maxEntries, sourceEvents int + batchBytes, spoolBytes, sourceBytes int64 + memoryRatio float64 + flushInterval, queryTimeout time.Duration + remoteLimit int64 + timezone *time.Location + conflictKey string +} + +func parseOptions(uri *url.URL, cfg *config.ChangefeedConfig) (options, error) { + o := options{workers: 4, batchRows: 2048, batchBytes: 1 << 20, spoolBytes: 10 << 30, sourceBytes: 256 << 20, sourceEvents: 128, maxEntries: 4096, memoryRatio: 0.02, flushInterval: time.Second, queryTimeout: 2 * time.Minute} + bad := func() (options, error) { + return options{}, errors.ErrSinkURIInvalid.GenWithStackByArgs("invalid Databend options; require exclusive=true, bounded positive quotas and supported incremental settings") + } + if uri == nil || (config.GetScheme(uri) != config.DatabendScheme && config.GetScheme(uri) != config.DatabendHTTPScheme && config.GetScheme(uri) != config.LakeScheme) || uri.Host == "" || uri.Fragment != "" { + return bad() + } + q, queryErr := url.ParseQuery(uri.RawQuery) + if queryErr != nil { + return bad() + } + allowed := map[string]bool{"exclusive": true, "workers": true, "batch-rows": true, "batch-bytes": true, "spool-bytes": true, "source-bytes": true, "source-events": true, "max-entries": true, "memory-ratio": true, "flush-interval": true, "query-timeout": true, "spool-dir": true, "warehouse": true, "remote-bytes-limit": true, "conflict-key": true} + for key, v := range q { + if !allowed[key] || len(v) != 1 { + return bad() + } + } + if q.Get("exclusive") != "true" { + return bad() + } + if q.Has("conflict-key") { + o.conflictKey = q.Get("conflict-key") + if strings.TrimSpace(o.conflictKey) == "" || strings.ContainsRune(o.conflictKey, 0) || len(o.conflictKey) > 256 { + return bad() + } + } + o.timezone = time.UTC + if cfg != nil { + if cfg.TimeZone != "" { + var err error + o.timezone, err = util.GetTimezone(cfg.TimeZone) + if err != nil { + return bad() + } + } + if cfg.EnableTableAcrossNodes || cfg.EnableSyncPoint || cfg.EnableActiveActive || cfg.IsLowLatencyMode() { + return bad() + } + if cfg.Consistent != nil && cfg.Consistent.Level != nil && *cfg.Consistent.Level != "none" && *cfg.Consistent.Level != "" { + return bad() + } + if cfg.SinkConfig != nil { + if cfg.SinkConfig.TxnAtomicity != nil && *cfg.SinkConfig.TxnAtomicity != "none" && *cfg.SinkConfig.TxnAtomicity != "" { + return bad() + } + for _, r := range cfg.SinkConfig.DispatchRules { + if r != nil && (r.TargetSchema != "" || r.TargetTable != "") { + return bad() + } + } + if len(cfg.SinkConfig.ColumnSelectors) > 0 { + return bad() + } + } + } + for key, p := range map[string]*int{"workers": &o.workers, "batch-rows": &o.batchRows, "source-events": &o.sourceEvents, "max-entries": &o.maxEntries} { + if q.Has(key) { + n, err := strconv.Atoi(q.Get(key)) + if err != nil || n <= 0 { + return bad() + } + *p = n + } + } + for key, p := range map[string]*int64{"batch-bytes": &o.batchBytes, "spool-bytes": &o.spoolBytes, "source-bytes": &o.sourceBytes, "remote-bytes-limit": &o.remoteLimit} { + if q.Has(key) { + n, err := strconv.ParseInt(q.Get(key), 10, 64) + if err != nil || n <= 0 { + return bad() + } + *p = n + } + } + for key, p := range map[string]*time.Duration{"flush-interval": &o.flushInterval, "query-timeout": &o.queryTimeout} { + if q.Has(key) { + d, err := time.ParseDuration(q.Get(key)) + if err != nil || d <= 0 { + return bad() + } + *p = d + } + } + if q.Has("memory-ratio") { + n, err := strconv.ParseFloat(q.Get("memory-ratio"), 64) + if err != nil || !(n > 0 && n < 1) { + return bad() + } + o.memoryRatio = n + } + if o.workers > 64 || o.batchRows > 100000 || o.sourceEvents > 65536 || o.maxEntries > 65536 || o.batchBytes < 1024 || o.batchBytes > 64<<20 || o.spoolBytes < o.batchBytes*2 || o.sourceBytes < o.batchBytes || o.spoolBytes > 1<<50 { + return bad() + } + o.database = strings.Trim(uri.Path, "/") + if strings.Contains(o.database, "/") || strings.ContainsRune(o.database, 0) { + return bad() + } + o.username = "root" + if uri.User != nil { + o.username = uri.User.Username() + o.password, _ = uri.User.Password() + } + endpoint := *uri + endpoint.Path = "" + endpoint.User = nil + endpoint.RawQuery = "" + endpoint.RawPath = "" + endpoint.Scheme = "https" + if config.GetScheme(uri) == config.DatabendHTTPScheme { + endpoint.Scheme = "http" + } + o.endpoint = endpoint.String() + o.warehouse = q.Get("warehouse") + o.spoolDir = q.Get("spool-dir") + return o, nil +} diff --git a/downstreamadapter/sink/databend/config_test.go b/downstreamadapter/sink/databend/config_test.go new file mode 100644 index 0000000000..fc197c2743 --- /dev/null +++ b/downstreamadapter/sink/databend/config_test.go @@ -0,0 +1,67 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "net/url" + "testing" + + "github.com/pingcap/ticdc/pkg/config" + "github.com/stretchr/testify/require" +) + +func TestDatabendOptions(t *testing.T) { + uri, err := url.Parse("databend://user:pass@warehouse/lake?exclusive=true&workers=2&batch-bytes=4096&spool-bytes=16384") + require.NoError(t, err) + o, err := parseOptions(uri, nil) + require.NoError(t, err) + require.Equal(t, "https://warehouse", o.endpoint) + require.Equal(t, "lake", o.database) + require.Equal(t, 2, o.workers) + require.EqualValues(t, 16384, o.spoolBytes) + lakeURI, err := url.Parse("lake://user:pass@warehouse:443/default?exclusive=true&warehouse=default") + require.NoError(t, err) + lake, err := parseOptions(lakeURI, nil) + require.NoError(t, err) + require.Equal(t, "https://warehouse:443", lake.endpoint) + for _, raw := range []string{ + "databend://host", "databend://host?exclusive=false", "databend://host?exclusive=true&workers=0", "databend://host?exclusive=true&memory-ratio=NaN", "databend://host?exclusive=true&batch-bytes=4096&spool-bytes=4096", "databend://host?exclusive=true&raw-change=true", "databend://host?exclusive=true&workers=1&workers=2", "databend://host?exclusive=true&query-timeout=0s", + } { + u, err := url.Parse(raw) + require.NoError(t, err) + _, err = parseOptions(u, nil) + require.Error(t, err, raw) + } + _, err = parseOptions(uri, &config.ChangefeedConfig{EnableTableAcrossNodes: true}) + require.Error(t, err) + _, err = parseOptions(uri, &config.ChangefeedConfig{EnableSyncPoint: true}) + require.Error(t, err) +} + +func TestConflictKeyOption(t *testing.T) { + for _, key := range []string{"id", "BusinessID", "a\"b"} { + u, err := url.Parse("lake://host/default?exclusive=true&conflict-key=" + url.QueryEscape(key)) + require.NoError(t, err) + o, err := parseOptions(u, &config.ChangefeedConfig{ForceReplicate: true}) + require.NoError(t, err) + require.Equal(t, key, o.conflictKey) + } + for _, raw := range []string{"", " ", "bad\x00name"} { + u, err := url.Parse("lake://host/default?exclusive=true&conflict-key=" + url.QueryEscape(raw)) + require.NoError(t, err) + _, err = parseOptions(u, nil) + require.Error(t, err) + } +} diff --git a/downstreamadapter/sink/databend/integration_test.go b/downstreamadapter/sink/databend/integration_test.go new file mode 100644 index 0000000000..c3b6206016 --- /dev/null +++ b/downstreamadapter/sink/databend/integration_test.go @@ -0,0 +1,94 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "context" + "net/url" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/stretchr/testify/require" +) + +// Explicit opt-in only. This test creates and drops its own unique database; +// never use a production warehouse or the running bend-cdc target for it. +func TestDatabendNativeLiveIntegration(t *testing.T) { + raw := os.Getenv("DATABEND_NATIVE_TEST_URI") + if raw == "" { + t.Skip("DATABEND_NATIVE_TEST_URI not configured; no live Databend evidence") + } + uri, err := url.Parse(raw) + require.NoError(t, err) + database := "ticdc_native_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + uri.Path = "/" + database + uri.RawPath = "" + q := uri.Query() + q.Set("exclusive", "true") + q.Set("workers", "1") + q.Set("flush-interval", "10ms") + q.Set("spool-dir", t.TempDir()) + uri.RawQuery = q.Encode() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + require.NoError(t, Verify(ctx, &config.ChangefeedConfig{}, uri)) + s, err := New(ctx, common.NewChangeFeedIDWithName(database, "default"), &config.ChangefeedConfig{}, uri, 0) + require.NoError(t, err) + t.Cleanup(func() { + s.Close() + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, _ = s.client.mutate(cleanupCtx, "DROP DATABASE IF EXISTS "+quoteIdentifier(database), nil) + }) + done := runRuntime(t, s) + var callbacks atomic.Int64 + apply := func(e *commonEvent.DMLEvent) { + e.PostTxnFlushed = []func(){func() { callbacks.Add(1) }} + before := callbacks.Load() + s.AddDMLEvent(e) + require.Eventually(t, func() bool { return callbacks.Load() == before+1 }, 30*time.Second, 10*time.Millisecond) + } + info := nativeTestInfo() + apply(nativeTestEvent(info, 100, 2)) + rows, err := s.client.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM "+quoteIdentifier(database)+`."t"`, nil) + require.NoError(t, err) + require.Equal(t, "2", *rows[0][0]) + deleted := nativeTestEvent(info, 130, 1) + deleted.RowTypes[0] = common.RowTypeDelete + apply(deleted) + // A historical UPSERT must not resurrect a retained tombstone. + apply(nativeTestEvent(info, 100, 1)) + rows, err = s.client.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM "+quoteIdentifier(database)+`."t"`, nil) + require.NoError(t, err) + require.Equal(t, "1", *rows[0][0]) + ddl := &commonEvent.DDLEvent{Type: byte(model.ActionTruncateTable), TableInfo: info, FinishedTs: 150} + require.NoError(t, s.WriteBlockEvent(ddl)) + apply(nativeTestEvent(info, 120, 2)) // Applied epoch rejects old DML without projection. + apply(nativeTestEvent(info, 160, 1)) + require.NoError(t, s.WriteBlockEvent(ddl)) // Applied replay must not truncate the new row. + rows, err = s.client.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM "+quoteIdentifier(database)+`."t"`, nil) + require.NoError(t, err) + require.Equal(t, "1", *rows[0][0]) + s.Close() + require.ErrorIs(t, <-done, context.Canceled) +} diff --git a/downstreamadapter/sink/databend/lane.go b/downstreamadapter/sink/databend/lane.go new file mode 100644 index 0000000000..0905c9a2be --- /dev/null +++ b/downstreamadapter/sink/databend/lane.go @@ -0,0 +1,337 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "context" + "encoding/json" + "time" + + "github.com/pingcap/ticdc/pkg/errors" + codec "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/sink/spool" +) + +type spoolRow struct { + ID string `json:"id"` + Payload string `json:"payload"` + CommitTs uint64 `json:"commit_ts"` + Action rowAction `json:"action"` +} +type pendingFragment struct { + entry *spool.Entry + completion *fragmentCompletion + count int + bytes int64 +} +type pendingTable struct { + state *tableState + fragments []pendingFragment + rows int + bytes int64 +} +type tableLane struct { + sink *Sink + input chan workItem + schemas *schemaManager + batches map[string]*pendingTable + entries int +} + +func (l *tableLane) run(ctx context.Context) error { + timer := time.NewTicker(l.sink.opts.flushInterval) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return context.Cause(ctx) + case <-timer.C: + if err := l.flushAll(ctx); err != nil { + return err + } + case item := <-l.input: + var err error + switch { + case item.event != nil: + err = l.encode(ctx, &item) + case item.ddl != nil: + err = l.sink.statistics.RecordDDLExecution(func() (string, error) { return "databend", l.schemas.applyDDL(ctx, item.ddl) }) + default: + err = l.flushAll(ctx) + } + if err != nil { + l.sink.stop(err) + err = context.Cause(l.sink.ctx) + } + if item.done != nil { + item.done <- err + } + if err != nil { + return err + } + } + } +} + +func (l *tableLane) encode(ctx context.Context, item *workItem) (err error) { + event := item.event + item.event = nil + released := false + release := func() { + if !released { + released = true + l.sink.releaseSource(item.charge) + } + } + defer release() + s, err := schemaFromNative(event.TableInfo, l.sink.opts.database, l.sink.opts.conflictKey) + if err != nil { + return err + } + state, skip, err := l.schemas.prepareDML(ctx, s, event.CommitTs) + if err != nil { + return err + } + enqueue, flush := event.DetachPostCallbacks() + tracker := newEventCompletion(flush) + defer func() { + if err != nil { + tracker.fail() + } + }() + if skip { + event = nil + release() + enqueue() + tracker.seal() + return nil + } + var blob []byte + rows := 0 + publish := func(last bool) error { + if rows == 0 { + if last { + enqueue() + } + return nil + } + blob = append(blob, ']') + fragment, ok := tracker.register() + if !ok { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("fragment registered after completion") + } + var callback func() + if last { + callback = enqueue + } + err := l.enqueue(ctx, state, blob, rows, fragment, callback) + blob = nil + rows = 0 + return err + } + count := int32(0) + for { + if ctx.Err() != nil { + return context.Cause(ctx) + } + change, ok := event.GetNextRow() + if !ok { + break + } + row, err := normalizeChange(event.TableInfo, s, change, event.CommitTs, l.sink.opts.timezone) + if err != nil { + return err + } + encoded, err := json.Marshal(spoolRow{row.id, row.payload, row.commitTs, row.action}) + if err != nil { + return errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot encode spool row") + } + if int64(len(encoded)+2) > l.sink.opts.batchBytes { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("single encoded row exceeds batch byte limit") + } + if rows > 0 && (int64(len(blob)+len(encoded)+2) > l.sink.opts.batchBytes || rows >= l.sink.opts.batchRows) { + if err := publish(false); err != nil { + return err + } + } + if rows == 0 { + blob = append(blob, '[') + } else { + blob = append(blob, ',') + } + blob = append(blob, encoded...) + rows++ + count++ + } + if count != event.Len() { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("incomplete native row iteration") + } + event = nil + release() // Only callback closures and immutable bounded fragments remain. + if err := publish(true); err != nil { + return err + } + tracker.seal() + return nil +} + +func (l *tableLane) enqueue(ctx context.Context, state *tableState, blob []byte, count int, completion *fragmentCompletion, enqueue func()) error { + msg := &codec.Message{Value: blob} // Never attach source success to spool.Callback. + msgs := []*codec.Message{msg} + if spool.AccountingBytes(msgs) > l.sink.opts.spoolBytes { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("encoded fragment exceeds spool quota") + } + for { + action, entry, err := l.sink.spool.TryEnqueue(msgs, enqueue) + if err != nil { + return err + } + if action == spool.EnqueueActionAcceptedOversized { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("spool size invariant violated") + } + if action == spool.EnqueueActionWaitDiskQuota { + // Drain every local table, not just the large event. Other lanes keep + // draining on their own timers; released segment files remain charged. + if err := l.flushAll(ctx); err != nil { + return err + } + if err := l.sink.spool.WaitForDiskQuota(ctx, msgs); err != nil { + return err + } + continue + } + key := schemaKey(state.schema) + batch := l.batches[key] + if batch == nil { + batch = &pendingTable{state: state} + l.batches[key] = batch + } + batch.fragments = append(batch.fragments, pendingFragment{entry, completion, count, int64(len(blob))}) + batch.rows += count + batch.bytes += int64(len(blob)) + l.entries++ + if batch.bytes >= l.sink.opts.batchBytes || batch.rows >= l.sink.opts.batchRows { + return l.flush(ctx, key) + } + if l.entries >= l.sink.opts.maxEntries { + return l.flushAll(ctx) + } + return nil + } +} + +func (l *tableLane) flushAll(ctx context.Context) error { + for key := range l.batches { + if err := l.flush(ctx, key); err != nil { + return err + } + } + return nil +} + +func (l *tableLane) flush(ctx context.Context, key string) error { + batch := l.batches[key] + if batch == nil { + return nil + } + // Bound remote retained-raw admission by monitoring only. The measurement is + // neither a cross-capture reservation nor an absolute physical-cost ceiling. + if l.sink.opts.remoteLimit > 0 { + rows, err := l.sink.client.selectRows(ctx, "SELECT COALESCE(SUM(data_compressed_size), 0)::VARCHAR FROM system.tables WHERE database = ? AND ends_with(name, '_cdc_raw')", []any{batch.state.schema.database}) + if err != nil { + return err + } + n, err := parseRemoteBytes(rows) + if err != nil { + return err + } + if n >= l.sink.opts.remoteLimit { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("remote raw monitoring threshold reached; expand budget or reinitialize manually") + } + } + var rows []normalizedRow + for _, f := range batch.fragments { + reader, err := l.sink.spool.NewMessageReader(f.entry) + if err != nil { + return err + } + _, data, _, ok, err := reader.Next() + if err != nil { + return err + } + if !ok { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("empty spool fragment") + } + var decoded []spoolRow + if err := json.Unmarshal(data, &decoded); err != nil { + return errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot decode spool fragment") + } + if len(decoded) != f.count { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("incomplete spool fragment") + } + for _, r := range decoded { + rows = append(rows, normalizedRow{r.ID, r.Payload, r.CommitTs, r.Action}) + } + } + var result applyResult + err := l.sink.statistics.RecordBatchExecution(func() (int, int64, error) { + var err error + result, err = batch.state.writer.applyBatch(ctx, rows, time.Now()) + return len(rows), batch.bytes, err + }) + if result.applied { + delete(l.batches, key) + l.entries -= len(batch.fragments) + // Reclaim space separately from source completion; OPEN events keep only + // counters, never already-applied payloads. Release never invokes success. + for _, f := range batch.fragments { + l.sink.spool.Release(f.entry) + f.completion.applied() + } + } + return err +} + +func (l *tableLane) cleanup() { + for _, b := range l.batches { + for _, f := range b.fragments { + f.completion.parent.fail() + l.sink.spool.Release(f.entry) + } + } + clear(l.batches) + for { + select { + case item := <-l.input: + if item.event != nil { + l.sink.releaseSource(item.charge) + } + default: + return + } + } +} + +// parseRemoteBytes rejects malformed measurements instead of treating them as 0. +func parseRemoteBytes(rows [][]*string) (int64, error) { + if len(rows) != 1 || len(rows[0]) != 1 || rows[0][0] == nil { + return 0, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid remote raw measurement") + } + var n int64 + if err := json.Unmarshal([]byte(*rows[0][0]), &n); err != nil || n < 0 { + return 0, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid remote raw byte count") + } + return n, nil +} diff --git a/downstreamadapter/sink/databend/live_fault_test.go b/downstreamadapter/sink/databend/live_fault_test.go new file mode 100644 index 0000000000..a6352be9dc --- /dev/null +++ b/downstreamadapter/sink/databend/live_fault_test.go @@ -0,0 +1,405 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" + "github.com/stretchr/testify/require" +) + +// liveResponseCut delegates to the REAL warehouse, then discards a chosen +// Succeeded response at the client transport boundary. It never simulates SQL +// execution or invents terminal success. A before-send cut is used only to prove +// partial raw INSERTs; recovery is permitted only after this test knows all old +// requests have either not been sent or reached a matching real terminal state. +// This is response-loss/process-state replacement injection, not a server crash. +type liveResponseCut struct { + base http.RoundTripper + matches func(string, []any) bool + at int + before bool + mu sync.Mutex + posts int + cutID string + terminal string + fired bool +} + +func (f *liveResponseCut) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Method == http.MethodPost { + r, err := req.GetBody() + if err != nil { + return nil, err + } + var payload struct { + SQL string `json:"sql"` + Params []any `json:"params"` + } + err = json.NewDecoder(r).Decode(&payload) + _ = r.Close() + if err != nil { + return nil, err + } + f.mu.Lock() + if f.matches(payload.SQL, payload.Params) { + f.posts++ + if f.posts == f.at { + f.cutID = req.Header.Get("X-DATABEND-QUERY-ID") + } + } + cutBefore := f.before && f.cutID == req.Header.Get("X-DATABEND-QUERY-ID") + if cutBefore { + f.fired = true + } + f.mu.Unlock() + if cutBefore { + return nil, io.ErrUnexpectedEOF + } + } + response, err := f.base.RoundTrip(req) + if err != nil { + return nil, err + } + data, err := io.ReadAll(response.Body) + _ = response.Body.Close() + if err != nil { + return nil, err + } + response.Body = io.NopCloser(bytes.NewReader(data)) + var page queryPage + if json.Unmarshal(data, &page) == nil { + f.mu.Lock() + cut := !f.before && !f.fired && page.ID != "" && page.ID == f.cutID && page.State == "Succeeded" + if cut { + f.terminal, f.fired = page.ID, true + } + f.mu.Unlock() + if cut { + _ = response.Body.Close() + return nil, io.ErrUnexpectedEOF + } + } + return response, nil +} + +func liveFaultEnvironment(t *testing.T) (context.Context, options, *httpClient) { + t.Helper() + raw := os.Getenv("DATABEND_NATIVE_TEST_URI") + if raw == "" { + t.Skip("explicit DATABEND_NATIVE_TEST_URI is required; no real warehouse evidence") + } + u, err := url.Parse(raw) + require.NoError(t, err) + u.Path = "/ticdc_fault_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + u.RawPath = "" + q := u.Query() + q.Set("exclusive", "true") + q.Set("workers", "1") + q.Set("batch-rows", "2") + q.Set("flush-interval", "10ms") + q.Set("spool-dir", t.TempDir()) + u.RawQuery = q.Encode() + o, err := parseOptions(u, &config.ChangefeedConfig{}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + t.Cleanup(cancel) + client := newHTTPClient(o) + r, err := client.mutate(ctx, "CREATE DATABASE "+quoteIdentifier(o.database), nil) + require.NoError(t, err) + require.Equal(t, committed, r.state) + t.Cleanup(func() { + cleanup, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + r, err := client.mutate(cleanup, "DROP DATABASE IF EXISTS "+quoteIdentifier(o.database), nil) + require.NoError(t, err) + require.Equal(t, committed, r.state) + }) + t.Log("owned real Lake database", o.database) + return ctx, o, client +} + +func liveRuntime(ctx context.Context, t *testing.T, o options, name string) *Sink { + t.Helper() + u, err := url.Parse(os.Getenv("DATABEND_NATIVE_TEST_URI")) + require.NoError(t, err) + u.Path, u.RawPath = "/"+o.database, "" + q := u.Query() + q.Set("exclusive", "true") + q.Set("workers", "1") + q.Set("flush-interval", "10ms") + q.Set("spool-dir", t.TempDir()) + u.RawQuery = q.Encode() + s, err := New(ctx, common.NewChangeFeedIDWithName(name, "default"), &config.ChangefeedConfig{}, u, 0) + require.NoError(t, err) + t.Cleanup(s.Close) + return s +} + +func liveNamedInfo(name string) *common.TableInfo { + base := nativeTestInfo() + return common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr(name), PKIsHandle: true, Columns: base.GetColumns()}) +} + +func liveCount(ctx context.Context, t *testing.T, c *httpClient, name string, expected string) { + t.Helper() + rows, err := c.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM "+name, nil) + require.NoError(t, err) + require.Equal(t, expected, *rows[0][0]) +} + +func TestDatabendNativeLiveRawPartialAndMergeResponseLoss(t *testing.T) { + ctx, o, admin := liveFaultEnvironment(t) + t.Run("partial_raw_before_second_insert", func(t *testing.T) { + info := liveNamedInfo("partial_raw") + schema, err := schemaFromNative(info, o.database, "") + require.NoError(t, err) + client := newHTTPClient(o) + m := newSchemaManager(client, o) + state, _, err := m.prepareDML(ctx, schema, 100) + require.NoError(t, err) + cut := &liveResponseCut{base: client.transport.Transport, at: 2, before: true, matches: func(sql string, _ []any) bool { return strings.HasPrefix(sql, "INSERT INTO "+schema.rawName()) }} + client.transport.Transport = cut + var rows []normalizedRow + for i := range 4 { + r, err := normalizeNativeRow(schema, insertAction, 100, map[string]any{"id": i, "value": "row"}) + require.NoError(t, err) + rows = append(rows, r) + } + applied, err := state.writer.applyBatch(ctx, rows, time.Now()) + require.Error(t, err) + require.False(t, applied.applied) + require.True(t, cut.fired) + require.Equal(t, 2, cut.posts, "unknown raw must not be retried") + liveCount(ctx, t, admin, schema.rawName(), "2") + liveCount(ctx, t, admin, schema.targetName(), "0") + // Replacement process state; the second request was never sent and the + // first request had a matching terminal success. Reapply ALL source rows. + m = newSchemaManager(admin, o) + state, _, err = m.prepareDML(ctx, schema, 100) + require.NoError(t, err) + applied, err = state.writer.applyBatch(ctx, rows, time.Now()) + require.NoError(t, err) + require.True(t, applied.applied) + liveCount(ctx, t, admin, schema.targetName(), "4") + liveCount(ctx, t, admin, schema.rawName(), "4") + }) + t.Run("real_merge_committed_terminal_response_discarded", func(t *testing.T) { + info := liveNamedInfo("merge_lost") + s := liveRuntime(ctx, t, o, "merge_lost") + schema, err := schemaFromNative(info, o.database, "") + require.NoError(t, err) + cut := &liveResponseCut{base: s.client.transport.Transport, at: 1, matches: func(sql string, _ []any) bool { + return strings.HasPrefix(sql, "MERGE INTO "+schema.targetName()+" a USING") + }} + s.client.transport.Transport = cut + done := runRuntime(t, s) + var flushed atomic.Int64 + e := nativeTestEvent(info, 100, 2) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + select { + case err := <-done: + require.Error(t, err) + case <-time.After(90 * time.Second): + t.Fatal("unknown real MERGE did not terminate old sink lifecycle") + } + s.Close() + require.Zero(t, flushed.Load(), "commit without client evidence must not flush") + require.False(t, s.IsNormal()) + require.True(t, cut.fired) + require.NotEmpty(t, cut.terminal, "test must know matching server terminal BEFORE recovery") + require.Equal(t, 1, cut.posts) + t.Log("discarded real MERGE Succeeded response", cut.terminal) + liveCount(ctx, t, admin, schema.targetName(), "2") + m := newSchemaManager(admin, o) + state, _, err := m.prepareDML(ctx, schema, 100) + require.NoError(t, err) + var rows []normalizedRow + for i := range 2 { + r, err := normalizeNativeRow(schema, insertAction, 100, map[string]any{"id": i, "value": "payload"}) + require.NoError(t, err) + rows = append(rows, r) + } + applied, err := state.writer.applyBatch(ctx, rows, time.Now()) + require.NoError(t, err) + require.True(t, applied.applied) + liveCount(ctx, t, admin, schema.targetName(), "2") + liveCount(ctx, t, admin, schema.rawName(), "2") + }) +} + +func TestDatabendNativeLiveDDLInterruptionRecovery(t *testing.T) { + ctx, o, admin := liveFaultEnvironment(t) + for _, kind := range []string{"create", "alter", "truncate", "drop"} { + cuts := []string{"pending", "target", "raw", "applied"} + if kind == "alter" { + cuts = []string{"pending", "add", "rename", "modify", "raw", "applied"} + } + if kind == "truncate" { + cuts = []string{"pending", "drop_raw", "truncate", "raw", "applied"} + } + for _, point := range cuts { + t.Run(kind+"/"+point, func(t *testing.T) { + info := liveNamedInfo(kind + "_" + point) + schema, err := schemaFromNative(info, o.database, "") + require.NoError(t, err) + s := liveRuntime(ctx, t, o, schema.table) + client := s.client + m := s.lanes[0].schemas + if kind != "create" { + state, _, err := m.prepareDML(ctx, schema, 100) + require.NoError(t, err) + r, err := normalizeNativeRow(schema, insertAction, 100, map[string]any{"id": 1, "value": "old"}) + require.NoError(t, err) + _, err = state.writer.applyBatch(ctx, []normalizedRow{r}, time.Now()) + require.NoError(t, err) + } + e := &commonEvent.DDLEvent{TableInfo: info, FinishedTs: 150} + switch kind { + case "create": + e.Type = byte(model.ActionCreateTable) + case "truncate": + e.Type = byte(model.ActionTruncateTable) + case "drop": + e.Type = byte(model.ActionDropTable) + case "alter": + e.Type = byte(model.ActionMultiSchemaChange) + e.Query = "ALTER TABLE t ADD COLUMN extra BIGINT NULL, RENAME COLUMN value TO label, MODIFY COLUMN label TEXT NULL" + cols := append([]*model.ColumnInfo(nil), info.GetColumns()...) + copyValue := *cols[1] + copyValue.Name = ast.NewCIStr("label") + cols[1] = ©Value + cols = append(cols, &model.ColumnInfo{ID: 3, Name: ast.NewCIStr("extra"), FieldType: *types.NewFieldType(mysql.TypeLonglong), State: model.StatePublic}) + e.TableInfo = common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr(schema.table), PKIsHandle: true, Columns: cols}) + } + cut := &liveResponseCut{base: client.transport.Transport, at: 1, matches: func(sql string, args []any) bool { + switch point { + case "pending", "applied": + return strings.HasPrefix(sql, "MERGE INTO "+schema.journalName()) && len(args) == 3 && args[1] == (point == "applied") + case "target": + if kind == "drop" { + return sql == "DROP TABLE IF EXISTS "+schema.targetName() + } + return strings.HasPrefix(sql, "CREATE TABLE IF NOT EXISTS "+schema.targetName()+" (") + case "raw": + if kind == "drop" { + return sql == "DROP TABLE IF EXISTS "+schema.rawName() + } + return strings.HasPrefix(sql, "CREATE TABLE IF NOT EXISTS "+schema.rawName()+" (") + case "drop_raw": + return sql == "DROP TABLE IF EXISTS "+schema.rawName() + case "truncate": + return sql == "TRUNCATE TABLE "+schema.targetName() + default: + return strings.HasPrefix(sql, "ALTER TABLE "+schema.targetName()+" "+strings.ToUpper(point)+" COLUMN ") + } + }} + client.transport.Transport = cut + var callbacks atomic.Int64 + e.PostTxnFlushed = []func(){func() { callbacks.Add(1) }} + done := runRuntime(t, s) + require.Error(t, s.WriteBlockEvent(e)) + select { + case err := <-done: + require.Error(t, err) + case <-time.After(90 * time.Second): + t.Fatal("unknown DDL did not terminate old sink lifecycle") + } + s.Close() + require.Zero(t, callbacks.Load(), "unknown DDL must not flush") + require.True(t, cut.fired) + require.Equal(t, 1, cut.posts) + require.NotEmpty(t, cut.terminal) + t.Log("cut after real committed DDL", point, cut.terminal) + // The test has independent matching terminal proof for the old + // request. Replace ALL local schema/DDL state; reuse only Lake data. + m = newSchemaManager(admin, o) + require.NoError(t, m.applyDDL(ctx, e)) + post, err := schemaFromNative(e.TableInfo, o.database, "") + require.NoError(t, err) + r, err := m.loadEpoch(ctx, post) + require.NoError(t, err) + require.EqualValues(t, 150, r.epoch) + require.True(t, r.applied) + if kind == "drop" { + exists, err := m.targetExists(ctx, post) + require.NoError(t, err) + require.False(t, exists) + rows, err := admin.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM system.tables WHERE database=? AND name IN (?, ?)", []any{post.database, post.table, post.table + "_cdc_raw"}) + require.NoError(t, err) + require.Equal(t, "0", *rows[0][0]) + return + } + state, skip, err := m.prepareDML(ctx, post, 160) + require.NoError(t, err) + require.False(t, skip) + payload := map[string]any{"id": 2, "value": "new"} + if kind == "alter" { + payload = map[string]any{"id": 2, "label": "new", "extra": 7} + } + row, err := normalizeNativeRow(post, insertAction, 160, payload) + require.NoError(t, err) + _, err = state.writer.applyBatch(ctx, []normalizedRow{row}, time.Now()) + require.NoError(t, err) + // Replaying an applied DDL must not clear new-generation data. + require.NoError(t, m.applyDDL(ctx, e)) + expected := "1" + if kind == "alter" { + expected = "2" + } + liveCount(ctx, t, admin, post.targetName(), expected) + projection := "id::VARCHAR, value::VARCHAR" + want := [][]string{{"2", "new"}} + if kind == "alter" { + projection = "id::VARCHAR, label::VARCHAR, COALESCE(extra::VARCHAR, 'SQL_NULL')" + want = [][]string{{"1", "old", "SQL_NULL"}, {"2", "new", "7"}} + } + actual, err := admin.selectRows(ctx, "SELECT "+projection+" FROM "+post.targetName()+" ORDER BY id", nil) + require.NoError(t, err) + values := make([][]string, len(actual)) + for i, row := range actual { + for _, cell := range row { + require.NotNil(t, cell) + values[i] = append(values[i], *cell) + } + } + require.Equal(t, want, values) + _, skip, err = m.prepareDML(ctx, post, 120) + require.NoError(t, err) + require.True(t, skip, "old DML must not enter post-DDL schema") + }) + } + } +} diff --git a/downstreamadapter/sink/databend/metrics.go b/downstreamadapter/sink/databend/metrics.go new file mode 100644 index 0000000000..1a0ae3e66b --- /dev/null +++ b/downstreamadapter/sink/databend/metrics.go @@ -0,0 +1,46 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + sinkmetrics "github.com/pingcap/ticdc/downstreamadapter/sink/metrics" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/sink/spool" +) + +func newSpoolMetrics(id common.ChangeFeedID) *spool.Metrics { + k, f := id.Keyspace(), id.Name() + return &spool.Metrics{ + MemoryBytes: sinkmetrics.DatabendBufferBytes.WithLabelValues(k, f, "memory"), + DiskBytes: sinkmetrics.DatabendBufferBytes.WithLabelValues(k, f, "disk"), + PendingPostEnqueue: sinkmetrics.DatabendBufferEntries.WithLabelValues(k, f, "enqueue"), + DiskQuotaWaiters: sinkmetrics.DatabendBufferEntries.WithLabelValues(k, f, "waiters"), + SegmentCount: sinkmetrics.DatabendBufferEntries.WithLabelValues(k, f, "segments"), + DiskQuotaWait: sinkmetrics.DatabendSpoolDuration.WithLabelValues(k, f), + LoadedBytes: sinkmetrics.DatabendSpoolLoad.WithLabelValues(k, f), + RotatedCount: sinkmetrics.DatabendSpoolRotations.WithLabelValues(k, f), + Close: func() { + for _, kind := range []string{"memory", "disk"} { + sinkmetrics.DatabendBufferBytes.DeleteLabelValues(k, f, kind) + } + for _, kind := range []string{"enqueue", "waiters", "segments"} { + sinkmetrics.DatabendBufferEntries.DeleteLabelValues(k, f, kind) + } + sinkmetrics.DatabendSpoolDuration.DeleteLabelValues(k, f) + sinkmetrics.DatabendSpoolLoad.DeleteLabelValues(k, f) + sinkmetrics.DatabendSpoolRotations.DeleteLabelValues(k, f) + }, + } +} diff --git a/downstreamadapter/sink/databend/native.go b/downstreamadapter/sink/databend/native.go new file mode 100644 index 0000000000..3b2a9179e5 --- /dev/null +++ b/downstreamadapter/sink/databend/native.go @@ -0,0 +1,263 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/chunk" +) + +func schemaFromNative(info *common.TableInfo, database, conflictKey string) (tableSchema, error) { + if info == nil { + return tableSchema{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("missing authoritative table metadata") + } + if info.IsPartitionTable() { + return tableSchema{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("partition dispatchers are outside the single-table writer proof; partition DDL is unsupported") + } + s := tableSchema{database: info.GetSchemaName(), table: info.GetTableName(), primaryKey: info.GetPrimaryKeyColumnNames(), source: info.GetSchemaName() + "." + info.GetTableName()} + if database != "" { + s.database = database + } + if strings.HasSuffix(s.table, "_cdc_raw") || strings.HasSuffix(s.table, "_cdc_schema_epoch") { + return tableSchema{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("source table uses a reserved CDC suffix") + } + for _, c := range info.GetColumns() { + if !common.IsColCDCVisible(c) { + continue + } + typ, enc, err := nativeColumnType(c) + if err != nil { + return tableSchema{}, err + } + def, expr, err := nativeDefault(c, enc) + if err != nil { + return tableSchema{}, err + } + s.columns = append(s.columns, column{name: c.Name.O, encoding: enc, sourceID: c.ID, typ: typ, nullable: !mysql.HasNotNullFlag(c.GetFlag()) || c.IsGenerated(), defaultSQL: def, defaultExpr: expr}) + } + if conflictKey != "" { + // Match bend-cdc's persisted single-column override. Never silently + // replace a composite key with only one of its members. + if len(s.primaryKey) > 1 { + return tableSchema{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("conflict-key cannot override a composite source key") + } + actual := "" + for _, col := range s.columns { + if strings.EqualFold(col.name, conflictKey) { + actual = col.name + break + } + } + if actual == "" { + return tableSchema{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("conflict-key must name a visible source column") + } + s.primaryKey = []string{actual} + } + return s, s.validate() +} + +func nativeColumnType(c *model.ColumnInfo) (string, columnEncoding, error) { + signed := func(a, b string) string { + if mysql.HasUnsignedFlag(c.GetFlag()) { + return b + } + return a + } + switch c.GetType() { + case mysql.TypeTiny: + return signed("INT8", "UINT8"), scalarColumn, nil + case mysql.TypeShort: + return signed("INT16", "UINT16"), scalarColumn, nil + case mysql.TypeInt24, mysql.TypeLong: + return signed("INT32", "UINT32"), scalarColumn, nil + case mysql.TypeLonglong: + return signed("INT64", "UINT64"), scalarColumn, nil + case mysql.TypeYear: + return "INT16", scalarColumn, nil + case mysql.TypeFloat: + return "FLOAT32", scalarColumn, nil + case mysql.TypeDouble: + return "FLOAT64", scalarColumn, nil + case mysql.TypeNewDecimal: + p, scale := c.GetFlen(), c.GetDecimal() + if p <= 0 || p > 76 || scale < 0 || scale > p { + return "", 0, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid decimal metadata") + } + return fmt.Sprintf("DECIMAL(%d,%d)", p, scale), scalarColumn, nil + case mysql.TypeDate, mysql.TypeNewDate: + return "DATE", scalarColumn, nil + case mysql.TypeDatetime, mysql.TypeTimestamp: + return "TIMESTAMP", scalarColumn, nil + case mysql.TypeDuration, mysql.TypeEnum, mysql.TypeSet: + return "VARCHAR", scalarColumn, nil + case mysql.TypeBit: + return "UINT64", scalarColumn, nil + case mysql.TypeJSON: + return "VARIANT", jsonColumn, nil + case mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: + if c.GetCharset() == "binary" { + return "BINARY", binaryColumn, nil + } + return "VARCHAR", scalarColumn, nil + default: + return "", 0, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported native column type") + } +} + +func nativeValues(info *common.TableInfo, row chunk.Row, location *time.Location) (map[string]any, error) { + if location == nil { + location = time.UTC + } + values := make(map[string]any, len(info.GetColumns())) + for _, col := range info.GetColumns() { + if !common.IsColCDCVisible(col) { + continue + } + offset, ok := info.GetRowColumnsOffset()[col.ID] + if !ok || offset >= row.Len() { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("row does not match authoritative schema") + } + d := row.GetDatum(offset, &col.FieldType) + var v any + if !d.IsNull() { + switch col.GetType() { + case mysql.TypeEnum: + v = d.GetMysqlEnum().Name + case mysql.TypeSet: + v = d.GetMysqlSet().Name + case mysql.TypeBit: + n, err := d.GetMysqlBit().ToInt(types.DefaultStmtNoWarningContext) + if err != nil { + return nil, errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid bit datum") + } + v = strconv.FormatUint(n, 10) + case mysql.TypeJSON: + v = d.GetMysqlJSON().String() + case mysql.TypeNewDecimal: + v = d.GetMysqlDecimal().String() + case mysql.TypeDate, mysql.TypeNewDate, mysql.TypeDatetime: + v = d.GetMysqlTime().String() + case mysql.TypeTimestamp: + stamp := d.GetMysqlTime() + if !stamp.IsZero() { + if err := stamp.ConvertTimeZone(location, time.UTC); err != nil { + return nil, errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot normalize timestamp timezone") + } + } + v = stamp.String() + case mysql.TypeDuration: + v = d.GetMysqlDuration().String() + case mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: + v = d.GetBytes() + default: + text, err := d.ToString() + if err != nil { + return nil, errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot format native datum") + } + v = text + } + } + values[col.Name.O] = v + } + return values, nil +} + +func normalizeChange(info *common.TableInfo, schema tableSchema, change commonEvent.RowChange, ts uint64, location *time.Location) (normalizedRow, error) { + action := rowAction(change.RowType.String()) + image := change.Row + if change.RowType == common.RowTypeDelete { + image = change.PreRow + } + values, err := nativeValues(info, image, location) + if err != nil { + return normalizedRow{}, err + } + row, err := normalizeNativeRow(schema, action, ts, values) + if err != nil { + return normalizedRow{}, err + } + if change.RowType == common.RowTypeUpdate { + before, err := nativeValues(info, change.PreRow, location) + if err != nil { + return normalizedRow{}, err + } + old, err := normalizeNativeRow(schema, deleteAction, ts, before) + if err != nil { + return normalizedRow{}, err + } + if old.id != row.id { + return normalizedRow{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("conflict key changed in an unsplit UPDATE; use a stable conflict-key or split change mode") + } + } + return row, nil +} + +func sqlString(s string) string { + return "'" + strings.ReplaceAll(strings.ReplaceAll(s, `\`, `\\`), "'", "''") + "'" +} + +func nativeDefault(c *model.ColumnInfo, encoding columnEncoding) (string, bool, error) { + value := c.GetDefaultValue() + if c.IsGenerated() || value == nil { + return "", c.DefaultIsExpr, nil + } + var text string + switch v := value.(type) { + case string: + text = v + case []byte: + text = string(v) + case bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + text = fmt.Sprint(v) + default: + return "", false, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported column default representation") + } + expr := c.DefaultIsExpr + if (c.GetType() == mysql.TypeTimestamp || c.GetType() == mysql.TypeDatetime) && strings.HasPrefix(strings.ToUpper(text), "CURRENT_TIMESTAMP") { + expr = true + } + if expr { + return "", true, nil + } // Initial DML carries every value explicitly. + if c.GetType() == mysql.TypeBit { + n, err := types.BinaryLiteral([]byte(text)).ToInt(types.DefaultStmtNoWarningContext) + if err != nil { + return "", false, errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid bit default") + } + return strconv.FormatUint(n, 10), false, nil + } + if encoding == binaryColumn { + return "unhex(" + sqlString(hex.EncodeToString([]byte(text))) + ")", false, nil + } + if encoding == jsonColumn { + if !json.Valid([]byte(text)) { + return "", false, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid JSON default") + } + return "parse_json(" + sqlString(text) + ")", false, nil + } + return sqlString(text), false, nil +} diff --git a/downstreamadapter/sink/databend/native_test.go b/downstreamadapter/sink/databend/native_test.go new file mode 100644 index 0000000000..fc83420538 --- /dev/null +++ b/downstreamadapter/sink/databend/native_test.go @@ -0,0 +1,282 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "encoding/hex" + "encoding/json" + "strconv" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/chunk" + "github.com/stretchr/testify/require" +) + +func nativeTestInfo() *common.TableInfo { + id := types.NewFieldType(mysql.TypeLonglong) + id.SetFlag(mysql.PriKeyFlag | mysql.NotNullFlag) + value := types.NewFieldType(mysql.TypeVarchar) + value.SetCharset("utf8mb4") + return common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr("t"), PKIsHandle: true, Columns: []*model.ColumnInfo{{ID: 1, Name: ast.NewCIStr("id"), FieldType: *id, State: model.StatePublic}, {ID: 2, Name: ast.NewCIStr("value"), FieldType: *value, State: model.StatePublic}}}) +} + +func nativeTestEvent(info *common.TableInfo, ts uint64, n int) *commonEvent.DMLEvent { + e := commonEvent.NewDMLEvent(common.NewDispatcherID(), 1, ts-1, ts, info) + e.Rows = chunk.NewChunkWithCapacity(info.GetFieldSlice(), n) + for i := range n { + e.Rows.AppendInt64(0, int64(i)) + e.Rows.AppendString(1, "payload") + e.RowTypes = append(e.RowTypes, common.RowTypeInsert) + } + e.Length = int32(n) + return e +} + +func TestNativeDatumAdapterAndRawChangeContract(t *testing.T) { + info := nativeTestInfo() + enumType := types.NewFieldType(mysql.TypeEnum) + enumType.SetElems([]string{"2", "x"}) + info = common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr("t"), PKIsHandle: true, Columns: []*model.ColumnInfo{info.GetColumns()[0], {ID: 2, Name: ast.NewCIStr("value"), FieldType: *enumType, State: model.StatePublic}}}) + s, err := schemaFromNative(info, "", "") + require.NoError(t, err) + chk := chunk.NewChunkWithCapacity(info.GetFieldSlice(), 2) + chk.AppendInt64(0, 1) + chk.AppendEnum(1, types.Enum{Name: "2", Value: 1}) + chk.AppendInt64(0, 2) + chk.AppendEnum(1, types.Enum{Name: "x", Value: 2}) + row, err := normalizeChange(info, s, commonEvent.RowChange{Row: chk.GetRow(0), RowType: common.RowTypeInsert}, 100, time.UTC) + require.NoError(t, err) + var payload map[string]any + require.NoError(t, json.Unmarshal([]byte(row.payload), &payload)) + require.Equal(t, "2", payload["value"]) + _, err = normalizeChange(info, s, commonEvent.RowChange{Row: chk.GetRow(1), PreRow: chk.GetRow(0), RowType: common.RowTypeUpdate}, 100, time.UTC) + require.ErrorContains(t, err, "split change mode") + // The normal mode keeps split DELETE and INSERT as distinct, ordered actions. + del, err := normalizeChange(info, s, commonEvent.RowChange{PreRow: chk.GetRow(0), RowType: common.RowTypeDelete}, 100, time.UTC) + require.NoError(t, err) + ins, err := normalizeChange(info, s, commonEvent.RowChange{Row: chk.GetRow(1), RowType: common.RowTypeInsert}, 100, time.UTC) + require.NoError(t, err) + require.Equal(t, "1", del.id) + require.Equal(t, "2", ins.id) +} + +func TestNativeTypeMapping(t *testing.T) { + for _, tc := range []struct { + typ byte + unsigned bool + want string + }{ + {mysql.TypeTiny, false, "INT8"}, {mysql.TypeTiny, true, "UINT8"}, {mysql.TypeShort, false, "INT16"}, {mysql.TypeLong, true, "UINT32"}, {mysql.TypeLonglong, true, "UINT64"}, {mysql.TypeYear, false, "INT16"}, {mysql.TypeDuration, false, "VARCHAR"}, {mysql.TypeDate, false, "DATE"}, {mysql.TypeTimestamp, false, "TIMESTAMP"}, {mysql.TypeJSON, false, "VARIANT"}, + } { + ft := types.NewFieldType(tc.typ) + if tc.unsigned { + ft.SetFlag(mysql.UnsignedFlag) + } + typ, _, err := nativeColumnType(&model.ColumnInfo{FieldType: *ft}) + require.NoError(t, err) + require.Equal(t, tc.want, typ) + } + ft := types.NewFieldType(mysql.TypeNewDecimal) + ft.SetFlen(65) + ft.SetDecimal(30) + typ, _, err := nativeColumnType(&model.ColumnInfo{FieldType: *ft}) + require.NoError(t, err) + require.Equal(t, "DECIMAL(65,30)", typ) + ft = types.NewFieldType(mysql.TypeBlob) + ft.SetCharset("binary") + typ, enc, err := nativeColumnType(&model.ColumnInfo{FieldType: *ft}) + require.NoError(t, err) + require.Equal(t, "BINARY", typ) + require.Equal(t, binaryColumn, enc) +} + +func TestNativeDatumValuesPreserveExactRepresentations(t *testing.T) { + base := nativeTestInfo().GetColumns()[0] + makeColumn := func(id int64, name string, tp byte) *model.ColumnInfo { + ft := types.NewFieldType(tp) + return &model.ColumnInfo{ID: id, Name: ast.NewCIStr(name), FieldType: *ft, State: model.StatePublic} + } + binary := makeColumn(2, "binary", mysql.TypeBlob) + binary.SetCharset("binary") + decimal := makeColumn(3, "decimal", mysql.TypeNewDecimal) + decimal.SetFlen(65) + decimal.SetDecimal(30) + document := makeColumn(4, "document", mysql.TypeJSON) + set := makeColumn(5, "set_label", mysql.TypeSet) + elems := make([]string, 64) + for i := range elems { + elems[i] = strconv.Itoa(i) + } + set.SetElems(elems) + stamp := makeColumn(6, "stamp", mysql.TypeTimestamp) + stamp.SetDecimal(6) + null := makeColumn(7, "sql_null", mysql.TypeJSON) + jsonNull := makeColumn(8, "json_null", mysql.TypeJSON) + info := common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr("typed"), PKIsHandle: true, Columns: []*model.ColumnInfo{base, binary, decimal, document, set, stamp, null, jsonNull}}) + s, err := schemaFromNative(info, "", "") + require.NoError(t, err) + chk := chunk.NewChunkWithCapacity(info.GetFieldSlice(), 1) + chk.AppendInt64(0, 1) + data := make([]byte, 256) + for i := range data { + data[i] = byte(i) + } + chk.AppendBytes(1, data) + dec := new(types.MyDecimal) + require.NoError(t, dec.FromString([]byte("12345678901234567890.123456789"))) + chk.AppendMyDecimal(2, dec) + doc, err := types.ParseBinaryJSONFromString(`{"uint":18446744073709551615}`) + require.NoError(t, err) + chk.AppendJSON(3, doc) + chk.AppendSet(4, types.Set{Name: "63", Value: uint64(1) << 63}) + chk.AppendTime(5, types.NewTime(types.FromDate(2026, 10, 25, 1, 2, 3, 123456), mysql.TypeTimestamp, 6)) + chk.AppendNull(6) + nilDoc, err := types.ParseBinaryJSONFromString("null") + require.NoError(t, err) + chk.AppendJSON(7, nilDoc) + row, err := normalizeChange(info, s, commonEvent.RowChange{Row: chk.GetRow(0), RowType: common.RowTypeInsert}, 100, time.UTC) + require.NoError(t, err) + var payload map[string]any + require.NoError(t, json.Unmarshal([]byte(row.payload), &payload)) + require.Equal(t, hex.EncodeToString(data), payload["binary"]) + require.Equal(t, "12345678901234567890.123456789", payload["decimal"]) + require.Equal(t, doc.String(), payload["document"]) + require.Equal(t, "63", payload["set_label"]) + require.Equal(t, "2026-10-25 01:02:03.123456", payload["stamp"]) + require.Nil(t, payload["sql_null"]) + require.Equal(t, "null", payload["json_null"]) +} + +func TestNativeDefaultEncoding(t *testing.T) { + col := nativeTestInfo().GetColumns()[1].Clone() + col.DefaultValue = []byte("a'\\b") + def, expr, err := nativeDefault(col, scalarColumn) + require.NoError(t, err) + require.False(t, expr) + require.Equal(t, `'a''\\b'`, def) + def, expr, err = nativeDefault(col, binaryColumn) + require.NoError(t, err) + require.False(t, expr) + require.Equal(t, `unhex('61275c62')`, def) + ft := types.NewFieldType(mysql.TypeBit) + col.FieldType = *ft + require.NoError(t, col.SetDefaultValue(string([]byte{0xff, 0xff}))) + def, expr, err = nativeDefault(col, scalarColumn) + require.NoError(t, err) + require.False(t, expr) + require.Equal(t, "65535", def) + ft = types.NewFieldType(mysql.TypeTimestamp) + col.FieldType = *ft + col.DefaultValue = "CURRENT_TIMESTAMP" + def, expr, err = nativeDefault(col, scalarColumn) + require.NoError(t, err) + require.True(t, expr) + require.Empty(t, def) +} + +func TestNativeTimestampUsesSourceTimezoneButDatetimeKeepsWallClock(t *testing.T) { + id := nativeTestInfo().GetColumns()[0] + timestampType := types.NewFieldType(mysql.TypeTimestamp) + timestampType.SetDecimal(6) + datetimeType := types.NewFieldType(mysql.TypeDatetime) + datetimeType.SetDecimal(6) + info := common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr("time_values"), PKIsHandle: true, Columns: []*model.ColumnInfo{id, {ID: 2, Name: ast.NewCIStr("stamp"), FieldType: *timestampType, State: model.StatePublic}, {ID: 3, Name: ast.NewCIStr("wall"), FieldType: *datetimeType, State: model.StatePublic}}}) + schema, err := schemaFromNative(info, "", "") + require.NoError(t, err) + chk := chunk.NewChunkWithCapacity(info.GetFieldSlice(), 1) + chk.AppendInt64(0, 1) + chk.AppendTime(1, types.NewTime(types.FromDate(2026, 10, 25, 8, 2, 3, 456789), mysql.TypeTimestamp, 6)) + chk.AppendTime(2, types.NewTime(types.FromDate(2026, 10, 25, 8, 2, 3, 456789), mysql.TypeDatetime, 6)) + row, err := normalizeChange(info, schema, commonEvent.RowChange{Row: chk.GetRow(0), RowType: common.RowTypeInsert}, 100, time.FixedZone("UTC+8", 8*60*60)) + require.NoError(t, err) + var values map[string]any + require.NoError(t, json.Unmarshal([]byte(row.payload), &values)) + require.Equal(t, "2026-10-25 00:02:03.456789", values["stamp"]) + require.Equal(t, "2026-10-25 08:02:03.456789", values["wall"]) +} + +// TiCDC's GetPrimaryKeyColumnNames intentionally falls back to a NOT NULL +// UNIQUE index. Do not describe or accidentally narrow this to declared PKs. +func TestNativeImplicitUniqueConflictKey(t *testing.T) { + base := nativeTestInfo() + value := *base.GetColumns()[1] + value.Offset = 1 + id := *base.GetColumns()[0] + id.FieldType = *types.NewFieldType(mysql.TypeLonglong) + id.SetFlag(mysql.NotNullFlag) + info := common.WrapTableInfo("lake", &model.TableInfo{ + ID: 2, Name: ast.NewCIStr("implicit_key"), + Columns: []*model.ColumnInfo{&id, &value}, + Indices: []*model.IndexInfo{{ + ID: 1, Name: ast.NewCIStr("uk_id"), Unique: true, State: model.StatePublic, + Columns: []*model.IndexColumn{{Name: id.Name, Offset: 0, Length: types.UnspecifiedLength}}, + }}, + }) + schema, err := schemaFromNative(info, "", "") + require.NoError(t, err) + require.Equal(t, []string{"id"}, schema.primaryKey) + nullableID := id + nullableID.SetFlag(0) + info = common.WrapTableInfo("lake", &model.TableInfo{ + ID: 3, Name: ast.NewCIStr("nullable_unique"), Columns: []*model.ColumnInfo{&nullableID, &value}, + Indices: []*model.IndexInfo{{ + ID: 1, Name: ast.NewCIStr("uk_id"), Unique: true, State: model.StatePublic, + Columns: []*model.IndexColumn{{Name: id.Name, Offset: 0, Length: types.UnspecifiedLength}}, + }}, + }) + _, err = schemaFromNative(info, "", "") + require.Error(t, err, "nullable UNIQUE is not an implicit conflict key") +} + +func TestNativeConfiguredConflictKey(t *testing.T) { + base := nativeTestInfo() + id, value := *base.GetColumns()[0], *base.GetColumns()[1] + id.SetFlag(0) + value.Offset = 1 + info := common.WrapTableInfo("lake", &model.TableInfo{ID: 7, Name: ast.NewCIStr("keyless"), Columns: []*model.ColumnInfo{&id, &value}}) + _, err := schemaFromNative(info, "", "") + require.Error(t, err) + schema, err := schemaFromNative(info, "", "ID") + require.NoError(t, err) + require.Equal(t, []string{"id"}, schema.primaryKey) + row, err := normalizeNativeRow(schema, insertAction, 100, map[string]any{"id": 1, "value": "v"}) + require.NoError(t, err) + require.NotEmpty(t, row.id) + _, err = normalizeNativeRow(schema, insertAction, 100, map[string]any{"id": nil, "value": "v"}) + require.Error(t, err, "the configured column may be nullable in source metadata, but identity values must not be NULL") + _, err = schemaFromNative(info, "", "missing") + require.Error(t, err) + // The explicit single-column override also matches bend-cdc on keyed tables. + schema, err = schemaFromNative(base, "", "VALUE") + require.NoError(t, err) + require.Equal(t, []string{"value"}, schema.primaryKey) + composite := common.WrapTableInfo("lake", &model.TableInfo{ID: 8, Name: ast.NewCIStr("composite"), Columns: []*model.ColumnInfo{&id, &value}, Indices: []*model.IndexInfo{{ID: 1, Primary: true, Unique: true, State: model.StatePublic, Columns: []*model.IndexColumn{{Name: id.Name, Offset: 0}, {Name: value.Name, Offset: 1}}}}}) + _, err = schemaFromNative(composite, "", "id") + require.ErrorContains(t, err, "composite") + // Arbitrary logical keys are not necessarily split by TiCDC's PK/UK scanner. + // A mutated key in an unsplit UPDATE must stop instead of orphaning old rows. + schema, err = schemaFromNative(info, "", "id") + require.NoError(t, err) + event := nativeTestEvent(info, 100, 2) + _, err = normalizeChange(info, schema, commonEvent.RowChange{PreRow: event.Rows.GetRow(0), Row: event.Rows.GetRow(1), RowType: common.RowTypeUpdate}, 100, time.UTC) + require.ErrorContains(t, err, "stable conflict-key") +} diff --git a/downstreamadapter/sink/databend/row.go b/downstreamadapter/sink/databend/row.go new file mode 100644 index 0000000000..3991c3452c --- /dev/null +++ b/downstreamadapter/sink/databend/row.go @@ -0,0 +1,245 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package databend owns the native Databend sink implementation. Its row and SQL +// layer is ported from bend-cdc's TiDB path, without the Canal JSON decoder. +package databend + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "math" + "sort" + "strings" + "unicode/utf8" + + "github.com/pingcap/ticdc/pkg/errors" +) + +type columnEncoding uint8 + +const ( + scalarColumn columnEncoding = iota + binaryColumn + jsonColumn +) + +type column struct { + name string + encoding columnEncoding + sourceID int64 + typ string + nullable bool + defaultSQL string + defaultExpr bool +} + +// tableSchema describes the target projection, not an inferred source schema. +type tableSchema struct { + database string + source string + table string + columns []column + primaryKey []string +} + +func (s tableSchema) validate() error { + validName := func(name string) bool { return name != "" && !strings.ContainsRune(name, 0) } + if !validName(s.database) || !validName(s.table) || len(s.columns) == 0 || len(s.primaryKey) == 0 { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("table metadata and a primary key are required") + } + columns := make(map[string]struct{}, len(s.columns)) + folded := make(map[string]struct{}, len(s.columns)) + for _, col := range s.columns { + if !validName(col.name) || col.encoding > jsonColumn { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid column metadata") + } + lower := strings.ToLower(col.name) + if _, ok := folded[lower]; ok { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("duplicate column name") + } + folded[lower] = struct{}{} + columns[col.name] = struct{}{} + } + keys := make(map[string]struct{}, len(s.primaryKey)) + for _, key := range s.primaryKey { + if _, ok := columns[key]; !ok { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("primary key is missing from columns") + } + if _, ok := keys[key]; ok { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("duplicate primary key column") + } + keys[key] = struct{}{} + } + return nil +} + +func (s tableSchema) canonicalKeys() []string { + keys := append([]string(nil), s.primaryKey...) + sort.Slice(keys, func(i, j int) bool { return strings.ToLower(keys[i]) < strings.ToLower(keys[j]) }) + return keys +} + +type rowAction string + +const ( + insertAction rowAction = "insert" + updateAction rowAction = "update" + deleteAction rowAction = "delete" +) + +// normalizedRow is an owned, immutable raw payload. The SQL writer cannot +// accidentally run Canal ordinal/Latin-1 decoding on this representation again. +type normalizedRow struct { + id string + payload string + commitTs uint64 + action rowAction +} + +// normalizeNativeRow accepts labels for ENUM/SET, original bytes for BINARY and +// exact JSON documents. The DMLEvent adapter must extract these from typed datum +// values; this function deliberately does not infer ordinal encodings. +func normalizeNativeRow(schema tableSchema, action rowAction, commitTs uint64, values map[string]any) (normalizedRow, error) { + if err := schema.validate(); err != nil { + return normalizedRow{}, err + } + if commitTs == 0 || commitTs > math.MaxInt64 { + return normalizedRow{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("commit TSO cannot be represented in raw BIGINT") + } + if action != insertAction && action != updateAction && action != deleteAction { + return normalizedRow{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported row action") + } + data := make(map[string]any, len(values)) + for _, col := range schema.columns { + value, exists := values[col.name] + if !exists { + if action == deleteAction { + continue // A tombstone needs the old primary key, not an after image. + } + return normalizedRow{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("incomplete after image") + } + normalized, err := normalizeNativeValue(col.encoding, value) + if err != nil { + return normalizedRow{}, err + } + data[col.name] = normalized + } + for name := range values { + if _, exists := data[name]; !exists { + return normalizedRow{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("value has no column metadata") + } + } + id, err := conflictIdentity(data, schema.canonicalKeys()) + if err != nil { + return normalizedRow{}, err + } + payload, err := json.Marshal(data) + if err != nil { + return normalizedRow{}, errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot encode normalized row") + } + return normalizedRow{id: id, payload: string(payload), commitTs: commitTs, action: action}, nil +} + +func normalizeNativeValue(encoding columnEncoding, value any) (any, error) { + if value == nil { + return nil, nil + } + if encoding == binaryColumn { + switch v := value.(type) { + case []byte: + return hex.EncodeToString(v), nil + case string: + return hex.EncodeToString([]byte(v)), nil // Original bytes, not Canal Latin-1. + default: + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("binary datum must contain bytes") + } + } + if encoding == jsonColumn { + var document []byte + switch v := value.(type) { + case string: + document = []byte(v) + case []byte: + document = v + case json.RawMessage: + document = v + default: + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("JSON datum must contain an exact document") + } + if !json.Valid(document) { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid JSON document") + } + return string(document), nil // Includes JSON null, distinct from SQL NULL. + } + switch v := value.(type) { + case string: + if !utf8.ValidString(v) { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid text encoding") + } + return strings.Clone(v), nil + case []byte: + if !utf8.Valid(v) { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid text encoding") + } + return string(v), nil + case bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + return value, nil + case json.Number: + text := string(v) + if len(text) == 0 || !json.Valid([]byte(text)) || (text[0] != '-' && (text[0] < '0' || text[0] > '9')) { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid exact JSON number") + } + return json.Number(strings.Clone(string(v))), nil + case float32: + if !math.IsNaN(float64(v)) && !math.IsInf(float64(v), 0) { + return v, nil + } + case float64: + if !math.IsNaN(v) && !math.IsInf(v, 0) { + return v, nil + } + } + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported native scalar datum") +} + +// conflictIdentity preserves bend-cdc's single-key text and canonical composite +// JSON shape. It intentionally contains no arrival time, batch ID or local seq. +func conflictIdentity(values map[string]any, keys []string) (string, error) { + type keyPart struct { + Name string `json:"name"` + Value string `json:"value"` + } + parts := make([]keyPart, 0, len(keys)) + for _, key := range keys { + value, ok := values[key] + if !ok || value == nil { + return "", errors.ErrDatabendInvalidData.GenWithStackByArgs("primary key is missing or null") + } + text := fmt.Sprint(value) + if len(keys) == 1 { + if strings.TrimSpace(text) == "" { + return "", errors.ErrDatabendInvalidData.GenWithStackByArgs("empty primary key") + } + return text, nil + } + parts = append(parts, keyPart{Name: key, Value: text}) + } + encoded, err := json.Marshal(parts) + if err != nil { + return "", errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot encode primary key") + } + return string(encoded), nil +} diff --git a/downstreamadapter/sink/databend/row_test.go b/downstreamadapter/sink/databend/row_test.go new file mode 100644 index 0000000000..2e64d59905 --- /dev/null +++ b/downstreamadapter/sink/databend/row_test.go @@ -0,0 +1,140 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "encoding/hex" + "encoding/json" + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func testSchema() tableSchema { + return tableSchema{database: "lake", table: "orders", columns: []column{{name: "id"}, {name: "value"}}, primaryKey: []string{"id"}} +} + +func testRow(t *testing.T, action rowAction, ts uint64, id int) normalizedRow { + t.Helper() + row, err := normalizeNativeRow(testSchema(), action, ts, map[string]any{"id": id, "value": "payload"}) + require.NoError(t, err) + return row +} + +func TestNativeRowNormalizesExactlyOnce(t *testing.T) { + schema := tableSchema{ + database: "lake", table: "native_values", + columns: []column{ + {name: "id"}, + {name: "enum_label"}, + {name: "set_label"}, + {name: "binary", encoding: binaryColumn}, + {name: "document", encoding: jsonColumn}, + {name: "sql_null", encoding: jsonColumn}, + {name: "json_null", encoding: jsonColumn}, + {name: "decimal"}, + }, + primaryKey: []string{"id"}, + } + bytes := make([]byte, 256) + for i := range bytes { + bytes[i] = byte(i) + } + wantHex := hex.EncodeToString(bytes) + document := `{"uint":18446744073709551615,"decimal":12345678901234567890.123456789}` + row, err := normalizeNativeRow(schema, insertAction, 123, map[string]any{ + "id": uint64(math.MaxUint64), "enum_label": "2", "set_label": "last", + "binary": bytes, "document": json.RawMessage(document), "sql_null": nil, "json_null": "null", + "decimal": "12345678901234567890.123456789", + }) + require.NoError(t, err) + clear(bytes) // The payload must no longer reference source chunk bytes. + var values map[string]json.RawMessage + require.NoError(t, json.Unmarshal([]byte(row.payload), &values)) + require.JSONEq(t, `"2"`, string(values["enum_label"]), "a numeric ENUM label is not an ordinal") + require.JSONEq(t, `"last"`, string(values["set_label"])) + require.JSONEq(t, `"`+wantHex+`"`, string(values["binary"])) + require.Equal(t, "18446744073709551615", string(values["id"])) + require.Equal(t, "18446744073709551615", row.id) + var actualDocument string + require.NoError(t, json.Unmarshal(values["document"], &actualDocument)) + require.Equal(t, document, actualDocument) + require.Equal(t, "null", string(values["sql_null"])) + require.Equal(t, `"null"`, string(values["json_null"])) + require.JSONEq(t, `"12345678901234567890.123456789"`, string(values["decimal"])) +} + +func TestNativeRowRetainsCompositeIdentity(t *testing.T) { + schema := tableSchema{ + database: "lake", table: "events", + columns: []column{{name: "tenant"}, {name: "id"}}, primaryKey: []string{"tenant", "id"}, + } + row, err := normalizeNativeRow(schema, deleteAction, 100, map[string]any{"tenant": "a|b", "id": int64(42)}) + require.NoError(t, err) + require.Equal(t, `[{"name":"id","value":"42"},{"name":"tenant","value":"a|b"}]`, row.id) + schema.primaryKey = []string{"id", "tenant"} + replayed, err := normalizeNativeRow(schema, deleteAction, 100, map[string]any{"tenant": "a|b", "id": "42"}) + require.NoError(t, err) + require.Equal(t, row.id, replayed.id, "typed and Canal-style string keys retain the baseline identity") +} + +func TestNativeRowRejectsInvalidInputs(t *testing.T) { + for _, tc := range []struct { + name string + action rowAction + ts uint64 + values map[string]any + }{ + {"zero TSO", insertAction, 0, map[string]any{"id": 1, "value": "v"}}, + {"TSO overflow", insertAction, math.MaxUint64, map[string]any{"id": 1, "value": "v"}}, + {"unknown action", "replace", 1, map[string]any{"id": 1, "value": "v"}}, + {"missing after image", updateAction, 1, map[string]any{"id": 1}}, + {"null key", insertAction, 1, map[string]any{"id": nil, "value": "v"}}, + {"empty key", insertAction, 1, map[string]any{"id": "", "value": "v"}}, + {"extra column", insertAction, 1, map[string]any{"id": 1, "value": "v", "extra": 1}}, + {"non finite", insertAction, 1, map[string]any{"id": 1, "value": math.Inf(1)}}, + {"invalid number", insertAction, 1, map[string]any{"id": 1, "value": json.Number("not-a-number")}}, + {"invalid UTF8", insertAction, 1, map[string]any{"id": 1, "value": string([]byte{0xff})}}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := normalizeNativeRow(testSchema(), tc.action, tc.ts, tc.values) + require.Error(t, err) + }) + } + _, err := normalizeNativeRow(testSchema(), deleteAction, 1, map[string]any{"id": 1}) + require.NoError(t, err, "a tombstone need not invent a full after image") + for _, encoding := range []columnEncoding{binaryColumn, jsonColumn} { + _, err := normalizeNativeValue(encoding, 123) + require.Error(t, err) + } + _, err = normalizeNativeValue(jsonColumn, "not-json") + require.Error(t, err) +} + +func TestTableMetadataValidation(t *testing.T) { + for _, change := range []func(*tableSchema){ + func(s *tableSchema) { s.database = "" }, + func(s *tableSchema) { s.table = "bad\x00name" }, + func(s *tableSchema) { s.primaryKey = nil }, + func(s *tableSchema) { s.primaryKey = []string{"missing"} }, + func(s *tableSchema) { s.primaryKey = []string{"id", "id"} }, + func(s *tableSchema) { s.columns = append(s.columns, column{name: "ID"}) }, + } { + s := testSchema() + change(&s) + require.Error(t, s.validate()) + } +} diff --git a/downstreamadapter/sink/databend/runtime_test.go b/downstreamadapter/sink/databend/runtime_test.go new file mode 100644 index 0000000000..0d1bd44edb --- /dev/null +++ b/downstreamadapter/sink/databend/runtime_test.go @@ -0,0 +1,497 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/sink/spool" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/stretchr/testify/require" +) + +// apiFixture models only the HTTP boundary and journal persistence. It does not +// execute Databend SQL and must not be cited as proof of server atomicity. +type apiFixture struct { + mu sync.Mutex + columns []columnDef + epoch uint64 + applied bool + plan string + sql []string + rawRows int + merges int + truncates int + unknownMerge bool + unknownRawAt int + targetMissing bool + remoteBytes string + maxAddTime string + failAfterTruncate bool +} + +func newAPIFixture(t *testing.T) (*apiFixture, *httptest.Server) { + s, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + f := &apiFixture{columns: toSchemaDef(s).Columns} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + SQL string `json:"sql"` + Params []json.RawMessage `json:"params"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + w.WriteHeader(400) + return + } + f.mu.Lock() + defer f.mu.Unlock() + f.sql = append(f.sql, req.SQL) + data := [][]string{} + switch { + case strings.HasPrefix(req.SQL, "SELECT epoch"): + if f.epoch > 0 { + data = [][]string{{strconv.FormatUint(f.epoch, 10), strconv.FormatBool(f.applied), f.plan}} + } + case strings.HasPrefix(req.SQL, "SELECT COALESCE(SUM(data_compressed_size)"): + value := f.remoteBytes + if value == "" { + value = "0" + } + data = [][]string{{value}} + case strings.HasPrefix(req.SQL, "SELECT COUNT(*)::VARCHAR FROM system.tables"): + value := "1" + if f.targetMissing { + value = "0" + } + data = [][]string{{value}} + case strings.HasPrefix(req.SQL, "CREATE TABLE") && !strings.Contains(req.SQL, "_cdc_"): + if f.targetMissing { + f.targetMissing = false + var p schemaPlan + _ = json.Unmarshal([]byte(f.plan), &p) + if len(p.Schema.Columns) > 0 { + f.columns = p.Schema.Columns + } + } + case strings.HasPrefix(req.SQL, "DESCRIBE"): + for _, c := range f.columns { + nullable := "NO" + if c.Nullable { + nullable = "YES" + } + data = append(data, []string{c.Name, c.Type, nullable}) + } + case strings.HasPrefix(req.SQL, "MERGE") && strings.Contains(req.SQL, "_cdc_schema_epoch"): + var e uint64 + var a bool + var p string + _ = json.Unmarshal(req.Params[0], &e) + _ = json.Unmarshal(req.Params[1], &a) + _ = json.Unmarshal(req.Params[2], &p) + if f.failAfterTruncate && a { + f.failAfterTruncate = false + w.WriteHeader(503) + return + } + f.epoch, f.applied, f.plan = e, a, p + case strings.HasPrefix(req.SQL, "INSERT"): + f.rawRows += len(req.Params) / 6 + if f.unknownRawAt > 0 && f.rawRows >= f.unknownRawAt { + w.WriteHeader(503) + return + } + case strings.HasPrefix(req.SQL, "MERGE"): + if f.unknownMerge { + w.WriteHeader(503) + return + } + f.merges++ + case strings.HasPrefix(req.SQL, "TRUNCATE"): + f.truncates++ + f.rawRows = 0 + case strings.HasPrefix(req.SQL, "DROP TABLE") && strings.Contains(req.SQL, "_cdc_raw"): + f.rawRows = 0 + case strings.HasPrefix(req.SQL, "ALTER TABLE"): + var plan schemaPlan + _ = json.Unmarshal([]byte(f.plan), &plan) + f.columns = plan.Schema.Columns + case strings.HasPrefix(req.SQL, "SELECT COALESCE(MAX(add_time)"): + data = [][]string{{f.maxAddTime}} + case strings.HasPrefix(req.SQL, "SELECT ?"): + data = [][]string{{"1"}} + } + _ = json.NewEncoder(w).Encode(map[string]any{"id": r.Header.Get("X-DATABEND-QUERY-ID"), "state": "Succeeded", "data": data}) + })) + t.Cleanup(server.Close) + return f, server +} + +func runtimeSink(t *testing.T, server *httptest.Server, extra string) *Sink { + t.Helper() + uri, err := url.Parse(strings.Replace(server.URL, "http://", "databend+http://", 1) + "?exclusive=true&workers=1&flush-interval=1h&query-timeout=1s&spool-dir=" + url.QueryEscape(t.TempDir()) + extra) + require.NoError(t, err) + s, err := New(context.Background(), common.NewChangeFeedIDWithName("databend-test", "default"), &config.ChangefeedConfig{}, uri, 0) + require.NoError(t, err) + t.Cleanup(s.Close) + return s +} + +func runRuntime(t *testing.T, s *Sink) <-chan error { + t.Helper() + done := make(chan error, 1) + go func() { done <- s.Run(context.Background()) }() + return done +} + +func waitEnqueued(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatal("event not enqueued") + } +} + +func TestRuntimeBatchApplyAndDDLBarrier(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + done := runRuntime(t, s) + var flushed atomic.Int64 + for _, ts := range []uint64{100, 120} { + e := nativeTestEvent(nativeTestInfo(), ts, 2) + enqueued := make(chan struct{}) + e.PostTxnEnqueued = []func(){func() { close(enqueued) }} + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + waitEnqueued(t, enqueued) + } + require.Zero(t, flushed.Load(), "raw acceptance is not target application") + e := &commonEvent.DDLEvent{Type: byte(model.ActionAddIndex), TableInfo: nativeTestInfo(), FinishedTs: 150} + require.NoError(t, s.FlushDMLBeforeBlock(e)) + require.EqualValues(t, 2, flushed.Load()) + f.mu.Lock() + require.Equal(t, 4, f.rawRows) + require.Equal(t, 1, f.merges) + f.mu.Unlock() + require.NoError(t, s.WriteBlockEvent(e)) + f.mu.Lock() + require.EqualValues(t, 150, f.epoch) + require.True(t, f.applied) + for _, sql := range f.sql { + require.NotContains(t, sql, "STREAM") + } + f.mu.Unlock() + s.Close() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestRuntimeUnknownMergeFailsLifecycleWithoutFlush(t *testing.T) { + f, server := newAPIFixture(t) + f.unknownMerge = true + s := runtimeSink(t, server, "&batch-rows=1") + done := runRuntime(t, s) + var flushed atomic.Int64 + e := nativeTestEvent(nativeTestInfo(), 100, 1) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + select { + case err := <-done: + require.Error(t, err) + require.Equal(t, errors.ErrChangefeedUnretryable.RFCCode(), errors.ErrorCode(err)) + require.Contains(t, err.Error(), "request") + case <-time.After(5 * time.Second): + t.Fatal("unknown mutation did not terminate Run") + } + require.Zero(t, flushed.Load()) + require.False(t, s.IsNormal()) + f.mu.Lock() + n := len(f.sql) + f.mu.Unlock() + s.AddDMLEvent(nativeTestEvent(nativeTestInfo(), 120, 1)) + f.mu.Lock() + require.Len(t, f.sql, n) + f.mu.Unlock() +} + +func TestRuntimeLargeEventStreamsAndReclaimsBeforeSeal(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "&batch-bytes=1024&spool-bytes=4096&memory-ratio=0.01") + done := runRuntime(t, s) + var flushed atomic.Int64 + e := nativeTestEvent(nativeTestInfo(), 100, 150) + enqueued := make(chan struct{}) + e.PostTxnEnqueued = []func(){func() { close(enqueued) }} + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + waitEnqueued(t, enqueued) + require.NoError(t, s.FlushDMLBeforeBlock(&commonEvent.DDLEvent{})) + require.EqualValues(t, 1, flushed.Load()) + f.mu.Lock() + require.Equal(t, 150, f.rawRows) + require.Greater(t, f.merges, 1) + f.mu.Unlock() + s.Close() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestRuntimeFatalAdmissionAndCloseNeverFlush(t *testing.T) { + _, server := newAPIFixture(t) + t.Run("capacity", func(t *testing.T) { + s := runtimeSink(t, server, "&source-bytes=1048576") + e := nativeTestEvent(nativeTestInfo(), 100, 100000) + var flushed atomic.Int64 + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + require.False(t, s.IsNormal()) + err := s.Run(context.Background()) + require.Error(t, err) + require.Equal(t, errors.ErrChangefeedUnretryable.RFCCode(), errors.ErrorCode(err)) + require.Zero(t, flushed.Load()) + }) + t.Run("close pending", func(t *testing.T) { + s := runtimeSink(t, server, "") + done := runRuntime(t, s) + e := nativeTestEvent(nativeTestInfo(), 100, 1) + var flushed atomic.Int64 + enqueued := make(chan struct{}) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + e.PostTxnEnqueued = []func(){func() { close(enqueued) }} + s.AddDMLEvent(e) + waitEnqueued(t, enqueued) + s.Close() + require.ErrorIs(t, <-done, context.Canceled) + require.Zero(t, flushed.Load()) + }) +} + +func TestSpoolPressureDrainsOtherTablesWithoutSuccessOnDiscard(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "&batch-bytes=1024&spool-bytes=4096&memory-ratio=0.01") + lane := s.lanes[0] + var flush atomic.Int64 + tracker := newEventCompletion(func() { flush.Add(1) }) + for i := range 40 { + schema := testSchema() + schema.table = "table" + strconv.Itoa(i) + writer, err := newTableWriter(schema, func(context.Context, string, []any) (mutationResult, error) { + return mutationResult{state: committed}, nil + }, 10, 1, 0) + require.NoError(t, err) + row := testRow(t, insertAction, 100, i) + data, err := json.Marshal([]spoolRow{{row.id, row.payload, row.commitTs, row.action}}) + require.NoError(t, err) + fragment, ok := tracker.register() + require.True(t, ok) + require.NoError(t, lane.enqueue(context.Background(), &tableState{schema: schema, writer: writer}, data, 1, fragment, nil)) + } + require.Zero(t, flush.Load(), "already applied OPEN fragments do not acknowledge the whole event") + tracker.seal() + lane.cleanup() + require.Zero(t, flush.Load(), "fatal cleanup cannot acknowledge retained fragments") + require.EqualValues(t, 0, spool.AccountingBytes(nil)) +} + +func TestConcurrentAdmissionAndCloseReleaseAllSources(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + done := runRuntime(t, s) + info := nativeTestInfo() + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { s.AddDMLEvent(nativeTestEvent(info, 100, 1)) }) + } + s.Close() + wg.Wait() + require.ErrorIs(t, <-done, context.Canceled) + s.mu.Lock() + defer s.mu.Unlock() + require.Zero(t, s.sourceBytes) + require.Zero(t, s.sourceEvents) +} + +func TestInternalTimeoutRemainsFatalNotLifecycleCancel(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + err := errors.WrapError(errors.ErrDatabendExecution, context.DeadlineExceeded) + s.stop(err) + require.True(t, errors.Is(context.Cause(s.ctx), context.DeadlineExceeded)) + err = s.Run(context.Background()) + require.Error(t, err) + require.Equal(t, errors.ErrChangefeedUnretryable.RFCCode(), errors.ErrorCode(err)) +} + +func TestRuntimePartialRawCommitDoesNotConfirmOrRetry(t *testing.T) { + f, server := newAPIFixture(t) + f.unknownRawAt = 2 + s := runtimeSink(t, server, "&batch-rows=1") + done := runRuntime(t, s) + var flush atomic.Int64 + e := nativeTestEvent(nativeTestInfo(), 100, 3) + e.PostTxnFlushed = []func(){func() { flush.Add(1) }} + s.AddDMLEvent(e) + select { + case err := <-done: + require.Error(t, err) + case <-time.After(5 * time.Second): + t.Fatal("partial raw failure did not stop") + } + require.Zero(t, flush.Load()) + f.mu.Lock() + defer f.mu.Unlock() + require.Equal(t, 2, f.rawRows, "unknown raw SQL must not be resent or continue to the third fragment") + require.Equal(t, 1, f.merges, "only the first fragment had a confirmed MERGE") +} + +func TestRuntimeUsesRetainedTimestampAfterRestart(t *testing.T) { + f, server := newAPIFixture(t) + f.maxAddTime = "2099-01-02 03:04:05.999999" + s := runtimeSink(t, server, "") + info := nativeTestInfo() + schema, err := schemaFromNative(info, "", "") + require.NoError(t, err) + state, skip, err := s.lanes[0].schemas.prepareDML(context.Background(), schema, 100) + require.NoError(t, err) + require.False(t, skip) + require.Equal(t, "2099-01-02 03:04:05.999999", state.writer.clock.last.Format("2006-01-02 15:04:05.000000")) + _, args := schema.insertRawSQL([]normalizedRow{testRow(t, deleteAction, 100, 1)}, &state.writer.clock, time.Now()) + require.Equal(t, "2099-01-02 03:04:06.000000", args[3], "recovery must not allocate a lower same-TSO tie breaker") +} + +func TestRuntimeLowTrafficAndTransportEpochDoesNotDiscardAdmittedData(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + s.opts.flushInterval = 10 * time.Millisecond + done := runRuntime(t, s) + var flushed atomic.Int64 + for _, epoch := range []uint64{1, 2} { + e := nativeTestEvent(nativeTestInfo(), 100+epoch, 1) + e.Epoch = epoch + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + } + require.Eventually(t, func() bool { return flushed.Load() == 2 }, 5*time.Second, time.Millisecond) + s.Close() + require.ErrorIs(t, <-done, context.Canceled) + require.EqualValues(t, 2, flushed.Load()) +} + +func TestRemoteRawMonitoringThresholdStopsBeforeNextInsert(t *testing.T) { + f, server := newAPIFixture(t) + f.remoteBytes = "100" + s := runtimeSink(t, server, "&remote-bytes-limit=100&batch-rows=1") + done := runRuntime(t, s) + var flushed atomic.Int64 + e := nativeTestEvent(nativeTestInfo(), 100, 1) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + select { + case err := <-done: + require.ErrorContains(t, err, "monitoring threshold") + case <-time.After(5 * time.Second): + t.Fatal("remote threshold did not stop admission") + } + require.Zero(t, flushed.Load()) + f.mu.Lock() + defer f.mu.Unlock() + require.Zero(t, f.rawRows) +} + +func TestRuntimeConfiguredConflictKeyReachesDMLAndJournal(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "&conflict-key=VaLuE") + done := runRuntime(t, s) + info := nativeTestInfo() + ddl := &commonEvent.DDLEvent{Type: byte(model.ActionCreateTable), TableInfo: info, FinishedTs: 90} + require.NoError(t, s.WriteBlockEvent(ddl)) + var flushed atomic.Int64 + e := nativeTestEvent(info, 100, 1) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + require.NoError(t, s.FlushDMLBeforeBlock(nil)) + require.EqualValues(t, 1, flushed.Load()) + f.mu.Lock() + var plan schemaPlan + require.NoError(t, json.Unmarshal([]byte(f.plan), &plan)) + require.Equal(t, []string{"value"}, plan.Schema.Keys) + require.Equal(t, 1, f.rawRows) + f.mu.Unlock() + s.Close() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestRuntimeSkippedDDLDrainsAndContinues(t *testing.T) { + for _, action := range []model.ActionType{model.ActionRenameTable, model.ActionRenameTables, model.ActionDropSchema} { + t.Run(action.String(), func(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + done := runRuntime(t, s) + var dml, ddl atomic.Int64 + e := nativeTestEvent(nativeTestInfo(), 100, 1) + e.PostTxnFlushed = []func(){func() { dml.Add(1) }} + s.AddDMLEvent(e) + // No TableInfo is intentional: DB/multi-table events need no new + // metadata/progress protocol just to be explicitly skipped. + ignored := &commonEvent.DDLEvent{Type: byte(action), SchemaName: "lake", TableName: "t", FinishedTs: 150, PostTxnFlushed: []func(){func() { ddl.Add(1) }}} + require.NoError(t, s.WriteBlockEvent(ignored)) + require.EqualValues(t, 1, dml.Load(), "prior DML must drain before skipped DDL callback") + require.EqualValues(t, 1, ddl.Load()) + f.mu.Lock() + require.Zero(t, f.epoch, "skipping must not invent a target schema epoch") + require.Equal(t, 1, f.merges) + before := len(f.sql) + f.mu.Unlock() + require.NoError(t, s.WriteBlockEvent(&commonEvent.DDLEvent{Type: byte(action), FinishedTs: 150})) + f.mu.Lock() + require.Len(t, f.sql, before, "no SQL must be executed for the skipped event itself") + f.mu.Unlock() + e = nativeTestEvent(nativeTestInfo(), 160, 1) + e.PostTxnFlushed = []func(){func() { dml.Add(1) }} + s.AddDMLEvent(e) + require.NoError(t, s.FlushDMLBeforeBlock(nil)) + require.EqualValues(t, 2, dml.Load()) + require.True(t, s.IsNormal()) + s.Close() + require.ErrorIs(t, <-done, context.Canceled) + }) + } +} + +func TestSkippedDDLDoesNotBypassUnknownDML(t *testing.T) { + f, server := newAPIFixture(t) + f.unknownMerge = true + s := runtimeSink(t, server, "") + done := runRuntime(t, s) + s.AddDMLEvent(nativeTestEvent(nativeTestInfo(), 100, 1)) + var flushed atomic.Int64 + ddl := &commonEvent.DDLEvent{Type: byte(model.ActionDropSchema), FinishedTs: 150, PostTxnFlushed: []func(){func() { flushed.Add(1) }}} + require.Error(t, s.WriteBlockEvent(ddl)) + require.Zero(t, flushed.Load()) + require.Error(t, <-done) + s.Close() +} diff --git a/downstreamadapter/sink/databend/schema.go b/downstreamadapter/sink/databend/schema.go new file mode 100644 index 0000000000..c15a9ed023 --- /dev/null +++ b/downstreamadapter/sink/databend/schema.go @@ -0,0 +1,707 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "context" + "encoding/json" + "fmt" + "math" + "reflect" + "strconv" + "strings" + "time" + + "github.com/pingcap/log" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + // Install TiDB AST literal constructors used by parser.New below. + _ "github.com/pingcap/tidb/pkg/types/parser_driver" + "go.uber.org/zap" +) + +// The only durable journal is the existing per-table schema epoch. Plan stores +// translated, bounded column operations so a pending DDL can resume before DML. +// It is not an applied DML ledger or an owner-fencing mechanism. +type columnDef struct { + Name string `json:"name"` + Type string `json:"type"` + Nullable bool `json:"nullable"` + Default string `json:"default,omitempty"` + Encoding columnEncoding `json:"encoding"` + SourceID int64 `json:"source_id"` + DefaultExpr bool `json:"default_expr,omitempty"` +} +type schemaDef struct { + Database string `json:"database"` + Table string `json:"table"` + Source string `json:"source"` + Columns []columnDef `json:"columns"` + Keys []string `json:"keys"` +} +type schemaStep struct { + Kind string `json:"kind"` + OldName string `json:"old_name,omitempty"` + Column columnDef `json:"column"` +} +type schemaPlan struct { + Kind string `json:"kind"` + Schema schemaDef `json:"schema"` + Steps []schemaStep `json:"steps,omitempty"` +} +type epochRecord struct { + epoch uint64 + applied bool + plan schemaPlan +} +type tableState struct { + schema tableSchema + epoch uint64 + writer *tableWriter + dropped bool +} +type schemaManager struct { + client *httpClient + opts options + tables map[string]*tableState +} + +func newSchemaManager(c *httpClient, o options) *schemaManager { + return &schemaManager{client: c, opts: o, tables: make(map[string]*tableState)} +} +func schemaKey(s tableSchema) string { return s.database + "\x00" + s.table } +func toSchemaDef(s tableSchema) schemaDef { + d := schemaDef{Database: s.database, Table: s.table, Source: s.source, Keys: s.primaryKey} + for _, c := range s.columns { + d.Columns = append(d.Columns, columnDef{c.name, c.typ, c.nullable, c.defaultSQL, c.encoding, c.sourceID, c.defaultExpr}) + } + return d +} + +func (d schemaDef) schema() tableSchema { + s := tableSchema{database: d.Database, table: d.Table, source: d.Source, primaryKey: d.Keys} + for _, c := range d.Columns { + s.columns = append(s.columns, column{name: c.Name, typ: c.Type, nullable: c.Nullable, defaultSQL: c.Default, encoding: c.Encoding, sourceID: c.SourceID, defaultExpr: c.DefaultExpr}) + } + return s +} + +func (c columnDef) definition() string { + result := quoteIdentifier(c.Name) + " " + c.Type + if c.Nullable { + result += " NULL" + } else { + result += " NOT NULL" + } + if c.Default != "" { + result += " DEFAULT " + c.Default + } + return result +} + +func (s tableSchema) createTargetSQL() string { + cols := make([]string, 0, len(s.columns)) + for _, c := range toSchemaDef(s).Columns { + cols = append(cols, c.definition()) + } + return "CREATE TABLE IF NOT EXISTS " + s.targetName() + " (" + strings.Join(cols, ", ") + ")" +} + +func (s tableSchema) journalName() string { + return quoteIdentifier(s.database) + "." + quoteIdentifier(s.table+"_cdc_schema_epoch") +} + +func (m *schemaManager) exec(ctx context.Context, sql string, args ...any) error { + result, err := m.client.mutate(ctx, sql, args) + if result.state == commitUnknown { + if err == nil { + err = errors.ErrDatabendCommitUnknown.GenWithStackByArgs() + } else { + err = errors.WrapError(errors.ErrDatabendCommitUnknown, err) + } + return errors.WrapChangefeedUnretryableErr(err) + } + if err != nil { + return err + } + if result.state != committed { + return errors.ErrDatabendExecution.GenWithStackByArgs() + } + return nil +} + +func (m *schemaManager) loadEpoch(ctx context.Context, s tableSchema) (epochRecord, error) { + if err := m.exec(ctx, "CREATE DATABASE IF NOT EXISTS "+quoteIdentifier(s.database)); err != nil { + return epochRecord{}, err + } + if err := m.exec(ctx, "CREATE TABLE IF NOT EXISTS "+s.journalName()+" (epoch BIGINT, applied BOOLEAN, plan VARCHAR)"); err != nil { + return epochRecord{}, err + } + rows, err := m.client.selectRows(ctx, "SELECT epoch, applied, plan FROM "+s.journalName(), nil) + if err != nil { + return epochRecord{}, err + } + if len(rows) == 0 { + return epochRecord{}, nil + } + if len(rows) != 1 || len(rows[0]) != 3 || rows[0][0] == nil || rows[0][1] == nil || rows[0][2] == nil { + return epochRecord{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid schema epoch journal") + } + e, err := strconv.ParseUint(*rows[0][0], 10, 64) + if err != nil || e == 0 || e > math.MaxInt64 { + return epochRecord{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid applied schema epoch") + } + applied, err := strconv.ParseBool(*rows[0][1]) + if err != nil { + return epochRecord{}, errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid epoch status") + } + r := epochRecord{epoch: e, applied: applied} + if err := json.Unmarshal([]byte(*rows[0][2]), &r.plan); err != nil { + return epochRecord{}, errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid epoch plan") + } + saved := r.plan.Schema.schema() + if err := saved.validate(); err != nil { + return epochRecord{}, err + } + if saved.database != s.database || saved.table != s.table || saved.source != s.source { + return epochRecord{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("target belongs to another source or schema journal") + } + return r, nil +} + +func (m *schemaManager) saveEpoch(ctx context.Context, s tableSchema, r epochRecord) error { + plan, err := json.Marshal(r.plan) + if err != nil { + return errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid schema plan") + } + return m.exec(ctx, "MERGE INTO "+s.journalName()+" t USING (SELECT ?::BIGINT AS epoch, ?::BOOLEAN AS applied, ?::VARCHAR AS plan) s ON true WHEN MATCHED THEN UPDATE * WHEN NOT MATCHED THEN INSERT *", int64(r.epoch), r.applied, string(plan)) +} + +func canonicalType(t string) string { + t = strings.ToUpper(strings.ReplaceAll(t, " ", "")) + if inner, ok := strings.CutPrefix(t, "NULLABLE("); ok { + t = strings.TrimSuffix(inner, ")") + } + switch t { + case "TINYINTUNSIGNED": + return "UINT8" + case "SMALLINTUNSIGNED": + return "UINT16" + case "INTUNSIGNED", "INTEGERUNSIGNED": + return "UINT32" + case "BIGINTUNSIGNED": + return "UINT64" + case "TINYINT": + return "INT8" + case "SMALLINT": + return "INT16" + case "INT", "INTEGER": + return "INT32" + case "BIGINT": + return "INT64" + case "STRING", "TEXT": + return "VARCHAR" + case "JSON": + return "VARIANT" + case "FLOAT": + return "FLOAT32" + case "DOUBLE": + return "FLOAT64" + } + return t +} + +func (m *schemaManager) targetExists(ctx context.Context, s tableSchema) (bool, error) { + rows, err := m.client.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM system.tables WHERE database = ? AND name = ?", []any{s.database, s.table}) + if err != nil { + return false, err + } + count, err := parseRemoteBytes(rows) + if err != nil { + return false, err + } + if count > 1 { + return false, errors.ErrDatabendInvalidData.GenWithStackByArgs("ambiguous target table metadata") + } + return count == 1, nil +} + +func (m *schemaManager) describe(ctx context.Context, s tableSchema) (map[string]columnDef, error) { + rows, err := m.client.selectRows(ctx, "DESCRIBE "+s.targetName(), nil) + if err != nil { + return nil, err + } + cols := make(map[string]columnDef, len(rows)) + for _, r := range rows { + if len(r) < 3 || r[0] == nil || r[1] == nil || r[2] == nil { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("incomplete target column metadata") + } + name := *r[0] + cols[name] = columnDef{Name: name, Type: canonicalType(*r[1]), Nullable: strings.EqualFold(*r[2], "YES") || strings.HasPrefix(strings.ToUpper(*r[1]), "NULLABLE(")} + } + return cols, nil +} + +func (m *schemaManager) verifyTarget(ctx context.Context, s tableSchema) error { + cols, err := m.describe(ctx, s) + if err != nil { + return err + } + if len(cols) != len(s.columns) { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("target column set differs from source metadata") + } + for _, c := range s.columns { + actual, ok := cols[c.name] + if !ok || canonicalType(actual.Type) != canonicalType(c.typ) || actual.Nullable != c.nullable { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("target type or nullability differs from source projection") + } + } + return nil +} + +func (m *schemaManager) initialize(ctx context.Context, s tableSchema) (*tableState, error) { + if state := m.tables[schemaKey(s)]; state != nil { + return state, nil + } + if len(m.tables) >= m.opts.maxEntries { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("table metadata capacity exhausted") + } + r, err := m.loadEpoch(ctx, s) + if err != nil { + return nil, err + } + if r.epoch > 0 && !r.applied { + if err := m.finishPlan(ctx, r); err != nil { + return nil, err + } + r.applied = true + } + effective := s + dropped := false + if r.epoch > 0 { + effective = r.plan.Schema.schema() + dropped = r.plan.Kind == "drop" + } + if !dropped { + if err := m.exec(ctx, effective.createTargetSQL()); err != nil { + return nil, err + } + if err := m.verifyTarget(ctx, effective); err != nil { + return nil, err + } + if err := m.exec(ctx, effective.createRawSQL()); err != nil { + return nil, err + } + } + writer, err := newTableWriter(effective, m.client.mutate, m.opts.batchRows, 3, r.epoch) + if err != nil { + return nil, err + } + if !dropped { + rows, err := m.client.selectRows(ctx, "SELECT COALESCE(MAX(add_time)::VARCHAR, '') FROM "+effective.rawName(), nil) + if err != nil { + return nil, err + } + if len(rows) != 1 || len(rows[0]) != 1 || rows[0][0] == nil { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid retained raw time watermark") + } + if text := *rows[0][0]; text != "" { + stamp, err := time.ParseInLocation("2006-01-02 15:04:05.999999", text, time.UTC) + if err != nil { + return nil, errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid retained raw timestamp") + } + writer.clock.last = stamp + } + } + state := &tableState{schema: effective, epoch: r.epoch, writer: writer, dropped: dropped} + m.tables[schemaKey(s)] = state + return state, nil +} + +func (m *schemaManager) prepareDML(ctx context.Context, s tableSchema, ts uint64) (*tableState, bool, error) { + state, err := m.initialize(ctx, s) + if err != nil { + return nil, false, err + } + if ts <= state.epoch { + return state, true, nil + } + if state.dropped || !reflect.DeepEqual(toSchemaDef(state.schema), toSchemaDef(s)) { + return nil, false, errors.ErrDatabendInvalidData.GenWithStackByArgs("DML schema is not the applied target schema; replay its DDL first") + } + return state, false, nil +} + +func allowedModify(from, to columnDef) bool { + a, b := canonicalType(from.Type), canonicalType(to.Type) + // Match the existing conservative widening family. No implicit narrowing or + // binary/JSON conversions are inferred from a successful server cast. + return a == b || a == "INT8" && (b == "INT32" || b == "INT64") || a == "INT32" && (b == "INT64" || b == "VARCHAR") +} + +func makeColumnSteps(query string, s tableSchema, current map[string]columnDef) ([]schemaStep, error) { + stmt, err := parser.New().ParseOneStmt(query, "", "") + if err != nil { + return nil, errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot parse incremental DDL") + } + alter, ok := stmt.(*ast.AlterTableStmt) + if !ok { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported column DDL") + } + desired := make(map[string]columnDef) + for _, c := range toSchemaDef(s).Columns { + desired[strings.ToLower(c.Name)] = c + } + foldedCurrent := make(map[string]columnDef, len(current)) + for name, c := range current { + foldedCurrent[strings.ToLower(name)] = c + } + current = foldedCurrent + isKey := func(name string) bool { + for _, k := range s.primaryKey { + if strings.EqualFold(k, name) { + return true + } + } + return false + } + parameterizedCharacterSafe := func(oldName string, def *ast.ColumnDef) bool { + if desired[strings.ToLower(def.Name.Name.O)].Type != "VARCHAR" || def.Tp.GetFlen() <= 0 { + return true + } + // Preserve bend-cdc's INT -> VARCHAR widening without opening arbitrary + // character narrowing. Every signed INT32 fits in 11 decimal characters. + // Same-target VARCHAR changes still cannot prove old source width. + return canonicalType(current[strings.ToLower(oldName)].Type) == "INT32" && + def.Tp.GetType() == mysql.TypeVarchar && def.Tp.GetFlen() >= 11 + } + var steps []schemaStep + for _, spec := range alter.Specs { + add := func(kind, old, name string) error { + oldKey, nameKey := strings.ToLower(old), strings.ToLower(name) + c := desired[nameKey] + if (kind == "add" || kind == "modify") && c.DefaultExpr { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("expression default DDL is unsupported") + } + if kind == "drop" { + c = columnDef{Name: name} + } else if c.Name == "" { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("DDL references an invisible or missing column") + } + if (kind == "drop" || kind == "rename") && (isKey(old) || isKey(name)) { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("changing a conflict key is unsupported") + } + prev, present := current[oldKey] + if present { + old = prev.Name + } + if kind == "drop" { + c.Name = old + } + switch kind { + case "add": + if _, ok := current[nameKey]; ok { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("column already exists before new DDL") + } + if !c.Nullable && c.Default == "" { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("ADD NOT NULL requires a literal source default") + } + case "drop", "rename", "modify": + if !present { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("old column is missing before new DDL") + } + } + if kind == "modify" && (!allowedModify(prev, c) || (isKey(old) && len(s.primaryKey) > 1)) { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported type transition or composite key modification") + } + if kind == "rename" { + if _, ok := current[nameKey]; ok { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("rename target already exists") + } + delete(current, oldKey) + } + switch kind { + case "drop": + delete(current, oldKey) + case "rename": + prev.Name = c.Name + current[nameKey] = prev // RENAME does not also change the old type. + default: + current[nameKey] = c + } + steps = append(steps, schemaStep{Kind: kind, OldName: old, Column: c}) + return nil + } + switch spec.Tp { + case ast.AlterTableAddColumns: + for _, c := range spec.NewColumns { + explicitDefault := false + for _, opt := range c.Options { + if opt.Tp == ast.ColumnOptionDefaultValue { + explicitDefault = true + } + } + col := desired[strings.ToLower(c.Name.Name.O)] + if !explicitDefault && !col.Nullable && col.Type == "VARCHAR" && col.Default == "" { + col.Default = "''" + desired[strings.ToLower(c.Name.Name.O)] = col + } + if err := add("add", "", c.Name.Name.O); err != nil { + return nil, err + } + } + case ast.AlterTableDropColumn: + if err := add("drop", spec.OldColumnName.Name.O, spec.OldColumnName.Name.O); err != nil { + return nil, err + } + case ast.AlterTableRenameColumn: + if err := add("rename", spec.OldColumnName.Name.O, spec.NewColumnName.Name.O); err != nil { + return nil, err + } + case ast.AlterTableModifyColumn: + if len(spec.NewColumns) != 1 { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid MODIFY") + } + def := spec.NewColumns[0] + if !parameterizedCharacterSafe(def.Name.Name.O, def) { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("parameterized character MODIFY may narrow source data") + } + name := spec.NewColumns[0].Name.Name.O + if err := add("modify", name, name); err != nil { + return nil, err + } + case ast.AlterTableChangeColumn: + if len(spec.NewColumns) != 1 { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid CHANGE") + } + old, name := spec.OldColumnName.Name.O, spec.NewColumns[0].Name.Name.O + if !parameterizedCharacterSafe(old, spec.NewColumns[0]) { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("parameterized character CHANGE may narrow source data") + } + if old != name { + if err := add("rename", old, name); err != nil { + return nil, err + } + } + if err := add("modify", name, name); err != nil { + return nil, err + } + case ast.AlterTableAddConstraint: + if spec.Constraint == nil || spec.Constraint.Tp == ast.ConstraintPrimaryKey { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("changing primary keys is unsupported") + } + case ast.AlterTableDropIndex, ast.AlterTableRenameIndex, ast.AlterTableIndexInvisible: // Secondary indexes have no target projection. + default: + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported DDL; target has not been changed") + } + } + return steps, nil +} + +func (m *schemaManager) finishPlan(ctx context.Context, r epochRecord) error { + s := r.plan.Schema.schema() + switch r.plan.Kind { + case "drop": + if err := m.exec(ctx, "DROP TABLE IF EXISTS "+s.rawName()); err != nil { + return err + } + if err := m.exec(ctx, "DROP TABLE IF EXISTS "+s.targetName()); err != nil { + return err + } + case "truncate", "create", "alter", "index": + if err := m.exec(ctx, s.createTargetSQL()); err != nil { + return err + } + if r.plan.Kind == "truncate" { + if err := m.exec(ctx, "DROP TABLE IF EXISTS "+s.rawName()); err != nil { + return err + } + if err := m.exec(ctx, "TRUNCATE TABLE "+s.targetName()); err != nil { + return err + } + } + for _, step := range r.plan.Steps { + cols, err := m.describe(ctx, s) + if err != nil { + return err + } + old, present := cols[step.OldName] + newCol, renamed := cols[step.Column.Name] + suffix := "" + switch step.Kind { + case "add": + if renamed { + if canonicalType(newCol.Type) != canonicalType(step.Column.Type) || newCol.Nullable != step.Column.Nullable { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("pending ADD has conflicting target representation") + } + continue + } + suffix = "ADD COLUMN " + step.Column.definition() + case "drop": + if !present { + continue + } + suffix = "DROP COLUMN " + quoteIdentifier(step.OldName) + case "rename": + if !present && renamed { + continue + } + if !present || renamed { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("pending RENAME is ambiguous") + } + suffix = "RENAME COLUMN " + quoteIdentifier(step.OldName) + " TO " + quoteIdentifier(step.Column.Name) + case "modify": + if !present || !allowedModify(old, step.Column) { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("pending MODIFY has conflicting target representation") + } + suffix = "MODIFY COLUMN " + step.Column.definition() + default: + return errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid durable DDL operation") + } + if err := m.exec(ctx, "ALTER TABLE "+s.targetName()+" "+suffix); err != nil { + return err + } + } + if err := m.verifyTarget(ctx, s); err != nil { + return err + } + if err := m.exec(ctx, s.createRawSQL()); err != nil { + return err + } + default: + return errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported durable DDL plan") + } + r.applied = true + return m.saveEpoch(ctx, s, r) +} + +func (m *schemaManager) applyDDL(ctx context.Context, e *commonEvent.DDLEvent) error { + if e.IsBootstrap { + return nil + } // Metadata bootstrap is not proof of an applied DDL. + if e.NotSync || e.TiDBOnly { + return nil + } + if e.FinishedTs == 0 || e.FinishedTs > math.MaxInt64 { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid DDL FinishedTs") + } + switch model.ActionType(e.Type) { + case model.ActionRenameTable, model.ActionRenameTables, model.ActionDropSchema: + // Compatibility with bend-cdc's non-projecting path. The outer sink + // has already drained prior DML; only these explicitly opted-out DDLs + // succeed without a target mutation or an invented applied epoch. + log.Warn("Databend sink skipped source DDL and kept target objects unchanged", + zap.String("ddlType", model.ActionType(e.Type).String()), + zap.String("schema", e.SchemaName), zap.String("table", e.TableName), + zap.Uint64("finishedTs", e.FinishedTs)) + return nil + } + if model.ActionType(e.Type) == model.ActionCreateSchema { + db := e.SchemaName + if m.opts.database != "" { + db = m.opts.database + } + if db == "" || strings.ContainsRune(db, 0) { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid schema name") + } + return m.exec(ctx, "CREATE DATABASE IF NOT EXISTS "+quoteIdentifier(db)) + } + if e.TableInfo == nil { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("DDL without table metadata is outside the incremental projection") + } + s, err := schemaFromNative(e.TableInfo, m.opts.database, m.opts.conflictKey) + if err != nil { + return err + } + r, err := m.loadEpoch(ctx, s) + if err != nil { + return err + } + if r.epoch > e.FinishedTs || r.epoch == e.FinishedTs && r.applied { + return nil + } + if r.epoch > 0 && !r.applied { + if err := m.finishPlan(ctx, r); err != nil { + return err + } + delete(m.tables, schemaKey(s)) + if r.epoch == e.FinishedTs { + return nil + } + } + plan := schemaPlan{Schema: toSchemaDef(s)} + switch model.ActionType(e.Type) { + case model.ActionCreateTable, model.ActionRecoverTable: + plan.Kind = "create" + case model.ActionTruncateTable: + plan.Kind = "truncate" + case model.ActionDropTable: + plan.Kind = "drop" + case model.ActionAddIndex, model.ActionDropIndex, model.ActionRenameIndex, model.ActionAlterIndexVisibility: + plan.Kind = "index" + case model.ActionAddColumn, model.ActionDropColumn, model.ActionModifyColumn, model.ActionMultiSchemaChange: + plan.Kind = "alter" + exists, err := m.targetExists(ctx, s) + if err != nil { + return err + } + if !exists { + // An actual DDL is sufficient authoritative metadata to create an + // empty projection; this is not historical-table discovery. + stmt, parseErr := parser.New().ParseOneStmt(e.Query, "", "") + if parseErr != nil { + return errors.WrapError(errors.ErrDatabendInvalidData, parseErr, "cannot parse incremental DDL") + } + alter, ok := stmt.(*ast.AlterTableStmt) + if !ok { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported incremental column DDL") + } + for _, spec := range alter.Specs { + switch spec.Tp { + case ast.AlterTableAddColumns, ast.AlterTableDropColumn, ast.AlterTableModifyColumn, ast.AlterTableChangeColumn, ast.AlterTableRenameColumn: + default: + return errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported DDL on missing projection") + } + } + plan.Kind = "create" + break + } + current, err := m.describe(ctx, s) + if err != nil { + return err + } + plan.Steps, err = makeColumnSteps(e.Query, s, current) + if err != nil { + return err + } + default: + return errors.ErrDatabendInvalidData.GenWithStackByArgs(fmt.Sprintf("unsupported DDL action %d", e.Type)) + } + // No new mutation is issued for rejected DDLs. Callers have already drained + // every affected lane; all old SQL must be terminated before a manual resume. + r = epochRecord{epoch: e.FinishedTs, plan: plan} + if err := m.saveEpoch(ctx, s, r); err != nil { + return err + } + if err := m.finishPlan(ctx, r); err != nil { + return err + } + delete(m.tables, schemaKey(s)) + return nil +} diff --git a/downstreamadapter/sink/databend/schema_test.go b/downstreamadapter/sink/databend/schema_test.go new file mode 100644 index 0000000000..8ab1f4ae68 --- /dev/null +++ b/downstreamadapter/sink/databend/schema_test.go @@ -0,0 +1,207 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "context" + "encoding/json" + "strings" + "testing" + + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/stretchr/testify/require" +) + +func TestColumnDDLTranslationIsConservative(t *testing.T) { + s, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + s.columns = append(s.columns, column{name: "extra", typ: "INT64", nullable: true, sourceID: 3}) + current := func() map[string]columnDef { + return map[string]columnDef{"id": {Name: "id", Type: "INT64"}, "value": {Name: "value", Type: "VARCHAR", Nullable: true}} + } + steps, err := makeColumnSteps("ALTER TABLE t ADD COLUMN extra BIGINT", s, current()) + require.NoError(t, err) + require.Len(t, steps, 1) + require.Equal(t, "add", steps[0].Kind) + for _, sql := range []string{"ALTER TABLE t RENAME COLUMN id TO value", "ALTER TABLE t DROP PRIMARY KEY", "ALTER TABLE t ADD COLUMN missing INT", "ALTER TABLE t TRUNCATE PARTITION p0"} { + _, err := makeColumnSteps(sql, s, current()) + require.Error(t, err, sql) + } + s.columns[2].nullable = false + _, err = makeColumnSteps("ALTER TABLE t ADD COLUMN extra BIGINT NOT NULL", s, current()) + require.Error(t, err) + s.columns[2].defaultSQL = "'42'" + _, err = makeColumnSteps("ALTER TABLE t ADD COLUMN extra BIGINT NOT NULL DEFAULT 42", s, current()) + require.NoError(t, err) + s.columns[2].defaultExpr = true + _, err = makeColumnSteps("ALTER TABLE t ADD COLUMN extra BIGINT", s, current()) + require.Error(t, err) +} + +func TestPendingTruncateRecoveryAndAppliedReplay(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + m := s.lanes[0].schemas + info := nativeTestInfo() + schema, err := schemaFromNative(info, "", "") + require.NoError(t, err) + _, skip, err := m.prepareDML(context.Background(), schema, 100) + require.NoError(t, err) + require.False(t, skip) + f.mu.Lock() + require.Zero(t, f.epoch, "bootstrap must not invent an applied epoch") + f.failAfterTruncate = true + f.mu.Unlock() + ddl := &commonEvent.DDLEvent{Type: byte(model.ActionTruncateTable), TableInfo: info, FinishedTs: 150} + require.Error(t, m.applyDDL(context.Background(), ddl)) + f.mu.Lock() + require.EqualValues(t, 150, f.epoch) + require.False(t, f.applied) + require.Equal(t, 1, f.truncates) + f.mu.Unlock() + // Simulate a fresh sink only AFTER the old request has a verified terminal + // state (the fixture failed before saving applied). Transport cancellation is + // not that proof in production. + m = newSchemaManager(s.client, s.opts) + _, skip, err = m.prepareDML(context.Background(), schema, 120) + require.NoError(t, err) + require.True(t, skip) + f.mu.Lock() + require.True(t, f.applied) + require.Equal(t, 2, f.truncates) + f.mu.Unlock() + _, skip, err = m.prepareDML(context.Background(), schema, 160) + require.NoError(t, err) + require.False(t, skip) + require.NoError(t, m.applyDDL(context.Background(), ddl)) + f.mu.Lock() + require.Equal(t, 2, f.truncates, "applied replay must not clear newer data") + f.mu.Unlock() +} + +func TestPendingColumnResumeBeforeNewDML(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + schema, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + schema.columns = append(schema.columns, column{name: "extra", typ: "INT64", nullable: true, sourceID: 3}) + plan := schemaPlan{Kind: "alter", Schema: toSchemaDef(schema), Steps: []schemaStep{{Kind: "add", Column: columnDef{Name: "extra", Type: "INT64", Nullable: true}}}} + raw, err := json.Marshal(plan) + require.NoError(t, err) + f.mu.Lock() + f.epoch = 150 + f.applied = false + f.plan = string(raw) + f.mu.Unlock() + m := s.lanes[0].schemas + _, skip, err := m.prepareDML(context.Background(), schema, 160) + require.NoError(t, err) + require.False(t, skip) + f.mu.Lock() + require.True(t, f.applied) + require.Len(t, f.columns, 3) + f.mu.Unlock() + _, skip, err = m.prepareDML(context.Background(), schema, 140) + require.NoError(t, err) + require.True(t, skip) +} + +func TestAddNotNullTextKeepsBaselineImplicitDefault(t *testing.T) { + s, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + s.columns = append(s.columns, column{name: "text", typ: "VARCHAR", sourceID: 3}) + current := func() map[string]columnDef { + return map[string]columnDef{"id": {Name: "id", Type: "INT64"}, "value": {Name: "value", Type: "VARCHAR", Nullable: true}} + } + steps, err := makeColumnSteps("ALTER TABLE t ADD COLUMN text TEXT NOT NULL", s, current()) + require.NoError(t, err) + require.Equal(t, "''", steps[0].Column.Default) + _, err = makeColumnSteps("ALTER TABLE t ADD COLUMN text TEXT NOT NULL DEFAULT NULL", s, current()) + require.Error(t, err) +} + +func TestChangeColumnValidatesOldTypeBeforePending(t *testing.T) { + s, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + s.columns[1].name = "NewName" + s.columns[1].typ = "INT8" + current := map[string]columnDef{"id": {Name: "id", Type: "INT64"}, "value": {Name: "value", Type: "INT64", Nullable: true}} + _, err = makeColumnSteps("ALTER TABLE t CHANGE COLUMN VALUE newname TINYINT", s, current) + require.Error(t, err, "rename must not masquerade as a completed type change") + s.columns[1].typ = "INT64" + current["value"] = columnDef{Name: "value", Type: "INT32", Nullable: true} + steps, err := makeColumnSteps("ALTER TABLE t CHANGE COLUMN VALUE newname BIGINT", s, current) + require.NoError(t, err) + require.Len(t, steps, 2) + require.Equal(t, "value", steps[0].OldName) + require.Equal(t, "NewName", steps[0].Column.Name) + require.Equal(t, "NewName", steps[1].OldName) +} + +func TestActualDDLCanCreateMissingProjectionWithoutFullImport(t *testing.T) { + f, server := newAPIFixture(t) + f.targetMissing = true + s := runtimeSink(t, server, "") + e := &commonEvent.DDLEvent{Type: byte(model.ActionModifyColumn), TableInfo: nativeTestInfo(), FinishedTs: 150, Query: "ALTER TABLE t MODIFY COLUMN value TEXT"} + require.NoError(t, s.lanes[0].schemas.applyDDL(context.Background(), e)) + f.mu.Lock() + defer f.mu.Unlock() + require.False(t, f.targetMissing) + require.True(t, f.applied) + require.EqualValues(t, 150, f.epoch) + var plan schemaPlan + require.NoError(t, json.Unmarshal([]byte(f.plan), &plan)) + require.Equal(t, "create", plan.Kind) + require.Empty(t, plan.Steps) +} + +// Lake DESCRIBE uses SQL aliases rather than internal UINT* spelling. +func TestCanonicalDescribeUnsignedTypes(t *testing.T) { + for source, want := range map[string]string{"TINYINT UNSIGNED": "UINT8", "SMALLINT UNSIGNED": "UINT16", "INT UNSIGNED": "UINT32", "BIGINT UNSIGNED": "UINT64", "Nullable(BIGINT UNSIGNED)": "UINT64", "Decimal(65, 30)": "DECIMAL(65,30)"} { + require.Equal(t, want, canonicalType(source)) + } +} + +func TestParameterizedCharacterWideningKeepsBaselineIntTransition(t *testing.T) { + for _, tc := range []struct { + query, from string + allowed bool + }{ + {"ALTER TABLE t MODIFY COLUMN value VARCHAR(200) NOT NULL DEFAULT '7'", "INT32", true}, + {"ALTER TABLE t MODIFY COLUMN value VARCHAR(11)", "INT32", true}, + {"ALTER TABLE t MODIFY COLUMN value VARCHAR(10)", "INT32", false}, + {"ALTER TABLE t MODIFY COLUMN value CHAR(200)", "INT32", false}, + {"ALTER TABLE t MODIFY COLUMN value VARCHAR(200)", "VARCHAR", false}, + {"ALTER TABLE t MODIFY COLUMN value VARCHAR(200)", "INT64", false}, + {"ALTER TABLE t CHANGE COLUMN value label VARCHAR(200)", "INT32", true}, + } { + t.Run(tc.query+"/"+tc.from, func(t *testing.T) { + schema, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + if strings.Contains(tc.query, "CHANGE") { + schema.columns[1].name = "label" + } + _, err = makeColumnSteps(tc.query, schema, map[string]columnDef{ + "id": {Name: "id", Type: "INT64"}, "value": {Name: "value", Type: tc.from, Nullable: true}, + }) + if tc.allowed { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} diff --git a/downstreamadapter/sink/databend/sink.go b/downstreamadapter/sink/databend/sink.go new file mode 100644 index 0000000000..7796c711bc --- /dev/null +++ b/downstreamadapter/sink/databend/sink.go @@ -0,0 +1,278 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "context" + "hash/fnv" + "net/url" + "sync" + "sync/atomic" + + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/pkg/sink/spool" + "github.com/pingcap/ticdc/pkg/writelease" + "golang.org/x/sync/errgroup" +) + +type workItem struct { + event *commonEvent.DMLEvent + charge int64 + ddl *commonEvent.DDLEvent + done chan error +} + +// Sink owns a bounded source admission pool, shared spool and a fixed number +// of serial table lanes. No transport epoch owns data after admission. +type Sink struct { + opts options + client *httpClient + spool *spool.Spool + lanes []*tableLane + statistics *metrics.Statistics + ctx context.Context + cancel context.CancelCauseFunc + normal atomic.Bool + stopOnce, cleanupOnce sync.Once + mu sync.Mutex + sourceBytes int64 + sourceEvents int + started bool + closed bool + done chan struct{} + ddlMu sync.Mutex +} + +func Verify(ctx context.Context, cfg *config.ChangefeedConfig, uri *url.URL) error { + o, err := parseOptions(uri, cfg) + if err != nil { + return err + } + c := newHTTPClient(o) + defer c.transport.CloseIdleConnections() + rows, err := c.selectRows(ctx, "SELECT ?::BIGINT", []any{int64(1)}) + if err != nil { + return err + } + if len(rows) != 1 || len(rows[0]) != 1 || rows[0][0] == nil || *rows[0][0] != "1" { + return errors.ErrDatabendInvalidData.GenWithStackByArgs("server must support HTTP bound parameters and terminal query states") + } + return nil +} + +func New(ctx context.Context, id common.ChangeFeedID, cfg *config.ChangefeedConfig, uri *url.URL, keyspaceID uint32) (*Sink, error) { + o, err := parseOptions(uri, cfg) + if err != nil { + return nil, err + } + buffer, err := spool.New(id, spool.WithRootDir(o.spoolDir), spool.WithDirectoryNamespace("databend-sink-spool", config.GetGlobalServerConfig().AdvertiseAddr), spool.WithMetrics(newSpoolMetrics(id)), spool.WithDiskQuotaBytes(o.spoolBytes), spool.WithSegmentBytes(min(64<<20, o.spoolBytes/4)), spool.WithMemoryRatio(o.memoryRatio)) + if err != nil { + return nil, err + } + ctx, cancel := context.WithCancelCause(ctx) + s := &Sink{opts: o, client: newHTTPClient(o), spool: buffer, ctx: ctx, cancel: cancel, done: make(chan struct{}), statistics: metrics.NewStatistics(id, keyspaceID, "databend")} + s.normal.Store(true) + for i := 0; i < o.workers; i++ { + s.lanes = append(s.lanes, &tableLane{sink: s, input: make(chan workItem, o.sourceEvents), schemas: newSchemaManager(s.client, o), batches: make(map[string]*pendingTable)}) + } + return s, nil +} +func (s *Sink) SinkType() common.SinkType { return common.DatabendSinkType } +func (s *Sink) IsNormal() bool { return s.normal.Load() && s.ctx.Err() == nil } +func (s *Sink) BatchCount() int { return min(s.opts.sourceEvents, s.opts.batchRows) } +func (s *Sink) BatchBytes() int { return int(min(s.opts.sourceBytes/2, s.opts.batchBytes)) } +func (s *Sink) SetWriteGate(gate *writelease.Gate) { s.client.gate.Store(gate) } +func (s *Sink) SetTableSchemaStore(*commonEvent.TableSchemaStore) {} +func (s *Sink) AddCheckpointTs(uint64) {} + +// stop marks an internal failure as unretryable, even when its cause is a +// timeout. Only stopLifecycle is used for externally requested cancellation. +func (s *Sink) stop(err error) { + s.stopLifecycle(errors.WrapChangefeedUnretryableErr(err)) +} + +func (s *Sink) stopLifecycle(err error) { + s.stopOnce.Do(func() { s.normal.Store(false); s.cancel(err) }) +} + +func (s *Sink) lane(database, table string) *tableLane { + h := fnv.New64a() + _, _ = h.Write([]byte(database + "\x00" + table)) + return s.lanes[h.Sum64()%uint64(len(s.lanes))] +} + +func (s *Sink) releaseSource(charge int64) { + s.mu.Lock() + s.sourceBytes -= charge + s.sourceEvents-- + s.mu.Unlock() +} + +func (s *Sink) AddDMLEvent(e *commonEvent.DMLEvent) { + if !s.IsNormal() { + return + } + if e == nil || e.TableInfo == nil || e.Rows == nil { + s.stop(errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid native DML event")) + return + } + // Shared chunks are conservatively charged in full for each admitted event. + // This can reject early but cannot hide source retention outside the pool. + charge := e.Rows.MemoryUsage() + max(e.GetSize(), int64(0)) + int64(len(e.RowTypes))*8 + int64(len(e.RowKeys))*24 + int64(len(e.Checksum))*64 + 1024 + for _, key := range e.RowKeys { + charge += int64(len(key)) + } + s.mu.Lock() + if !s.IsNormal() || charge > s.opts.sourceBytes-s.sourceBytes || s.sourceEvents >= s.opts.sourceEvents { + s.mu.Unlock() + s.stop(errors.ErrDatabendInvalidData.GenWithStackByArgs("source admission capacity exhausted; old sink lifecycle must terminate")) + return + } + s.sourceBytes += charge + s.sourceEvents++ + db := e.TableInfo.GetSchemaName() + if s.opts.database != "" { + db = s.opts.database + } + lane := s.lane(db, e.TableInfo.GetTableName()) + select { + case lane.input <- workItem{event: e, charge: charge}: + s.mu.Unlock() + default: + s.sourceBytes -= charge + s.sourceEvents-- + s.mu.Unlock() + s.stop(errors.ErrDatabendInvalidData.GenWithStackByArgs("source admission queue exhausted")) + } +} + +func (s *Sink) waitWork(lane *tableLane, item workItem) error { + item.done = make(chan error, 1) + select { + case <-s.ctx.Done(): + return context.Cause(s.ctx) + case lane.input <- item: + } + select { + case <-s.ctx.Done(): + return context.Cause(s.ctx) + case err := <-item.done: + return err + } +} + +func (s *Sink) FlushDMLBeforeBlock(commonEvent.BlockEvent) error { + if !s.IsNormal() { + return context.Cause(s.ctx) + } + for _, lane := range s.lanes { + if err := s.waitWork(lane, workItem{}); err != nil { + return err + } + } + return nil +} + +func (s *Sink) WriteBlockEvent(event commonEvent.BlockEvent) error { + s.ddlMu.Lock() + defer s.ddlMu.Unlock() + if err := s.FlushDMLBeforeBlock(event); err != nil { + return err + } + e, ok := event.(*commonEvent.DDLEvent) + if !ok { + err := errors.ErrDatabendInvalidData.GenWithStackByArgs("syncpoints are not supported") + s.stop(err) + return err + } + db, table := e.SchemaName, e.TableName + if e.TableInfo != nil { + db, table = e.TableInfo.GetSchemaName(), e.TableInfo.GetTableName() + } + if s.opts.database != "" { + db = s.opts.database + } + if err := s.waitWork(s.lane(db, table), workItem{ddl: e}); err != nil { + return err + } + event.PostFlush() + return nil +} + +func (s *Sink) cleanup() { + s.cleanupOnce.Do(func() { + // Pair with AddDMLEvent's admission critical section. Once normal is + // false, no producer may publish behind cleanup's final queue drain. + s.mu.Lock() + s.closed = true + s.mu.Unlock() + for _, lane := range s.lanes { + lane.cleanup() + } + s.spool.Close() + s.client.transport.CloseIdleConnections() + s.statistics.Close() + }) +} + +func (s *Sink) Run(ctx context.Context) error { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return context.Cause(s.ctx) + } + if s.started { + s.mu.Unlock() + return errors.ErrDatabendInvalidData.GenWithStackByArgs("sink lifecycle cannot be reused") + } + s.started = true + s.mu.Unlock() + defer close(s.done) + defer s.cleanup() + stop := context.AfterFunc(ctx, func() { s.stopLifecycle(context.Cause(ctx)) }) + defer stop() + g, _ := errgroup.WithContext(s.ctx) + for _, lane := range s.lanes { + g.Go(func() error { + err := lane.run(s.ctx) + if err != nil { + s.stop(err) + } + return err + }) + } + _ = g.Wait() + s.normal.Store(false) + // The first cause already distinguishes a normal external stop from an + // unretryable internal failure. No nested timeout can downgrade the latter. + return context.Cause(s.ctx) +} + +func (s *Sink) Close() { + s.stopLifecycle(context.Canceled) + s.mu.Lock() + s.closed = true + started := s.started + s.mu.Unlock() + if started { + <-s.done + } else { + s.cleanup() + } +} diff --git a/downstreamadapter/sink/databend/sql.go b/downstreamadapter/sink/databend/sql.go new file mode 100644 index 0000000000..3cf729e7cd --- /dev/null +++ b/downstreamadapter/sink/databend/sql.go @@ -0,0 +1,126 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "fmt" + "strings" + "time" +) + +func quoteIdentifier(name string) string { + return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` +} + +func (s tableSchema) targetName() string { + return quoteIdentifier(s.database) + "." + quoteIdentifier(s.table) +} + +func (s tableSchema) rawName() string { + return quoteIdentifier(s.database) + "." + quoteIdentifier(s.table+"_cdc_raw") +} + +func (s tableSchema) createRawSQL() string { + return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( +id VARCHAR(255), table_name VARCHAR(255), raw_data JSON, +add_time TIMESTAMP, event_pos BIGINT, action VARCHAR(20))`, s.rawName()) +} + +// addTimeClock assigns timestamps in the precision actually sent to Databend. +// Comparing nanoseconds before formatting microseconds would lose tie ordering. +type addTimeClock struct { + last time.Time +} + +func (c *addTimeClock) next(candidate time.Time) time.Time { + candidate = candidate.UTC().Truncate(time.Microsecond) + if !c.last.IsZero() && !candidate.After(c.last) { + candidate = c.last.Add(time.Microsecond) + } + c.last = candidate + return candidate +} + +func (s tableSchema) insertRawSQL(rows []normalizedRow, clock *addTimeClock, now time.Time) (string, []any) { + placeholders := make([]string, len(rows)) + args := make([]any, 0, len(rows)*6) + for i, row := range rows { + placeholders[i] = "(?, ?, ?, ?, ?, ?)" + args = append(args, row.id, s.table, row.payload, + clock.next(now).Format("2006-01-02 15:04:05.000000"), int64(row.commitTs), string(row.action)) + } + return "INSERT INTO " + s.rawName() + + " (id, table_name, raw_data, add_time, event_pos, action) VALUES " + strings.Join(placeholders, ", "), args +} + +func (s tableSchema) mergeSQL(epoch uint64) string { + projections := make([]string, 0, len(s.columns)) + for _, col := range s.columns { + // A source name is a literal JSON object key, not a JSONPath. Quoting + // it as an SQL identifier still leaves embedded quotes unescaped in + // Databend's get_by_keypath conversion. GET uses the exact string key. + path := "get(s.raw_data, " + sqlString(col.name) + ")" + alias := quoteIdentifier(col.name) + switch col.encoding { + case binaryColumn: + projections = append(projections, fmt.Sprintf("unhex(%s) AS %s", path, alias)) + case jsonColumn: + projections = append(projections, fmt.Sprintf( + "CASE WHEN %[1]s IS NULL OR is_null_value(%[1]s) THEN NULL ELSE parse_json(%[1]s::VARCHAR) END AS %[2]s", path, alias)) + default: + projections = append(projections, path+" AS "+alias) + } + } + // Keep the operation discriminator separate from arbitrary source names. + operation := "__ticdc_operation" + for { + collision := false + for _, c := range s.columns { + if strings.EqualFold(c.name, operation) { + collision = true + break + } + } + if !collision { + break + } + operation += "_" + } + op := quoteIdentifier(operation) + joins := make([]string, 0, len(s.primaryKey)) + for _, key := range s.canonicalKeys() { + quoted := quoteIdentifier(key) + joins = append(joins, "a."+quoted+" = b."+quoted) + } + // The source is deliberately raw, not a stream. This is bend-cdc's strict + // ordering fallback, including latest tombstones and the applied DDL fence. + return fmt.Sprintf(`MERGE INTO %s a USING ( +SELECT %s, s.action AS %s FROM %s s +WHERE s.event_pos > %d AND NOT EXISTS ( +SELECT 1 FROM %s h WHERE h.id = s.id AND ( +h.event_pos > s.event_pos OR (h.event_pos = s.event_pos AND h.add_time > s.add_time))) +QUALIFY ROW_NUMBER() OVER (PARTITION BY s.id ORDER BY s.event_pos DESC, s.add_time DESC) = 1 +) b ON %s +WHEN MATCHED AND b.%s != 'delete' THEN UPDATE * +WHEN MATCHED AND b.%s = 'delete' THEN DELETE +WHEN NOT MATCHED AND b.%s != 'delete' THEN INSERT *`, + s.targetName(), strings.Join(projections, ", "), op, s.rawName(), epoch, s.rawName(), strings.Join(joins, " AND "), op, op, op) +} + +func (s tableSchema) compactRawSQL() string { + return fmt.Sprintf(`DELETE FROM %s AS stale WHERE EXISTS ( +SELECT 1 FROM %s AS newer WHERE newer.id = stale.id AND ( +newer.event_pos > stale.event_pos OR (newer.event_pos = stale.event_pos AND newer.add_time > stale.add_time)))`, s.rawName(), s.rawName()) +} diff --git a/downstreamadapter/sink/databend/sql_test.go b/downstreamadapter/sink/databend/sql_test.go new file mode 100644 index 0000000000..39c6932b3e --- /dev/null +++ b/downstreamadapter/sink/databend/sql_test.go @@ -0,0 +1,92 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestRawInsertUsesParametersAndStoredPrecision(t *testing.T) { + schema := testSchema() + rows := []normalizedRow{ + testRow(t, deleteAction, 100, 1), testRow(t, insertAction, 100, 1), testRow(t, updateAction, 100, 1), + } + var clock addTimeClock + now := time.Date(2026, 10, 24, 1, 2, 3, 456789001, time.UTC) + statement, args := schema.insertRawSQL(rows, &clock, now) + require.Equal(t, 18, len(args)) + require.Equal(t, 3, strings.Count(statement, "(?, ?, ?, ?, ?, ?)")) + require.NotContains(t, statement, "payload") + require.Equal(t, []any{"2026-10-24 01:02:03.456789", "2026-10-24 01:02:03.456790", "2026-10-24 01:02:03.456791"}, []any{args[3], args[9], args[15]}) + require.Equal(t, "delete", args[5]) + require.Equal(t, "insert", args[11]) + _, later := schema.insertRawSQL(rows[:1], &clock, now.Add(-time.Hour)) + require.Equal(t, "2026-10-24 01:02:03.456792", later[3], "clock rollback must not reverse source tie order") +} + +func TestFullRawMergeSQLKeepsBaselineSemantics(t *testing.T) { + schema := tableSchema{ + database: `lake"db`, table: `order"s`, + columns: []column{{name: "tenant"}, {name: "id"}, {name: `raw"bytes`, encoding: binaryColumn}, {name: "json", encoding: jsonColumn}}, + primaryKey: []string{"tenant", "id"}, + } + require.NoError(t, schema.validate()) + statement := schema.mergeSQL(500) + for _, want := range []string{ + `MERGE INTO "lake""db"."order""s" a`, + `FROM "lake""db"."order""s_cdc_raw" s`, + `s.event_pos > 500`, `h.event_pos > s.event_pos`, `h.add_time > s.add_time`, + `PARTITION BY s.id ORDER BY s.event_pos DESC, s.add_time DESC`, + `a."id" = b."id" AND a."tenant" = b."tenant"`, + `unhex(get(s.raw_data, 'raw"bytes'))`, `is_null_value(get(s.raw_data, 'json'))`, `parse_json(get(s.raw_data, 'json')::VARCHAR)`, + `WHEN MATCHED AND b."__ticdc_operation" = 'delete' THEN DELETE`, + `WHEN NOT MATCHED AND b."__ticdc_operation" != 'delete' THEN INSERT *`, + } { + require.Contains(t, statement, want) + } + require.NotContains(t, statement, "_cdc_stream") + require.NotContains(t, statement, "LIMIT") + require.NotContains(t, statement, "TRY_") + compact := schema.compactRawSQL() + require.Contains(t, compact, `newer.id = stale.id`) + require.Contains(t, compact, `newer.event_pos > stale.event_pos`) + require.NotContains(t, compact, "NOW()", "latest tombstones are not TTL deleted") + require.NotContains(t, compact, ">=", "equal physical replays must not delete every latest row") + require.NotContains(t, schema.createRawSQL(), "STREAM") +} + +func TestMergeOperationAliasDoesNotCollideWithSourceColumn(t *testing.T) { + schema := testSchema() + schema.columns = append(schema.columns, column{name: "action"}, column{name: "__ticdc_operation"}) + statement := schema.mergeSQL(0) + require.Contains(t, statement, `get(s.raw_data, 'action') AS "action"`) + require.Contains(t, statement, `s.action AS "__ticdc_operation_"`) + require.Contains(t, statement, `b."__ticdc_operation_" = 'delete'`) +} + +func TestMergeProjectionUsesLiteralJSONKeyNotKeyPath(t *testing.T) { + schema := testSchema() + schema.columns = append(schema.columns, column{name: `a"b`}, column{name: "a.b"}, column{name: "O'Reilly"}, column{name: `a\b`}) + sql := schema.mergeSQL(0) + require.Contains(t, sql, `get(s.raw_data, 'a"b') AS "a""b"`) + require.Contains(t, sql, `get(s.raw_data, 'a.b') AS "a.b"`) + require.Contains(t, sql, `get(s.raw_data, 'O''Reilly') AS "O'Reilly"`) + require.Contains(t, sql, `get(s.raw_data, 'a\\b') AS "a\b"`) + require.NotContains(t, sql, "s.raw_data:") +} diff --git a/downstreamadapter/sink/databend/writer.go b/downstreamadapter/sink/databend/writer.go new file mode 100644 index 0000000000..068149a3e8 --- /dev/null +++ b/downstreamadapter/sink/databend/writer.go @@ -0,0 +1,160 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "context" + "encoding/json" + "math" + "sync" + "time" + + "github.com/pingcap/ticdc/pkg/errors" +) + +type commitState uint8 + +const ( + commitUnknown commitState = iota // Missing evidence must never mean success. + committed + notCommitted +) + +// mutationResult is produced at the client boundary, before any inner retry. +// observationErr is valid only after commit has independently been confirmed. +type mutationResult struct { + state commitState + retryable bool + observationErr error +} + +// executeMutation must read the server terminal state, not just the first result +// page. The HTTP adapter supplies it; transport errors alone must not +// be labelled notCommitted. Arguments and SQL must not be included in errors. +type executeMutation func(context.Context, string, []any) (mutationResult, error) + +type applyResult struct { + // applied remains true if only subsequent raw compaction fails. A worker + // must not undo/redo already applied source completion to repair maintenance. + applied bool +} + +// tableWriter is a serial raw/merge lane. It owns no source callbacks; its worker +// completes fragments only when applyResult.applied is true. +type tableWriter struct { + mu sync.Mutex + schema tableSchema + execute executeMutation + batchRows int + maxAttempts int + epoch uint64 + clock addTimeClock + written uint64 + merged uint64 + failed error +} + +func newTableWriter(schema tableSchema, execute executeMutation, batchRows, maxAttempts int, epoch uint64) (*tableWriter, error) { + if err := schema.validate(); err != nil { + return nil, err + } + if execute == nil || batchRows <= 0 || maxAttempts <= 0 || epoch > math.MaxInt64 { + return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid table writer configuration") + } + // Keep metadata owned even if the caller reuses its slices for a later DDL. + schema.columns = append([]column(nil), schema.columns...) + schema.primaryKey = append([]string(nil), schema.primaryKey...) + return &tableWriter{schema: schema, execute: execute, batchRows: batchRows, maxAttempts: maxAttempts, epoch: epoch}, nil +} + +// applyBatch requires the target/raw tables and the durable schema epoch to +// have been initialized by the table worker. It preserves source row order and +// never writes a row at/before the applied epoch into the new projection. +func (w *tableWriter) applyBatch(ctx context.Context, rows []normalizedRow, now time.Time) (applyResult, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.failed != nil { + return applyResult{}, w.failed + } + // Validate the complete batch before issuing any mutation. + for _, row := range rows { + if row.id == "" || !json.Valid([]byte(row.payload)) || row.commitTs == 0 || row.commitTs > math.MaxInt64 || + (row.action != insertAction && row.action != updateAction && row.action != deleteAction) { + return applyResult{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid normalized row") + } + } + pending := make([]normalizedRow, 0, len(rows)) + for _, row := range rows { + if row.commitTs > w.epoch { + pending = append(pending, row) + } + } + if len(pending) == 0 { + // Only a proven applied DDL fence permits this no-op, not schema bootstrap. + return applyResult{applied: true}, nil + } + for start := 0; start < len(pending); start += w.batchRows { + end := min(start+w.batchRows, len(pending)) + statement, args := w.schema.insertRawSQL(pending[start:end], &w.clock, now) + if err := w.runMutation(ctx, statement, args); err != nil { + w.failed = err + return applyResult{}, err + } + w.written++ + } + if err := w.runMutation(ctx, w.schema.mergeSQL(w.epoch), nil); err != nil { + w.failed = err + return applyResult{}, err + } + w.merged = w.written + result := applyResult{applied: true} + if err := w.runMutation(ctx, w.schema.compactRawSQL(), nil); err != nil { + w.failed = err + return result, err + } + return result, nil +} + +func (w *tableWriter) runMutation(ctx context.Context, statement string, args []any) error { + for attempt := 0; attempt < w.maxAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return errors.WrapError(errors.ErrDatabendExecution, err) + } + result, err := w.execute(ctx, statement, args) + switch result.state { + case committed: + if err != nil { + // Contradictory execution evidence is not an observation error. + return errors.WrapError(errors.ErrDatabendCommitUnknown, err) + } + // A confirmed commit is not undone by result statistics parsing. + return nil + case notCommitted: + if !result.retryable || attempt+1 == w.maxAttempts { + if err != nil { + return errors.WrapError(errors.ErrDatabendExecution, err) + } + return errors.ErrDatabendExecution.GenWithStackByArgs() + } + default: + // Check before transient-error handling: no unknown request is resent. + if err != nil { + return errors.WrapError(errors.ErrDatabendCommitUnknown, err) + } + return errors.ErrDatabendCommitUnknown.GenWithStackByArgs() + } + } + return errors.ErrDatabendExecution.GenWithStackByArgs() +} diff --git a/downstreamadapter/sink/databend/writer_test.go b/downstreamadapter/sink/databend/writer_test.go new file mode 100644 index 0000000000..b843941550 --- /dev/null +++ b/downstreamadapter/sink/databend/writer_test.go @@ -0,0 +1,176 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package databend + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestTableWriterAppliesOnlyAfterAllRawAndMerge(t *testing.T) { + var statements []string + execute := func(_ context.Context, statement string, args []any) (mutationResult, error) { + statements = append(statements, statement) + if strings.HasPrefix(statement, "INSERT") { + require.Len(t, args, 6) + } + return mutationResult{state: committed}, nil + } + w, err := newTableWriter(testSchema(), execute, 1, 3, 0) + require.NoError(t, err) + result, err := w.applyBatch(context.Background(), []normalizedRow{ + testRow(t, deleteAction, 100, 1), testRow(t, insertAction, 100, 2), + }, time.Now()) + require.NoError(t, err) + require.True(t, result.applied) + require.Len(t, statements, 4) + require.True(t, strings.HasPrefix(statements[0], "INSERT")) + require.True(t, strings.HasPrefix(statements[1], "INSERT")) + require.True(t, strings.HasPrefix(statements[2], "MERGE")) + require.True(t, strings.HasPrefix(statements[3], "DELETE")) + require.EqualValues(t, 2, w.written) + require.Equal(t, w.written, w.merged) +} + +func TestTableWriterStopsUnknownBeforeInnerRetry(t *testing.T) { + for _, failAt := range []int{1, 2, 3} { + t.Run([]string{"", "first raw", "second raw", "merge"}[failAt], func(t *testing.T) { + calls := 0 + execute := func(context.Context, string, []any) (mutationResult, error) { + calls++ + if calls == failAt { + return mutationResult{state: commitUnknown, retryable: true}, context.DeadlineExceeded + } + return mutationResult{state: committed}, nil + } + w, err := newTableWriter(testSchema(), execute, 1, 10, 0) + require.NoError(t, err) + rows := []normalizedRow{testRow(t, insertAction, 100, 1), testRow(t, updateAction, 120, 2)} + result, err := w.applyBatch(context.Background(), rows, time.Now()) + require.Error(t, err) + code, ok := errors.RFCCode(err) + require.True(t, ok) + require.Equal(t, errors.ErrDatabendCommitUnknown.RFCCode(), code) + require.False(t, result.applied) + require.Equal(t, failAt, calls, "unknown must not reach an inner retry, MERGE, or compaction") + require.Zero(t, w.merged) + _, err = w.applyBatch(context.Background(), rows, time.Now()) + require.Error(t, err) + require.Equal(t, failAt, calls, "a stopped lane cannot take a new batch") + }) + } +} + +func TestTableWriterRetriesOnlyConfirmedNotCommitted(t *testing.T) { + calls := 0 + execute := func(context.Context, string, []any) (mutationResult, error) { + calls++ + if calls == 1 { + return mutationResult{state: notCommitted, retryable: true}, errors.ErrDatabendInvalidData.GenWithStackByArgs("confirmed rollback") + } + return mutationResult{state: committed, observationErr: errors.ErrDatabendInvalidData.GenWithStackByArgs("statistics unavailable")}, nil + } + w, err := newTableWriter(testSchema(), execute, 1, 2, 0) + require.NoError(t, err) + result, err := w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 100, 1)}, time.Now()) + require.NoError(t, err) + require.True(t, result.applied) + require.Equal(t, 4, calls, "one proven rollback retry, then raw/MERGE/compaction; no statistics retry") + + calls = 0 + w, err = newTableWriter(testSchema(), func(context.Context, string, []any) (mutationResult, error) { + calls++ + return mutationResult{state: notCommitted, retryable: true}, errors.ErrDatabendInvalidData.GenWithStackByArgs("confirmed rollback") + }, 1, 2, 0) + require.NoError(t, err) + result, err = w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 100, 1)}, time.Now()) + require.Error(t, err) + require.False(t, result.applied) + require.Equal(t, 2, calls) +} + +func TestTableWriterDoesNotRevokeAppliedOnCompactionFailure(t *testing.T) { + calls := 0 + execute := func(context.Context, string, []any) (mutationResult, error) { + calls++ + if calls == 3 { + return mutationResult{state: notCommitted}, errors.ErrDatabendInvalidData.GenWithStackByArgs("compaction denied") + } + return mutationResult{state: committed}, nil + } + w, err := newTableWriter(testSchema(), execute, 10, 1, 0) + require.NoError(t, err) + result, err := w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 100, 1)}, time.Now()) + require.Error(t, err) + require.True(t, result.applied, "MERGE committed; maintenance does not roll it back") + require.Equal(t, w.written, w.merged) + _, err = w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 120, 2)}, time.Now()) + require.Error(t, err) + require.Equal(t, 3, calls) +} + +func TestTableWriterRejectsMissingOrContradictoryCommitEvidence(t *testing.T) { + for _, tc := range []struct { + name string + result mutationResult + err error + }{ + {name: "missing terminal evidence"}, + {name: "contradictory evidence", result: mutationResult{state: committed}, err: errors.ErrDatabendInvalidData.GenWithStackByArgs("execution failed")}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + w, err := newTableWriter(testSchema(), func(context.Context, string, []any) (mutationResult, error) { + calls++ + return tc.result, tc.err + }, 10, 3, 0) + require.NoError(t, err) + result, err := w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 100, 1)}, time.Now()) + require.Error(t, err) + code, ok := errors.RFCCode(err) + require.True(t, ok) + require.Equal(t, errors.ErrDatabendCommitUnknown.RFCCode(), code) + require.False(t, result.applied) + require.Equal(t, 1, calls) + }) + } +} + +func TestTableWriterAppliedEpochAndValidation(t *testing.T) { + calls := 0 + w, err := newTableWriter(testSchema(), func(context.Context, string, []any) (mutationResult, error) { + calls++ + return mutationResult{state: committed}, nil + }, 10, 1, 100) + require.NoError(t, err) + result, err := w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 100, 1)}, time.Now()) + require.NoError(t, err) + require.True(t, result.applied) + require.Zero(t, calls) + _, err = w.applyBatch(context.Background(), []normalizedRow{{id: "1", payload: "not-json", commitTs: 120, action: insertAction}}, time.Now()) + require.Error(t, err) + require.Zero(t, calls) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result, err = w.applyBatch(ctx, []normalizedRow{testRow(t, insertAction, 120, 1)}, time.Now()) + require.Error(t, err) + require.False(t, result.applied) + require.Zero(t, calls) +} diff --git a/downstreamadapter/sink/metrics/databend.go b/downstreamadapter/sink/metrics/databend.go new file mode 100644 index 0000000000..3f446893ef --- /dev/null +++ b/downstreamadapter/sink/metrics/databend.go @@ -0,0 +1,29 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +var ( + DatabendBufferBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: "ticdc", Subsystem: "databend", Name: "buffer_bytes", Help: "Local Databend memory and physical spool bytes."}, []string{"namespace", "changefeed", "kind"}) + DatabendBufferEntries = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: "ticdc", Subsystem: "databend", Name: "buffer_entries", Help: "Local Databend pending enqueue, segment and quota waiter counts."}, []string{"namespace", "changefeed", "kind"}) + DatabendSpoolDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{Namespace: "ticdc", Subsystem: "databend", Name: "spool_wait_seconds", Help: "Databend spool quota wait time.", Buckets: prometheus.ExponentialBuckets(0.001, 2, 20)}, []string{"namespace", "changefeed"}) + DatabendSpoolLoad = prometheus.NewHistogramVec(prometheus.HistogramOpts{Namespace: "ticdc", Subsystem: "databend", Name: "spool_load_bytes", Help: "Databend encoded bytes loaded from local spool.", Buckets: prometheus.ExponentialBuckets(1024, 4, 9)}, []string{"namespace", "changefeed"}) + DatabendSpoolRotations = prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: "ticdc", Subsystem: "databend", Name: "spool_rotations_total", Help: "Databend local spool segment rotations."}, []string{"namespace", "changefeed"}) +) + +func InitDatabendMetrics(registry *prometheus.Registry) { + registry.MustRegister(DatabendBufferBytes, DatabendBufferEntries, DatabendSpoolDuration, DatabendSpoolLoad, DatabendSpoolRotations) +} diff --git a/downstreamadapter/sink/sink.go b/downstreamadapter/sink/sink.go index a62aea599e..d1d04b6c44 100644 --- a/downstreamadapter/sink/sink.go +++ b/downstreamadapter/sink/sink.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/ticdc/downstreamadapter/sink/blackhole" "github.com/pingcap/ticdc/downstreamadapter/sink/cloudstorage" + "github.com/pingcap/ticdc/downstreamadapter/sink/databend" "github.com/pingcap/ticdc/downstreamadapter/sink/kafka" "github.com/pingcap/ticdc/downstreamadapter/sink/mysql" "github.com/pingcap/ticdc/downstreamadapter/sink/pulsar" @@ -72,6 +73,8 @@ func New(ctx context.Context, cfg *config.ChangefeedConfig, changefeedID common. return pulsar.New(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive, keyspaceID) case config.S3Scheme, config.FileScheme, config.GCSScheme, config.GSScheme, config.AzblobScheme, config.AzureScheme, config.CloudStorageNoopScheme: return cloudstorage.New(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive, cfg.EnableTableAcrossNodes, nil, keyspaceID) + case config.DatabendScheme, config.DatabendHTTPScheme, config.LakeScheme: + return databend.New(ctx, changefeedID, cfg, sinkURI, keyspaceID) case config.BlackHoleScheme: return blackhole.New(changefeedID, keyspaceID) } @@ -97,6 +100,8 @@ func Verify(ctx context.Context, cfg *config.ChangefeedConfig, changefeedID comm return pulsar.Verify(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive) case config.S3Scheme, config.FileScheme, config.GCSScheme, config.GSScheme, config.AzblobScheme, config.AzureScheme, config.CloudStorageNoopScheme: return cloudstorage.Verify(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive, cfg.EnableTableAcrossNodes) + case config.DatabendScheme, config.DatabendHTTPScheme, config.LakeScheme: + return databend.Verify(ctx, cfg, sinkURI) case config.BlackHoleScheme: return nil } diff --git a/downstreamadapter/sink/sink_test.go b/downstreamadapter/sink/sink_test.go index 48c347da9a..4a2244f10f 100644 --- a/downstreamadapter/sink/sink_test.go +++ b/downstreamadapter/sink/sink_test.go @@ -15,9 +15,11 @@ package sink import ( "context" + "net/url" "testing" "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/config" "github.com/stretchr/testify/require" ) @@ -75,3 +77,19 @@ func requireInvalidSinkURIError(t *testing.T, err error) { require.Contains(t, errMsg, `parse ""`) require.Contains(t, errMsg, "invalid URL escape") } + +func TestDatabendFactoryRegistration(t *testing.T) { + cfg := &config.ChangefeedConfig{SinkURI: "databend+http://127.0.0.1:1?exclusive=true&spool-dir=" + url.QueryEscape(t.TempDir())} + s, err := New(context.Background(), cfg, common.NewChangefeedID(common.DefaultKeyspaceName), common.DefaultKeyspaceID) + require.NoError(t, err) + defer s.Close() + require.Equal(t, common.DatabendSinkType, s.SinkType()) + store := commonEvent.NewTableSchemaStore(nil, s.SinkType(), false) + s.SetTableSchemaStore(store) + require.False(t, config.IsStorageScheme("databend")) + require.False(t, config.IsMySQLCompatibleScheme("databend")) + cfg.SinkURI = "databend://user:verysecure@localhost?exclusive=false" + _, err = New(context.Background(), cfg, common.NewChangefeedID(common.DefaultKeyspaceName), common.DefaultKeyspaceID) + require.Error(t, err) + require.NotContains(t, err.Error(), "verysecure") +} diff --git a/pkg/common/event/table_schema_store.go b/pkg/common/event/table_schema_store.go index 588f33b7fa..0cfe0b701c 100644 --- a/pkg/common/event/table_schema_store.go +++ b/pkg/common/event/table_schema_store.go @@ -91,7 +91,7 @@ func newTableSchemaStoreRequirements( updateTableIDs: false, needTableNames: true, } - case commonType.CloudStorageSinkType, commonType.BlackHoleSinkType: + case commonType.CloudStorageSinkType, commonType.BlackHoleSinkType, commonType.DatabendSinkType: return tableSchemaStoreRequirements{ needTableIDs: false, updateTableIDs: false, diff --git a/pkg/common/event/table_schema_store_test.go b/pkg/common/event/table_schema_store_test.go index fd56f35adc..2aa18f407e 100644 --- a/pkg/common/event/table_schema_store_test.go +++ b/pkg/common/event/table_schema_store_test.go @@ -253,3 +253,12 @@ func TestTableSchemaStoreNonMysqlTableIDsBootstrapOnly(t *testing.T) { require.ElementsMatch(t, []int64{10, 20}, store.GetAllNormalTableIds()) } + +func TestTableSchemaStoreWhenDatabendSink(t *testing.T) { + store := NewTableSchemaStore(nil, common.DatabendSinkType, false) + require.NotNil(t, store) + require.Nil(t, store.TableIDStore) + require.Nil(t, store.tableNameStore) + store.AddEvent(&DDLEvent{FinishedTs: 100}) + require.Empty(t, store.GetAllTableIds()) +} diff --git a/pkg/common/types.go b/pkg/common/types.go index 638d75305d..82e718786e 100644 --- a/pkg/common/types.go +++ b/pkg/common/types.go @@ -344,6 +344,7 @@ const ( CloudStorageSinkType BlackHoleSinkType RedoSinkType + DatabendSinkType ) type RowType byte diff --git a/pkg/config/sink_protocol.go b/pkg/config/sink_protocol.go index 113d66eab9..ed76ed65ea 100644 --- a/pkg/config/sink_protocol.go +++ b/pkg/config/sink_protocol.go @@ -114,6 +114,12 @@ func (p Protocol) String() string { const ( // KafkaScheme indicates the scheme is kafka. KafkaScheme = "kafka" + // DatabendScheme is the native HTTPS warehouse sink. + DatabendScheme = "databend" + // LakeScheme is the TiDB Cloud Lake HTTPS endpoint alias. + LakeScheme = "lake" + // DatabendHTTPScheme is the explicit plaintext local/test warehouse sink. + DatabendHTTPScheme = "databend+http" // KafkaSSLScheme indicates the scheme is kafka+ssl. KafkaSSLScheme = "kafka+ssl" // BlackHoleScheme indicates the scheme is blackhole. diff --git a/pkg/errors/databend.go b/pkg/errors/databend.go new file mode 100644 index 0000000000..41d7d88ffb --- /dev/null +++ b/pkg/errors/databend.go @@ -0,0 +1,27 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package errors + +import "github.com/pingcap/errors" + +var ( + ErrDatabendProtocol = errors.Normalize("Databend request %s: %s", errors.RFCCodeText("CDC:ErrDatabendProtocol")) + ErrDatabendInvalidData = errors.Normalize( + "invalid Databend sink data: %s", errors.RFCCodeText("CDC:ErrDatabendInvalidData")) + ErrDatabendExecution = errors.Normalize( + "Databend mutation failed", errors.RFCCodeText("CDC:ErrDatabendExecution")) + ErrDatabendCommitUnknown = errors.Normalize( + "Databend mutation commit state is unknown", errors.RFCCodeText("CDC:ErrDatabendCommitUnknown")) +) diff --git a/pkg/errors/helper.go b/pkg/errors/helper.go index 19a6bc05f6..8a76b559f3 100644 --- a/pkg/errors/helper.go +++ b/pkg/errors/helper.go @@ -102,6 +102,8 @@ func RFCCode(err error) (errors.RFCErrorCode, bool) { } var changefeedUnRetryableErrors = []*errors.Error{ + ErrDatabendInvalidData, + ErrDatabendCommitUnknown, ErrExpressionColumnNotFound, ErrExpressionParseFailed, ErrSchemaSnapshotNotFound, diff --git a/pkg/metrics/downstream_type.go b/pkg/metrics/downstream_type.go index 399337cbe6..c960036802 100644 --- a/pkg/metrics/downstream_type.go +++ b/pkg/metrics/downstream_type.go @@ -59,6 +59,8 @@ func downstreamTypeFromScheme(scheme string) string { return config.AzblobScheme case config.CloudStorageNoopScheme: return config.CloudStorageNoopScheme + case config.DatabendScheme, config.DatabendHTTPScheme, config.LakeScheme: + return config.DatabendScheme case config.BlackHoleScheme: return config.BlackHoleScheme case "": diff --git a/pkg/metrics/downstream_type_test.go b/pkg/metrics/downstream_type_test.go index 50cf8d718d..d3ac34ac43 100644 --- a/pkg/metrics/downstream_type_test.go +++ b/pkg/metrics/downstream_type_test.go @@ -45,6 +45,9 @@ func TestDownstreamTypeFromSinkURI(t *testing.T) { {sinkURI: "azblob://bucket/prefix", expectedLabel: "azblob"}, {sinkURI: "noop://bucket/prefix", expectedLabel: "noop"}, {sinkURI: "blackhole:///", expectedLabel: "blackhole"}, + {sinkURI: "databend://warehouse", expectedLabel: "databend"}, + {sinkURI: "lake://warehouse:443/default", expectedLabel: "databend"}, + {sinkURI: "databend+http://127.0.0.1:8000", expectedLabel: "databend"}, {sinkURI: "127.0.0.1:3306", expectedLabel: "unknown"}, {sinkURI: "mysql://[::1", expectedLabel: "unknown"}, {sinkURI: "", expectedLabel: "unknown"}, diff --git a/pkg/metrics/init.go b/pkg/metrics/init.go index 79e5cedc68..a6b0133aaa 100644 --- a/pkg/metrics/init.go +++ b/pkg/metrics/init.go @@ -44,6 +44,7 @@ func InitMetrics(registry *prometheus.Registry) { kafka.InitMetrics(registry) gc.InitMetrics(registry) metrics.InitCloudStorageMetrics(registry) + metrics.InitDatabendMetrics(registry) initRedoMetrics(registry) initDDLMetrics(registry) } diff --git a/pkg/sink/spool/spool.go b/pkg/sink/spool/spool.go index 4ec7f90410..bd90faf98b 100644 --- a/pkg/sink/spool/spool.go +++ b/pkg/sink/spool/spool.go @@ -1010,3 +1010,9 @@ func calculateEntrySize(msgs []*common.Message) entrySize { } return size } + +// AccountingBytes returns the encoded size charged by TryEnqueue. Callers may +// reject oversized entries before the optional in-memory bypass is considered. +func AccountingBytes(msgs []*common.Message) int64 { + return calculateEntrySize(msgs).accountingBytes +} diff --git a/pkg/sink/spool/spool_test.go b/pkg/sink/spool/spool_test.go index 5f92e615a7..696f54c504 100644 --- a/pkg/sink/spool/spool_test.go +++ b/pkg/sink/spool/spool_test.go @@ -1031,3 +1031,9 @@ func TestTryEnqueueReturnsOversizedActionWithInMemoryEntry(t *testing.T) { require.True(t, entry.InMemory()) require.False(t, entry.IsSpilled()) } + +func TestAccountingBytesMatchesAdmissionEncoding(t *testing.T) { + messages := []*common.Message{{Key: []byte("key"), Value: []byte("value")}, {Value: []byte("second")}} + require.EqualValues(t, len(serializeMessages(messages)), AccountingBytes(messages)) + require.Zero(t, AccountingBytes(nil)) +} diff --git a/tests/integration_tests/databend_native/.gitignore b/tests/integration_tests/databend_native/.gitignore new file mode 100644 index 0000000000..c18dd8d83c --- /dev/null +++ b/tests/integration_tests/databend_native/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/tests/integration_tests/databend_native/README.md b/tests/integration_tests/databend_native/README.md new file mode 100644 index 0000000000..4847c4776e --- /dev/null +++ b/tests/integration_tests/databend_native/README.md @@ -0,0 +1,301 @@ +# Native Databend / Lake acceptance test + +## One-command local E2E (no external Lake required) + +```sh +make integration_test_databend +# Or use the integration-case entry point: +bash tests/integration_tests/databend_native/run.sh databend +# Faster, selected scenario (repeat --suite to select more): +python3 tests/integration_tests/databend_native/local.py --suite acceptance +``` + +Prerequisites: a running Docker engine (Linux or Docker Desktop on macOS), TiUP, +the `mysql` CLI, Python 3 and the repository's Go/build dependencies. No cloud +credentials, manually running database or S3/MinIO service is required. Allow +enough free disk for the Go build and source cluster; 8 GiB Docker memory is +recommended. The runner does not install tools or start the Docker daemon. + +`local.py` follows the existing integration tests' owned-background-service +pattern, but reuses these Python assertions rather than a MySQL downstream diff: + +1. Build current-branch TiCDC using `make cdc` (or use `E2E_CDC_BINARY`). +2. Start a unique Databend **meta + query** all-in-one background container, + pinned to `v1.2.925-patch-13` and its multi-platform image digest. Use local + filesystem storage, not cloud object storage; publish HTTP only on loopback + with a random port and temporary random credentials. +3. Start an owned TiUP v8.5.1 source (one PD/TiKV/TiDB, no TiFlash/monitor) and + a new-architecture TiCDC capture. Wait for actual SQL and owner readiness. +4. Run the existing acceptance, DDL, explicit-key, skipped-DDL, four rejection + probes and real-Databend response-loss tests. The acceptance suite includes + a drained capture restart, disk-spool readback and full value comparisons. +5. On success, failure or interruption, stop only this run's processes/container + and clean its unique TiUP tag. Preserve private logs and per-suite results; + remove the temporary URI file. The container's target data is ephemeral and + is removed even on failure, unlike external-warehouse mode below. + +The summary path is printed at startup and exit. `summary.json` records source +commit/dirty status, build/server versions, image identity, suite exits and +cleanup. Detailed case assertions remain in each suite's `result.json`; fault +assertions are in `fault.log`. A rejected-scope PASS proves fail-closed behavior, +not support. Local fs-backed Databend proves SQL/adapter behavior, not cloud S3, +cloud gateway routing, cost, distributed HA or production capacity. + +Optional environment: `DATABEND_E2E_IMAGE` to explicitly test another image, +`TIDB_E2E_VERSION` for the source version, `E2E_PORT_OFFSET` (default 10000) for +source ports, `E2E_CDC_BINARY` for an already-built capture, and `OUT_DIR` for an +existing artifacts parent directory. Occupied source ports fail rather than +attaching to someone else's cluster. The runner intentionally ignores inherited +`LAKE_URI_FILE`, `DATABEND_NATIVE_TEST_URI`, capture address and source-port +settings, so it cannot accidentally use an external target. + +Service-free lifecycle checks: + +```sh +python3 -m unittest discover -s tests/integration_tests/databend_native -p 'test_*.py' +``` + +### Local execution evidence (2026-10-26) + +Executed on macOS arm64 with Docker Desktop, TiDB v8.5.1 and the pinned Databend +v1.2.925-patch-13 image; TiCDC was built from `457c2f9bd` plus this test-only dirty +tree. `make integration_test_databend` (reusing that current-branch binary via +`E2E_CDC_BINARY`) passed **all nine suites**: 27 Python case assertions and three +real-backend Go tests, including 19 DDL response-loss boundaries. Acceptance +observed 41 disk-spool readbacks and a successful drained capture restart. All +suite-owned database/feed cleanup and runner process/container cleanup passed. +After fixing canonical-path cleanup, the default build path was rerun separately +with `local.py --suite acceptance` (no `E2E_CDC_BINARY`): build, all nine acceptance +cases and automatic cleanup passed again. Independent checks found neither old +nor restarted capture PIDs, no owned Databend containers, and no URI file. +The service-free lifecycle suite passed seven tests. Linux/amd64 CI execution was +not performed in this run; the image is multi-platform, not evidence of a tested +second platform. + +The initial v1.2.852-nightly probe failed real parameter binding; it is **not** a +supported backend established by this test. No SQL string-interpolation fallback +was introduced. Startup now checks binding before creating the source/capture. +The managed tests also exposed and fixed fixture issues: hard-coded +restart port, owner election preceding coordinator readiness, and TIME_WAIT +blocking immediate reruns, plus canonical-path comparison during cleanup on +macOS (`/var` versus `/private/var`). These are test-lifecycle fixes, not TiCDC +framework changes. + +## External authorized warehouse mode + +Opt-in integration test for **a local TiUP TiDB + current-branch new-architecture +TiCDC + a real, explicitly authorized Lake test warehouse**. It uses only the +Python standard library and a local `mysql` CLI; no Dumpling or S3 is involved. + +The runner creates a random `native_e2e_*` database and a matching changefeed, +compares actual TiDB rows against actual Lake rows, and removes only those owned +resources after success. On failure, it keeps its isolated database/feed and a +private result file for diagnosis. Never point it at a production source/target. + +## Prepare + +1. Build `cmd/cdc` from this branch. Use `--newarch` and UTC for the test capture. + Any compatibility release label injected into this local build is **not** an + official upstream release; record the source commit and dirty-tree status. +2. Start TiUP with one TiDB/PD/TiKV and no TiFlash/monitor. For example: + + ```sh + tiup playground v8.5.1 --tag native-lake-test \ + --db 1 --pd 1 --kv 1 --tiflash 0 --ticdc 0 \ + --without-monitor --port-offset 10000 + ``` + +3. Start the built CDC with `--newarch --tz UTC`, local PD + `http://127.0.0.1:12379`, API `127.0.0.1:18300`, and a private test data dir. + The source is always `127.0.0.1`; `MYSQL_PORT` defaults to `14000`. +4. Put the authorized Lake URI in a **0600** file outside the repository. The + runner reads `LAKE_URI_FILE`; never put the URI/password on a command line, + into a shared report, or into Git. `lake://` is the HTTPS sink alias. +5. On low-free-space development laptops, TiDB v8.5.1's fast/distributed ADD + INDEX can reject a tiny test table before CDC sees any DDL. For this **owned + local TiUP only**, use legacy index construction if necessary: + + ```sql + SET GLOBAL tidb_ddl_enable_fast_reorg=OFF; + SET GLOBAL tidb_enable_dist_task=OFF; + ``` + + Record this source-environment adjustment. Do not change these settings on + a shared/production source just to make this test pass. + +## Run + +```sh +umask 077 +export LAKE_URI_FILE=/absolute/path/to/private/lake-uri +export E2E_RUN_DIR=/absolute/path/to/new/private/run-directory +export CDC_SERVER=http://127.0.0.1:18300 +export MYSQL_PORT=14000 +# Optional process-restart scenario: only the exact owned local binary is stopped. +export E2E_CDC_BINARY=/absolute/path/to/current-branch-cdc +export E2E_CDC_DATA_DIR=/absolute/path/to/the/cdc/data-dir +export E2E_PD=http://127.0.0.1:12379 +python3 tests/integration_tests/databend_native/run.py +``` + +Use a fresh run directory each time. Dedicated source and target database names +are chosen automatically; the URI's original database (e.g. `default`) is not +used as the test database. Existing tables there are never altered or dropped. + +The current suite covers: + +- DML insert/update/delete, nonclustered PK updates, secondary UK updates, + same-transaction delete/reinsert and composite keys. +- Quoted table/column names, `action` and internal-discriminator name collisions. +- Signed/unsigned integer boundaries, DECIMAL(65,30), FLOAT/DOUBLE, CHAR/TEXT, + binary/BLOB/VARBINARY padding/bytes, BIT(64), numeric-label ENUM, 64-element SET, + exact JSON integers, SQL NULL versus JSON null, dates, microsecond + datetime/timestamp/time, YEAR, and visible STORED generated columns. + VIRTUAL generated columns must not appear in the target. +- ADD/default/backfill, RENAME, integer widening, DROP column, secondary indexes, + TRUNCATE/new DML and DROP TABLE/raw cleanup. +- A 1,200-row transaction larger than the **64 KiB** disk quota, with a small + memory hot set. A live spool read histogram must show disk readback; this is + not inferred solely from configured thresholds. +- Optional drained process restart: wait for a positive source transaction TSO + checkpoint, pause, verify the owned CDC process, terminate it, create a source + DML/DDL backlog, restart, wait through coordinator bootstrap, resume without + overriding checkpoint, then compare the recovered state. This is **not** + a claim of in-flight ambiguous-query failover or multi-capture HA. + +Comparison is key-based, not dependent on differing numeric/VARCHAR ORDER BY +semantics. JSON null is observed explicitly with `is_null_value`; FLOAT is read +at sufficient source precision and compared with tolerance. All other integer, +Decimal, text and binary comparisons are exact (binary is canonicalized as hex). + +## Failure inspection and cleanup + +`result.json` records individual cases, versions, target schema and restart/spool +evidence. It excludes credentials. Logs and temporary credentials stay in the +private run directory, not the repository. + +First inspect a failure and ensure the old CDC queries are terminated. To remove +just the owned failed run: + +```sh +python3 tests/integration_tests/databend_native/run.py --cleanup +``` + +Use the same environment/run directory. Cleanup validates the generated namespace +and matching feed, pauses a healthy feed before deletion, and never drops the +URI's original database. The runner does not stop an externally launched TiUP or +CDC after the test: stop your owned local processes separately. Do not confuse a +successful source/target test with fault-injection, cost/SLO, unsupported-DDL or +production-readiness validation. + +## Extended DDL and baseline parity probes + +`extended.py` uses the same local services/private environment and a fresh +`E2E_RUN_DIR`. It verifies defaults and nullability with actual target DESCRIBE, +not only row counts. Its positive suite also covers a NOT NULL UNIQUE handle +without a declared PK and sufficiently wide `INT -> VARCHAR(200)`. + +```sh +python3 tests/integration_tests/databend_native/extended.py +# Separate fresh run directories for each intentionally unsupported probe: +python3 tests/integration_tests/databend_native/extended.py --reject partition +python3 tests/integration_tests/databend_native/extended.py --reject no-pk +python3 tests/integration_tests/databend_native/extended.py --reject narrow +python3 tests/integration_tests/databend_native/extended.py --reject expression-default +``` + +A rejection PASS means **stable failed feed + nonadvancing checkpoint + unchanged +existing target (or no new projection)**, NOT feature support. The `no-pk` probe +forces replication and writes a source row, so upstream eligibility filtering +cannot masquerade as sink rejection. A NOT NULL UNIQUE table is a positive test, +not a `no-pk` rejection. Optional `--unique-only` / `--varchar-only` isolate those +positive cases for diagnosis. + +`baseline.py` runs an ephemeral Go overlay against the **clean, fixed b648162** +independent bend-cdc module. It changes no baseline files. Set `BEND_CDC_DIR` to +that checkout, retain the same private URI-file environment, and use a fresh run +directory. `baseline_parity_test.go.txt` is the reviewable overlay fixture. + +```sh +export BEND_CDC_DIR=/absolute/path/to/webapi/bend-cdc +python3 tests/integration_tests/databend_native/baseline.py +``` + +These baseline probes exercise its actual Canal/schema-to-Lake core, not the +entire S3 listing/decoding path. In particular they assert RENAME / DROP DATABASE +are **not projected**, rather than treating a nil error as feature support. + +## Real warehouse response-loss injection + +`live_fault_test.go` is opt-in. All SQL executes on the real authorized warehouse. +A delegating HTTP RoundTripper drops a matching real Succeeded response before +the sink sees it; it does NOT mock SQL success. For partial raw writes it blocks +the second INSERT before sending, after the first real INSERT committed. + +```sh +python3 - <<'PY' +import os, pathlib, subprocess +p = pathlib.Path(os.environ['LAKE_URI_FILE']) +assert not p.stat().st_mode & 0o077 + +env = os.environ.copy() +env['DATABEND_NATIVE_TEST_URI'] = p.read_text().strip() +env.setdefault('GOMAXPROCS', '4') +subprocess.run([ + 'go', 'test', '-p', '2', '-race', './downstreamadapter/sink/databend', + '-run', 'TestDatabendNativeLive(RawPartialAndMergeResponseLoss|DDLInterruptionRecovery)$', + '-v', '-count=1', '-timeout=20m', +], env=env, check=True) +PY +``` + +Coverage: partial raw recovery; committed MERGE with lost terminal response and +zero source flush; 19 DDL commit boundaries across CREATE/ALTER/TRUNCATE/DROP, +zero early DDL callbacks, durable pending/applied recovery, full recovered values, +and old-DML/applied-DDL replay fences. Each test creates/drops a random +`ticdc_fault_test_*` database. + +Recovery is allowed because the test independently knows the old matching query +reached terminal success (or was never sent), and has ended the old sink +lifecycle. Schema-manager state is rebuilt inside the test process; this is not +a SIGKILL-at-every-instruction test, mid-execution server failure, or automatic +HA proof. A real arbitrary UNKNOWN still requires manual terminal/exclusivity +verification before resume. + + +## Explicit conflict key and skipped DDL + +These are current positive acceptance scenarios (not the historical rejection +probes). Use the same local setup and a new E2E_RUN_DIR for each command: + +```sh +python3 tests/integration_tests/databend_native/options.py --manual-key +python3 tests/integration_tests/databend_native/options.py +``` + +The first adds `conflict-key=ID` to the feed URI and enables API +`replica_config.force_replicate=true`. It verifies a genuinely keyless table's +INSERT/UPDATE/DELETE/ADD COLUMN, persisted effective identity, and an override of +a different single native PK. Source columns resolve case insensitively. + +This is a feed-wide **single-column override**, matching bend-cdc, not automatic +uniqueness discovery: the operator must guarantee unique, non-NULL, stable values. +Do not change the key on an existing target without reinitialization. Composite +source keys cannot be replaced; absent/virtual columns and NULL row identities +are rejected. An unsplit UPDATE that changes the configured identity fails +closed; this version does not synthesize a new delete/insert protocol for arbitrary +mutable logical keys. A truly ineligible source table still needs TiCDC's existing +`force-replicate = true` (TOML) / `force_replicate` (API) setting. + +The second proves RENAME TABLE / DROP DATABASE are explicitly skipped with a +warning, the checkpoint advances, previous target objects/data remain, and later +source DML/DDL still applies. Rename does not move historical rows: later DML +uses the new target name and may lazily create that projection. Dropping the +source database does not delete the target database; recreating the same names +does not automatically clear retained target rows. Reinitialize separately if +an empty projection is required. Only these requested DDLs +are skipped; unsafe column DDL and prior UNKNOWN writes still stop the task. + +Partition dispatchers remain rejected: the existing lane only serializes within +one process, not between captures owning different physical partitions. These +options do not claim automatic multi-capture ownership/fencing support. diff --git a/tests/integration_tests/databend_native/baseline.py b/tests/integration_tests/databend_native/baseline.py new file mode 100644 index 0000000000..20e61d586a --- /dev/null +++ b/tests/integration_tests/databend_native/baseline.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Read-only Go overlay for real Lake parity probes of fixed bend-cdc b648162. + +Uses the same private LAKE_URI_FILE/E2E_RUN_DIR as run.py; no baseline files are +changed. Requires the independent bend-cdc module at BEND_CDC_DIR. +""" + +import json +import os +import pathlib +import subprocess +import urllib.parse + + +def main(): + credential = pathlib.Path(os.environ["LAKE_URI_FILE"]) + if credential.stat().st_mode & 0o077: + raise RuntimeError("LAKE_URI_FILE must have private 0600 permissions") + raw = credential.read_text().strip() + uri = urllib.parse.urlparse(raw) + root = pathlib.Path(os.environ["E2E_RUN_DIR"]) + root.mkdir(parents=True, exist_ok=True) + os.chmod(root, 0o700) + log = root / "bend-live-parity.log" + if log.exists(): + raise RuntimeError("refusing to overwrite previous baseline evidence") + repo = pathlib.Path(os.environ["BEND_CDC_DIR"]).resolve() + head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=repo, text=True + ).strip() + if head != "b6481621021e82aee659ad88e8e6c00f6dd11906": + raise RuntimeError( + "baseline checkout must be fixed b648162, not HEAD of another version" + ) + dirty = subprocess.check_output( + ["git", "status", "--porcelain"], cwd=repo, text=True + ) + if dirty: + raise RuntimeError("baseline must be clean; overlay does not modify checkout") + fixture = pathlib.Path(__file__).with_name("baseline_parity_test.go.txt").resolve() + overlay = root / "bend-overlay.json" + overlay.write_text( + json.dumps( + { + "Replace": { + str(repo / "pkg/source/tidb/zz_native_parity_overlay_test.go"): str( + fixture + ) + } + } + ) + ) + q = urllib.parse.parse_qs(uri.query) + # This authorized endpoint exposes the Basic-auth Query API directly. + q.update(login=["disable"], sslmode=["enable"]) + env = os.environ.copy() + env["TIDB_DDL_INTEGRATION_DSN"] = uri._replace( + scheme="databend", query=urllib.parse.urlencode(q, doseq=True) + ).geturl() + env.setdefault("GOMAXPROCS", "4") + with log.open("w") as output: + os.chmod(log, 0o600) + result = subprocess.run( + [ + "go", + "test", + "-p", + "2", + "-tags", + "integration", + "-overlay", + str(overlay), + "./pkg/source/tidb", + "-run", + "^(TestBendParityBaselineLive|TestDatabendSchemaIndexWarningAndTruncate)$", + "-v", + "-count=1", + "-timeout=12m", + ], + cwd=repo, + env=env, + stdout=output, + stderr=subprocess.STDOUT, + ) + # Driver errors are private, but scrub literal credentials from evidence too. + text = log.read_text().replace(raw, "[REDACTED_URI]") + for password in (uri.password, urllib.parse.unquote(uri.password or "")): + if password: + text = text.replace(password, "[REDACTED]") + log.write_text(text) + print("BASELINE", "PASS" if result.returncode == 0 else "FAIL", log) + raise SystemExit(result.returncode) + + +if __name__ == "__main__": + main() diff --git a/tests/integration_tests/databend_native/baseline_parity_test.go.txt b/tests/integration_tests/databend_native/baseline_parity_test.go.txt new file mode 100644 index 0000000000..bdadb4a3f2 --- /dev/null +++ b/tests/integration_tests/databend_native/baseline_parity_test.go.txt @@ -0,0 +1,120 @@ +package tidb + +import ( + "context" + "database/sql" + "fmt" + _ "github.com/datafuselabs/databend-go" + "github.com/stretchr/testify/require" + "os" + "testing" + "time" +) + +// Temporary test overlay against the untouched fixed bend-cdc checkout. +func TestBendParityBaselineLive(t *testing.T) { + dsn := os.Getenv("TIDB_DDL_INTEGRATION_DSN") + if dsn == "" { + t.Skip("requires authorized Lake") + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute) + defer cancel() + db, err := sql.Open("databend", dsn) + require.NoError(t, err) + defer db.Close() + name := fmt.Sprintf("bend_parity_%d", time.Now().UnixNano()) + _, err = db.ExecContext(ctx, "CREATE DATABASE "+name) + require.NoError(t, err) + defer func() { + _, err := db.ExecContext(context.Background(), "DROP DATABASE IF EXISTS "+name) + require.NoError(t, err) + }() + t.Log("owned baseline database", name) + sink, err := NewDatabendMergeSink(&DatabendMergeSinkConfig{DSN: dsn, TargetDatabase: name, AllowDelete: true, MergeInterval: time.Hour}) + require.NoError(t, err) + defer sink.Close() + id := TableID{Database: "source", Table: "partitioned"} + columns := []ColumnSchema{{Name: "id", Type: "INT", IsPk: "true", Nullable: "false"}, {Name: "v", Type: "INT", Nullable: "true"}} + apply := func(t *testing.T, epoch int64, query string) { + require.NoError(t, sink.ApplySchema(ctx, id, &SchemaMetaFile{TableVersion: epoch, Query: query, Columns: columns})) + } + t.Run("logical_partition_metadata_and_DML", func(t *testing.T) { + apply(t, 100, "CREATE TABLE partitioned(id INT PRIMARY KEY,v INT) PARTITION BY HASH(id) PARTITIONS 2") + _, err := sink.WriteBatch(ctx, id, []*CanalJSONMessage{{Type: "INSERT", PKNames: []string{"id"}, Data: []map[string]interface{}{{"id": "1", "v": "12"}}, TiDBExtension: &CanalJSONTiDBExtension{CommitTS: 110}}}) + require.NoError(t, err) + require.NoError(t, sink.Flush(ctx, id)) + var count int + require.NoError(t, db.QueryRowContext(ctx, "SELECT count(*) FROM "+name+".partitioned").Scan(&count)) + require.Equal(t, 1, count) + }) + t.Run("rename_is_skipped_not_projected", func(t *testing.T) { + apply(t, 120, "RENAME TABLE partitioned TO renamed") + var count int + require.NoError(t, db.QueryRowContext(ctx, "SELECT count(*) FROM system.tables WHERE database=? AND name='renamed'", name).Scan(&count)) + require.Zero(t, count) + require.NoError(t, db.QueryRowContext(ctx, "SELECT count(*) FROM "+name+".partitioned").Scan(&count)) + require.Equal(t, 1, count) + }) + t.Run("drop_database_is_skipped_not_projected", func(t *testing.T) { + apply(t, 130, "DROP DATABASE source") + var count int + require.NoError(t, db.QueryRowContext(ctx, "SELECT count(*) FROM "+name+".partitioned").Scan(&count)) + require.Equal(t, 1, count) + }) + t.Run("default_and_nullability", func(t *testing.T) { + columns[1].Nullable = "false" + v := "7" + columns[1].Default = &v + apply(t, 140, "ALTER TABLE partitioned MODIFY COLUMN v INT NOT NULL DEFAULT 7") + columns[1].Nullable = "true" + columns[1].Default = nil + apply(t, 150, "ALTER TABLE partitioned MODIFY COLUMN v INT NULL DEFAULT NULL") + rows, err := db.QueryContext(ctx, "DESCRIBE "+name+".partitioned") + require.NoError(t, err) + defer rows.Close() + var observed bool + for rows.Next() { + var col, typ, nullable, def, extra string + require.NoError(t, rows.Scan(&col, &typ, &nullable, &def, &extra)) + if col == "v" { + t.Log("baseline default/nullability", typ, nullable, def) + require.Equal(t, "YES", nullable) + require.Equal(t, "NULL", def) + observed = true + } + } + require.NoError(t, rows.Err()) + require.True(t, observed) + }) + t.Run("int_to_parameterized_varchar", func(t *testing.T) { + columns[1].Type = "VARCHAR(200)" + v := "7" + columns[1].Default = &v + apply(t, 160, "ALTER TABLE partitioned MODIFY COLUMN v VARCHAR(200) NULL DEFAULT '7'") + var value string + require.NoError(t, db.QueryRowContext(ctx, "SELECT v FROM "+name+".partitioned").Scan(&value)) + require.Equal(t, "12", value) + }) + t.Run("no_PK_requires_explicit_conflictKey", func(t *testing.T) { + plain, err := NewDatabendMergeSink(&DatabendMergeSinkConfig{DSN: dsn, TargetDatabase: name}) + require.NoError(t, err) + defer plain.Close() + table := TableID{Database: "source", Table: "no_pk"} + meta := &SchemaMetaFile{TableVersion: 100, Query: "CREATE TABLE no_pk(id INT,v INT)", Columns: []ColumnSchema{{Name: "id", Type: "INT"}, {Name: "v", Type: "INT"}}} + require.Error(t, plain.ApplySchema(ctx, table, meta)) + explicit, err := NewDatabendMergeSink(&DatabendMergeSinkConfig{DSN: dsn, TargetDatabase: name, ConflictKey: "id", AllowDelete: true, MergeInterval: time.Hour}) + require.NoError(t, err) + defer explicit.Close() + require.NoError(t, explicit.ApplySchema(ctx, table, meta)) + _, err = explicit.WriteBatch(ctx, table, []*CanalJSONMessage{{Type: "INSERT", Data: []map[string]interface{}{{"id": "1", "v": "22"}}, TiDBExtension: &CanalJSONTiDBExtension{CommitTS: 110}}}) + require.NoError(t, err) + require.NoError(t, explicit.Flush(ctx, table)) + var count int + require.NoError(t, db.QueryRowContext(ctx, "SELECT count(*) FROM "+name+".no_pk").Scan(&count)) + require.Equal(t, 1, count) + }) + t.Run("multi_column_is_rejected_before_execution", func(t *testing.T) { + _, err := SchemaChanges(&SchemaMetaFile{Query: "ALTER TABLE partitioned ADD COLUMN a INT,ADD COLUMN b INT", Columns: []ColumnSchema{{Name: "a", Type: "INT"}, {Name: "b", Type: "INT"}}}) + require.ErrorContains(t, err, "multi-operation") + }) +} diff --git a/tests/integration_tests/databend_native/extended.py b/tests/integration_tests/databend_native/extended.py new file mode 100644 index 0000000000..e23a99e20a --- /dev/null +++ b/tests/integration_tests/databend_native/extended.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Opt-in real TiUP/Lake DDL matrix and explicit unsupported-scope probes. + +Run after run.py's documented local setup. Each invocation owns a fresh random +source/target database/feed. A rejected probe PASS means rejection was verified, +not that the corresponding feature is supported. +""" + +import argparse +import json +import time + +from run import Run, quote + + +class ExtendedRun(Run): + def __init__(self, force_replicate=False): + self.force_replicate = force_replicate + super().__init__() + + def api(self, path, data=None, method=None): + if path == "/api/v2/changefeeds" and data and self.force_replicate: + data["replica_config"]["force_replicate"] = True + return super().api(path, data, method) + + def schema(self, table, column, nullable, default=None): + self.check_feed() + rows = self.lake("DESCRIBE " + quote(self.database) + "." + quote(table)) + row = next(r for r in rows if r[0] == column) + assert row[2] == ("YES" if nullable else "NO"), row + if default is not None: + actual = row[3].strip("()' ") + assert actual.upper() == default.upper(), (column, actual, default) + self.results.setdefault("schema_observations", []).append(row) + self.save() + + def unique_without_pk(self): + db = "`" + self.database + "`" + self.local( + f"CREATE TABLE {db}.implicit_key(id INT NOT NULL UNIQUE,v INT);" + f"INSERT INTO {db}.implicit_key VALUES(1,5),(3,8)" + ) + self.simple("implicit_key", ["id", "v"]) + self.local(f"UPDATE {db}.implicit_key SET id=2,v=6 WHERE id=1") + self.simple("implicit_key", ["id", "v"]) + self.local(f"DELETE FROM {db}.implicit_key WHERE id=2") + self.simple("implicit_key", ["id", "v"]) + self.results["implicit_key_config"] = { + "force_replicate": self.feed_status() + .get("config", {}) + .get("force_replicate", False), + "declared_primary_key": False, + "not_null_unique_key": "id", + } + self.save() + + def int_to_varchar(self): + db = "`" + self.database + "`" + self.local( + f"CREATE TABLE {db}.varchar_widen(id INT PRIMARY KEY,v INT NOT NULL DEFAULT 7);INSERT INTO {db}.varchar_widen VALUES(1,-2147483648)" + ) + self.simple("varchar_widen", ["id", "v"]) + self.local( + f"ALTER TABLE {db}.varchar_widen MODIFY COLUMN v VARCHAR(200) NOT NULL DEFAULT '7';INSERT INTO {db}.varchar_widen VALUES(2,'0012')" + ) + self.simple("varchar_widen", ["id", "v"]) + self.schema("varchar_widen", "v", False, "7") + + def tests(self): + self.case( + "NOT NULL UNIQUE without declared PK, including key change", + self.unique_without_pk, + ) + self.case( + "INT to sufficiently wide VARCHAR(200) with literal default", + self.int_to_varchar, + ) + db = "`" + self.database + "`" + table = db + ".matrix" + self.local( + f"CREATE TABLE {table}(id INT PRIMARY KEY,n INT NOT NULL DEFAULT 1,s VARCHAR(32),tiny TINYINT);" + f"INSERT INTO {table}(id,s,tiny) VALUES(1,'seed',12)" + ) + self.simple("matrix", ["id", "n", "s", "tiny"]) + + def execute(sql, names): + self.local(sql) + self.simple("matrix", names) + + names = ["id", "n", "s", "tiny"] + + def default_change(): + execute( + f"ALTER TABLE {table} MODIFY COLUMN n INT NOT NULL DEFAULT 7;" + f"INSERT INTO {table}(id,s) VALUES(2,'default7')", + names, + ) + self.schema("matrix", "n", False, "7") + + self.case( + "same-type MODIFY literal default preserves old values", default_change + ) + + def nullable(): + execute( + f"ALTER TABLE {table} MODIFY COLUMN n INT NULL DEFAULT NULL;" + f"INSERT INTO {table}(id,s) VALUES(3,'null')", + names, + ) + self.schema("matrix", "n", True, "NULL") + + self.case("NOT NULL to NULL with explicit DEFAULT NULL", nullable) + + def not_null(): + execute(f"UPDATE {table} SET n=8 WHERE n IS NULL", names) + execute( + f"ALTER TABLE {table} MODIFY COLUMN n INT NOT NULL DEFAULT 9;" + f"INSERT INTO {table}(id) VALUES(4)", + names, + ) + self.schema("matrix", "n", False, "9") + + self.case("NULL to NOT NULL after removing source nulls", not_null) + + def implicit_nullable(): + execute( + f"ALTER TABLE {table} MODIFY COLUMN n BIGINT DEFAULT 11;" + f"INSERT INTO {table}(id,n) VALUES(5,NULL)", + names, + ) + self.schema("matrix", "n", True, "11") + + self.case( + "MODIFY complete definition implicitly removes NOT NULL", implicit_nullable + ) + + def default_removed(): + execute( + f"ALTER TABLE {table} MODIFY COLUMN n BIGINT NULL;" + f"INSERT INTO {table}(id) VALUES(6)", + names, + ) + self.schema("matrix", "n", True, "NULL") + + self.case( + "MODIFY omitted DEFAULT removes previous source default", default_removed + ) + + def widening(): + execute(f"ALTER TABLE {table} MODIFY COLUMN tiny INT", names) + execute( + f"ALTER TABLE {table} MODIFY COLUMN tiny BIGINT;" + f"UPDATE {table} SET tiny=9223372036854775806 WHERE id=1", + names, + ) + execute(f"ALTER TABLE {table} MODIFY COLUMN s TEXT", names) + + self.case("TINYINT to INT to BIGINT and VARCHAR to TEXT", widening) + + def add_defaults(): + execute( + f"ALTER TABLE {table} ADD COLUMN amount DECIMAL(20,6) NOT NULL DEFAULT 1.234567", + names + ["amount"], + ) + execute( + f"ALTER TABLE {table} ADD COLUMN optional INT NULL DEFAULT NULL", + names + ["amount", "optional"], + ) + self.schema("matrix", "optional", True, "NULL") + execute( + f"ALTER TABLE {table} ADD COLUMN note VARCHAR(64) NOT NULL DEFAULT 'O\\'Reilly\\\\path'", + names + ["amount", "optional", "note"], + ) + + self.case("ADD numeric/text/NULL literal defaults and backfill", add_defaults) + + def change(): + execute( + f"ALTER TABLE {table} CHANGE COLUMN tiny counter BIGINT NULL DEFAULT 17;" + f"INSERT INTO {table}(id) VALUES(7)", + ["id", "n", "s", "counter", "amount", "optional", "note"], + ) + self.schema("matrix", "counter", True, "17") + + self.case("CHANGE rename plus full default/nullability definition", change) + + def multi(): + execute( + f"ALTER TABLE {table} ADD COLUMN extra INT NOT NULL DEFAULT 42," + " DROP COLUMN optional, RENAME COLUMN note TO message;" + f"UPDATE {table} SET extra=43 WHERE id=1", + ["id", "n", "s", "counter", "amount", "extra", "message"], + ) + + self.case("multi-operation ADD/DROP/RENAME final projection", multi) + + def reject(self, name): + db = "`" + self.database + "`" + target = quote(self.database) + '."probe"' + before = None + if name not in ("partition", "no-pk"): + self.local( + f"CREATE TABLE {db}.probe(id INT PRIMARY KEY,v INT DEFAULT 1);" + f"INSERT INTO {db}.probe VALUES(1,7)" + ) + self.simple("probe", ["id", "v"]) + before = self.lake("SELECT * FROM " + target) + sql = { + "partition": f"CREATE TABLE {db}.probe(id INT PRIMARY KEY,v INT) PARTITION BY HASH(id) PARTITIONS 2", + "no-pk": f"CREATE TABLE {db}.probe(id INT,v INT)", + "narrow": f"ALTER TABLE {db}.probe MODIFY COLUMN v TINYINT", + "expression-default": f"ALTER TABLE {db}.probe ADD COLUMN ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP", + }[name] + self.local(sql) + if name in ("partition", "no-pk"): + self.local(f"INSERT INTO {db}.probe VALUES(1,5)") + deadline = time.monotonic() + 65 + while time.monotonic() < deadline: + status = self.feed_status() + if status.get("state") == "failed": + break + time.sleep(0.5) + else: + raise AssertionError( + "probe did not fail closed: " + + self.sanitize( + json.dumps( + {k: status.get(k) for k in ("state", "error", "checkpoint_ts")} + ) + ) + ) + # Verify a stable failed lifecycle, not only a transient reported error. + checkpoint = status["checkpoint_ts"] + time.sleep(2) + after = self.feed_status() + assert after.get("state") == "failed", self.sanitize(after) + assert after["checkpoint_ts"] == checkpoint + self.results["rejection"] = { + "case": name, + "state": after["state"], + "checkpoint_ts": checkpoint, + "error": self.sanitize(after.get("error")), + } + if before is not None: + assert self.lake("SELECT * FROM " + target) == before + else: + assert self.lake( + "SELECT COUNT(*)::VARCHAR FROM system.tables WHERE database=? AND name='probe'", + [self.database], + ) == [["0"]] + self.save() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--reject", + choices=[ + "partition", + "no-pk", + "narrow", + "expression-default", + ], + ) + parser.add_argument("--unique-only", action="store_true") + parser.add_argument("--varchar-only", action="store_true") + args = parser.parse_args() + run = ExtendedRun(force_replicate=args.reject == "no-pk") + try: + run.setup() + if args.reject: + run.case( + "expected rejection: " + args.reject, lambda: run.reject(args.reject) + ) + elif args.varchar_only: + run.case( + "INT to sufficiently wide VARCHAR(200) with literal default", + run.int_to_varchar, + ) + elif args.unique_only: + run.case( + "NOT NULL UNIQUE without declared PK, including key change", + run.unique_without_pk, + ) + else: + run.tests() + run.cleanup() + run.results["status"] = "PASS" + run.save() + except Exception as error: + run.results["status"] = "FAIL" + run.results["error"] = run.sanitize(error) + run.save() + print("ERROR", run.sanitize(error), flush=True) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/tests/integration_tests/databend_native/local.py b/tests/integration_tests/databend_native/local.py new file mode 100755 index 0000000000..4366bf7564 --- /dev/null +++ b/tests/integration_tests/databend_native/local.py @@ -0,0 +1,483 @@ +#!/usr/bin/env python3 +"""Owned local TiUP + TiCDC + Docker Databend lifecycle for native sink E2E. + +Unlike run.py's external-warehouse mode, this runner always creates its own +loopback-only Databend, credentials, source and capture. No cloud URI is read. +""" + +import argparse +import hashlib +import json +import os +import pathlib +import shlex +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import time +import urllib.request +import uuid + +HERE = pathlib.Path(__file__).resolve().parent +REPO = HERE.parents[2] +# Fixed multi-architecture release, not a moving latest/nightly tag. +IMAGE = ( + "datafuselabs/databend:v1.2.925-patch-13@sha256:" + "7f93f005aa0d626ebc7554d142b0a6b6ebb90e085b0d5bd0d97470084c42d41b" +) +SUITES = { + "acceptance": ["run.py"], + "ddl": ["extended.py"], + "manual-key": ["options.py", "--manual-key"], + "skipped-ddl": ["options.py"], + **{ + f"reject-{name}": ["extended.py", "--reject", name] + for name in ("partition", "no-pk", "narrow", "expression-default") + }, + "fault": None, +} + + +def wait_ready(check, process, timeout, description): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process is not None and process.poll() is not None: + raise RuntimeError( + description + " exited before becoming ready; inspect logs" + ) + try: + if check(): + return + except (OSError, ValueError, RuntimeError, subprocess.SubprocessError): + pass + time.sleep(0.5) + raise RuntimeError(description + " readiness timeout; inspect logs") + + +def free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +class Local: + def __init__(self, root, image, version): + self.root = root + self.image = image + self.version = version + self.processes = [] + self.container = "ticdc-databend-e2e-" + uuid.uuid4().hex[:12] + self.container_created = False + self.tag = "native-e2e-" + uuid.uuid4().hex[:12] + self.env = os.environ.copy() + # Never inherit a cloud target or another runner's capture/restart options. + for key in ( + "LAKE_URI_FILE", + "DATABEND_NATIVE_TEST_URI", + "E2E_CDC_BINARY", + "E2E_CDC_DATA_DIR", + "E2E_PD", + "CDC_SERVER", + "MYSQL_PORT", + ): + self.env.pop(key, None) + self.summary = {"status": "RUNNING", "suites": []} + self.binary = None + self.offset = int(os.environ.get("E2E_PORT_OFFSET", "10000")) + if not 0 <= self.offset <= 30000: + raise RuntimeError("E2E_PORT_OFFSET must be in [0,30000]") + + def command(self, args, **kwargs): + kwargs.setdefault("timeout", 900) + return subprocess.run(args, check=True, env=self.env, **kwargs) + + def start(self, args, log): + with (self.root / log).open("wb") as output: + process = subprocess.Popen( + args, + env=self.env, + stdout=output, + stderr=subprocess.STDOUT, + start_new_session=True, + cwd=REPO, + ) + self.processes.append(process) + return process + + def save(self): + (self.root / "summary.json").write_text( + json.dumps(self.summary, indent=2) + "\n" + ) + + def prepare(self): + for program in ("docker", "tiup", "mysql", "go"): + if not shutil.which(program): + raise RuntimeError("missing local prerequisite: " + program) + self.command( + ["docker", "info"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + # Fixed source ports must not silently attach to an existing playground. + for base in (4000, 10080, 2379, 2380, 20160, 20180): + with socket.socket() as sock: + # Permit immediate reruns after our previous cluster closes; + # TIME_WAIT is not a live listener. No SO_REUSEPORT is enabled. + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", base + self.offset)) + self.summary["source_commit"] = self.command( + ["git", "-C", str(REPO), "rev-parse", "HEAD"], + capture_output=True, + text=True, + ).stdout.strip() + self.summary["source_dirty"] = bool( + self.command( + ["git", "-C", str(REPO), "status", "--porcelain"], + capture_output=True, + text=True, + ).stdout.strip() + ) + supplied = os.environ.get("E2E_CDC_BINARY") + if supplied: + self.binary = pathlib.Path(supplied).resolve(strict=True) + else: + self.binary = self.root / "cdc" + build = self.start(["make", "cdc"], "build.log") + if build.wait(timeout=900): + raise RuntimeError( + "current-branch TiCDC build failed; inspect build.log" + ) + shutil.copy2(REPO / "bin/cdc", self.binary) + self.summary["cdc_binary_supplied"] = bool(supplied) + self.summary["cdc_version"] = self.command( + [str(self.binary), "version"], capture_output=True, text=True + ).stdout.strip() + with self.binary.open("rb") as binary: + self.summary["cdc_sha256"] = hashlib.file_digest( + binary, "sha256" + ).hexdigest() + print("Starting owned local Databend background container", flush=True) + with (self.root / "pull.log").open("w") as log: + self.command( + ["docker", "pull", self.image], stdout=log, stderr=subprocess.STDOUT + ) + self.summary["databend_image"] = json.loads( + self.command( + ["docker", "image", "inspect", self.image], + capture_output=True, + text=True, + ).stdout + )[0]["Id"] + self.summary["databend_image_tag"] = self.image + # No host network, no cloud storage, no MinIO download, no exposed password. + password = uuid.uuid4().hex + self.env["QUERY_DEFAULT_PASSWORD"] = password + self.command( + [ + "docker", + "create", + "--name", + self.container, + "--label", + "ticdc.e2e=" + self.tag, + "--publish", + "127.0.0.1::8000", + "--env", + "QUERY_STORAGE_TYPE=fs", + "--env", + "QUERY_DEFAULT_USER=ticdc_e2e", + "--env", + "QUERY_DEFAULT_PASSWORD", + self.image, + ], + stdout=subprocess.DEVNULL, + ) + self.container_created = True + self.command(["docker", "start", self.container], stdout=subprocess.DEVNULL) + address = self.command( + ["docker", "port", self.container, "8000/tcp"], + capture_output=True, + text=True, + ).stdout.strip() + uri = f"databend+http://ticdc_e2e:{password}@{address}/default?exclusive=true" + credentials = self.root / "local-uri" + credentials.write_text(uri) + credentials.chmod(0o600) + self.env["LAKE_URI_FILE"] = str(credentials) + self.env["DATABEND_NATIVE_TEST_URI"] = uri + self.env.pop("QUERY_DEFAULT_PASSWORD") + self.env["MYSQL_PORT"] = str(4000 + self.offset) + self.env["E2E_RUN_DIR"] = str(self.root / "readiness") + # Reuse the exact HTTP/terminal-state adapter used by the assertions. + from run import Run + + old_env = os.environ.copy() + try: + os.environ.update(self.env) + probe = Run() + finally: + os.environ.clear() + os.environ.update(old_env) + wait_ready(lambda: probe.lake("SELECT 1") == [["1"]], None, 90, "Databend") + self.summary["databend_version"] = probe.lake("SELECT version()")[0][0] + if probe.lake("SELECT ?::BIGINT", [1]) != [["1"]]: + raise RuntimeError("Databend must support Query API bound parameters") + print("Starting owned local TiDB/PD/TiKV", flush=True) + source = self.start( + [ + "tiup", + "playground", + self.version, + "--tag", + self.tag, + "--db", + "1", + "--pd", + "1", + "--kv", + "1", + "--tiflash", + "0", + "--ticdc", + "0", + "--without-monitor", + "--host", + "127.0.0.1", + "--port-offset", + str(self.offset), + ], + "tiup.log", + ) + wait_ready(lambda: probe.local("SELECT 1") == "1", source, 180, "TiDB") + # Owned source only: legacy index construction works on low-space laptops. + probe.local( + "SET GLOBAL tidb_ddl_enable_fast_reorg=OFF;SET GLOBAL tidb_enable_dist_task=OFF" + ) + self.summary["source_version"] = probe.local("SELECT version()") + self.summary["source_adjustments"] = [ + "tidb_ddl_enable_fast_reorg=OFF", + "tidb_enable_dist_task=OFF", + ] + port = free_port() + self.env.update( + CDC_SERVER=f"http://127.0.0.1:{port}", + E2E_CDC_BINARY=str(self.binary), + E2E_CDC_DATA_DIR=str(self.root / "cdc-data"), + E2E_PD=f"http://127.0.0.1:{2379 + self.offset}", + ) + capture = self.start( + [ + str(self.binary), + "server", + "--newarch", + "--tz", + "UTC", + "--pd", + self.env["E2E_PD"], + "--addr", + f"127.0.0.1:{port}", + "--data-dir", + self.env["E2E_CDC_DATA_DIR"], + "--log-file", + str(self.root / "cdc.log"), + ], + "cdc-console.log", + ) + + def capture_ready(): + with urllib.request.urlopen( + self.env["CDC_SERVER"] + "/api/v2/status", timeout=5 + ) as response: + if not json.load(response).get("is_owner", False): + return False + # Owner election precedes coordinator bootstrap. Do not submit the + # first create request while the read API still returns HTTP 500. + with urllib.request.urlopen( + self.env["CDC_SERVER"] + "/api/v2/changefeeds", timeout=5 + ) as response: + return response.status == 200 + + wait_ready(capture_ready, capture, 90, "TiCDC") + self.save() + + def suites(self, names): + for name in names: + self.env["E2E_RUN_DIR"] = str(self.root / name) + if name == "fault": + args = [ + "go", + "test", + "-p", + "2", + "-race", + "./downstreamadapter/sink/databend", + "-run", + "TestDatabendNativeLive", + "-v", + "-count=1", + "-timeout=20m", + ] + else: + args = [sys.executable, str(HERE / SUITES[name][0]), *SUITES[name][1:]] + print("Running " + name, flush=True) + started = time.monotonic() + child = self.start(args, name + ".log") + # Reap the old capture while acceptance's owned restart waits for exit. + while child.poll() is None: + for process in self.processes: + process.poll() + time.sleep(0.2) + record = { + "name": name, + "exit_code": child.returncode, + "seconds": round(time.monotonic() - started, 2), + } + self.summary["suites"].append(record) + self.save() + if child.returncode: + raise RuntimeError( + name + " failed; inspect " + str(self.root / (name + ".log")) + ) + + def cleanup(self): + # A restarted capture is launched by run.py, not this process. Check both + # its generated PID path and exact binary before signalling it. + restarted = self.root / "acceptance/restart-pid" + errors = [] + if restarted.exists(): + try: + pid = int(restarted.read_text()) + command = subprocess.run( + ["ps", "-p", str(pid), "-o", "command="], + capture_output=True, + text=True, + ) + argv = shlex.split(command.stdout.strip()) + if command.returncode == 0: + data_dir = ( + argv[argv.index("--data-dir") + 1] + if "--data-dir" in argv + and argv.index("--data-dir") + 1 < len(argv) + else "" + ) + if ( + len(argv) < 2 + or pathlib.Path(argv[0]).resolve() != self.binary.resolve() + or argv[1] != "server" + or pathlib.Path(data_dir).resolve() + != (self.root / "cdc-data").resolve() + ): + raise RuntimeError("refusing to stop unowned restarted capture") + os.kill(pid, signal.SIGTERM) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.2) + else: + raise RuntimeError("restarted capture did not terminate") + except Exception as error: + errors.append(str(error)) + for process in reversed(self.processes): + try: + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=35) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + except ProcessLookupError: + pass + except Exception as error: + errors.append("process cleanup: " + type(error).__name__) + # TiUP may have already exited after a child startup failure. Its unique + # tag confines cleanup to this run; never use broad pkill/tiup clean --all. + if (self.root / "tiup.log").exists(): + try: + result = subprocess.run( + ["tiup", "clean", self.tag], + env=self.env, + capture_output=True, + text=True, + timeout=60, + ) + (self.root / "tiup-clean.log").write_text(result.stdout + result.stderr) + if result.returncode: + errors.append("TiUP tag cleanup failed") + except Exception as error: + errors.append("TiUP cleanup: " + type(error).__name__) + if self.container_created: + try: + with (self.root / "databend.log").open("wb") as log: + subprocess.run( + ["docker", "logs", self.container], + stdout=log, + stderr=subprocess.STDOUT, + timeout=30, + ) + except Exception as error: + errors.append("Databend logs: " + type(error).__name__) + try: + result = subprocess.run( + ["docker", "rm", "--force", "--volumes", self.container], + capture_output=True, + text=True, + timeout=45, + ) + if result.returncode: + errors.append("Databend container cleanup failed") + except Exception as error: + errors.append("Databend cleanup: " + type(error).__name__) + (self.root / "local-uri").unlink(missing_ok=True) + self.summary["cleanup"] = "PASS" if not errors else errors + if errors: + self.summary["status"] = "FAIL" + self.save() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--suite", + choices=SUITES, + action="append", + help="repeat to select suites; default all", + ) + args = parser.parse_args() + os.umask(0o077) + root = pathlib.Path( + tempfile.mkdtemp(prefix="ticdc-databend-local-", dir=os.environ.get("OUT_DIR")) + ) + print("Local E2E artifacts: " + str(root), flush=True) + runner = Local( + root, + os.environ.get("DATABEND_E2E_IMAGE", IMAGE), + os.environ.get("TIDB_E2E_VERSION", "v8.5.1"), + ) + + def interrupted(signum, _frame): + raise InterruptedError("local E2E interrupted by signal " + str(signum)) + + signal.signal(signal.SIGTERM, interrupted) + signal.signal(signal.SIGINT, interrupted) + try: + runner.prepare() + runner.suites(args.suite or list(SUITES)) + runner.summary["status"] = "PASS" + except Exception as error: + # Do not include inherited env, query SQL or credentials in the summary. + runner.summary["status"] = "FAIL" + runner.summary["error"] = type(error).__name__ + ": " + str(error) + print(runner.summary["error"], file=sys.stderr) + finally: + runner.cleanup() + print("RESULT " + str(root / "summary.json"), flush=True) + return 0 if runner.summary["status"] == "PASS" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/integration_tests/databend_native/options.py b/tests/integration_tests/databend_native/options.py new file mode 100644 index 0000000000..b83f0c501e --- /dev/null +++ b/tests/integration_tests/databend_native/options.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Real-source acceptance for conflict-key and explicitly skipped source DDL. + +Uses the same private environment and isolated ownership rules as run.py. +""" + +import argparse +import json +import time +import urllib.parse + +from run import Run, quote + + +class OptionsRun(Run): + def __init__(self, manual=False): + self.manual = manual + super().__init__() + if manual: + q = urllib.parse.parse_qs(self.parsed.query) + q["conflict-key"] = ["ID"] # Match authoritative source casing. + self.parsed = self.parsed._replace( + query=urllib.parse.urlencode(q, doseq=True) + ) + + def api(self, path, data=None, method=None): + if self.manual and path == "/api/v2/changefeeds" and data: + data["replica_config"]["force_replicate"] = True + return super().api(path, data, method) + + def drained_marker(self): + marker = int( + self.local( + "BEGIN;SELECT COUNT(*) FROM mysql.tidb;SELECT @@tidb_current_ts;COMMIT" + ).splitlines()[-1] + ) + assert marker > 0 + deadline = time.monotonic() + 75 + while self.check_feed()["checkpoint_ts"] < marker: + if time.monotonic() > deadline: + raise AssertionError("checkpoint did not pass skipped DDL") + time.sleep(0.3) + self.results.setdefault("drained_markers", []).append(marker) + self.save() + + def test_manual(self): + db = "`" + self.database + "`" + self.local( + f"CREATE TABLE {db}.keyless(id BIGINT,v VARCHAR(64));INSERT INTO {db}.keyless VALUES(1,'one'),(2,'two')" + ) + self.simple("keyless", ["id", "v"]) + self.local( + f"UPDATE {db}.keyless SET v='updated' WHERE id=1;DELETE FROM {db}.keyless WHERE id=2;INSERT INTO {db}.keyless VALUES(3,'three')" + ) + self.simple("keyless", ["id", "v"]) + self.local( + f"ALTER TABLE {db}.keyless ADD COLUMN extra BIGINT NULL;INSERT INTO {db}.keyless VALUES(4,'four',42)" + ) + self.simple("keyless", ["id", "v", "extra"]) + journal = self.lake( + "SELECT plan FROM " + quote(self.database) + '."keyless_cdc_schema_epoch"' + ) + + plan = json.loads(journal[0][0]) + assert plan["schema"]["keys"] == ["id"], plan + self.results["manual_key"] = { + "source_primary_key": False, + "source_unique_key": False, + "effective_key": plan["schema"]["keys"], + "force_replicate": self.feed_status()["config"].get("force_replicate"), + } + # bend-cdc also permits replacing a single native PK, not a composite. + self.local( + f"CREATE TABLE {db}.override_key(other_id INT PRIMARY KEY,id BIGINT,v VARCHAR(64));INSERT INTO {db}.override_key VALUES(10,1,'one'),(20,2,'two')" + ) + self.simple("override_key", ["other_id", "id", "v"]) + self.local( + f"UPDATE {db}.override_key SET v='changed' WHERE other_id=10;DELETE FROM {db}.override_key WHERE other_id=20" + ) + self.simple("override_key", ["other_id", "id", "v"]) + self.drained_marker() + self.save() + + def test_rename(self): + db = "`" + self.database + "`" + target = quote(self.database) + self.local( + f"CREATE TABLE {db}.original(id INT PRIMARY KEY,v VARCHAR(64));INSERT INTO {db}.original VALUES(1,'old')" + ) + self.simple("original", ["id", "v"]) + self.local(f"RENAME TABLE {db}.original TO {db}.renamed") + self.drained_marker() + assert self.lake("SELECT id::VARCHAR,v FROM " + target + '."original"') == [ + ["1", "old"] + ] + assert self.lake( + "SELECT COUNT(*)::VARCHAR FROM system.tables WHERE database=? AND name='renamed'", + [self.database], + ) == [["0"]] + # No historical rows are moved. Subsequent DML gets a new projection. + self.local( + f"UPDATE {db}.renamed SET v='new' WHERE id=1;INSERT INTO {db}.renamed VALUES(2,'inserted')" + ) + self.simple("renamed", ["id", "v"]) + assert self.lake("SELECT id::VARCHAR,v FROM " + target + '."original"') == [ + ["1", "old"] + ] + self.results["rename"] = { + "source_renamed": True, + "old_target_preserved": True, + "new_target_populated_by_subsequent_dml": True, + } + self.save() + + def test_drop_database(self): + db = "`" + self.database + "`" + target = quote(self.database) + before = self.lake( + "SELECT id::VARCHAR,v FROM " + target + '."renamed" ORDER BY id' + ) + self.local(f"DROP DATABASE {db}") + self.drained_marker() + assert ( + self.lake("SELECT id::VARCHAR,v FROM " + target + '."renamed" ORDER BY id') + == before + ) + assert self.lake("SELECT id::VARCHAR,v FROM " + target + '."original"') == [ + ["1", "old"] + ] + self.local( + f"CREATE DATABASE {db};CREATE TABLE {db}.after_drop(id INT PRIMARY KEY,v INT);INSERT INTO {db}.after_drop VALUES(1,42)" + ) + self.simple("after_drop", ["id", "v"]) + self.drained_marker() + self.results["drop_database"] = { + "target_database_preserved": True, + "existing_target_data_preserved": True, + "subsequent_source_database_recreation_and_new_table_dml": True, + } + self.save() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--manual-key", action="store_true") + args = parser.parse_args() + run = OptionsRun(manual=args.manual_key) + try: + run.setup() + if args.manual_key: + run.case( + "force replicated keyless DML/DDL and single native key override", + run.test_manual, + ) + else: + run.case( + "RENAME skipped, old target preserved and subsequent DML continues", + run.test_rename, + ) + run.case( + "DROP DATABASE skipped, checkpoint and subsequent DML continue", + run.test_drop_database, + ) + run.cleanup() + run.results["status"] = "PASS" + run.save() + except Exception as error: + run.results["status"] = "FAIL" + run.results["error"] = run.sanitize(error) + run.save() + print("ERROR", run.sanitize(error), flush=True) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/tests/integration_tests/databend_native/run.py b/tests/integration_tests/databend_native/run.py new file mode 100755 index 0000000000..abcce2b12a --- /dev/null +++ b/tests/integration_tests/databend_native/run.py @@ -0,0 +1,769 @@ +#!/usr/bin/env python3 +"""Opt-in TiUP -> native TiCDC -> isolated Lake database acceptance test. + +Requires an already-running local TiUP and current-branch CDC server. Credentials +are read from LAKE_URI_FILE (mode 0600), never argv or repository config. Only the +random native_e2e_* database and this run's changefeed are created/dropped. +""" + +import base64 +import datetime +import decimal +import json +import os +import pathlib +import re +import signal +import shlex +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + + +def quote(name): + return '"' + name.replace('"', '""') + '"' + + +class Run: + def __init__(self, existing=False): + credential_file = pathlib.Path(os.environ["LAKE_URI_FILE"]) + if credential_file.stat().st_mode & 0o077: + raise RuntimeError("LAKE_URI_FILE must have private 0600 permissions") + self.uri = credential_file.read_text().strip() + self.parsed = urllib.parse.urlparse(self.uri) + self.root = pathlib.Path(os.environ["E2E_RUN_DIR"]) + self.root.mkdir(parents=True, exist_ok=True) + os.chmod(self.root, 0o700) + self.cdc = os.environ.get("CDC_SERVER", "http://127.0.0.1:18300") + self.port = os.environ.get("MYSQL_PORT", "14000") + self.database = ( + "native_e2e_" + + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S_") + + uuid.uuid4().hex[:8] + ) + self.feed = self.database.replace("_", "-") + # Plain HTTP is only enabled by the sink's explicit test scheme. + scheme = "http" if self.parsed.scheme == "databend+http" else "https" + self.base = scheme + "://" + self.parsed.netloc.rsplit("@", 1)[-1] + self.results = { + "database": self.database, + "changefeed": self.feed, + "cases": [], + "started_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + } + self.feed_created = False + self.remote_created = False + if existing: + saved = json.loads((self.root / "result.json").read_text()) + database = saved["database"] + if not re.fullmatch( + r"native_e2e_[0-9]{8}_[0-9]{6}_[0-9a-f]{8}", database + ) or saved["changefeed"] != database.replace("_", "-"): + raise RuntimeError( + "refusing cleanup outside this runner's isolated namespace" + ) + self.results = saved + self.database = database + self.feed = saved["changefeed"] + self.remote_created = True + self.feed_created = True + else: + if (self.root / "result.json").exists(): + raise RuntimeError( + "E2E_RUN_DIR already contains a result; use a new directory or --cleanup" + ) + self.save() + + def sanitize(self, text): + return ( + str(text) + .replace(self.uri, "[REDACTED_URI]") + .replace(self.parsed.password or "\0", "[REDACTED]") + ) + + def save(self): + path = self.root / "result.json" + path.write_text(json.dumps(self.results, indent=2, ensure_ascii=False)) + os.chmod(path, 0o600) + + def local(self, sql): + proc = subprocess.run( + [ + "mysql", + "-h", + "127.0.0.1", + "-P", + self.port, + "-u", + "root", + "--default-character-set=utf8mb4", + "-N", + "-B", + "--raw", + "-e", + "SET time_zone='+00:00';" + sql, + ], + text=True, + capture_output=True, + timeout=40, + ) + if proc.returncode: + raise RuntimeError("local SQL failed: " + self.sanitize(proc.stderr)) + return proc.stdout.strip() + + def api(self, path, data=None, method=None): + req = urllib.request.Request( + self.cdc + path, + data=None if data is None else json.dumps(data).encode(), + method=method, + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=60) as response: + raw = response.read() + return json.loads(raw) if raw else {} + except urllib.error.HTTPError as error: + raise RuntimeError( + "CDC HTTP %d: %s" % (error.code, self.sanitize(error.read().decode())) + ) from None + + def lake(self, sql, params=None): + """No mutation retries: an ambiguous response stops the test for inspection.""" + query_id = str(uuid.uuid4()) + auth = ( + urllib.parse.unquote(self.parsed.username) + + ":" + + urllib.parse.unquote(self.parsed.password) + ) + headers = { + "Authorization": "Basic " + base64.b64encode(auth.encode()).decode(), + "Content-Type": "application/json", + "X-DATABEND-WAREHOUSE": urllib.parse.parse_qs(self.parsed.query).get( + "warehouse", ["default"] + )[0], + "X-DATABEND-QUERY-ID": query_id, + } + body = { + "sql": sql, + "session": {"database": "default", "settings": {"timezone": "UTC"}}, + "pagination": {"wait_time_secs": 1, "max_rows_per_page": 1024}, + } + if params is not None: + body["params"] = params + req = urllib.request.Request( + self.base + "/v1/query", data=json.dumps(body).encode(), headers=headers + ) + rows = [] + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(req, timeout=40) as response: + result = json.load(response) + route = response.headers.get("X-DATABEND-ROUTE-HINT") + except urllib.error.HTTPError as error: + raise RuntimeError( + "Lake HTTP %d request %s: %s" + % (error.code, query_id, self.sanitize(error.read(1024).decode())) + ) from None + if result.get("state") == "Failed" or result.get("error"): + raise RuntimeError( + "Lake SQL request %s failed: %s" + % (query_id, self.sanitize(result.get("error"))) + ) + if result.get("id") != query_id: + raise RuntimeError("Lake query identity mismatch: " + query_id) + rows.extend(result.get("data") or []) + path = result.get("next_uri") + if result.get("state") == "Succeeded" and ( + not path or path.endswith("/final") + ): + final = result.get("final_uri") + if final and final.startswith("/v1/query/"): + try: + with urllib.request.urlopen( + urllib.request.Request(self.base + final, headers=headers), + timeout=5, + ) as response: + response.read() + except (OSError, urllib.error.HTTPError): + pass # Result cleanup does not revoke confirmed query success. + return rows + if not path or not path.startswith("/v1/query/") or path.startswith("//"): + raise RuntimeError("Missing terminal evidence for " + query_id) + headers["X-DATABEND-STICKY-NODE"] = result.get("node_id", "") + headers["X-DATABEND-NODE-ID"] = result.get("node_id", "") + if route: + headers["X-DATABEND-ROUTE-HINT"] = route + req = urllib.request.Request(self.base + path, headers=headers) + time.sleep(0.1) + raise RuntimeError("Lake query terminal state timeout: " + query_id) + + def feed_status(self): + return self.api("/api/v2/changefeeds/" + self.feed) + + def check_feed(self): + status = self.feed_status() + if status.get("state") in ("failed", "error", "warning") or status.get("error"): + raise RuntimeError( + "changefeed unhealthy: " + + self.sanitize( + json.dumps( + { + "state": status.get("state"), + "error": status.get("error"), + "checkpoint_ts": status.get("checkpoint_ts"), + } + ) + ) + ) + return status + + def case(self, name, action): + started = time.monotonic() + record = {"name": name} + try: + action() + record.update(status="PASS", seconds=round(time.monotonic() - started, 3)) + print("PASS", name, record["seconds"], flush=True) + except Exception as error: + record.update( + status="FAIL", + seconds=round(time.monotonic() - started, 3), + error=self.sanitize(error), + ) + print("FAIL", name, record["error"], flush=True) + raise + finally: + self.results["cases"].append(record) + self.save() + + def compare(self, table, columns, numeric=(), json_columns=(), timeout=75): + # JSON_OBJECT keeps control characters/newlines and SQL NULL unambiguous. + source = ( + "SELECT JSON_OBJECT(" + + ",".join("'%s',%s" % (name, expr) for name, expr, _ in columns) + + ") FROM `%s`.`%s` ORDER BY id" % (self.database, table) + ) + expected = [ + json.loads(line) for line in self.local(source).splitlines() if line + ] + target = ( + "SELECT " + + ",".join(expr for _, _, expr in columns) + + " FROM " + + quote(self.database) + + "." + + quote(table) + + " ORDER BY id" + ) + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + self.check_feed() + try: + data = self.lake(target) + actual = [dict(zip([c[0] for c in columns], row)) for row in data] + + # Databend may bind ORDER BY id to the projected VARCHAR alias. + # Compare the key-indexed set, never numeric-vs-text sort order. + def key(row): + return str(row["id"]), str(row.get("tenant", "")) + + expected.sort(key=key) + actual.sort(key=key) + if len(expected) != len(actual): + raise AssertionError( + "row count expected %d actual %d" % (len(expected), len(actual)) + ) + for left, right in zip(expected, actual): + for name in left: + a, b = left[name], right[name] + if a is None or b is None: + assert a is b, (name, a, b) + elif name in json_columns: + assert json.loads( + a, parse_float=decimal.Decimal + ) == json.loads(b, parse_float=decimal.Decimal), ( + name, + a, + b, + ) + elif name in numeric: + da, db = decimal.Decimal(str(a)), decimal.Decimal(str(b)) + assert abs(da - db) <= max( + abs(da), decimal.Decimal(1) + ) * decimal.Decimal("0.000001"), (name, a, b) + else: + assert str(a) == str(b), (name, a, b) + return + except (RuntimeError, AssertionError) as error: + last = self.sanitize(error) + time.sleep(0.5) + raise AssertionError("target did not converge: " + last) + + def simple(self, table, names): + self.compare( + table, + [ + (name, "CAST(`%s` AS CHAR)" % name, quote(name) + "::VARCHAR") + for name in names + ], + ) + + def setup(self): + self.results["source_version"] = self.local("SELECT version()") + self.results["cdc_status"] = self.api("/api/v2/status") + self.results["lake_version"] = self.lake("SELECT version(), ?::BIGINT", [1])[0] + self.local("CREATE DATABASE `%s`" % self.database) + self.lake("CREATE DATABASE " + quote(self.database)) + self.remote_created = True + q = urllib.parse.parse_qs(self.parsed.query) + q.update( + { + "exclusive": ["true"], + "workers": ["1"], + "batch-rows": ["64"], + "batch-bytes": ["32768"], + "flush-interval": ["200ms"], + "spool-bytes": ["65536"], + "memory-ratio": ["0.001"], + "source-bytes": ["268435456"], + "source-events": ["256"], + "spool-dir": [str(self.root / "spool")], + } + ) + sink = self.parsed._replace( + path="/" + self.database, query=urllib.parse.urlencode(q, doseq=True) + ).geturl() + self.api( + "/api/v2/changefeeds", + { + "changefeed_id": self.feed, + "sink_uri": sink, + "replica_config": { + "filter": {"rules": [self.database + ".*"]}, + "sink": {"transaction_atomicity": "none"}, + "scheduler": {"enable_table_across_nodes": False}, + }, + }, + ) + self.feed_created = True + self.save() + time.sleep(2) + self.check_feed() + + def tests(self): + db = "`" + self.database + "`" + self.case( + "DML insert/update/delete and nonclustered PK/UK changes", + lambda: self.test_keys(db), + ) + self.case( + "Composite primary key and escaped identifiers", + lambda: self.test_composite(db), + ) + self.case( + "Quoted identifiers and operation discriminator collision", + lambda: self.test_identifiers(db), + ) + self.case( + "Column scalar/binary/decimal/JSON/ENUM/SET/temporal fidelity", + lambda: self.test_types(db), + ) + self.case("DDL add/default/rename/widen/drop", lambda: self.test_ddl(db)) + self.case( + "TRUNCATE new generation and subsequent DML", lambda: self.test_truncate(db) + ) + self.case( + "Large transaction and low traffic batching", lambda: self.test_batch(db) + ) + if os.environ.get("E2E_CDC_BINARY"): + self.case( + "Graceful CDC process restart with paused DML/DDL backlog", + lambda: self.test_restart(db), + ) + self.case("DROP TABLE cleanup", lambda: self.test_drop(db)) + self.results["last_status"] = self.feed_status() + # Never persist the full sink URI returned by a future API version. + self.results["last_status"] = json.loads( + self.sanitize(json.dumps(self.results["last_status"])) + ) + self.save() + + def test_keys(self, db): + self.local( + f"CREATE TABLE {db}.keys_test (id BIGINT PRIMARY KEY NONCLUSTERED,email VARCHAR(128) UNIQUE, value VARCHAR(255)); INSERT INTO {db}.keys_test VALUES (1,'a','first'),(2,'b','second')" + ) + self.simple("keys_test", ["id", "email", "value"]) + self.local(f"UPDATE {db}.keys_test SET email='c',value='changed' WHERE id=1") + self.simple("keys_test", ["id", "email", "value"]) + self.local(f"UPDATE {db}.keys_test SET id=3 WHERE id=1") + self.simple("keys_test", ["id", "email", "value"]) + self.local( + f"BEGIN; DELETE FROM {db}.keys_test WHERE id=2; INSERT INTO {db}.keys_test VALUES (2,'new','reused'); COMMIT;" + ) + self.simple("keys_test", ["id", "email", "value"]) + self.local(f"DELETE FROM {db}.keys_test WHERE id=3") + self.simple("keys_test", ["id", "email", "value"]) + + def test_composite(self, db): + self.local( + f"CREATE TABLE {db}.composite_test(id INT, tenant VARCHAR(64), `action` VARCHAR(64), PRIMARY KEY(tenant,id)); INSERT INTO {db}.composite_test VALUES (1,'a|b','insert'),(2,'a|b','update')" + ) + self.simple("composite_test", ["id", "tenant", "action"]) + self.local( + f"UPDATE {db}.composite_test SET tenant='b|c',action='delete' WHERE id=1; DELETE FROM {db}.composite_test WHERE id=2" + ) + self.simple("composite_test", ["id", "tenant", "action"]) + + def test_identifiers(self, db): + self.local( + f"""CREATE TABLE {db}.`quoted"table` (id INT PRIMARY KEY,`a"b` VARCHAR(64),`__ticdc_operation` VARCHAR(64),`action` VARCHAR(64)); INSERT INTO {db}.`quoted"table` VALUES(1,'quoted-column','source-value','insert')""" + ) + self.simple('quoted"table', ["id", 'a"b', "__ticdc_operation", "action"]) + self.local( + f"""UPDATE {db}.`quoted"table` SET `action`='delete',`a"b`='changed' WHERE id=1""" + ) + self.simple('quoted"table', ["id", 'a"b', "__ticdc_operation", "action"]) + + def test_types(self, db): + sets = ",".join("'s%d'" % i for i in range(64)) + definitions = ( + """id INT PRIMARY KEY, s_tiny TINYINT, u_tiny TINYINT UNSIGNED, + s_small SMALLINT, u_small SMALLINT UNSIGNED,s_medium MEDIUMINT,u_medium MEDIUMINT UNSIGNED, + s_int INT,u_int INT UNSIGNED,s_big BIGINT,u_big BIGINT UNSIGNED, + d DECIMAL(65,30),f FLOAT,g DOUBLE, label ENUM('2','x'),labels SET(%s), + bytes BLOB,txt TEXT,doc JSON,sql_null JSON,json_null JSON, + dt DATETIME(6),ts TIMESTAMP(6) NULL,day DATE,duration TIME(6),yr YEAR,bits BIT(64),fixed_bytes BINARY(8),var_bytes VARBINARY(32),char_value CHAR(8),bool_value BOOLEAN, + generated_value BIGINT GENERATED ALWAYS AS (s_big + 1) STORED, + virtual_value BIGINT GENERATED ALWAYS AS (id * 2) VIRTUAL""" + % sets + ) + self.local(f"CREATE TABLE {db}.types_test({definitions})") + hexbytes = bytes(range(256)).hex() + self.local(f"""INSERT INTO {db}.types_test(id,s_tiny,u_tiny,s_small,u_small,s_medium,u_medium,s_int,u_int,s_big,u_big,d,f,g,label,labels,bytes,txt,doc,sql_null,json_null,dt,ts,day,duration,yr,bits,fixed_bytes,var_bytes,char_value,bool_value) + VALUES (1,-128,255,-32768,65535,-8388608,16777215,-2147483648,4294967295,9223372036854775806,18446744073709551615, + 12345678901234567890123456789012345.123456789012345678901234567890,1.234567,1.234567890123,'2','s63',UNHEX('{hexbytes}'), + '中文😀 quote\\\' slash\\\\ newline\\n',JSON_OBJECT('uint',18446744073709551615,'arr',JSON_ARRAY(NULL,TRUE),'label','2'),NULL,CAST('null' AS JSON), + '2026-10-25 01:02:03.123456','2026-10-25 01:02:03.654321','2026-10-25','-123:45:56.123456',2026,b'1111111111111111111111111111111111111111111111111111111111111111',UNHEX('000102ff'),UNHEX('00feff'),'char',TRUE); + INSERT INTO {db}.types_test(id) VALUES (2); + INSERT INTO {db}.types_test(id,s_tiny,u_tiny,s_small,u_small,s_medium,u_medium,s_int,u_int,s_big,u_big,d) + VALUES(3,127,0,32767,0,8388607,0,2147483647,0,-9223372036854775808,0,-99999999999999999999999999999999999.999999999999999999999999999999)""") + names = [ + "id", + "s_tiny", + "u_tiny", + "s_small", + "u_small", + "s_medium", + "u_medium", + "s_int", + "u_int", + "s_big", + "u_big", + "d", + "f", + "g", + "label", + "labels", + "txt", + "dt", + "ts", + "day", + "duration", + "yr", + "generated_value", + "char_value", + "bool_value", + ] + cols = [(n, "CAST(`%s` AS CHAR)" % n, quote(n) + "::VARCHAR") for n in names] + cols = [ + (name, "CAST(CAST(`%s` AS DOUBLE) AS CHAR)" % name, target) + if name in ("f", "g") + else (name, source, target) + for name, source, target in cols + ] + cols += [ + ("bytes", "HEX(bytes)", "upper(hex(" + quote("bytes") + "))"), + ("bits", "CAST(bits+0 AS CHAR)", quote("bits") + "::VARCHAR"), + ( + "fixed_bytes", + "HEX(fixed_bytes)", + "upper(hex(" + quote("fixed_bytes") + "))", + ), + ("var_bytes", "HEX(var_bytes)", "upper(hex(" + quote("var_bytes") + "))"), + ] + cols += [ + ( + n, + "CAST(`%s` AS CHAR)" % n, + "CASE WHEN is_null_value(" + + quote(n) + + ") THEN 'null' ELSE " + + quote(n) + + "::VARCHAR END", + ) + for n in ["doc", "sql_null", "json_null"] + ] + self.compare( + "types_test", + cols, + numeric=("f", "g"), + json_columns=("doc", "sql_null", "json_null"), + ) + described = self.lake("DESCRIBE " + quote(self.database) + '."types_test"') + self.results["types_describe"] = described + assert "virtual_value" not in [r[0] for r in described] + self.local( + f"UPDATE {db}.types_test SET label='x',labels='s0,s63',d=-1.000000000000000000000000000001,doc=JSON_OBJECT('changed',true),bytes=UNHEX('00ff'),txt=NULL WHERE id=1" + ) + self.compare( + "types_test", + cols, + numeric=("f", "g"), + json_columns=("doc", "sql_null", "json_null"), + ) + + def test_ddl(self, db): + self.local( + f"CREATE TABLE {db}.ddl_test(id INT PRIMARY KEY,value INT); INSERT INTO {db}.ddl_test VALUES(1,42)" + ) + self.simple("ddl_test", ["id", "value"]) + self.local( + f"ALTER TABLE {db}.ddl_test ADD COLUMN extra VARCHAR(64) NOT NULL DEFAULT 'hello'" + ) + self.simple("ddl_test", ["id", "value", "extra"]) + self.local(f"ALTER TABLE {db}.ddl_test RENAME COLUMN value TO amount") + self.simple("ddl_test", ["id", "amount", "extra"]) + self.local( + f"ALTER TABLE {db}.ddl_test MODIFY COLUMN amount BIGINT; UPDATE {db}.ddl_test SET amount=9223372036854775806 WHERE id=1" + ) + self.simple("ddl_test", ["id", "amount", "extra"]) + self.local( + f"ALTER TABLE {db}.ddl_test DROP COLUMN extra; INSERT INTO {db}.ddl_test VALUES(2,-12)" + ) + self.simple("ddl_test", ["id", "amount"]) + self.local( + f"CREATE INDEX amount_index ON {db}.ddl_test(amount);ALTER TABLE {db}.ddl_test DROP INDEX amount_index" + ) + self.local(f"ALTER TABLE {db}.ddl_test ADD COLUMN text_value TEXT NOT NULL") + self.simple("ddl_test", ["id", "amount", "text_value"]) + + def test_truncate(self, db): + self.local( + f"CREATE TABLE {db}.truncate_test(id INT PRIMARY KEY,value VARCHAR(64));INSERT INTO {db}.truncate_test VALUES(1,'old')" + ) + self.simple("truncate_test", ["id", "value"]) + self.local( + f"TRUNCATE TABLE {db}.truncate_test; INSERT INTO {db}.truncate_test VALUES(2,'new')" + ) + self.simple("truncate_test", ["id", "value"]) + + def test_batch(self, db): + self.local(f"CREATE TABLE {db}.batch_test(id INT PRIMARY KEY,value TEXT)") + values = ",".join("(%d,'%s')" % (i, "x" * 256) for i in range(1200)) + # Use stdin for large synthetic transactions, rather than shell/argv limits. + sql = ( + "SET time_zone='+00:00';BEGIN;INSERT INTO %s.batch_test VALUES%s;COMMIT;" + % (db, values) + ) + proc = subprocess.run( + ["mysql", "-h", "127.0.0.1", "-P", self.port, "-u", "root"], + input=sql, + text=True, + capture_output=True, + timeout=60, + ) + assert proc.returncode == 0, proc.stderr + self.simple("batch_test", ["id", "value"]) + self.local( + f"UPDATE {db}.batch_test SET value='tail' WHERE id=1199;DELETE FROM {db}.batch_test WHERE id<100" + ) + self.simple("batch_test", ["id", "value"]) + with urllib.request.urlopen(self.cdc + "/metrics", timeout=5) as response: + text = response.read().decode() + count = 0 + for line in text.splitlines(): + if ( + line.startswith("ticdc_databend_spool_load_bytes_count{") + and ('changefeed="' + self.feed + '"') in line + ): + count += float(line.rsplit(" ", 1)[-1]) + self.results["spool_disk_load_count"] = count + assert count > 0, "large transaction did not exercise disk spool" + + def wait_state(self, expected, timeout=60): + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + try: + status = self.feed_status() + if status.get("state") == expected: + return status + last = status.get("state", "missing state") + except (OSError, RuntimeError) as error: + # GET is safe to repeat while the restarted coordinator finishes + # bootstrap. Never retry an ambiguous mutation POST here. + last = self.sanitize(error) + time.sleep(0.3) + raise AssertionError("changefeed did not reach " + expected + ": " + last) + + def test_restart(self, db): + self.local( + f"CREATE TABLE {db}.restart_test(id INT PRIMARY KEY,value VARCHAR(64));INSERT INTO {db}.restart_test VALUES(1,'before'),(2,'keep')" + ) + self.simple("restart_test", ["id", "value"]) + marker = int( + self.local( + f"BEGIN;SELECT COUNT(*) FROM {db}.restart_test;SELECT @@tidb_current_ts;COMMIT" + ).splitlines()[-1] + ) + assert marker > 0, ( + "source checkpoint marker must be a real transaction read TSO" + ) + deadline = time.monotonic() + 60 + while self.check_feed().get("checkpoint_ts", 0) < marker: + if time.monotonic() > deadline: + raise AssertionError( + "checkpoint did not drain before controlled restart" + ) + time.sleep(0.3) + self.api("/api/v2/changefeeds/" + self.feed + "/pause", {}, "POST") + self.wait_state("stopped") + status = self.api("/api/v2/status") + pid = status["pid"] + binary = os.path.realpath(os.environ["E2E_CDC_BINARY"]) + command = subprocess.check_output( + ["ps", "-p", str(pid), "-o", "command="], text=True + ).strip() + argv = shlex.split(command) + if len(argv) < 2 or os.path.realpath(argv[0]) != binary or argv[1] != "server": + raise RuntimeError("refusing to stop an unowned CDC process") + os.kill(pid, signal.SIGTERM) + deadline = time.monotonic() + 45 + while True: + try: + os.kill(pid, 0) + except ProcessLookupError: + break + if time.monotonic() > deadline: + raise AssertionError("old local CDC process did not terminate") + time.sleep(0.2) + # This is a drained, exclusive restart, not ambiguous-query failover. + self.local( + f"UPDATE {db}.restart_test SET value='queued' WHERE id=1;INSERT INTO {db}.restart_test VALUES(3,'queued-insert');ALTER TABLE {db}.restart_test ADD COLUMN extra BIGINT NULL" + ) + console = (self.root / "restart-console.log").open("w") + # Keep the original owned capture's endpoint, including an ephemeral + # port assigned by local.py; never bind another runner's fixed port. + address = urllib.parse.urlparse(self.cdc).netloc + process = subprocess.Popen( + [ + binary, + "server", + "--newarch", + "--pd", + os.environ.get("E2E_PD", "http://127.0.0.1:12379"), + "--addr", + address, + "--advertise-addr", + address, + "--data-dir", + os.environ["E2E_CDC_DATA_DIR"], + "--log-file", + str(self.root / "restarted-cdc.log"), + "--tz", + "UTC", + ], + stdout=console, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + console.close() + (self.root / "restart-pid").write_text(str(process.pid)) + deadline = time.monotonic() + 60 + while True: + try: + if self.api("/api/v2/status").get("is_owner"): + break + except (OSError, RuntimeError): + pass + if time.monotonic() > deadline: + raise AssertionError("new local CDC did not become ready") + time.sleep(0.5) + self.wait_state("stopped") + self.api("/api/v2/changefeeds/" + self.feed + "/resume", {}, "POST") + self.wait_state("normal") + self.simple("restart_test", ["id", "value", "extra"]) + self.results["restart"] = { + "old_pid": pid, + "new_pid": process.pid, + "before_checkpoint": marker, + "after_checkpoint": self.feed_status().get("checkpoint_ts"), + } + self.save() + + def test_drop(self, db): + self.local( + f"CREATE TABLE {db}.drop_test(id INT PRIMARY KEY);INSERT INTO {db}.drop_test VALUES(1)" + ) + self.simple("drop_test", ["id"]) + self.local(f"DROP TABLE {db}.drop_test") + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + self.check_feed() + rows = self.lake( + "SELECT COUNT(*)::VARCHAR FROM system.tables WHERE database=? AND name IN (?,?)", + [self.database, "drop_test", "drop_test_cdc_raw"], + ) + if rows == [["0"]]: + return + time.sleep(0.5) + raise AssertionError("DROP left target/raw tables") + + def cleanup(self): + # Failures intentionally keep this isolated run for diagnosis. An operator + # can remove just this feed and native_e2e_* database after inspecting. + if self.feed_created: + status = self.feed_status() + if status.get("state") == "normal": + self.api("/api/v2/changefeeds/" + self.feed + "/pause", {}, "POST") + self.wait_state("stopped") + self.api("/api/v2/changefeeds/" + self.feed, method="DELETE") + if self.remote_created: + assert self.database.startswith("native_e2e_") + self.lake("DROP DATABASE IF EXISTS " + quote(self.database)) + self.local("DROP DATABASE IF EXISTS `%s`" % self.database) + self.results["cleanup"] = "PASS" + self.save() + + +if __name__ == "__main__": + run = Run(existing="--cleanup" in sys.argv[1:]) + if "--cleanup" in sys.argv[1:]: + run.cleanup() + print("CLEANED", run.database, flush=True) + sys.exit(0) + try: + run.setup() + run.tests() + run.cleanup() + run.results["status"] = "PASS" + run.save() + print("RESULT", str(run.root / "result.json"), flush=True) + except Exception as error: + run.results["status"] = "FAIL" + run.results["error"] = run.sanitize(error) + run.save() + print( + "ERROR", + run.sanitize(error), + "RESULT", + str(run.root / "result.json"), + flush=True, + ) + sys.exit(1) diff --git a/tests/integration_tests/databend_native/run.sh b/tests/integration_tests/databend_native/run.sh new file mode 100755 index 0000000000..797c77d77e --- /dev/null +++ b/tests/integration_tests/databend_native/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# The generic integration runner also discovers this case for other sink types. +set -euo pipefail +if [ "${1:-databend}" != databend ]; then + echo "databend_native: requires the databend sink suite; skipping" + exit 0 +fi +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$CUR/../../.." +exec python3 "$CUR/local.py" diff --git a/tests/integration_tests/databend_native/test_local.py b/tests/integration_tests/databend_native/test_local.py new file mode 100644 index 0000000000..082687bb44 --- /dev/null +++ b/tests/integration_tests/databend_native/test_local.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Service-free checks for the local E2E lifecycle and target selection.""" + +import json +import os +import pathlib +import shlex +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from local import Local, wait_ready +from run import Run + + +class LocalRunnerTest(unittest.TestCase): + def test_owned_background_process_is_stopped(self): + with tempfile.TemporaryDirectory() as directory: + runner = Local(pathlib.Path(directory), "test-image", "v8.5.1") + child = runner.start( + [sys.executable, "-c", "import time; time.sleep(60)"], "owned.log" + ) + try: + runner.cleanup() + self.assertIsNotNone(child.poll()) + self.assertEqual(runner.summary["cleanup"], "PASS") + finally: + if child.poll() is None: + child.kill() + child.wait() + + def test_never_inherits_cloud_or_other_capture(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch.dict( + os.environ, + { + "LAKE_URI_FILE": "/not/read/cloud-secret", + "DATABEND_NATIVE_TEST_URI": "must-not-use", + "CDC_SERVER": "https://not-local.invalid", + "E2E_CDC_BINARY": "/unowned/cdc", + "MYSQL_PORT": "1234", + }, + ), + ): + runner = Local(pathlib.Path(directory), "test-image", "v8.5.1") + for name in ( + "LAKE_URI_FILE", + "DATABEND_NATIVE_TEST_URI", + "CDC_SERVER", + "E2E_CDC_BINARY", + "MYSQL_PORT", + ): + self.assertNotIn(name, runner.env) + runner.summary["status"] = "PASS" + runner.cleanup() + self.assertEqual( + json.loads((runner.root / "summary.json").read_text())["cleanup"], + "PASS", + ) + + def test_explicit_http_scheme_only(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + credential = root / "uri" + credential.write_text( + "databend+http://user:password@127.0.0.1:18000/default" + ) + credential.chmod(0o600) + with patch.dict( + os.environ, + {"LAKE_URI_FILE": str(credential), "E2E_RUN_DIR": str(root / "http")}, + ): + self.assertEqual(Run().base, "http://127.0.0.1:18000") + credential.write_text("lake://user:password@lake.example/default") + with patch.dict( + os.environ, + {"LAKE_URI_FILE": str(credential), "E2E_RUN_DIR": str(root / "https")}, + ): + self.assertEqual(Run().base, "https://lake.example") + + def test_readiness_retries_service_errors_but_checks_exit(self): + attempts = iter([RuntimeError("not ready"), True]) + + def probe(): + value = next(attempts) + if isinstance(value, Exception): + raise value + return value + + with patch("local.time.sleep"): + wait_ready(probe, None, 1, "test-service") + process = subprocess.Popen(["true"]) + process.wait() + with self.assertRaisesRegex(RuntimeError, "exited before"): + wait_ready(lambda: True, process, 1, "test-service") + + def test_refuses_unowned_restart_pid_and_still_removes_credential(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / "acceptance").mkdir() + (root / "acceptance/restart-pid").write_text(str(os.getpid())) + (root / "local-uri").write_text("secret") + runner = Local(root, "test-image", "v8.5.1") + runner.binary = root / "unowned-cdc" + with patch("local.os.kill") as kill: + runner.cleanup() + kill.assert_not_called() + self.assertEqual(runner.summary["status"], "FAIL") + self.assertIn("unowned restarted capture", runner.summary["cleanup"][0]) + self.assertFalse((root / "local-uri").exists()) + + def test_restart_ownership_compares_canonical_paths(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / "acceptance").mkdir() + (root / "acceptance/restart-pid").write_text("12345") + (root / "sub").mkdir() + runner = Local(root, "test-image", "v8.5.1") + runner.binary = root / "sub/../cdc" + command = shlex.join( + [ + str(runner.binary.resolve()), + "server", + "--data-dir", + str(root / "cdc-data"), + ] + ) + with ( + patch( + "local.subprocess.run", + return_value=subprocess.CompletedProcess([], 0, stdout=command), + ), + patch("local.os.kill", side_effect=[None, ProcessLookupError]) as kill, + ): + runner.cleanup() + self.assertEqual(kill.call_args_list[0].args[0], 12345) + self.assertEqual(runner.summary["cleanup"], "PASS") + + def test_log_failure_does_not_skip_container_removal(self): + with tempfile.TemporaryDirectory() as directory: + runner = Local(pathlib.Path(directory), "test-image", "v8.5.1") + runner.container_created = True + + def execute(args, **kwargs): + if args[:2] == ["docker", "logs"]: + raise subprocess.TimeoutExpired(args, 30) + return subprocess.CompletedProcess(args, 0) + + with patch("local.subprocess.run", side_effect=execute) as run: + runner.cleanup() + self.assertEqual(run.call_args_list[-1].args[0][:2], ["docker", "rm"]) + self.assertEqual(runner.summary["status"], "FAIL") + self.assertIn("Databend logs", runner.summary["cleanup"][0]) + + +if __name__ == "__main__": + unittest.main() From f77e2e0ac6355720d003a556bfd09a99edfef5a2 Mon Sep 17 00:00:00 2001 From: jiangjianyuan Date: Mon, 26 Oct 2026 10:54:58 +0800 Subject: [PATCH 2/2] sink: unify Lake naming and address review feedback Use Lake package, scheme, metric and test entry-point names. Keep only one English final design and E2E report, without personal filesystem paths. Retain real backend protocol, image and legacy baseline identifiers. Backpressure shared source admission and full lane queues instead of failing ordinary contention. Preserve cancellation, bounded accounting and fail-closed handling of oversized events and unknown writes. Restrict plaintext URIs to loopback, detect live source listeners before setup, and support Python 3.10 with streaming hashing and early checks. Add regression tests and rerun all nine local E2E suites with a one-event shared source budget. --- Makefile | 8 +- ...databend-native-sink-design-review-v0.2.md | 1236 ----------------- ...databend-native-sink-design-review-v0.3.md | 447 ------ .../databend-native-sink-design-review.md | 395 ------ .../design/databend-native-sink-e2e-report.md | 131 -- ...atabend-native-sink-parity-fault-report.md | 198 --- docs/design/databend-native-sink-plan.md | 278 ---- .../databend-native-sink-review-prompt.md | 74 - ...tabend-native-sink-scope-review-comment.md | 94 -- docs/design/lake-sink-design.md | 508 +++++++ .../sink/{databend => lake}/client.go | 36 +- .../sink/{databend => lake}/client_test.go | 2 +- .../sink/{databend => lake}/completion.go | 2 +- .../{databend => lake}/completion_test.go | 2 +- .../sink/{databend => lake}/config.go | 14 +- .../sink/{databend => lake}/config_test.go | 24 +- .../{databend => lake}/integration_test.go | 8 +- .../sink/{databend => lake}/lane.go | 31 +- .../{databend => lake}/live_fault_test.go | 12 +- .../sink/{databend => lake}/metrics.go | 28 +- .../sink/{databend => lake}/native.go | 32 +- .../sink/{databend => lake}/native_test.go | 2 +- .../sink/{databend => lake}/row.go | 44 +- .../sink/{databend => lake}/row_test.go | 2 +- .../sink/{databend => lake}/runtime_test.go | 101 +- .../sink/{databend => lake}/schema.go | 94 +- .../sink/{databend => lake}/schema_test.go | 2 +- .../sink/{databend => lake}/sink.go | 81 +- .../sink/{databend => lake}/sql.go | 6 +- .../sink/{databend => lake}/sql_test.go | 2 +- .../sink/{databend => lake}/writer.go | 20 +- .../sink/{databend => lake}/writer_test.go | 16 +- downstreamadapter/sink/metrics/databend.go | 29 - downstreamadapter/sink/metrics/lake.go | 29 + downstreamadapter/sink/sink.go | 10 +- downstreamadapter/sink/sink_test.go | 12 +- pkg/common/event/table_schema_store.go | 2 +- pkg/common/event/table_schema_store_test.go | 4 +- pkg/common/types.go | 2 +- pkg/config/sink_protocol.go | 8 +- pkg/errors/helper.go | 4 +- pkg/errors/{databend.go => lake.go} | 14 +- pkg/metrics/downstream_type.go | 4 +- pkg/metrics/downstream_type_test.go | 6 +- pkg/metrics/init.go | 2 +- .../databend_native/README.md | 301 ---- .../.gitignore | 0 .../baseline.py | 0 .../baseline_parity_test.go.txt | 0 .../extended.py | 0 .../{databend_native => lake_native}/local.py | 79 +- .../options.py | 0 .../{databend_native => lake_native}/run.py | 8 +- .../{databend_native => lake_native}/run.sh | 4 +- .../test_local.py | 38 +- 55 files changed, 1013 insertions(+), 3473 deletions(-) delete mode 100644 docs/design/databend-native-sink-design-review-v0.2.md delete mode 100644 docs/design/databend-native-sink-design-review-v0.3.md delete mode 100644 docs/design/databend-native-sink-design-review.md delete mode 100644 docs/design/databend-native-sink-e2e-report.md delete mode 100644 docs/design/databend-native-sink-parity-fault-report.md delete mode 100644 docs/design/databend-native-sink-plan.md delete mode 100644 docs/design/databend-native-sink-review-prompt.md delete mode 100644 docs/design/databend-native-sink-scope-review-comment.md create mode 100644 docs/design/lake-sink-design.md rename downstreamadapter/sink/{databend => lake}/client.go (78%) rename downstreamadapter/sink/{databend => lake}/client_test.go (99%) rename downstreamadapter/sink/{databend => lake}/completion.go (99%) rename downstreamadapter/sink/{databend => lake}/completion_test.go (99%) rename downstreamadapter/sink/{databend => lake}/config.go (91%) rename downstreamadapter/sink/{databend => lake}/config_test.go (63%) rename downstreamadapter/sink/{databend => lake}/integration_test.go (94%) rename downstreamadapter/sink/{databend => lake}/lane.go (86%) rename downstreamadapter/sink/{databend => lake}/live_fault_test.go (97%) rename downstreamadapter/sink/{databend => lake}/metrics.go (51%) rename downstreamadapter/sink/{databend => lake}/native.go (81%) rename downstreamadapter/sink/{databend => lake}/native_test.go (99%) rename downstreamadapter/sink/{databend => lake}/row.go (74%) rename downstreamadapter/sink/{databend => lake}/row_test.go (99%) rename downstreamadapter/sink/{databend => lake}/runtime_test.go (83%) rename downstreamadapter/sink/{databend => lake}/schema.go (80%) rename downstreamadapter/sink/{databend => lake}/schema_test.go (99%) rename downstreamadapter/sink/{databend => lake}/sink.go (77%) rename downstreamadapter/sink/{databend => lake}/sql.go (97%) rename downstreamadapter/sink/{databend => lake}/sql_test.go (99%) rename downstreamadapter/sink/{databend => lake}/writer.go (88%) rename downstreamadapter/sink/{databend => lake}/writer_test.go (91%) delete mode 100644 downstreamadapter/sink/metrics/databend.go create mode 100644 downstreamadapter/sink/metrics/lake.go rename pkg/errors/{databend.go => lake.go} (55%) delete mode 100644 tests/integration_tests/databend_native/README.md rename tests/integration_tests/{databend_native => lake_native}/.gitignore (100%) rename tests/integration_tests/{databend_native => lake_native}/baseline.py (100%) rename tests/integration_tests/{databend_native => lake_native}/baseline_parity_test.go.txt (100%) rename tests/integration_tests/{databend_native => lake_native}/extended.py (100%) rename tests/integration_tests/{databend_native => lake_native}/local.py (87%) rename tests/integration_tests/{databend_native => lake_native}/options.py (100%) rename tests/integration_tests/{databend_native => lake_native}/run.py (99%) rename tests/integration_tests/{databend_native => lake_native}/run.sh (66%) rename tests/integration_tests/{databend_native => lake_native}/test_local.py (78%) diff --git a/Makefile b/Makefile index e4f5b8d24d..ec6a3106b8 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ generate-protobuf generate_mock \ cdc kafka_consumer storage_consumer pulsar_consumer filter_helper kafka_auth_server \ prepare_test_binaries \ - unit_test_in_verify_ci integration_test_build integration_test_build_fast integration_test_mysql integration_test_kafka integration_test_storage integration_test_pulsar integration_test_databend \ + unit_test_in_verify_ci integration_test_build integration_test_build_fast integration_test_mysql integration_test_kafka integration_test_storage integration_test_pulsar integration_test_lake \ generate-next-gen-grafana check-next-gen-grafana @@ -255,10 +255,10 @@ check_failpoint_ctl: tools/bin/failpoint-ctl integration_test: integration_test_mysql -# Self-contained local source/capture/Databend; does not require unrelated Kafka, +# Self-contained local source/capture/Lake; does not require unrelated Kafka, # Pulsar, MinIO, diff-inspector or downstream TiDB test binaries. -integration_test_databend: - python3 tests/integration_tests/databend_native/local.py +integration_test_lake: + python3 tests/integration_tests/lake_native/local.py integration_test_mysql: check_third_party_binary tests/integration_tests/run.sh mysql "$(CASE)" "$(START_AT)" diff --git a/docs/design/databend-native-sink-design-review-v0.2.md b/docs/design/databend-native-sink-design-review-v0.2.md deleted file mode 100644 index 30e3077abc..0000000000 --- a/docs/design/databend-native-sink-design-review-v0.2.md +++ /dev/null @@ -1,1236 +0,0 @@ -# RFC：TiCDC 原生 Databend Lake Warehouse Sink 详细设计与评审报告 - -| 项目 | 内容 | -|---|---| -| 文档版本 | v0.2，2026-10-24，吸收两份独立评审后的协议修订版 | -| 决策状态 | Proposed；存在生产阻塞项，不是 Ready for Production | -| 实现状态 | 仅设计文档；未新增 sink、未提交、未部署 | -| 用户已确定要求 | 原生 TiCDC sink；迁移 bend-cdc 核心能力；批量写 staging/MERGE；第一版必须有内存 + 限额本地磁盘 spool + 背压 | -| 分析对象 | Databend SQL warehouse；不泛指 Iceberg/Delta/Hudi 文件格式 | -| 主动评审目标 | 找出丢数据、状态回退、错误确认、DDL 重放、双写、内存/磁盘失控及不可恢复场景 | - -> 阅读提醒：本文将“源码事实”“拟定设计”“待验证假设”分开描述。章节完整不代表所有协议已证明。第 21 节列出阻塞项；任何 agent 都不应把它们自动视为已解决。 - -## 0. 评审导航与版本固定 - -### 0.1 两份源码基线 - -| 对象 | 本地仓库 | 固定 commit | -|---|---|---| -| bend-cdc | `/Users/jiangjianyuan/work/git/pingcap/webapi`，子目录 bend-cdc | `b6481621021e82aee659ad88e8e6c00f6dd11906` | -| 新架构 TiCDC | `/Users/jiangjianyuan/work/git/pingcap/ticdc` | `2bf6dacd0aec9ac2fad464600e47341f9aa951f2` | - -TiCDC remote 是 `git@github.com:sdojjy/tiflow.git`；被分析的新架构分支名为 `feat/create-paused-changefeed`。这是 fork 分支，不是上游最新主干，也没有证据证明它就是线上部署版本。 - -**当前 checkout 已切换**：本报告编写时，本地 TiCDC 是旧架构 `master` / `03ed2432bd7782198742d3aa20e07edcc47097e6`。因此新架构代码一律通过 `git show` 读取固定 commit；没有切分支，没有覆盖其他工作。不能在当前 HEAD 下搜索不到新接口，就认定报告引用错误；也不能拿旧架构接口直接实现本文。 - -复核方法（只读，无需 checkout）： - -```bash -git -C /Users/jiangjianyuan/work/git/pingcap/ticdc show \ - 2bf6dacd0aec9ac2fad464600e47341f9aa951f2:downstreamadapter/sink/sink.go -git -C /Users/jiangjianyuan/work/git/pingcap/ticdc show \ - 2bf6dacd0aec9ac2fad464600e47341f9aa951f2:pkg/sink/spool/spool.go -git -C /Users/jiangjianyuan/work/git/pingcap/webapi show \ - b6481621021e82aee659ad88e8e6c00f6dd11906:bend-cdc/pkg/sink/databend/databend_cdc.go -``` - -### 0.2 推荐阅读顺序 - -1. 目标与非目标（§1–2）。 -2. 安全不变量、事件与表状态机（§4–5）。 -3. 内存/磁盘预算和攒批（§6–7）。 -4. MERGE/恢复/DDL/所有权（§8–12）。 -5. 类型、配置、运行保障（§13–17）。 -6. 测试、上线、阻塞项、评审问题（§18–23）。 - -评审结论应给出具体执行时序和反例，不只判断“总体可行”。文末 TiCDC 源码索引用 T 编号，bend-cdc 用 C 编号，阻塞项用 B 编号，三者不混用。TiCDC 链接固定 commit；bend-cdc 本机链接须配合本节固定 commit 复核。 - -### 0.3 v0.2 评审处置与变更摘要 - -本轮只修订设计，不把新增协议描述当作实现/实验成功。两份评审中有价值的反例均进入验收；对无证据的事实断言和不充分修复，作如下限定： - -| 评审主题 | 处置 | 本版对应内容 | -|---|---|---| -| PK不变、二级UK更新也会拆成同TSO DELETE/INSERT | 接受；是普通UPDATE的支持契约问题，不只是PK变键 | §11.4，固定raw-change模式；B05 | -| DDL不能套用DML事务fence；KillQuery返回nil不是终态 | 接受 | §9.4、§12.2分mutation矩阵；B01/B03 | -| 增量分片提前确认/整事件回收死锁 | 接受 | §5.2、§7.4；新增B17 | -| 入口共享线程与共享chunk绕过预算 | 接受，需要跨层接入，不是只改sink | §6.7；新增B16 | -| raw/tombstone在维护成功时仍随历史键增长 | 接受 | §9.6；新增B18 | -| no-op MERGE必然不消费stream，是规范要求 | 不接受其普遍性断言；存在版本相关已知修复，必须测实际SQL/版本 | §9.7；B02 | -| enable-table-across-nodes=false等于禁用HA | 不接受；它限制拆表分布，不阻止整表迁移/接管 | §12.5；T15 | -| 固定sleep、进程panic或取消成功能保证旧SQL终止 | 不接受；只能在可信终态/真正fence后放行 | §12.2、§12.5 | -| Segment pinning、stream失效、bootstrap竞态完全遗漏 | 原版已识别但不可验收，细化并分别跟踪 | §6.8、§9.8、§10.6；B13–B15 | -| 必须用事务DAG归约/唯一约束journal/标记行推进stream | 不预设实现成立；先验证源净事件、目标能力与原子性 | §10–12；不盲选SQL tie-break/无界事务缓存 | - -新阻塞项编号保留评审建议的B13–B15，另外增加B16入口信用、B17完成封口、B18远端容量。既有B01–B12不重编号；“设计已明确”与“实现/实验已关闭”分别记录。 - -## 1. 背景、目标与成功标准 - -### 1.1 当前链路及代价 - -```text -TiDB → TiCDC 原生事件 → Canal JSON 编码 → S3/OSS 文件 - → bend-cdc LIST/GET/SQS → Canal JSON 解码 - → 元数据/类型处理 → Databend raw → stream → MERGE → target -``` - -中间链路有编码/解码、对象请求、文件扫描、网络流量及独立消费进程成本。它也提供较长的中间缓冲、审计与回放窗口；去掉对象存储不等于只删成本而不损失能力。 - -### 1.2 目标链路 - -```text -TiDB → TiCDC 原生 DMLEvent / DDLEvent - → 事件适配/元数据/有界 admission - → 内存热缓冲 → 必要时溢写限额本地 spool - → 同表同 schema epoch 攒批 → Databend raw INSERT - → append-only stream → MERGE → 目标表可见 - → 释放覆盖的 spool + 完成 flush callback → checkpoint 可推进 -``` - -目标是：保留核心同步正确性，降低增量中间链路成本；不是完全消除 JSON、磁盘或 Databend MERGE。第一版 raw_data 仍可用 JSON 作为受控行载体，但不再使用 Canal JSON envelope 传输。 - -### 1.3 成功标准 - -- 最终状态与源在同一已完成逻辑水位上一致；包括 DELETE、不发生旧数据复活。 -- source-to-target 按 at-least-once 传输，重复与重启后幂等收敛。 -- 新增可确认进度必须有目标应用成功的证据；正常路径要求MERGE成功，合法幂等no-op要求同等级的已支配/消费证据（§9.7），绝不以仅raw提交代替。 -- 支持权威建表、复合主键、批量 staging、核心类型映射和明确白名单 DDL。 -- 进程内存、spool 磁盘、节点总预算有界；满载后背压而不是无界积压。 -- 达到双方约定的吞吐、target-applied lag 和恢复时间；数值待压测/产品确认，不预设达标。 -- 固定相同数据量、下游和延迟 SLO 后，总成本实测低于旧链；不预先承诺节省百分比。 - -## 2. 范围、非目标与第一版支持矩阵 - -### 2.1 本设计包含 - -原生增量 sink、连接与鉴权、schema bootstrap、类型转换、内存/磁盘 spool、批处理、raw/stream/MERGE、checkpoint 回调、DDL 屏障与恢复、所有权/接管、配额、监控、灰度和回退。 - -### 2.2 非目标 - -- 不把 TiCDC 改成 MySQL/Postgres/MongoDB 通用捕获器。 -- 不把新 sink 当成自动全量导出器；全量初始化继续独立执行。 -- 不保证跨表或跨物理片段的源事务原子可见，也不保证读取者获得源数据库级一致快照。 -- 不承诺复现所有 TiDB/MySQL 约束、索引、collation、默认表达式和生成列计算行为。 -- 不支持外部应用同时改目标表/内部 raw/stream/journal。 -- 不以本地 spool 提供节点永久丢失后的恢复能力；它不是 WAL。 -- 不支持无限期下游停机、无限 raw 历史或无限磁盘预算。 - -### 2.3 第一版边界 - -| 能力 | 第一版目标 | 约束 | -|---|---|---| -| 非分区表，非空单列/复合 PK | 支持 | PK 列保留于完整 after image，明确 before image 契约 | -| 无 PK/nullable UK/force-replicate | 拒绝 | 不自动换用隐式 handle，也不静默 append-only | -| 分区表/table-across-nodes | 拒绝 | 未解决 logical table owner 汇聚前不开放 | -| 同逻辑目标表一个 owner | 必须 | 正常调度与异常接管都需成立 | -| PK不变、仅二级UK值更新 | 必须支持 | 固定raw-change事件模式，禁止继承默认拆分模式；见§11.4 | -| 主键值更新 | 验证门槛，未验收前不承诺支持 | 必须证明入口净变化、旧键删除与重放；不能假设所有变键都以UPDATE暴露而承诺可检测拒绝 | -| 主键定义改变 | 拒绝 | 改 PK 列集合/类型可能重定义 row identity | -| 历史空表自动建表 | 必须提供路径 | start-ts schema bootstrap 或全量初始化预建 | -| 列 DDL | 白名单 | ADD/DROP/RENAME 非键列，有限 MODIFY | -| TRUNCATE | 核心对齐项 | table incarnation 与中断恢复验收前不能上线 | -| DROP/RENAME TABLE/SCHEMA、复杂分区 DDL | 初期阻塞 | 不以 warning 代替正确处理 | -| 二级索引 DDL | 有条件 no-op | 已证明不改变行身份/投影后才确认 | -| 不支持的同步选项 | 创建时拒绝 | 如未经设计的 active-active、BDR、syncpoint，不能悄悄忽略 | - -“第一版目标”不等于实现完成；若某核心项仍未验证，只能发布带明确限制的 PoC，不能对外声称 bend-cdc 核心能力全量对齐。 - -## 3. 代码现状与迁移策略 - -### 3.1 bend-cdc 实际入口 - -事实：TiDB 路径是 `TiDBRunner → TiDBCDCManager → DatabendMergeSink → DatabendSink/CDCManager`（C01–C03）。不是简单 `DatabendSink.WriteBatch` 的固定 id INSERT/DELETE 实现。 - -| 已有职责 | 迁移策略 | -|---|---| -| 文件 discovery、SQS、Canal 解码、S3 key checkpoint | 删除这层职责，使用 TiCDC 原生事件与上游进度 | -| 每表 PK 推导、目标路由、epoch 管理 | 保留语义，从 TableInfo/DDL 获取权威数据 | -| raw 参数化 INSERT、stream、MERGE | 迁移核心算法，不照搬 goroutine/循环生命周期 | -| 乱序抑制、最新水位、DELETE tombstone | 必须保留 | -| 类型转换、binary/ENUM/SET/JSON 测试 | 保留测试语义，重写原生 datum 表示 | -| schema 白名单和 TRUNCATE | 保留并补充 TiCDC 原生恢复协议 | -| 每表连接池/后台循环、全局 Background context | 不照搬；改为受限共享资源与可取消生命周期 | -| Dumpling/CSV/COPY INTO | 独立初始化路径，接口共享 mapper，任务生命周期不并入 sink | - -事实：raw 是普通持久化表，附带 append-only stream；不是临时内存表。严格顺序模式查询 retained raw history；compaction 保留每个键最新版本，删除也保留墓碑（C03）。因此 raw 容量可能随**历史键集合**增长，而不只是当前目标行数。 - -### 3.2 TiCDC 现有接入点 - -T01 的 Sink 接口支持 AddDMLEvent、FlushDMLBeforeBlock、WriteBlockEvent、SetWriteGate、Run、Close。T02 提供 CommitTs、TableInfo、before/after 和 enqueue/flush 分离回调。 - -不能仅改 factory：需要 scheme/config、SinkType、TableSchemaStore、dispatcher、恢复、metrics 和测试等配套。冒充 MysqlSinkType 会触发 `*mysql.Sink` 强转（T07），不可采用。 - -### 3.3 复用现有 S3 spool 的边界 - -固定新架构基线已有 spool（T03–T05），具备内存热数据、segment 溢写、磁盘配额等待、释放和回调拆分。拟复用/提取通用能力，不直接复用 Canal 编码器或 S3 payload builder。 - -已发现必须审查的差异： - -1. `TryEnqueue` 对单 entry 大于磁盘配额有“内存接受后立即刷”的例外，不能当严格内存上限。 -2. `prepareWorkDir` 启动清空所属目录，证明这不是重启恢复 WAL。 -3. `acceptEntryLocked` 在溢写时先序列化 blob，存在额外瞬时内存；必须计入预算。 -4. S3 `buildPayload` 使用 bytes.Buffer 拼完整上传批次；新 sink 不能照搬后声称磁盘回读完全流式。 -5. 基线 S3 路径有 unlimited channel 等结构；不应复制到要求硬预算的新 sink。 -6. 现有 spool 预算不是现成的节点所有 changefeed 总预算，更不是全部磁盘消费者的全局预算。 - -## 4. 设计原则与安全不变量 - -| 编号 | 必须始终成立的约束 | -|---|---| -| I1 | 未有目标应用成功或合法幂等no-op完成证据的事件不得触发flush callback；仅raw提交、结果行/计数不构成证据 | -| I2 | 同事件重放不能使目标版本回退、重复插入业务行或复活已删除键 | -| I3 | 同逻辑目标表只有一个有效 merge/DDL owner；锁不能只存在单进程 | -| I4 | DDL fence 前的 DML 已应用；fence 后 DML 不使用旧投影；旧 DML 重放不能污染新 schema | -| I5 | collector交付/共享chunk、spool、队列、编码、回读、元数据预算有界；磁盘有单任务和节点总预算；远端raw有停写阈值 | -| I6 | 正在等待配额的生产者不能阻止负责 MERGE/回收的消费者前进 | -| I7 | 未物理释放的 segment 空间不能从磁盘预算中提前扣掉 | -| I8 | 失租/停止后的异步工作不能继续执行新的下游变更；在途旧请求必须被接管协议覆盖 | -| I9 | 最新键水位与 DELETE tombstone 不能被普通 TTL 清除 | -| I10 | schema bootstrap/version 与“某 DDL 前数据已应用”的 durable fence 不能混为一谈 | -| I11 | metrics/result parsing 失败不能被简单当成业务未提交；提交不确定也不能被简单当成功 | -| I12 | 所有“不支持”功能可诊断地拒绝/阻塞，不用静默降级代替正确性 | - -## 5. 组件划分与状态机 - -### 5.1 组件 - -```text -TiCDC dispatcher - │ DMLEvent/DDL - ▼ -DatabendSink adapter - ├─ capability/config validation - ├─ schema/route registry - ├─ budgeted admission + typed row normalizer - ├─ hot memory + disk spool + transaction completion tracker - ├─ per-table lane scheduler - │ └─ raw INSERT → MERGE → covered-batch completion - ├─ DDL barrier/recovery coordinator - └─ target client + write gate + owner fencing adapter - -Capture-wide BudgetManager / free-space guard - └─ 所有 Databend changefeed 共享，若与 S3/redo 共盘还需联合规划 -``` - -候选代码目录： - -- `/Users/jiangjianyuan/work/git/pingcap/ticdc/downstreamadapter/sink/databend/`：原生事件、callback、lane、barrier、生命周期。 -- `/Users/jiangjianyuan/work/git/pingcap/ticdc/pkg/sink/databend/`：配置、客户端、mapper、SQL、journal、恢复。 -- `/Users/jiangjianyuan/work/git/pingcap/ticdc/pkg/sink/spool/`：仅必要的通用预算与存储扩展，避免破坏 S3 行为。 -- `/Users/jiangjianyuan/work/git/pingcap/ticdc/tests/integration_tests/databend_sink/`：真实链路验证。 - -以上是新架构上的拟定目录，不表示当前旧架构 HEAD 已有这些代码。 - -### 5.2 事件/分片状态机 - -事件完成与分片生命周期必须分开，不再把“事件ACK后才释放全部entry”当顺序要求： - -```text -事件tracker：OPEN → SEALED → COMPLETED - └────────────→ FAILED / CANCELLED（未完成不确认） -分片：REGISTERED → STAGED → RAW_COMMITTED → APPLYING → APPLIED → RECLAIMED - └→ COMMIT_UNKNOWN → RECONCILING -APPLIED分片可独立回收；其源事件仍可处于OPEN或SEALED -``` - -APPLIED表示目标已包含该分片，或按§9.7证明它被已应用版本正确支配；不是只看到raw中有更高版本。回收失败不撤销APPLIED、不重复MERGE，但空间继续计费。 - -tracker协议（拟实现，非现有DMLEvent保证）： - -1. 单一编码producer持有OPEN状态；每个分片必须先原子登记、增加registered/pending，再发布给消费者。发布失败也不能把它伪装成已应用，需重试该分片或置FAILED。 -2. 只有编码遍历完整个源事件、没有尚未登记的行，才可Seal；Seal与失败/完成检查在同一状态同步域内。若先前分片均已完成但producer还OPEN，仍不得确认。 -3. 确认条件为 `SEALED && pending == 0 && !failed && CAS(completed,false,true)`；Seal与最后一个Apply竞争时由CAS保证只触发一次。零分片也必须Seal，过滤语义通过后才完成。 -4. 每个分片APPLIED转换也必须一次性；仅递减pending一次，并独立释放payload。只保留未完成分片的小描述符和有界tracker，不永久保存所有已回收分片的位图；每事件/每lane限制未完成窗口。 -5. DetachPostCallbacks不捕获DMLEvent，但其flush闭包自身没有once保护，只有enqueue有CAS（T02）；由tracker提供flush的一次性。失败/取消后迟到完成只能清理资源，不能把未完成事件改成成功。 -6. 一个大事件可边编码边应用/回收，累计编码量可以大于spool额度；并发驻留量、源chunk与元数据仍受§6约束。不能靠一次性复制整事件换取封口。 - -反例验收：暂停F2登记，让F1完成，此时flush次数必须为0；随后登记F2、Seal、应用F2后恰为1。再测试大于可用spool的事件、重复完成、Seal/Apply并发、发布失败与取消。这里的“事件”是TiCDC完成单位，不承诺跨表事务原子性。 - -### 5.3 表状态机 - -```text -DISCOVERED → BOOTSTRAPPING → ACTIVE -ACTIVE → DRAINING_BEFORE_DDL → DDL_PENDING → DDL_APPLIED → ACTIVE -任意状态 → FENCED / STOPPING / FAILED -重启：RECOVERING → 验证 owner/schema/journal/未完成 raw → ACTIVE -``` - -只有 ACTIVE 接受正常新 epoch DML；旧版本重放走受控验证/跳过流程，不以普通写入路径解释。 - -v0.2补充暂停/恢复转移，避免把所有阻塞都混成retry： - -| 状态 | 可继续的工作 | 恢复ACTIVE的必要条件 | -|---|---|---| -| LOCAL_RESOURCE_BLOCKED | 已接管分片的编码窗口/应用/回收与控制，受保留池约束 | 物理配额/信用回到低水位，未丢唤醒且无fatal IO | -| REMOTE_CAPACITY_BLOCKED | 已raw提交coverage的应用、安全维护与故障协调 | 扩额或安全清理后的新鲜测量+增长预留通过；旧chunk不能无界排队 | -| COMMIT_UNKNOWN | 同lane仅执行权威终态查询/协调 | 服务端提交/回滚证据及coverage核验；不能仅等到timeout | -| HANDOFF_BLOCKED / FENCED | 终态核验、释放自有资源；无新业务mutation | 新有效owner取得mutation排他，完整恢复后走RECOVERING | -| RECOVERY_REQUIRED | 按证据执行授权恢复，正常admission关闭 | journal/ledger/raw/stream/binding一致且unknown均已处置 | -| REINITIALIZATION_REQUIRED | 隔离旧目标、显式新快照初始化，不在旧lane继续写 | 新基线/新内部对象校验与显式切换；不是直接改回ACTIVE | - -多个阻塞原因并存时保存原因集合；释放一个配额不能清掉COMMIT_UNKNOWN或DDL fence。RESOURCE_BLOCKED是本地资源阻塞的通称，对外状态使用LOCAL_RESOURCE_BLOCKED区分远端容量。进程失败/取消仍遵守§15，不因解除某一局部阻塞自动恢复写入。 - -## 6. 内存、磁盘 spool 与预算 - -### 6.1 为什么第一版必须有 spool - -这是已确定需求。热数据留内存，下游抖动或待 MERGE 数据增加时溢写磁盘；磁盘不是无限缓存。防止写爆依靠**硬预算 + 可用空间保护 + 背压**，不是“用了磁盘就安全”。 - -### 6.2 预算对象 - -定义： - -- `M_hot`：spool 内存 payload。 -- `M_encode`：规范化、序列化和参数构造临时内存。 -- `M_read`：磁盘回读和 SQL 批次缓冲。 -- `M_meta`：entry、callback、表状态和事务 tracker。 -- `M_in`：尚未转交/释放的源 chunk 和入口队列。 -- `D_live`：segment 实际占用，包括已逻辑释放但尚未回收部分。 -- `D_reserved`：在途写预留，包括头部、对齐、允许的分配开销。 - -要求 admission 预留预算后才接受自有数据；不能只统计 M_hot 然后无限持有 M_in/M_meta。进程 RSS 还包括 TiCDC 其他组件与 Go/runtime 开销，应单独观测，不能保证等于 sink 配额。 - -单任务 `D_live + D_reserved ≤ D_feed`，所有任务共享 `ΣD ≤ D_capture`。每个批次先占有预算，再读回/编码；并发 worker 数乘以最大批次不能超过 M_read 总池。 - -### 6.3 磁盘 admission 算法 - -```text -1. 估计/增量计算 entry 的编码及落盘字节，申请内存/磁盘预算。 -2. 在 capture 共享锁下预留预算；不能持锁做网络 IO。 -3. 留内存或 append segment；失败回滚尚未使用的预留。 -4. 对实际文件分配/长度进行结算，超出预留视为实现错误或停止 admission。 -5. 额度不足：通知压力调度；等待仅发生在有信用约束的异步任务，不阻塞共享dynstream处理线程（§6.7）。 -6. 删除/截断成功后才释放实际磁盘额度并唤醒等待者。 -``` - -文件系统 guard 要考虑：statfs 可用空间、配置的保留量、未落盘预留、segment 分配开销及 inode。不同实例共享预算时必须原子预留；其他进程写同一文件系统无法由本组件完全控制，ENOSPC 仍需容错，不能承诺绝不发生。 - -建议 dedicated spool volume 或明确与 eventstore/redo/log 的共盘预算。仅限制 Databend spool 不会限制 TiCDC 其他组件的磁盘增长。 - -### 6.4 队列、callback 与超大事务 - -- 所有数据队列按**字节与元素数**同时限额；每个小事件也占 M_meta，避免零小 payload 绕过限制。 -- admission 前不能无界 detach/复制所有 chunk；增量编码和所有权转移需有明确时机。 -- callback 不得捕获整块原始 DMLEvent/chunk；使用独立 completion tracker（参考 T02），但要证明所有数据引用已解除。 -- 大事务按行/字节拆分；原子性不扩大为整个源事务，多片 MERGE 期间目标可能看到部分事务，这是公开限制。 -- 单行大于支持上限时终止该表/changefeed并保留错误，不采用任意内存直通。 -- 最大并行事务 tracker 与等待确认 entry 数另有限制，防止“数据落磁盘了，callback 仍把内存占满”。 - -### 6.5 高低水位和死锁规避 - -达到高水位时暂停新 admission，低水位后恢复，避免频繁抖动。具体水位按有效内存+磁盘预算定义，不盲用 S3 的比例。 - -MERGE、清理、lease/DDL 控制必须有独立可运行资源。配额等待者不得占住唯一 worker、表锁或整个 executor。额度不足可强制提前 MERGE，忽略通常的 merge-trigger-rows/interval；否则“额度满但没达到 MERGE 阈值”会永久等待。 - -### 6.6 文件生命周期与重启 - -- 目录按 sink-kind、capture/进程实例、changefeed 实例隔离；启动只能清理已确认非活跃的自身旧目录。 -- orphan 目录即使不用于恢复，也必须计入磁盘扫描与回收流程，不能因进程换 ID 每次留下 10 GiB。 -- MERGE 后 entry 逻辑释放不保证整个 segment 可删除，必须处理长寿命 entry 造成空间钉住；优先缩短 segment、密度控制或安全复制存活 entry。 -- 本地文件不要求承担 crash durability；重启仍从 checkpoint 重放。是否 fsync、何时 fsync要按临时 spool 语义确定，不能给出 WAL 级承诺。 -- checksum/长度校验发现运行中 spool 损坏：停止、不确认，从 checkpoint 恢复；不跳过坏记录继续。 - -### 6.7 collector→dispatcher→sink入口信用协议 - -**已确认接入事实**：BasicDispatcher同步调用AddDMLEvent，dynstream线程同步等待Handle返回；collector在调用HandleEvents之前已执行BatchDMLEvent.AssembleRows，多个DMLEvent可共享chunk（T12–T14）。因此原接口本身不能证明本设计的端到端有界性。 - -拟新增的能力协议如下，名称是概念接口，不表示当前代码已有： - -```text -ReceiveEnvelope(deliveryID, path, sessionEpoch, wireBytes, rowCount, decodedBound) - → collector TryReserveDelivery(bytes, elements, decodeExpansion) - 无信用：block该path并保留最多一个已计费delivery；返回共享线程 - 有信用：受限executor解码/AssembleRows，附SharedChunkLease - → dispatcher一次登记该delivery的全部tableProgress blocker - → sink TryTransfer(deliveryID, lease, events) - 接管：转移有界入口队列的所有权，快速返回；异步增量编码 - 暂停：delivery仍归上层，block该path；异步信用通知后重投同delivery - → 行数据复制到有预算的内部格式后逐步解除chunk引用 - → 最后一个使用者释放SharedChunkLease；归还decoded预算 -``` - -必须实现的细则： - -- 在线路收包/排队阶段就限制最大消息字节、元素数和同时在途delivery，不能等AssembleRows后才算账。发送端按事件边界/行数/字节切batch，协议协商硬上限；解码膨胀按保守上界预留并增量校验，超限拒绝/重建会话，不继续分配。 -- 传输编码大小不等于chunk内存。SharedChunkLease按实际分配容量保守计费**一次**，覆盖所有DMLEvent视图；每个事件视图和callback再计M_meta。不通过“这个事件只用了一行”只计那一行。 -- 原始chunk生命周期由lease所有者管理，sink不能擅自把共享Rows清空而破坏同batch其他视图。编码worker复制受限行窗口后归还引用；最后一个视图释放前全chunk仍计费。 -- 信用是额度凭证，不是“承诺之后归还”的无界债务。collector/dynstream已有内存控制若可复用，必须证明同一buffer单一拥有/跨层转移，无遗漏、无双计；接入前做分配链路审计。 -- TryTransfer未接管时不Detach回调、不发PostEnqueue，也不重复注册tableProgress。用稳定deliveryID和sessionEpoch记状态：RECEIVED/PROGRESS_REGISTERED/TRANSFERRED；重投仅重试转移。所有事件先加blocker再允许任何一个完成，沿用基线防checkpoint跳跃约束。 -- AddDMLEvent旧签名不能表达拒收/稍后重试。新增可选AdmissionSink capability和dispatcher/collector分支（或等价显式接口）；旧sink走原路径。不能仅改Databend.Add内部实现后宣称协议已接通。 -- 每path最多一个已交付等待batch，跨path有字节/数量总上限；阻塞的是该path，不是共享handler线程。只有在sink入口lease和元数据已接管后才可PostEnqueue；它仍不表示应用成功。 -- data信用与控制事件/DDL/lease通知分池，并为B表保留最小公平信用。只有A达到每path上限、全局仍有保留额度时，保证B可推进；全局预算耗尽时允许整个feed背压，但MERGE/回收/控制必须可运行。不承诺在全节点资源耗尽时无限保障B的新DML。 -- 释放信用时先完成可重入的状态更新，再通知可重试path，避免丢唤醒；session失效的旧delivery只释放，不能重投新owner。 - -`max-delivery-bytes`包括已解码chunk上界而非仅wire长度，`max-delivery-events`限制小事件元数据。若现有eventservice不能拆分单个过大原生事件，必须增加有序分片/完成标记协议或明确在接收前拒绝该输入；不能因为源事务大而无界AssembleRows。`max-row-bytes`不能代替这些上限。 - -本节扩大了原方案范围：B16未实现并通过共享线程公平性与真实内存实验前，不能宣称128/256MiB示例是端到端内存硬界。大源事务可支持为多个有界delivery，不要求所有源事务进一个chunk。 - -### 6.8 Segment pinning与压力排空协议 - -保留物理字节计费，不以逻辑Release数估算空间。拟给segment维护有界的存活引用→lane反向索引；压力时： - -1. BudgetManager以pressureEpoch合并通知，唤醒所有持有受压segment引用的lane，而不只唤醒最新申请者。 -2. lane跳过通常行数/时间阈值，刷出当前可应用分片；OPEN大事件的已应用片也可回收。DDL/COMMIT_UNKNOWN阻塞的lane记录原因，不能为释放空间越过barrier。 -3. 独立M_read/控制池执行排空；数据admission不能占用这部分保留量。按最老entry和公平轮转调度，避免热点饿死冷表。 -4. 首版优先验收“强制排空+物理回收”，不把segment搬迁当现成解法。若仍有长寿命pin且有必要搬迁，需预留maintenance disk/memory，先复制并校验、原子切换引用、等待读者退出，再删旧segment；复制期间两份都计费。不能在配额已满时临时多写一份。 -5. 不能排空/搬迁时进入RESOURCE_BLOCKED并告警，冻结新输入；通过恢复故障或人工扩额解除，不删除存活数据。可诊断的下游阻塞不等于无条件保证活性。 - -验收需覆盖热表A与冷表B、同segment混排、额度满但未达MERGE阈值、坏segment/删除失败、搬迁额度不足，以及大事件OPEN期间空间循环。原版已有时间触发，故不能把“冷表永远等5万行”作为既定事实；测试的是压力传播和保留资源是否真正有效。 - -## 7. 攒批、调度和确认语义 - -### 7.1 两层批处理 - -第一层按**目标表 + incarnation + schema version**攒 raw INSERT；任一条件触发：行数、编码字节、最长等待、DDL barrier、配额压力、优雅停止。 - -第二层合并若干 raw 批次再 MERGE；行数/时间/压力/barrier 均可触发。计时从最老未完成记录/批次入队起算，不能每次新事件到来都重置导致小表永不刷出。 - -上次示例 5000 行、8 MiB、1s raw flush，50000 行或 5s MERGE，只是压测起点，不是默认 SLO。真实延迟还包括排队、网络、SQL、重试、调度和 checkpoint 广播。 - -### 7.2 第一版采用保守的表级串行 lane - -拟定决策：同一逻辑目标表在一个 merge cycle 内，先提交选中的 raw 批次，再停止该表新的 raw INSERT，完成 MERGE及必要状态确认后，开启下一 cycle。新的源事件仍可进入受配额控制的 spool。 - -这样避免“MERGE 固定 stream 快照后，另一 raw writer 插入更高版本，被全历史过滤看到但并未一起应用”的复杂竞态。不同表有限并行。该决策以吞吐换证明简单，后续若并发 raw 与 MERGE，必须另写 snapshot/coverage 协议，不能直接去掉互斥。 - -多个 DMLEvent 可以共用一个 cycle；一个大事件也可跨 cycle，但回调等所有片段 APPLIED。数据 replay 事件可以被更新版本覆盖，不等于必须逐个中间状态呈现在目标。 - -### 7.3 覆盖集合与回调 - -每个cycle固定本地 `coverage = {entry/fragment IDs}`，其raw INSERT已明确成功，且不混入尚未应用DDL的schema。MERGE终态与消费覆盖证明满足§9后才将分片APPLIED；事件是否可回调还须§5.2的SEALED条件。 - -本地 generation/entry ID 是覆盖记账工具，不是持久化恢复身份，也不能当作 TiDB row version。进程重启后重新从 source checkpoint 构造完成关系。 - -`PostEnqueue` 只表示 sink 接管数据且满足背压策略，不能用于推进 durable checkpoint。`PostFlush` 才表示 target applied。沿用 detach 回调时要审查每个 hook 对动态流唤醒、流控与内存释放的影响。 - -### 7.4 释放与回收顺序 - -下游应用/消费已确认 → 分片一次性APPLIED → 归还其payload/entry并触发物理回收 → 更新tracker pending；满足SEALED完成条件后独立执行事件callback。上述状态更新需有同步与可重入保证；不要求磁盘unlink成功才回调，但未释放物理空间仍计费。 - -关键区别：F1已应用时即使事件仍OPEN、F2未登记或未应用，也可回收F1。tracker只保留已完成计数/有界未完成集合，不捕获F1数据；从而事件总大小可超过spool可用额度而不循环等待整事件ACK。 - -callback 不能重复执行；回收失败不撤销 APPLIED、不重复业务 SQL，继续保留磁盘计费。callback 后进程马上崩溃而上游 checkpoint 未持久化时，重放仍应安全。 - -### 7.5 对 TiCDC 接口的具体映射 - -| 接口 | 拟定职责 | 禁止事项 | -|---|---|---| -| AdmissionSink能力 / AddDMLEvent接入 | 先通过§6.7信用协议；原Add只接收已拥有入口lease的交付，快速转入异步编码 | 在共享处理线程等磁盘；未接管却发PostEnqueue；重复登记progress | -| FlushDMLBeforeBlock | 向受影响 lane 插入有序 flush marker，强制 marker 前 coverage 完成 | 拿着 admission 锁等待消费者；把本地排空当全局 DDL 完成 | -| WriteBlockEvent | 由协调后的指定执行者执行 DDL/recovery，成功才 PostFlush | 所有 capture 重复执行同一个 TRUNCATE | -| AddCheckpointTs | 接收已计算进度用于观测或另行设计的维护，不承担必要的首次 MERGE 触发 | 等 checkpoint 回调才 MERGE，导致进度闭环死锁 | -| SetWriteGate | 将许可检查接入每个实际 mutation 边界 | 只在 sink 构造时检查一次 | -| SetTableSchemaStore | 注册 schema 名称/范围依赖；补齐专属 SinkType 分支 | 假设名称清单就是可用的历史完整 schema | -| Run / Close | 生命周期、受限 worker、取消、正常 drain/失租区分 | detached Background worker 或无限等待 | - -必要关系是 `独立计时/行数/压力触发 MERGE → PostFlush → TiCDC 计算 checkpoint → AddCheckpointTs`，不能反过来依赖 AddCheckpointTs 推动第一步。全局 checkpoint 是已确认覆盖的聚合结果，不是单表 `MAX(commit_ts)`。 - -cycle 的控制伪代码（不表示现有接口已有该实现）： - -```text -entries = lane.selectBoundedEntries(schema, memoryReadBudget) -coverage = freeze(entries) -for batch in splitByRowsBytesParams(entries): - assertCurrentOwnerAndWriteGate() - rawInsertAndClassifyCommit(batch) # UNKNOWN: 停留协调，不越过 -freezeFurtherRawWritesForThisTable() -assertCurrentOwnerAndWriteGate() -result = mergeAndClassifyCommit(schema, incarnation, coverage) -if result in {APPLIED_CONFIRMED, NOOP_CONFIRMED} and coverageAndConsumptionProved(result): - markApplied(coverage) - releaseEntriesAndSchedulePhysicalReclaim() - tryCompleteSealedEventsOnce() -else: - keepCoverageUnackedAndReconcileOrFail() -``` - -原始输入、batch 和 coverage 的引用必须在各预算内；伪代码的 entries 不能实现为一次把磁盘积压全部读回内存。 - -## 8. 持久化数据模型与事件身份 - -### 8.1 隔离域 - -`source identity + changefeed instance + logical table ID + table incarnation + target mapping` 共同定义同步对象。复用 changefeed 名称或 DROP 后同名 CREATE 不能复用旧状态。 - -表名只是显示与路由名，不是唯一历史身份。内部对象名使用稳定 ID/hash 加可读后缀，避免用户表名长度/转义、同名表和 `_cdc_raw` 撞名。初始化必须验证内部对象 metadata 指向同一源与任务，不能见到 IF NOT EXISTS 就接管。 - -### 8.2 行身份与版本 - -- `row_key`：按 PK 列 ID、类型、长度前缀和值构造确定性编码；保留完整 canonical key。hash 仅用于索引/命名，不能以碰撞概率为零代替相等判断。 -- `commit_ts`：事务 TSO，使用无损整数表示，不能转 float64；选择 Databend UINT64 或经范围验证的 DECIMAL(20,0)。 -- `event_id`：上述隔离域、schema version、canonical key、commit_ts、净动作的确定性摘要/编码。 -- `payload_hash`：同 event_id 对应规范化 payload 相同；冲突时 fail-closed，不能通过到达时间选一条。 -- `add_time`：观测/清理辅助字段，绝不定义业务先后。 - -**commit_ts 不是每行唯一 ID**。同一事务不同键共享 TSO。相同键同一 TSO 的多条输入，必须通过已验证的源净变化契约归约；无法区分时不能用 dispatcher Seq、slice index、进程时钟创造伪顺序。 - -### 8.3 概念 raw schema（不是可直接执行的完整 DDL） - -```text -RawRecord { - source_id, changefeed_instance, logical_table_id, incarnation, - source_schema_version, - row_key, event_id, payload_hash, - commit_ts, - action: UPSERT | DELETE, - payload: exact JSON / typed normalized row, - ingest_time -} -``` - -第一版可以每目标表一套 raw/stream，但必须全任务共享 pool/budget；不要给每张表单独起一套无限后台 goroutine。 - -### 8.4 元数据/journal - -需要区分至少三类状态： - -1. `SchemaDescriptor`：列 ID、名称、类型、精度、PK、源 schema version、目标 fingerprint。 -2. `DDLJournal`：DDL ID/commitTs、before/after fingerprint、操作内容、pending/applied、incarnation。 -3. `OwnerRecord/Binding`：任务实例、表身份、target 映射、所有权代际/能力状态。 - -schema version 表示“如何解释行”，applied DDL fence 表示“这一边界前已处理到什么程度”；bootstrap 加载已有 schema 不自动证明旧 DML 已应用（I10）。本地 writeGen/mergedGen 和数据库每键水位同样不能互相替代。 - -### 8.5 applied witness与“更高raw版本已应用”的证明 - -retained raw本身是ingested watermark，不天然是applied watermark。若SQL过滤旧candidate依赖更高版本，必须能证明该更高版本已应用；否则I1不成立。 - -拟定首选证据为`AppliedCycle(operation_id, incarnation, schema_fence, coverage_digest, consumed_boundary)`，与target MERGE及stream消费**同一已验证事务提交**。若P0证实该组合可原子提交，恢复时用该证据关联operation ledger，而不是把本机mergedGen写到数据库后声称原子。APPLIED标记不存所有payload，长期保留/汇总与journal同受远端预算控制。 - -如果无法提供上述原子证据，不允许单独“MERGE成功后补写marker”冒充无窗口协议。可另行验证在单owner冻结的同schema/incarnation范围，对受影响key从retained raw重算最高有效净动作并在本cycle重新应用，避免把未证实higher raw仅用作过滤条件；跨schema历史不自动这样重投。该替代需要明确SQL快照、coverage和no-op消费证明，未验收就保持RECOVERY_REQUIRED，不静默降级。 - -因此持久化applied witness/等价重应用算法是B02/B03的一部分；新raw schema的event_id唯一性、目标一行内容或MAX(TSO)都不能替代它。 - -## 9. raw INSERT、stream、MERGE 与 compaction - -### 9.1 raw INSERT - -使用参数绑定，不拼接业务值；显式列清单和全限定表名，避免 USE/session 对连接池的隐式依赖。控制 SQL/请求字节和参数数；大 batch 在字节预算内拆分。 - -重试必须使用同一逻辑 event_id 与 payload。INSERT 成功响应丢失可能带来物理重复 raw 行；必须先设计 MERGE去重，不能假设数据库替我们提供唯一约束。 - -### 9.2 stream 的用途和能力前提 - -raw 的每次业务变化都 INSERT 一条带 action 的记录，因此 stream 使用 append-only；业务 DELETE 是 raw 的 INSERT tombstone，不依赖 stream 捕获 raw DELETE。raw 历史清理不应成为新业务删除。 - -公开文档确认有 append-only stream，且 CREATE STREAM 页面标注 Enterprise 能力；部署许可/可用性需预检，不能假定所有 Databend 版本/发行版都有。[CREATE STREAM](https://docs.databend.com/sql/sql-commands/ddl/stream/create-stream) - -公开指南描述 stream 消费与提交绑定、并发消费者有冲突限制；**这不等于已经证明本文 MERGE/事务/no-op 的全部语义**。[Stream workflow](https://docs.databend.com/guides/load-data/continuous-data-pipelines/stream) - -第一版默认要求 stream 能力。现有 bend-cdc 建 stream 失败时回退全 raw 查询（C03），新 sink 不应悄悄照搬：性能和覆盖语义改变，必须显式配置、单独验收，默认 fail-closed。 - -### 9.3 MERGE 算法草图 - -在单 owner、表 lane 已冻结 raw INSERT、schema 一致的前提下： - -1. 确定本 cycle coverage，验证每个 fragment 已 raw committed。 -2. 从该 stream 待消费 delta 选 candidate，并参考同 incarnation、允许 schema fence 下的 retained raw history。 -3. 按canonical key选最高commit_ts的有效净动作；必须先满足§11.4源事件模式/净变化契约。不能把默认UK拆分产生的合法同版本DELETE/UPSERT误判为源数据冲突。契约满足后才对剩余同版本异payload fail-closed，完全相同replay合并。 -4. 过滤低于 retained key watermark 的旧 replay；不能因 newer 仅存在于尚未应用的另一个未协调 writer 的 raw 就确认旧事件。 -5. 使用显式列投影转换目标值，按 typed PK 相等匹配：匹配 UPSERT 更新，匹配 DELETE 删除，不匹配 UPSERT 插入，不匹配 DELETE不插入。 -6. 正常写入和零物理修改分别按§9.4/§9.7取得终态、coverage与消费证据后标记APPLIED；不是要求目标行数必须大于0。缺证据进入COMMIT_UNKNOWN/停写，不发callback。 - -MERGE 的增删改语法由官方文档支持，但这里的去重/覆盖/fence 算法是拟定设计，需真实 SQL 验证。[MERGE](https://docs.databend.com/sql/sql-commands/dml/dml-merge) - -不能直接把本文草图当生产 SQL：特别要验证 raw self-reference 与 stream 读取是否共享正确快照、无匹配行或只含被抑制旧事件时 stream 是否前进、查询投影失败后是否完全回滚。C03 的既有 SQL可作起点，不是证明。 - -### 9.4 提交结果判定 - -#### 9.4.1 adapter返回的证据结构 - -拟定TargetClient返回下面的逻辑结构，而不是`executed bool`： - -```text -ExecutionEvidence { - operation_id, sql_digest, owner_generation, table_incarnation, - schema_fence, coverage_digest, mode: AUTOCOMMIT | EXPLICIT_TXN, - attempt_id, request_id, server_query_id?, server_transaction_id?, - terminal: SUCCEEDED | FAILED | ROLLED_BACK | UNKNOWN, - commit: COMMITTED | NOT_COMMITTED | UNKNOWN, - stream_before?, stream_after?, consumption_proof?, - error_class, server_error_code?, evidence_source, observation_counts?, - observed_at, server_version, driver_version -} -``` - -operation_id属于一次固定SQL/coverage；attempt_id区分物理发送。query_id不等于幂等键，重试标签也不自动覆盖DDL。request/attempt必须在发请求前登记；收到server ID立即补记，不能仅保存在即将关闭的Rows里。SQL digest不含业务原文；错误与证据日志脱敏。 - -结果分为`APPLIED_CONFIRMED`、`NOOP_CONFIRMED`、`NOT_APPLIED_CONFIRMED`、`COMMIT_UNKNOWN`。`OBSERVATION_FAILED`是附加观测状态，不覆盖前三种提交判定。显式事务中单条MERGE的SUCCEEDED仍非COMMITTED，必须另取COMMIT终态，且同连接/会话正确绑定。 - -#### 9.4.2 首选取得路径与错误分类 - -首版拟使用受控HTTP/driver adapter：保留原始响应状态/错误与query ID，持续读取/轮询到服务端成功终态；自动提交需证明该终态与提交绑定，显式事务还需COMMIT成功。只经database/sql拿到Rows不能提供全部证据；若v0.9.4不暴露必需信息，应增加窄接口或单独客户端，不能从私有字段猜测。具体endpoint、认证、终态字段、节点路由、保留期需在P0按实际server版本落实，未落实就禁止生产/自动接管。 - -| 观察结果 | 分类与动作 | -|---|---| -| 权威服务端确认语法/权限失败且无提交 | NOT_APPLIED_CONFIRMED，停止或按错误策略重试；不callback | -| 已收到结果页，但最终轮询报执行错误 | 不是统计失败；依据服务端提交/回滚语义判断，不能证明则UNKNOWN | -| 网络断开/超时/首响应丢失 | UNKNOWN，停止同lane后续mutation，查询已有query/request关联状态 | -| Rows值解码/类型转换错误 | 与提交证据分离；有独立COMMITTED证据才保留成功，否则UNKNOWN | -| 服务端成功终态已确认，仅操作计数解析失败 | APPLIED或NOOP保持，统计unavailable，禁止为补统计重做SQL | -| Kill/Close返回nil、查询列表中暂时不存在 | 不是提交/终止证明，保持UNKNOWN | -| 显式事务中语句成功、COMMIT响应丢失 | UNKNOWN，按transaction/query关联恢复；不能按语句成功确认 | - -已核实D01：v0.9.4 `waitForData`在有缓冲行时可退出，不强制ReadFinished；KillQuery丢弃请求重试错误并返回nil。C07则在QueryContext返回Rows后立即设executed。由此只能得出“现有标记不足以作为证明”，**不能声称已复现MERGE先返回数据再提交失败**。 - -#### 9.4.3 不确定结果、重启与找不到query - -1. 进入UNKNOWN即冻结该lane及可能冲突的DDL/compaction，不确认coverage。保留本地payload直至明确结果或受控停止；不得无限重发。 -2. 在旧owner仍有效时，用有界协调worker查询服务端权威终态；证明已提交则按原coverage继续，证明未提交/已回滚且旧请求不再运行才可重试。 -3. 发请求前把在途操作描述写入目标durable operation ledger并确认（或由P0证明等价外部持久化查询目录）。ledger状态写入本身也有提交不确定性：未明确登记成功不发送业务SQL；更新丢响应按相同operation_id读回核验。它只存小元数据，不存全部源payload，也不构成分布式锁。 -4. 重启先取得§12排他性，再扫描未终结ledger、关联query/transaction终态。若数据库不支持客户端指定/关联request ID，首响应丢失时仍可能无法定位SQL，此时自动恢复不可用，进入RECOVERY_REQUIRED；不能假装ledger消除了这个窗口。 -5. query记录过期、查不到、权限不足或只查到“当前不在运行”均不等于NOT_COMMITTED。可通过已证明原子提交的应用标记/消费元数据等替代证据协调；**目标中查到一行业务值或MAX(TSO)不足以证明整个coverage**。 -6. 无可信证据则fail-closed，运维确认所有旧查询终结并完成目标核验后恢复或重新初始化。超时只转人工状态，不自动转成功。 - -operation ledger的持久化/并发控制与ID查询路径尚未实测，B03必须以接口字段、实际请求响应、故障实验和重启证据关闭，不能以“已审driver”关闭。 - -### 9.5 compaction - -第一版同表 compaction 与 raw INSERT/MERGE 串行。只在已明确成功的 cycle 之后删除被支配历史；不能删除唯一最新水位/墓碑，也不能删除恢复 pending DDL 所需状态。 - -最新事件的物理重复行要有明确去重策略：仅删除严格更旧版本不能清除相同 event_id 的重复，长期重试会膨胀；用 ROW_NUMBER 删除相同行也不能误删全部。是否支持稳定物理 row ID、重写去重表或安全重建 stream，列为待验证项 B06。 - -compaction 失败独立记账和告警；APPLIED 不回退，但持续失败超过 backlog/空间阈值应停止 admission。raw 保留全 payload 的最新行成本高，后续可分离小 watermark/tombstone 表与短期原始 payload；第一版不借此增加跨表原子性依赖。 - -### 9.6 远端raw/watermark容量与成功运行下的停写边界 - -持续新PK INSERT→DELETE可让target接近空而raw每键保留一个墓碑；MERGE和compaction全部成功仍增长。因此新增独立于本地spool的远端预算，覆盖raw、历史watermark/tombstone、stream依赖保留快照、内部journal/ledger及物理重复。 - -拟定控制： - -- 每feed配置remote-max-bytes、remote-max-history-keys、检查周期和软/硬水位；部署必须给出有限值，不以0代表默认无限。 -- capacity observer使用真实版本支持的表统计/存储计量；明确active逻辑字节、retained snapshot与物理计费字节差异，不把COUNT(*)当物理占用。历史键数使用保守上界，不能低估删除键。 -- observer结果过旧或查询失败超过限期后停止新raw admission。新cycle先预留“最大可能增长”额度，再raw INSERT；并发表共享feed预算，禁止每表分别看一遍旧统计后各自超额写入。 -- 测量延迟、SQL失败残留、压缩/快照放大意味着应用阈值不是数据库物理硬quota。设置安全余量，预算覆盖已预留在途写；若要求绝对计费上限，还需warehouse/provider硬配额能力，未证实时不承诺。 -- 软水位告警并提高安全compaction优先级；到admission阈值进入REMOTE_CAPACITY_BLOCKED，停止新raw INSERT，允许已提交raw的MERGE、必要DDL恢复和安全清理。DDL自身要保留维护余量，不能借“维护”无限扩张。 -- 恢复仅有三种：显式扩额并重新测量；有证明地消除物理重复/被支配历史；从新一致快照S2重建独立目标与内部状态，校验后切换。重建需要额外预算与旧feed冻结/隔离，禁止原地删最新墓碑。 - -验收：固定target有效行数、持续新键churn，记录历史键、物理存储/计费增长、扫描量与成本；compaction均成功时达到预算仍必须可诊断停写，停止后新raw写入次数为0（已预留请求除外且计入余量）。B18跟踪停写控制，B06跟踪去重和保留算法。 - -### 9.7 no-op、已支配重放与stream消费 - -零目标行修改不等于失败，也不等于已经完成。确认分为： - -| 情况 | 完成所需证据 | -|---|---| -| 正常MERGE改写业务行 | 服务端COMMITTED终态 + 本cycle覆盖/stream消费契约成立 | -| 原coverage全部因更高版本而被抑制 | 更高版本已应用证明（不能仅raw存在）+本次终态/消费证明，返回NOOP_CONFIRMED | -| DELETE匹配不到目标或完全重复写 | 相同版本净动作/墓碑一致 +终态/消费证明，不要求操作计数>0 | -| 空stream但本地coverage非空 | 默认恢复异常；需证明coverage已由历史消费覆盖，否则不能确认 | -| 无待确认coverage的空cycle | 直接结束调度，不制造源事件callback | - -本版选择保守规则：所有已写入raw的coverage，在当前或可证明历史消费覆盖后才完成；不因为“raw中有更大TSO”提前确认。单owner恢复先解决遗留raw与未终结查询，再接入新cycle；不能用未应用的新raw支配旧事件。 - -**版本事实**：Databend已合并[#20671](https://github.com/databendlabs/databend/pull/20671)，修复某些no-op DML跳过目标提交时丢失stream offset更新的问题，并说明planner-level no_effect短路不在修复范围。这不是“所有版本空MERGE都不消费”的规范，也不证明部署已含补丁。 - -P0必须取得server build与补丁覆盖证据，分别测实际SQL的数据依赖过滤为空、常量空集/优化短路、仅旧重放、DELETE不命中、重复事件、autocommit与显式COMMIT/ROLLBACK。cursor证据采用真实版本可核验的metadata或等价前后delta实验；不能只看counts=0或假定一定有可查offset API。 - -若实际组合不能可靠消费，首版拒绝该server/SQL组合或修复服务端再验收。显式WITH CONSUME/标记表只能另行证明与MERGE、owner、coverage同一提交边界后采用；禁止在MERGE前独立消费，禁止向业务target塞伪业务行。没有消费证明时不callback，但必须告警/进入恢复态，不能静默无限循环重刷。 - -### 9.8 stream过期/丢失后的恢复顺序 - -检测stale/缺失/依赖snapshot丢失立即冻结lane，不静默新建空stream。先排除旧owner/未知SQL；核验目标incarnation、DDL journal、最新墓碑与尚未应用raw的完整性。若raw仍完整，需通过显式full-raw回放覆盖算法重建stream起点并验收，且不混旧schema；“CREATE新stream”本身不会补已存在raw。 - -缺任何恢复证据就进入REINITIALIZATION_REQUIRED，从新一致快照重建；从上游checkpoint重放不是充分条件，因为上次已确认但被目标外部破坏的数据可能不会重发。重建路径未实现前，首版只支持诊断停写+人工重新初始化，不承诺自动修复。B14专门验收，不能因已有fail-closed就标成自动恢复完成。 - -## 10. DDL、建表与 schema 演进 - -### 10.1 初始 schema 与历史空表 - -已有表不会因为新建 changefeed 而重新执行历史 CREATE TABLE。首条 DML 带 TableInfo 可懒建非空表，但永不变更的历史空表必须由 start-ts schema snapshot bootstrap 或独立初始化任务预建。 - -bootstrap 要与同一时点 schema 匹配：不能查询“当前最新 schema”再解释较早 DML。TableSchemaStore 中名称清单不等于完整列定义。具体元数据读取 API 接口与调度顺序需实现前落定（B04）。 - -建目标表使用权威列顺序、精度、NULL/default 规则；建后 DESCRIBE/fingerprint 校验。目标已存在时不盲目接管：结构、主键唯一性与初始化水位均需验证。未经初始化的既有目标可能本身有重复 PK，MERGE 不能自动修复这一前置问题。 - -### 10.2 正常 DDL 协议 - -```text -收到 DDL(D) - → 禁止受影响表的新 epoch 进入 raw 写路径 - → 本地 flush marker，强制所有旧 epoch entry MERGE完成 - → maintainer 完成受影响 dispatcher 的全局协调 - → 验证当前 owner 与 before fingerprint - → durable journal(PENDING, D, operation, before/after schema) - → 执行翻译后的 Databend DDL / 重置 - → 校验 after fingerprint - → journal(APPLIED) - → DDL PostFlush - → 放行新 schema DML -``` - -FlushDMLBeforeBlock 只证明它负责的本地队列完成，不能冒充全局屏障。DB/All 范围 DDL 必须枚举所有影响表；TableSchemaStore 需求分支需新增（T06）。NotSync/过滤 DDL 的语义必须显式接入。 - -### 10.3 DDL 白名单 - -| 操作 | 处理 | -|---|---| -| CREATE DATABASE/TABLE | 翻译并校验；不原样下推 TiDB DDL | -| ADD 非键列 | 字面量默认值/NULL 语义明确才支持；对已有行回填需与源一致 | -| DROP/RENAME 非键列 | 旧 epoch排空，更新 fingerprint 和投影；防止 drop 后同名 re-add 混列 | -| MODIFY | 沿用有限安全白名单；不因目标已碰巧同类型就跳过未验证变更 | -| 默认值/NULL 属性 | 区分未声明、DEFAULT NULL、空字符串和表达式；不支持表达式则拒绝 | -| 二级索引 | 证明无行投影/身份变化后 no-op，仍更新必要元数据 | -| PK 定义更改/复杂多操作 | 拒绝/阻塞 | -| RENAME/DROP TABLE/SCHEMA | 未实现前明确阻塞，不能仅 warning继续 | - -复用 C04/C05 白名单时要保留适用限制：已有 VARCHAR→TEXT、TINYINT→INT、INT→BIGINT、部分 INT→文本，并不等价于支持任意类型改写。 - -### 10.4 journal结构、恢复优先级与普通DDL转移 - -DDL identity使用`source/任务实例/逻辑表血缘/源DDL ID或可证明唯一的等价身份`，包含commit_ts、old/new physical table ID与DDL内容摘要。只用同名表或全局MAX(ddlTs)不可区分重建/多表DDL。journal记录before/after列ID fingerprint、drain fence证据、phase、owner generation、operation/query关联和新代际是否已开放。 - -不假设Databend支持journal上强制唯一约束；单有效owner下同DDL身份单调推进，读回重复内容必须一致，多owner排他由§12提供。journal写入也经§9.4分类，不明确持久化就不得越过该步骤；冲突/倒退/未知身份fail-closed。 - -恢复优先级固定为： - -```text -所有新mutation保持关闭 - → 取得owner排他并处置旧在途SQL - → 核验source/target绑定、journal/ledger完整性与schema snapshot - → 恢复PENDING DDL/TRUNCATE（先于任何旧DML普通解码/写入） - → 安装已证实的applied-DDL fence与incarnation映射 - → 对旧事件执行身份校验后的skip/recovery规则 - → 协调遗留raw/stream与未知cycle - → 完成当前schema bootstrap核验，开放正常增量 -``` - -| durable phase / 当前证据 | 允许动作 | 禁止动作 | -|---|---|---| -| 无journal且收到新DDL | drain并持久化PREPARED，成功后发送DDL | 先DDL后补journal | -| PREPARED +无未知SQL +可信before fingerprint | 按同DDL身份执行白名单DDL，记录执行attempt | 在旧DDL可能仍运行时重发 | -| EXECUTION_UNKNOWN | 查询服务端终态；只有after结构可唯一证明本DDL完成时才收敛 | 只见列不存在/目标同类型就猜成功 | -| 可信after fingerprint +执行终态/等价完成证据 | 写VERIFIED，再写APPLIED与drain fence，均确认持久化 | journal未确认即放行新schema DML | -| APPLIED +相同DDL重放 | 验证目标仍属于该代际，跳过业务DDL，只完成重放事件 | 重做ALTER/TRUNCATE | -| 旧schema DML重放 | 只有身份匹配且drain fence证明该前缀已应用/由TRUNCATE合法清除时跳过 | 仅凭当前schema更新时间丢DML | -| 元数据/目标不一致或无法判定中间状态 | RECOVERY_REQUIRED，维持写关闭 | 用IF EXISTS掩盖冲突继续 | - -fingerprint不是万能提交证明：TRUNCATE前后schema相同；有外部DDL时相同结构也可能来自别的操作。目标禁止外部写是前置条件，仍须记录mutation身份。多操作DDL首版不支持，不能拆成多句后只验最终schema而忽略中途失败。 - -TiCDC基线MySQL专属恢复查询/skip标记需抽窄RecoveryCapability或Databend专属分支，覆盖普通dispatcher和merge task入口（T07）。恢复返回的是明确skip范围、代际与证据，不是笼统`lastDDLTS`。该API/持久化实现仍由B04/B09阻塞，表格只固定允许的状态转移。 - -### 10.5 TRUNCATE逐步恢复算法 - -TRUNCATE改变源物理表ID/代际；目标名可能不变。新代际不得在APPLIED记录持久化前接收任何DML。优先新建带代际后缀的raw/stream、隔离旧对象，旧对象物理清理后移，不先删唯一恢复证据。 - -| phase | 正常动作 | 崩溃/响应丢失后的恢复 | -|---|---|---| -| R0 DRAINED | 全局旧DML排空；登记DDL身份、old/new incarnation、reset目标与drain证明 | 无可信drain证明则重新协调，不直接清空 | -| R1 PREPARED | durable记录新代际CLOSED；旧SQL已终结后发送目标TRUNCATE，登记operation | 先协调unknown旧TRUNCATE。只有证明新代际从未开放且无新数据写入、目标独占时，才可在旧SQL终结后重复reset；否则停写 | -| R2 TARGET_RESET | 终态确认后记目标reset完成；创建新代际raw/stream并验证binding | 不从“表为空”推导旧query已终结；新对象存在须核验identity，不能见名接管 | -| R3 NEW_BOUND | 持久化新binding/schema/fence，旧raw标retired而非可消费 | 新代际仍CLOSED；补完缺失journal后读回一致性 | -| R4 APPLIED | 所有前置步骤完成且持久化APPLIED，再允许DDL回调/开放新代际 | 相同DDL只skip；即使checkpoint还在TRUNCATE之前也不得再清target | -| R5 ACTIVE | 新代际可写；旧raw/stream仅由受fence维护清理 | APPLIED/绑定冲突或丢失时停写核验，绝不通过再TRUNCATE“修复” | - -安全依据是“新代际写入发生在可信APPLIED之后”，不是TRUNCATE天然幂等。R1的可重做条件必须由journal、排他所有权与未开放协议共同证明，不能只依赖本机布尔值。若目标/journal被外部删除或查询无法确认，算法选择阻塞而不是猜测。 - -验证每个R0–R5之前/之后崩溃、步骤提交后响应丢失、journal更新不确定、旧owner晚到TRUNCATE。重点：R5写入@510后从@490重放@500 TRUNCATE，@510不得被再次清空。即使@510理论上可能重放，也不把这当允许重复破坏的理由。 - -### 10.6 历史空表bootstrap与DDL交叉 - -初始化取一致schema snapshot S并建立bootstrap epoch，表清单与列定义必须属于同一个S；枚举和分页期间收到的DDL按源顺序缓存于有界队列/上游保留，不能最新schema补齐旧清单。snapshot完成并持久化binding前，相关表不开放DML写入。 - -若DDL(D>S)先到而某空表尚未建好：该表先以S建表/核验，再走正常D的drain/journal；若采用直接建D后schema的优化，必须证明跳过区间无未应用数据，首版不默认采用。schema读取失败/历史版本不可用就停止bootstrap，不退回查询最新DESCRIBE。 - -测试无历史CREATE重放的空表、分页时ADD/DROP列、同名列drop/re-add、DDL后首批DML、bootstrap中断重启。B15验证时序，B04验证获取API与dispatcher恢复接入，两者不能互相代替。 - -## 11. 主键、净变化与乱序处理 - -### 11.1 普通更新和删除 - -INSERT/UPDATE 归一成完整 after-image UPSERT；DELETE 使用旧 PK 与 tombstone。完整 image 要由源契约保证，不得将缺失字段自动填 NULL。 - -同键较大 commit_ts 胜出；DELETE 也参加比较。收到 `UPSERT@100, DELETE@200, delayed UPSERT@150` 后，目标必须仍不存在，raw 保留至少 DELETE@200 的版本水位。 - -### 11.2 主键值更新 - -必须处理旧键消失和新键存在。源入口可能将变键 UPDATE 拆成 DELETE/INSERT，也可能暴露 before/after；两种都要真实测试。 - -同事务多行交换键/复用键需要按最终 KV净状态归约，不能简单规定“最后到达赢”，也不能未证明就让 DELETE 总比 INSERT 新。若源不能提供确定净结果，第一版必须拒绝该能力而不是静默留脏行（B05）。 - -### 11.3 collation 与身份 - -canonical key 的字节相等、源 PK 约束相等与 Databend SQL JOIN 相等可能不同。测试大小写、Unicode、尾空格、binary/text 键、DECIMAL 规范化和 NULL。 - -不能只在 raw 用区分大小写的字符串 key、target 用另一种 collation，然后宣称相等语义一致。第一版可限制到已验证 PK 类型/collation 集合；拒绝规则必须在创建/发现新表时可执行。 - -### 11.4 原生事件模式：二级UK、真实变键与净变化 - -源码事实（T11）：outputRawChangeEvent=false时，IsUKChanged对索引列/unique标志列比较，更新二级唯一键也可能SplitUpdate；INSERT可被缓存到DELETE之后甚至跨分片输出。T12的初始化只为CloudStorage/Kafka读取该开关，新SinkType若不加分支就保持false。 - -**本版决策：Databend固定要求outputRawChangeEvent=true，作为内部事件契约，不开放随意切回false的调优项。** 必须贯通manager→SharedInfo→dispatcher注册/重注册→eventservice→scanner；bootstrap、迁移、重试/重建session、兼容版本都校验实际生效。无法协商/确认则拒绝启动，不以sink端猜DELETE优先级补救。排除会改写/过滤DML两半的未经设计选项。 - -| 输入/场景 | 规范化规则与限制 | -|---|---| -| PK相同，二级UK从a→b | 保留完整before/after UPDATE，输出同PK一个UPSERT@C；不制造DELETE@C | -| 同PK普通列UPDATE | 完整after UPSERT；不完整image拒绝，不用NULL补齐 | -| UPDATE明确携带oldPK!=newPK | 只有已验证该源表示和事务净契约后输出旧键DELETE+新键UPSERT；不以到达顺序解决同TSO冲突 | -| clustered PK/record handle变更以独立KV DELETE/INSERT出现 | raw=true未必合并成UPDATE；必须验证提交时按record key净结果及跨分片契约，不能假定旧/new可配对 | -| 同事务反复修改/复用同键 | 先证明源提交事件已是确定净结果；若还需归约，必须有稳定事务结束边界/有界或可溢写归约状态，不能把DMLEvent结束等同完整事务结束 | -| 同key/TSO的相反净动作或不同payload在契约后仍存在 | 停止并保留诊断，不用ROW_NUMBER任意选一条，也不规定DELETE/UPSERT一律胜出 | - -因此，raw=true解决的是**避免人为UK拆分**，不是对真实PK变化、同事务多键替换和所有冲突归约的万能证明。第一版在真实TiDB事件与重启测试通过前不宣称支持变键；如果输入无法区分“变键”与普通DELETE/INSERT，就不能承诺可检测地禁用PK UPDATE，必须收紧可保证的源使用范围或补源端契约/能力后再发布。 - -测试用例先验证源SQL合法性。不能直接把可能违反即时唯一约束的单句PK互换当作可复现负载；选择TiDB实际允许的中间键/多语句事务与最终KV事件，保存事件dump。必须覆盖PK不变仅UK更新、raw开关true/false对照、两半强制跨事件/cycle/重启、同事务多行相同TSO、真实handle变更。不能为了证明确定性而在sink构造虚假的行序号。 - -此决策应纳入scheme创建时校验及DDL新增唯一索引后的持续校验，B05的关闭条件同时覆盖普通UPDATE与可选变键，不再缩写成“PK更新测试通过”。 - -## 12. 所有权、失租和故障接管 - -### 12.1 为什么进程 mutex 不够 - -每个 TiCDC capture 都可能建立 sink，同表/同 stream 不能有两个独立 merge owner。即使表跨节点调度关闭，正常迁移、旧节点网络分区及未结束 SQL 仍可能重叠。 - -T08 的 write gate 检查新请求是否可以发出，但请求通过检查后失租，服务端可能仍在运行。`ctx.Cancel()` 或客户端断连不等于服务端已回滚。 - -### 12.2 所有权协议要求 - -正常 handoff：旧 owner停 admission → drain/停止提交 → 确认其在途 mutation 的终态 → 释放 owner → 新 owner取得新 generation → 恢复 journal/raw → 放行。 - -异常接管：必须有目标侧原子 fencing/冲突机制，或可证实旧 SQL 已终止的机制，再开放新 owner。旧 DDL/compaction/stream 消费尤其危险；仅依靠幂等 row event_id 不能保护 schema 和 truncate。 - -候选A仅适用于可证明同事务原子冲突的mutation，不能泛化到DDL。官方说明DDL会提交当前多语句事务，所以先检查owner再DDL不提供该保护。[BEGIN/DDL限制](https://docs.databend.com/sql/sql-commands/ddl/transaction/begin) - -| mutation类别 | 首版排他要求 | A候选是否可直接覆盖 | -|---|---|---| -| raw INSERT /目标MERGE | 单owner lane；接管前旧query终态确认；候选事务fence需验证隔离/写冲突及提交原子性 | 仅待P0证明，普通SELECT owner再写不够 | -| journal/operation ledger/binding DML | 与owner代际关联，单调、读回一致；旧owner写元数据也须被排除 | 同上，元数据表不自动是锁 | -| CREATE/ALTER/TRUNCATE/DROP | 控制面串行授权 +所有相关旧SQL已终结 +本DDL终态与journal恢复 | 否，不把DDL塞BEGIN当fence | -| stream创建/替换/重建 | 与DDL同级排他；先冻结消费和raw写，验证未丢coverage | 否 | -| raw compaction/退役对象清理 | 独立登记mutation；只处理已证实retired/被支配对象，排除旧owner维护SQL | 若为事务DML需另测，不能默认 | - -候选B要求服务端终态确认，而非“调用取消成功”。D01的KillQuery丢弃错误，返回nil可能根本未取消；即使取消请求被接收，也要确认旧query不再能提交。未知request ID、服务端查询记录过期、节点不可达、取消接口受阻都保持HANDOFF_BLOCKED。 - -正常handoff与异常接管都使用同一mutation registry，涵盖业务与维护SQL。需要证明查询枚举/登记完整，不能只查当前客户端连接或某一个query ID。旧owner已退出只防止未来发送，不能证明已发送SQL终止。 - -**不同mutation的保护均未经过实际server验证，B01仍为生产阻塞项。** 外部锁可序列化新授权，不能撤销已发送SQL；先读owner表再MERGE也不原子。首版可先做受控非HA PoC，但必须满足§12.5，不把候选描述标成已落地。 - -### 12.3 mutation 边界 - -raw INSERT、MERGE、CREATE/ALTER/TRUNCATE、journal写入、stream 创建/重建、compaction和内部对象清理都必须检查 write gate/owner。验证连接不应随便创建数据库等副作用；需要区分只读预检与用户授权初始化。 - -### 12.4 多任务和映射冲突 - -同库同表的绑定注册必须跨 changefeed 生效，不是每 capture 一个 map。规则变更、大小写折叠、前后缀、同名源库映射都可能撞目标。未能实现可靠排他时要求运维静态独占并在重启核验 metadata；这仍不是完整生产自动化方案。 - -### 12.5 未闭合fencing时的PoC运行边界 - -`scheduler.enable-table-across-nodes=false`只禁止一表拆span跨节点，并不禁止整表调度、故障接管或维护迁移（T15）。因此它是必要规模限制，不是“禁用HA”的实现。 - -受控PoC必须增加Databend恢复准入：任何新capture/session owner在取得授权后仍保持写gate关闭，完成registry扫描/旧query终态核验/DDL恢复后才开放。未实现跨节点准入时只能在隔离单capture实验环境运行，禁用自动替换/重启写入并要求人工恢复；不能只配置一个false就接入生产调度。 - -失租立即阻止本机新mutation并取消worker；进程fail-stop可作纵深措施，不构成服务端fence。禁止用固定sleep>query timeout或驱动Kill nil作为安全接管证据。无法证明query超时上界、取消终态与枚举完整性时保持fail-closed。 - -真实故障验收：阻断取消/终态接口并延迟旧TRUNCATE，新owner必须始终拒绝写；恢复可达并取得可信终态后，才完成恢复并放行。控制面锁、客户端write gate、目标query终态三者角色分别记录，不混为一个“lease有效”布尔值。 - -## 13. 类型映射与 schema/value 统一 - -### 13.1 统一 mapper - -一套 mapper 同时服务 schema bootstrap、DDL、raw 编码和 MERGE projection；输入是 FieldType/flags/Flen/Decimal/Elems/列 ID，不是样本值。沿用 C03/C06 的测试案例，避免各路径字符串映射漂移。 - -| 源类型 | 目标原则 | 必测边界 | -|---|---|---| -| signed integer | 保留精确宽度或无损提升 | 最小/最大值，负值 | -| unsigned integer | 使用对应 UINT，BIGINT→UINT64 | 大于 2^63−1，最大 2^64−1 | -| DECIMAL(p,s) | 精确保留 p/s，不经 float64 | DECIMAL(65,0)、最大 scale、负数/前导零 | -| BIT(n) | UINT64 | BIT(64) 全 1，NULL | -| FLOAT/DOUBLE | 明确非有限值策略 | NaN/Inf 支持差异、精度 | -| BINARY/BLOB | 原始 bytes→HEX或已验证二进制参数 | 0x00–0xFF、空 bytes、NULL | -| ENUM/SET | typed value/Elems→约定字符串表示 | 空值、非法 ordinal、声明顺序、逗号/转义 | -| JSON | 精确 JSON 文本或 typed value，不经通用 float64 map | 大整数、SQL NULL vs JSON null、数组/嵌套 | -| DATE/DATETIME/TIMESTAMP | 明确时区/精度/范围；DATETIME不隐式平移 | 零日期、DST、微秒、超范围 | -| TIME/YEAR | TIME 字符串，YEAR 小整数 | 负 TIME、超过 24h、零 YEAR | -| TEXT/CHAR/VARCHAR | 字符集解码与目标字符语义明确 | 引号、反斜杠、控制字符、Unicode | -| VIRTUAL/STORED generated | VIRTUAL 不写；STORED写物化值 | 初建表、DDL后、仅生成列 | -| vector/geometry/其他 | 未定义前拒绝或显式安全映射 | 不静默 infer VARCHAR | - -原生 binary 不能套 Canal Latin-1 还原；原生 ENUM label 不能再当 Canal ordinal 解码。JSON 验证不应 unmarshal 成 float64 导致精度损失。 - -### 13.2 raw payload 与 projection - -第一版可用规范化 JSON：decimal/大整数字符串或精确数字文本必须约定一种表示；binary采用明确 HEX marker/列类型；SQL NULL 使用单独 null语义。不要让同一字段有时 raw bytes、有时 base64、有时 Latin-1 字符串。 - -所有值转换失败必须停止对应应用，不以 TRY_CAST 静默填 NULL；可选容错模式不属于第一版。缺少字段与字段值 NULL必须可区分，必要时带列 presence，避免不完整行覆盖。 - -### 13.3 默认值与目标侧行为 - -源值优先于目标 default；CDC INSERT已有完整数据时不重新计算 CURRENT_TIMESTAMP。ADD COLUMN 的历史回填只在明确兼容时执行；普通建表 default 的翻译不能成为源时间语义漂移来源。 - -## 14. 配置接口草案 - -**全部 Databend 新字段均未实现，以下不是当前可运行配置。** 数字是测试起点，不是已承诺默认值。 - -连接标识：`databend://HOST:PORT/DB_NAME?sslmode=require`。DB_USER/TOKEN应由受保护配置或运行时 credential provider注入；具体接口待安全设计，避免把真实 token 写进日志、提交记录或 CLI 参数。Bend-cdc 已有 runtime AccessTokenLoader，可借鉴但要接入 TiCDC生命周期。 - -```toml -[filter] -rules = ["DB_NAME.*"] - -[scheduler] -enable-table-across-nodes = false - -[sink.databend-config] -auto-create-table = true -# 镜像模式固定处理DELETE；不开放allow-delete=false -# outputRawChangeEvent内部强制true，不提供切换为false的用户项 - -batch-max-rows = 5000 -batch-max-bytes = 8388608 # 8 MiB，编码后数据预算；请求开销另计 -flush-interval = "1s" -merge-trigger-rows = 50000 -merge-interval = "5s" -merge-concurrency = 4 # 跨表，同表串行 - -spool-memory-quota = 67108864 # 64 MiB 热数据，不等于整个 sink RSS -spool-disk-quota = 10737418240 # 10 GiB，含维护预留，单 changefeed -spool-maintenance-reserve = 67108864 # 64 MiB，包含在磁盘总额内，不可用于普通admission -max-inflight-bytes = 268435456 # 256 MiB，含入口/热数据/编码/读回/元数据及控制预留 -max-row-bytes = 8388608 # 规范化后的单行含key/元数据；大事件应拆分 -max-delivery-bytes = 33554432 # 32 MiB，解码后chunk容量预算；wire另有协议上限 -max-delivery-events = 1024 # 元数据数量限制，不代表允许一次交付1024个最大事件 - -remote-max-bytes = 1099511627776 # 1 TiB示例admission预算，不是已证明的物理计费硬quota -remote-max-history-keys = 100000000 # 1亿历史键示例；按实际行宽/成本重新设定 -remote-check-interval = "30s" -remote-observation-max-age = "2m" # 过期后停新raw，不能永久盲写 -``` - -节点级拟定（server配置；字段名尚未实现/定稿）： - -```toml -[sink-spool] -base-dir = "/PATH_TO_DEDICATED_SPOOL" -total-disk-quota = 107374182400 # 100 GiB，capture 所有相关任务共享 -min-free-disk-bytes = 21474836480 # 至少保留 20 GiB，容量按节点实际调整 -``` - -配置校验：非负/上下界、单行与batch预算、内存池及并发可行性、磁盘总额/余量、未知字段拒绝、不支持功能拒绝。max-inflight必须包含编码和读回池，不能让四个 worker各额外分配无限 batch。 - -256MiB只是便于预算拆分的示例，不是已验证RSS保证。例如热64、入口32、编码32、读回64、元数据16、控制48MiB，总和256；真实分池根据最大row/delivery/request放大、并发数校验。任何一个8MiB行若连同参数/协议开销不能装进预留，必须提前拒绝或调小有效行上限，不能临时超额分配。sync.Pool只是复用机制，不是预算；超大buffer不能无限留在pool。 - -远端软水位与admission安全阈值由上述预算扣除观测误差、增长预留和维护余量推导，生产配置需给出推导结果；示例没有承诺1TiB成本封顶。每个pool和阈值的计费定义/保守放大系数应在启动诊断中展示，但不泄漏业务数据。 - -v0.2决定镜像模式固定处理DELETE，不再开放`allow-delete=false`；旧计划/客户端传此不支持项时显式拒绝，不静默忽略。删除版本和墓碑按统一规则保留。非镜像append模式不属于第一版。 - -更改配额下降到当前用量以下：停止新 admission、继续消费者排空，不直接删文件。改变目标映射、PK策略、spool根目录、类型映射等不默认支持在线生效，需停止和显式迁移。 - -## 15. 错误、生命周期与安全 - -### 15.1 错误分类 - -| 错误 | 行为 | -|---|---| -| 临时网络/限流 | 有限指数退避 + jitter，保持原 event_id，不确认 | -| 类型/权限/语法/不支持 DDL | 永久或人工介入状态，保留首因，停止对应 feed | -| COMMIT_UNKNOWN | 等待/查询终态/协调，不能盲目与新 owner 并发重试 | -| token过期 | 受控刷新并重试；刷新失败停止，不打印 token | -| spool配额满 | 背压并主动调度 MERGE，不是丢数据错误 | -| 入口信用耗尽 | block所属path，快速返回共享线程；等待信用重投，不重复progress登记 | -| 远端预算达到/观测过期 | REMOTE_CAPACITY_BLOCKED，停新raw；保留排空与安全维护路径 | -| stream失效/应用证据缺失 | RECOVERY_REQUIRED或REINITIALIZATION_REQUIRED，不静默重建空stream | -| ENOSPC/文件损坏/IO失败 | 停 admission，明确告警；不能假装写入成功 | -| compaction/回收失败 | 业务成功不回退，空间继续计费；达到运维阈值停止新输入 | -| metrics解析失败 | 业务终态独立；观测标不可用，不重复执行已提交业务 | - -只保留一层主要 retry预算，避免内外两层循环把错误隐藏几十分钟。所有重试可取消，受单操作 timeout和feed级最大恢复策略约束。 - -### 15.2 正常关闭 vs 失租 - -正常关闭：停止 admission，按设定期限尝试 drain/MERGE，成功的确认，超时未完成不确认，再取消并关闭；不能无限 Background flush。 - -失租/强制终止：不执行“为了收尾再写一次”的新 mutation。停止工作，收集在途状态并交给接管协议。取消、等待和pool关闭需幂等；不得边清 spool边还有 worker读写。 - -### 15.3 安全与权限 - -TLS默认要求验证证书；最小权限覆盖目标 DML、必要建表/ALTER、内部元数据/stream，不要求仓库管理员。只读 Verify和有副作用初始化分离。标识符全量转义，业务值参数绑定；不同任务内部对象权限隔离。 - -本地 spool含业务明文可能敏感：受限目录权限、独占卷/宿主盘加密、保留与清理策略；不能因为“临时文件”就忽略数据保护。清理仅限实例拥有目录，不跟随不可信符号链接。 - -## 16. 可观测性、容量与成本模型 - -### 16.1 指标分层 - -- input accepted/bytes、spool memory/physical disk/reserved/metadata、排队最老年龄。 -- raw committed rows/bytes、raw待应用批次、MERGE周期耗时和终态。 -- target applied progress vs source resolved/checkpoint，分别展示；MAX(commitTs)不等于覆盖水位。 -- quota/free-space waiting、ENOSPC、segment reclaim failure/orphan bytes、每feed公平性。 -- DDL pending duration、owner generation/fenced、commit unknown query数、token刷新失败。 -- target mutation counts和source event counts分开，失败读取计数时 unavailable，不编造 0。 -- delivery credit/共享chunk已分配字节、blocked path、重投/重复转移、progress登记次数。 -- OPEN/SEALED tracker数、pending分片、Seal等待年龄、duplicate completion、已应用未回收bytes。 -- remote active/retained/reserved bytes、历史键上界、观测年龄、远端预算停写原因。 -- no-op confirmed/消费证据缺失、ledger未终结operation与服务端终态查询失败。 - -标签避免无限高基数；事件原始值、PK、token不写常规日志。异常样本需脱敏、本机保存、访问受控。 - -### 16.2 容量估算 - -若平均编码输入速率为 R bytes/s、有效本地可用配额为 D，粗略缓冲时间 `T ≈ D / R`；编码放大、元数据、未回收segment与已有积压会缩短窗口。多feed还受共享预算和公平分配约束。 - -spool只是有限吸收抖动。长时间停机后必须在上游 GC/日志保留范围内恢复；容量不够要背压，不能承诺不停机/零延迟。内部 eventstore等其他磁盘也要计入运维容量。 - -### 16.3 成本对照 - -固定输入变更率/行宽/更新热点/表数/目标可见延迟后比较： - -```text -旧成本 = TiCDC编码 + S3请求/容量/网络 + bend-cdc消费计算 - + Databend raw/MERGE/compaction + 运维资源 -新成本 = TiCDC原生适配/spool IO/资源增量 - + Databend raw/MERGE/compaction + 网络 + 运维资源 -``` - -必须记录 TiCDC CPU、RSS、磁盘吞吐/写放大、spool空间、Databend credit/query cost、MERGE次数、raw扫读量、小表对象数。不能仅用少一个 deployment证明总成本下降。 - -## 17. 全量初始化与 start-ts 接续 - -sink只做增量。独立初始化协议： - -1. 选择可重现一致快照 S，固定 schema S。 -2. 在保留窗口允许范围内启动/准备从 S接续的changefeed，确保增量不丢；可停留在受控bootstrap状态，不能让内存无限积压。 -3. Dumpling导出、COPY INTO目标，使用同一mapper；全量失败可按任务批次恢复。 -4. 校验全量数据/目标结构/元数据绑定，登记 baseline S。 -5. 放行约定边界之后的增量；确认 TiCDC实际 start-ts 是排除还是包含边界，不能靠口头假设。 - -初始化期间既要容忍 DDL，也要明确是否禁止 DDL或按快照版本隔离；不能查询最新 schema处理旧快照。全量导入不能在增量写完后覆盖回旧状态。必要的全量S3文件与增量链路去S3不矛盾。 - -## 18. 恢复矩阵与反例 - -| 中断点 | 允许的残留 | 预期恢复 | 不允许做什么 | -|---|---|---|---| -| 内存/磁盘 spool 后，raw前 | 本地临时文件 | 从checkpoint重放 | 把spool落盘当确认 | -| raw成功、响应丢失 | 相同event_id重复raw | 协调后幂等重试 | 新增本机时间当业务版本 | -| raw成功、MERGE前 | raw积压/stream增量 | 回放源并重新应用 | 只依赖丢失的pendingEvents内存map | -| MERGE明确失败 | target未推进 | 重试/人工修复 | 触发PostFlush | -| MERGE/COMMIT结果不明 | SQL可能仍执行 | COMMIT_UNKNOWN协调 | 新owner立即并发DDL/清理 | -| MERGE成功、callback前 | target已更新 | 重放不回退 | 清除最新墓碑 | -| callback后checkpoint保存前 | 重复source事件 | 幂等收敛 | 把重复当新写入时间更晚版本 | -| MERGE成功、compaction失败 | raw大、spool可能已释放 | 保留APPLIED，重试维护/限制新输入 | 重复业务操作修复统计 | -| entry释放、segment删除失败 | 真实磁盘仍占用 | 继续计费、重试回收/告警 | 配额假释放继续写满 | -| DDL pending后 | journal存在、目标旧schema | fingerprint确定后执行 | 跳过检查直接applied | -| ALTER成功、journal未applied | 目标新schema | reconcile并隔离旧DML | 旧行按新列名写入 | -| applied TRUNCATE后重启 | 新代际数据可能已存在 | 跳过同DDL，恢复正确代际 | 再次清空目标 | -| capture永久丢失 | 本地spool丢失 | 新owner在fence后从checkpoint重放 | 依赖本地磁盘保证数据可恢复 | -| stream过期/内部对象被删 | 恢复证据缺失 | 停止并进入显式重建协议 | 静默新建空stream丢旧raw | -| F1已应用、F2未登记 | OPEN tracker、F1可已回收 | 不回调；重启从checkpoint重放整个源事件 | 只看registered==applied就确认 | -| delivery已登记progress但未转入sink | 上层仍有lease/blocker | 同session按deliveryID重投转移，或取消恢复 | 再加一遍progress、丢chunk计费 | -| no-op成功但消费未知 | target未改/stream可能未推进 | 核验版本和消费证据，未证明就阻塞 | counts=0当失败循环重刷或当成功直接确认 | -| 远端持续新键churn | target小、raw墓碑持续增大 | 到远端预算停新raw，显式扩额/重建 | 以本地spool已排空宣称容量安全 | - -关键反例： - -- DELETE后延迟INSERT复活：检验I2/I9。 -- 高于MERGE快照的新raw抑制旧candidate却未应用：检验§7串行lane和coverage。 -- 生产者拿着表锁等磁盘，MERGE也要同锁：检验I6。 -- 同一segment一个长事务钉住空间，其余全释放：检验I7和物理回收。 -- 失租旧TRUNCATE晚于新owner写入提交：检验I3/I8，幂等DML不能解决。 -- 初始schema updateTS被误当已应用DDL fence导致漏首批数据：检验I10。 - -## 19. 测试与验收计划 - -### 19.1 分层验证 - -| 层级 | 验证内容 | 证据要求 | -|---|---|---| -| 单测/property | mapper、key规范化、状态机、预算、回调exactly-once-in-process | 固定seed与随机序列，包含错误路径 | -| mock客户端 | commit unknown、cancel、权限、retry、回收失败 | 精确检查未完成事件不确认 | -| 真实Databend | stream/MERGE、no-op、事务、session、原子恢复 | 固定server/driver版本、SQL记录与最终表状态 | -| TiDB/TiCDC E2E | 原生before/after、DDL、快照边界、回放、故障调度 | 固定版本与事件时序，checksum/count/schema | -| 并发/资源 | race、内存、磁盘、并发feed、满盘恢复 | 实际RSS/allocated bytes、死锁与泄漏检查 | -| 灰度/成本 | 影子目标、同SLO、费用与可见延迟 | 不用本地单测替代线上或计费证据 | - -### 19.2 必测清单 - -1. 空批、低流量、热点更新、大表、小表海量、混合表大小、整库空表。 -2. PK/复合PK、变键、多行同事务、换键、同TSO净变化；不支持表在发现时拒绝。 -3. 所有§13边界值、字段缺失vsNULL、case/collation、生成列与DDL后mapper刷新。 -4. insert/update/delete/reinsert、乱序、重复、同event_id异payload冲突。 -5. 每个§18中断点；网络断开、超时、服务端SQL继续运行、节点失租。 -6. PENDING/APPLIED DDL、无后续DML、drop/re-add同名列、TRUNCATE多步中断。 -7. 多feed共享配额、热/冷表公平性、segment pin、孤儿清理、ENOSPC/inode耗尽。 -8. 配额达到但merge-trigger未达到、等待者被取消、长事务超过可用spool、单行超限。 -9. stream没新数据、只有重复或被抑制旧数据、no-op MERGE、SELECT是否消费、回滚后cursor。 -10. 正常Close、失租Close、快速重复start/stop、token刷新、所有权切换、目标被外部改写。 -11. 同名不同库映射、目标schema已存在但不兼容、内部对象属于别的任务。 -12. 快照S前后写入与DDL、源GC保留不足、stream失效和恢复失败。 - -### 19.2.1 v0.2新增的可执行验收矩阵 - -下面每项均未执行；它们是实现/P0必须交付的测试，不是本文已通过的证据。 - -| ID | 注入点与负载 | 必须断言 | 关闭项 | -|---|---|---|---| -| V01 | 固定PK仅改二级UK，分别raw=true/false，切分cycle并在两半间重启 | 正式模式只输出正确净变化；最终仅新值，无版本冲突/迟到DELETE破坏；迁移重注册仍raw=true | B05 | -| V02 | F1先APPLIED，暂停F2登记；并发Seal/Apply/重复完成 | OPEN时callback=0，全集完成后=1；已APPLIED分片可提前回收 | B17 | -| V03 | 累计事件大于spool，可用空间小于事件总量，多个有界delivery | 编码/应用/回收循环前进；所有分片未完成前不确认；入口chunk/元数据不越预算 | B08/B16/B17 | -| V04 | A/B共用dynstream线程；A用尽path信用，预留B信用且下游健康 | A handler快速返回；B DML/DDL在测试设定的有界期限推进；重投progress/lease各一次 | B16 | -| V05 | 多DMLEvent共享大chunk，wire小但解码膨胀；断session/取消 | chunk容量只计一次且释放恰一次，无丢计/双计；超限在分配前拒绝；RSS剖析含临时内存 | B08/B16 | -| V06 | 热/冷表共享segment；配额满，冷表未达普通阈值；unlink/搬迁失败 | 受压引用lane均被唤醒；保留池可排空；不假释放物理quota，无无限busy loop | B07/B13 | -| V07 | 首结果页后注入服务端失败/传输断开；提交后响应丢失；计数解析失败 | 前两种无独立证据不ACK；已提交统计失败不重做；UNKNOWN可跨重启协调或明确停写 | B03 | -| V08 | 阻断Kill接口，延迟旧TRUNCATE/compaction，新owner请求接管 | Kill nil不放行，固定sleep不放行；取得可信终态后才能恢复写 | B01 | -| V09 | 实际no-op SQL各形态，旧/含修复build对照；事务提交/回滚 | 目标可0修改但消费证据与callback正确；无planner短路漏消费；重试标签不偷消费下一批 | B02 | -| V10 | R0–R5每步前后崩溃；R5写新数据后回放旧TRUNCATE | 新数据不被再清空；unknown query先协调；journal缺失/冲突不猜测 | B04/B09 | -| V11 | stream依赖snapshot过期/内部对象删除 | 可诊断停写；恢复不跳过遗留raw；不足证据进入显式重初始化 | B10/B14 | -| V12 | schema S分页时并发空表ADD/DROP/re-add，bootstrap中断 | 每表按正确历史schema接DDL；无首批漏写/错列 | B04/B15 | -| V13 | target恒定小、持续新键churn，compaction始终成功 | 远端预算触发停新raw；统计过期也停写；不删最新墓碑；多表在途增长预留一致 | B06/B18 | - -V07的“服务端先出数据后失败”可先用mock验证客户端分类，但真实MERGE是否可出现该时序须另记录，不把mock当服务端复现。V04对B的期限应由固定压测配置定义；不在全局资源耗尽时作无法兑现的公平性承诺。 - -### 19.3 属性/模型测试建议 - -构建内存reference model：按源净事件顺序维护 PK→值/墓碑，随机插入重复、乱序、cancel、重启和DDL边界，比较“完成全部已确认覆盖后”的目标最终状态。 - -断言：确认集合不超出已证明APPLIED；每个回调最多一次；预算无负数/超额预留；无未计费文件；取消后无新mutation；旧DDL/旧epoch不污染新代际。 - -### 19.4 发布门槛 - -必须关闭所有会造成数据丢失、错误确认、双owner、不可恢复DDL、资源无界的阻塞项。性能不足可以限制规模后发布PoC；正确性缺口不能只通过缩短flush interval掩盖。 - -## 20. 上线、回退与交付拆分 - -### 20.1 阶段 - -| 阶段 | 交付物 | 退出标准 | -|---|---|---| -| P0语义验证 | 原生事件契约、tracker、入口信用、终态/fence四份协议；server/driver/SQL能力实验 | V01/V02/V04/V07/V08/V09设计/原型证据；无法证明fence只能限制隔离单capture实验，不能开放生产HA | -| P1纵切PoC | 原生scheme/config、跨层信用接入、SEALED tracker、spool、单表PK批写/MERGE、远端预算 | 受限单owner故障重放与资源有界，包含UK UPDATE与大事件；不能跳过端到端入口预算 | -| P2核心对齐 | 复合PK/类型/建表/空表、DDLjournal/恢复/TRUNCATE | 支持矩阵所有已承诺项完成E2E | -| P3灰度 | 影子目标、切换/回退、容量与成本结果 | 正确性、可见延迟、恢复和费用均验收 | -| P4可选扩展 | 分区、跨节点汇聚、typed staging、高并行raw | 单独RFC与不变量证明 | - -此前“PoC约1–2周”的估算未包含完整共享配额/owner/DDL/真实故障覆盖，不能当本详细范围的排期承诺。先做P0再按已关闭风险重估。 - -### 20.2 灰度切换 - -旧链继续写旧目标，新sink写独立影子目标，不能共用raw/stream/journal。固定初始化S和对照水位，验证目标checksum/count/schema与业务查询，连续观察后切读，再停旧链。 - -影子对照必须明确水位：任意时刻比较两个异步目标的行数可能自然不同，不直接当错误。新sink不保证全changefeed在同一时刻的snapshot read，需要在受控drain/已对齐边界比较。 - -### 20.3 回退 - -保留旧链数据与checkpoint覆盖窗口；先让旧链追平到验证边界再切读。不把新sink内部raw/journal当旧bend-cdc文件游标，不直接复用内部表。 - -升级涉及raw schema、key编码、schema journal或spool格式需版本化。临时spool可丢弃重放不等于目标journal可丢；回退二进制必须理解已有持久化元数据，否则阻止启动。 - -## 21. 已知不足、待验证假设与生产阻塞项 - -| ID | 风险/未闭合问题 | 当前拟定措施 | 关闭所需证据 | -|---|---|---|---| -| B01 | 各mutation的fencing/旧在途SQL与新owner重叠未证明 | §12分DML/DDL/stream/维护排他；取消返回值不是证据 | V08及完整registry、正常迁移/异常接管；非HA准入不能只配across-nodes=false | -| B02 | stream+MERGE提交/快照/no-op/cursor/applied witness未实测 | 固定build及补丁、§8.5/§9.7覆盖和消费证明 | V09、实际SQL与版本矩阵、atomic witness或等价算法，含planner短路 | -| B03 | QueryContext/Rows/COMMIT不确定结果判定与重启定位 | §9.4证据结构、错误分类、durable operation ledger；UNKNOWN停写 | V07、真实endpoint/字段/保留期、首响应丢失/查不到query及ledger自身不确定结果 | -| B04 | start-ts完整schema bootstrap/空表与恢复API接入未定稿 | 权威snapshot与窄recovery接口 | 既有空表、schema变更并发、重启E2E | -| B05 | 普通二级UK更新拆分与PK/换键净事件契约 | §11.4固定raw=true贯通注册；真实handle变更另验 | V01、迁移重注册、源合法事务dump与checksum；不能只测试PK UPDATE | -| B06 | raw相同event_id物理重复与墓碑长期容量 | 逻辑去重+安全compaction，禁止TTL删最新 | 等值重复清理SQL、stream不丢数据、增长基准 | -| B07 | node级预算与其他磁盘消费者共盘 | 全feed共享budget+free guard，建议独占卷 | 多feed/多组件满载测试、物理字节计费 | -| B08 | 源chunk/序列化/blob/readback/callback瞬时内存 | 分阶段预算，禁止无界channel和oversized旁路 | 内存剖析、极端行/事务压力与race测试 | -| B09 | DDL journal不是现成原生恢复协议，TRUNCATE多步 | before/after fingerprint、generation、skip/recovery接入 | 所有DDL中断点/旧DML重放测试 | -| B10 | STREAM企业能力、许可、保留窗口/失效 | 明确capability校验，默认不静默fallback | 实际warehouse版本/权限/许可与失效演练 | -| B11 | 跨表原子性不提供，目标读取可能见部分事务 | 产品明确语义，影子比对在受控边界 | 使用方接受与文档/测试一致 | -| B12 | 性能与成本尚无数据；保守串行lane可能限制热点吞吐 | P3同SLO benchmark，先保正确性 | 固定负载和费用证据 | -| B13 | 跨lane segment pin与压力/回收活性 | §6.8反向引用、压力广播、保留排空池；搬迁需双份预算 | V06，热/冷表、删除失败、资源不足不假释放；不以原版有定时器就认为已解决 | -| B14 | stream失效后的安全恢复/重新初始化 | §9.8冻结、证据核验；不够证据只允许显式重初始化 | V11，full-raw恢复若未实现不得宣称自动修复 | -| B15 | 历史空表bootstrap与DDL/schema竞态 | §10.6同一S快照、bootstrap epoch、顺序DDL | V12，分页/同名列重建/中断，不能以最新schema兜底 | -| B16 | collector→dynstream→sink入口信用与共享chunk责任 | §6.7新增capability、deliveryID、progress只登记一次、跨层lease | V03/V04/V05、协议batch上限、兼容旧sink、真实分配/RSS与控制事件推进 | -| B17 | 动态分片全集封口与独立回收 | §5.2 OPEN/SEALED/COMPLETED、先登记后发布、once完成 | V02/V03及race/失败/取消；flush闭包自身不是once | -| B18 | 成功维护下远端历史键增长/观测过期 | §9.6有限远端预算、增长预留、停写/扩额/重建流程 | V13，active/retained计量与费用，不能借删除最新墓碑降量 | - -本轮B01–B18均未因“写了设计”关闭。B01–B10、B13–B18若影响受支持场景的正确性/资源边界，必须在该场景发布前用实现和实验关闭；不能以缩短flush interval或新增风险编号代替。B11需明确语义接受，B12需按固定SLO验收。首版范围缩减必须可执行，不能写一个无法在源事件上识别的“禁用PK更新”开关冒充保护。 - -## 22. 替代方案与取舍 - -| 方案 | 优点 | 代价/拒绝原因 | -|---|---|---| -| 保留S3+bend-cdc | 变动小，有对象存储缓冲/回放 | 中间编码/消费成本仍在，可作为fallback | -| TiCDC纯内存直写 | 实现较简单 | 已被用户否决；无法满足第一版磁盘spool要求 | -| 原生sink + 本地spool + raw/stream/MERGE | 复用核心语义、去中间链、有限吸收抖动 | owner/DDL/提交/预算仍需正式实现；本文选择 | -| raw落盘立即确认，后台异步MERGE | 易释放本地资源、吞吐可能更高 | 需要独立durable任务恢复、applied水位和接管；第一版不选 | -| 每条直接UPSERT/DELETE目标 | 链路短 | 小SQL多，成本高；旧事件防回退和删除墓碑仍需解决 | -| 每batch临时表然后MERGE并删除全部staging | 局部逻辑简单 | 丢失跨批次watermark/tombstone，迟到重放可能回退 | -| 多capture直接共享同一stream | 表面并发高 | ownership/覆盖与DDL竞态未解决，不选 | -| 无stream，全raw扫描MERGE | 不依赖stream许可 | 扫描成本和coverage策略不同，需独立验收，非静默fallback | -| typed staging/独立watermark表 | 可能减少JSON与raw历史成本 | schema/事务复杂度高，放后续独立优化 | - -## 23. 发给 reviewer 的具体问题和输出约定 - -请优先寻找以下反例，每条提供触发条件、事件序列、违反的不变量、影响和最小修正： - -1. MERGE后才确认的coverage定义，在重放、空stream和旧事件被抑制时是否充分？ -2. 单表raw/MERGE串行是否仍存在进程间/接管竞态？什么条件才能真正排除？ -3. 本地磁盘配额计费能否被segment pin、回收失败、孤儿文件、其他capture绕过？ -4. 超大事务、callback tracker和chunk引用会不会让spool有界而RSS仍无界？ -5. 配额不足时是否有消费者等待生产者、DDL等待callback、callback等待DDL的循环？ -6. retained raw历史与目标schema变更是否会混用epoch？初始bootstrap是否可能错误跳过DML？ -7. PK变键/同事务换键是否存在无法靠commitTs消歧的合法输入？ -8. 有无在不提供目标侧fence时仍安全自动HA的证明？如果没有，应缩减什么承诺？ -9. DDL结果不确定、TRUNCATE部分完成和旧owner晚到SQL如何收敛？ -10. stream无业务行更新时是否一定消费？许可/失效/保留期会不会让恢复停摆？ -11. 同event_id重复物理记录的compaction是否可能删光水位，或无限累积？ -12. 固定镜像DELETE语义后，已有目标数据、collation、full-image缺失等前置条件是否仍不可执行或存在漏洞？ -13. 此设计是否真的完整覆盖bend-cdc“建表、批写、MERGE、类型改写、DDL”核心能力？遗漏请逐项列出。 -14. 配置数量/共享模块改动是否过度设计？能否在不破坏不变量下缩小首版？ - -输出建议：`结论(接受/修改后接受/拒绝) → P0/P1/P2发现 → 每项证据/时序/修复 → 发布阻塞 → 可后移优化`。不要以测试没跑作为全部结论；需要指出该测试要证明哪个语义。也不要替本文补全缺失假设后默默给通过。 - -## 24. 证据、实际验证与可复核性 - -### 24.1 已执行与未执行 - -- 本会话前序已执行 bend-cdc：`go test ./pkg/sink/databend ./pkg/source/tidb -count=1`，两包通过；这是已有代码的本地测试结果，不是新sink验证。 -- 未配置真实Databend测试连接，相关集成测试按条件跳过。未跑真实stream/事务、TiDB/TiCDC新sink E2E、故障注入、benchmark或部署。 -- 本次新增/修改Markdown并检查格式、代码证据引用；未实现新配置或SQL，未构建TiCDC，未创建分支/提交/PR。 -- 编写时读取的是固定Git对象与当前bend-cdc源码；公开文档仅用于能力与限制背景，不能代替部署版本/真实运行证据。 - -### 24.2 新架构TiCDC固定源码索引 - -以下链接基于已用git对象读取的commit；远程仓库可见性受reviewer权限影响，无法打开时使用§0的git show。引用的固定路径是Git对象路径，不要求当前旧架构checkout存在对应文件。 - -- **T01**：[Sink接口与factory](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/sink.go)。 -- **T02**：[DMLEvent、原生行与回调](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/dml_event.go)。 -- **T03**：[Spool、TryEnqueue、oversized与目录生命周期](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/sink/spool/spool.go)。 -- **T04**：[S3 buffer manager与完整payload构造](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/cloudstorage/buffer_manager.go)。 -- **T05**：[S3 writer、写入后回调与release](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/cloudstorage/writer.go)。 -- **T06**：[TableSchemaStore的sink-type需求](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/table_schema_store.go)。 -- **T07**：[dispatcher manager的MySQL恢复分支](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go)；[merge task恢复分支](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/task.go)。 -- **T08**：[capture写许可gate](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/writelease/write_gate.go)。 -- **T09**：[DDL原生事件](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/ddl_event.go)；[dispatcher屏障](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatcher/basic_dispatcher.go)。 -- **T10**：[S3 writers创建spool及上游队列](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/cloudstorage/dml_writers.go)。 -- **T11**：[UPDATE的UK拆分，L970–1014](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/eventservice/event_scanner.go#L970-L1014)；[IsUKChanged，L219–259](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/codec.go#L219-L259)。 -- **T12**：[raw-change初始化，L320–350](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go#L320-L350)；[dispatcher同步交付与progress先登记，L295–335](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatcher/basic_dispatcher.go#L295-L335)；[session注册传播，L590–655](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/eventcollector/dispatcher_session.go#L590-L655)。 -- **T13**：[dynstream同步Handle与path blocking，L340–356](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/utils/dynstream/stream.go#L340-L356)。 -- **T14**：[collector提前AssembleRows并交付，L360–395](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/eventcollector/dispatcher_stat.go#L360-L395)。 -- **T15**：[across-nodes配置定义，L34–38](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/config/scheduler_config.go#L34-L38)；[控制初始span拆分，L201–208](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/maintainer/span/span_controller.go#L201-L208)。 - -### 24.3 bend-cdc源码索引 - -下列本机链接在本报告编写时的bend-cdc commit上有效；异地reviewer应按§0的bend-cdc基线读取同路径的Git对象，不能以未来HEAD替代。C编号只表示源码，不是§21的B编号阻塞项。 - -- **C01**：[TiDBRunner](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/service/tidb_runner.go)。 -- **C02**：[DatabendMergeSink、PK/schema/转换](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go)。 -- **C03**：[CDCManager、raw/stream/MERGE/compaction](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go)。 -- **C04**：[DDL翻译](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/schema_ddl.go)；[schema元数据与建表](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/schema_meta.go)。 -- **C05**：[列变更白名单](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/schema_change.go)；[epoch](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/schema_epoch.go)；[TRUNCATE](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/truncate.go)。 -- **C06**：[类型与连接](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend.go)；[TiDB值转换](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/tidb_values.go)。 -- **C07**：[MERGE结果判定](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/merge_result.go)。 -- **C08**：[applied watermark](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/applied_watermark.go)；[Dumpling初始化](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/dumpling_executor.go)。 - -### 24.4 官方文档证据边界 - -1. [MERGE语法](https://docs.databend.com/sql/sql-commands/dml/dml-merge)：支持source/subquery驱动的增删改；不证明源端幂等算法。 -2. [CREATE STREAM](https://docs.databend.com/sql/sql-commands/ddl/stream/create-stream)：append-only选项、Enterprise标记；需实际部署许可验证。 -3. [Stream工作流](https://docs.databend.com/guides/load-data/continuous-data-pipelines/stream)：提交消费与并发限制的公开说明；MERGE/no-op组合仍需实验。 -4. [Transaction](https://docs.databend.com/sql/sql-commands/ddl/transaction)：事务控制入口；不据此假设所有DDL/owner校验支持所需隔离与原子性。 -5. [BEGIN中DDL限制](https://docs.databend.com/sql/sql-commands/ddl/transaction/begin)：DDL提交当前多语句事务；不能泛化DML事务fence到DDL。 -6. [no-op stream offset修复#20671](https://github.com/databendlabs/databend/pull/20671)：已合并版本相关修复，明确存在未覆盖的planner短路；不证明目标部署已包含，也不是所有空MERGE的规范定义。 - -### 24.5 驱动证据与本轮验证边界 - -- **D01**：bend-cdc go.mod引用databend-go v0.9.4；本机读取[waitForData，rows.go L27–50](/Users/jiangjianyuan/go/pkg/mod/github.com/datafuselabs/databend-go@v0.9.4/rows.go#L27-L50)、[KillQuery，client.go L746–767](/Users/jiangjianyuan/go/pkg/mod/github.com/datafuselabs/databend-go@v0.9.4/client.go#L746-L767)。异地reviewer需获取相同版本，不把本机路径当远程可用。 -- 本次读取文件SHA256：rows.go `065c09c10a3a7b334da795949de1b717a5fd0014000d55be80e5fbd4fb5da6b9`；client.go `b3d9b42925ae4d584e84afbc244bfa97096eb5591eda257ecd2747894a35aecf`。这是文件内容指纹，不代表已运行驱动集成实验。 -- v0.2只增加协议/状态转移/验收条件并更新评审入口；V01–V13均未实际执行。Markdown结构、TOML语法、文件/Git对象引用检查与新sink正确性验证严格分开。 -- 不把独立评审的P0/P1标签当复现证据；例如no-op规范、PK互换SQL合法性、Kill接口能力、永久丢数据时序，都需按本文限制重新核验。没有创建新review agent、没有切分支、没有修改业务Go代码。 - ---- - -**最终建议**：方向可行，进入独立评审和P0验证；不要直接宣布可生产实施。优先要求reviewer击穿owner fencing、提交结果判定、DDL恢复和资源硬预算这四条关键路径，而不是只讨论字段命名或批大小。 diff --git a/docs/design/databend-native-sink-design-review-v0.3.md b/docs/design/databend-native-sink-design-review-v0.3.md deleted file mode 100644 index 6e3768308d..0000000000 --- a/docs/design/databend-native-sink-design-review-v0.3.md +++ /dev/null @@ -1,447 +0,0 @@ -# RFC v0.3：TiCDC 原生 Databend Sink——直接 raw 批次与最小接入 - -| 项目 | 结论 | -|---|---| -| 版本 / 日期 | v0.3 / 2026-10-24 | -| 状态 | Proposed；唯一候选主路径，待实现及实验，不是生产就绪 | -| 目标 | 将 bend-cdc 的建表、类型转换、批写 staging、MERGE、必要 DDL/恢复迁入原生 sink，去掉增量 Canal JSON→S3→消费者链 | -| 改动约束 | 核心位于 Databend sink；外部仅必要注册、raw-change、schema/recovery、失败生命周期接入 | -| 首版运行范围 | 隔离单 capture、静态独占目标、人工核验旧 SQL 后恢复的受控 PoC；不承诺自动 HA | -| 全量工具范围 | 不迁入 Dumpling/CSV/COPY INTO，不实现或编排全量初始化；只处理 TiCDC 交付的增量事件及必要表元数据 | -| 缓冲 | 有界入口/内存 + 限额本地 spool + 现有 PostEnqueue 背压 | -| 应用主路径 | manifest → raw → 本批主键 K → retained history H → 每键净动作 S → MERGE → flush callback | -| 不采用 | stream cursor、消费 hint/witness、普通 DML durable applied ledger、新 delivery/信用协议、progress 迁移、跨 capture 强一致预算服务 | -| 实现状态 | 未实现新 sink，未执行真实 Databend/新 sink 故障实验 | - -**本文替代 v0.2 作为本轮唯一实施候选。** [v0.2 历史设计](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review-v0.2.md)只供风险分析，不再构成并行实施要求；其中 B01–B18/V01–V13 保留为历史编号。当前仅跟踪本文 Q1–Q7、A01–A12，不把删掉旧协议当作其风险已通过实验。 - -## 0. 固定基线与本轮裁决 - -| 对象 | 仓库 | 固定 commit | -|---|---|---| -| 新架构 TiCDC | `/Users/jiangjianyuan/work/git/pingcap/ticdc` | `2bf6dacd0aec9ac2fad464600e47341f9aa951f2` | -| bend-cdc | `/Users/jiangjianyuan/work/git/pingcap/webapi` 下 `bend-cdc` | `b6481621021e82aee659ad88e8e6c00f6dd11906` | - -TiCDC origin 为 `git@github.com:sdojjy/tiflow.git`，上述是 fork 新架构基线，不代表上游/生产最新版本。编写时 checkout 是旧架构 master / `03ed2432bd7782198742d3aa20e07edcc47097e6`；只通过 Git 对象读取新架构，没有切分支。 - -```bash -git -C /Users/jiangjianyuan/work/git/pingcap/ticdc show \ - 2bf6dacd0aec9ac2fad464600e47341f9aa951f2:downstreamadapter/sink/sink.go -``` - -本轮采纳三项裁决： - -1. 采用“manifest 明确成员 + K/H/S + 直接 MERGE”，不再把 stream 消费范围当确认边界；普通 DML 不默认要求 durable applied ledger。 -2. 复用现有 dispatcher/progress/callback 生命周期；transport reset 不丢 sink 已接管数据。接管失败必须让旧生命周期停止，不能只 reset session 接着跑。 -3. 自动 HA 和严格 feed 全局远端额度移出受控 PoC 承诺；保留旧 SQL 排他检查、容量监控和可诊断停止，不以缩小范围掩盖数据错误。 - -## 1. 流程、职责与核心约束 - -```text -TiCDC dispatcher（现有 progress 登记） - → AddDMLEvent 非阻塞接管到有界入口 - → 异步按权威 schema/主键转换原生行 - → 内存热缓冲，必要时溢写限额 spool - → 同表/同 incarnation/同 raw_generation 攒批并封口 manifest - → 批量 INSERT raw，核验所有 manifest 成员到齐 - → 独占该表 lane：冻结其他 raw/DDL/compaction - → K = manifest 主键；H = K 对应 retained raw；S = 每键最高版本净动作 - → MERGE target USING S，取得提交成功终态 - → 本批分片 APPLIED，独立释放本地数据 - → 源事件 SEALED 且全部分片完成，flush callback 一次 - → TiCDC 原有机制推进 checkpoint -``` - -明确约束： - -- raw 是普通持久化 staging 表，不是 TEMP，也不是本地 spool。raw INSERT 成功不等于目标应用完成。 -- commit_ts 是事务 TSO，不是每行唯一 ID。版本不使用到达时间、批次序号或进程时钟。 -- 同逻辑目标表首版一条串行 lane；跨表有限 worker；没有外部应用改 target/raw/journal。 -- 不承诺跨表/跨分片事务原子可见。不实现、调用或编排全量导出/导入，不自动扫描源表历史行；目标基线由使用方负责，不构成sink工作包。 -- 不确认未应用的数据、不伪造 flush 清进度；所有“不支持”操作显式失败。 -- 限制 sink 自持内存与磁盘，不宣称硬限制全 TiCDC RSS；源 chunk 在进入 sink 前的分配不是本组件能消除的。 - -### 1.1 核心功能迁移 - -| bend-cdc 职责 | 原生 sink 处理 | -|---|---| -| Canal JSON 解码、文件 discovery、SQS、S3 游标 | 不迁移，改吃原生事件与现有 checkpoint | -| 主键、库表路由、建表、类型转换 | 按 TableInfo/列 ID/FieldType 适配，复用既有语义与测试案例 | -| raw、stream、MERGE | 保留 raw/MERGE；stream 不作为首版必要依赖 | -| 乱序、重试、最新版本和删除墓碑 | 必须保留，使用 K/H/S 实际应用最高有效版本 | -| DDL 白名单、pending/applied、TRUNCATE | 保留必要持久化恢复，接入现有 DDL 屏障 | -| Dumpling/CSV/COPY INTO | 不迁移、不引入依赖，也不开发独立全量初始化模块或共享全量mapper接口 | - -## 2. 接入边界与首版支持范围 - -| 修改位置 | 允许范围 | -|---|---| -| Databend sink 新包 | adapter、buffer/spool、manifest、tracker、raw/MERGE SQL、mapper、DDL journal、错误与指标 | -| scheme/type/config/factory/metrics | 新 sink 正常注册与校验，不伪装 MySQL SinkType | -| dispatcher manager/session 注册 | Databend 强制 raw-change=true 的小分支，贯通初建/重注册 | -| TableSchemaStore / recovery | 必要类型分支及专属恢复结果；沿用既有 dispatcher 创建/DDL 协调 | -| manager fatal 错误生命周期 | 先验证既有关闭路径；确有缺口才加有限失败分支，不引入通用交付协议 | -| collector/dynstream/TableProgress/调度核心 | 不改 delivery/信用/重投/progress 迁移协议；如确实不能接入，先给最小阻塞证据 | - -首版支持：非分区表、非空单列/复合 PK、完整行镜像、普通增删改与二级 UK 值更新、权威建表、有限非键列 DDL、通过恢复测试的 TRUNCATE。 - -首版不承诺：无 PK/nullable UK/force-replicate、分区或一表跨节点、多任务撞同一目标、复杂 DDL/PK 定义变更、active-active/BDR/syncpoint、外部写入、全自动 HA。也不要求历史空表自动发现/预建；没有增量事件的历史表可以暂不出现在目标，不能默认为会重放历史CREATE。 - -PK/handle 值变化及同事务复用键必须按真实源事件验证；不能假设全部变键都能识别为 UPDATE。未通过测试时不宣称支持，也不拿一个实际上无法识别所有变键的开关冒充拒绝保护。 - -## 3. 原生事件、类型转换与内部数据 - -### 3.1 原生事件模式 - -固定基线在 raw-change=false 时会因 UK 变化拆 UPDATE，PK 不变也可能产生同 key/TSO 的 DELETE/INSERT。Databend 内部固定 true，不开放用户切回 false；校验 manager→SharedInfo→注册/重注册→eventservice 实际生效。未知旧协议拒绝，不靠 SQL 任意选动作补救。 - -- INSERT / PK 不变 UPDATE：完整 after-image UPSERT。 -- DELETE：旧 PK + tombstone。 -- 同 key/TSO 完全相同净事件：重放，逻辑去重。 -- 同 key/TSO 异动作/异 payload：先检查源净变化契约，未解释的冲突在 MERGE 前失败；不能 ROW_NUMBER 任意决胜，也不能规定 DELETE 永远胜出。 -- raw=true 不等于真实 record-handle 变化已归约。测试先确认源 SQL 合法，再保存实际事件与最终键集合。 - -### 3.2 类型与 schema - -一个 mapper 同时服务建表、raw 编码、MERGE projection、DDL。输入是列 ID/FieldType/flags/精度/Elems，不按样本值猜类型。 - -| 类别 | 要求 | -|---|---| -| signed/unsigned、BIT | 无损整数，验证 UINT64/BIT64 边界，commit_ts 不转浮点 | -| DECIMAL | 明确目标精度能力，精确文本/参数，不经 float64；不支持范围拒绝 | -| binary/blob | 原生 bytes 用固定 HEX/已验证参数；不用 Canal Latin-1 还原 | -| ENUM/SET | 按原生 datum/Elems 转换，不重复做 Canal ordinal 解码 | -| JSON | 保留大整数，区分 SQL NULL/JSON null/字段缺失 | -| DATE/DATETIME/TIMESTAMP/TIME/YEAR | 时区/微秒/零值/负 TIME/超范围策略明确，不隐式修正 | -| 字符串/PK collation | 源键相等、canonical key 与 target JOIN 语义须一致，限制到已验证集合 | -| generated | VIRTUAL 不写，STORED 使用物化值；不把目标生成逻辑当源语义替代 | - -转换失败停止，不 TRY_CAST 静默写 NULL。ADD COLUMN 历史回填/default 必须与源相容；复杂表达式未定义前拒绝。 - -### 3.3 最小内部记录 - -```text -RawRow { - source_binding, logical_table, incarnation, raw_generation, - source_schema_version, row_key, commit_ts, - action: UPSERT | DELETE, - event_id, payload_hash, payload, - batch_id -} -``` - -row_key 使用确定类型/长度编码，保留完整键；hash 不代替相等性。event_id 基于源身份、代际、规范键、TSO、净动作等稳定信息，不依赖重启后重分批。batch_id 只标识一次逻辑 raw 批次，同批重试复用；重启可重新分批，业务幂等依赖事件身份与版本，不要求重建原 batch_id。 - -## 4. 有界接管、正常背压与失败生命周期 - -### 4.1 正常接管 - -```text -dispatcher 按现状登记该组事件的 progress - → AddDMLEvent 尝试非阻塞接管 - → 成功：sink 保留事件与回调的责任，有界入口中的引用计费 - → worker 增量编码/spool，不把整个事务一次复制 - → 根据水位/驻留引用与剩余额度，延迟或触发原有 PostEnqueue - → raw/MERGE 后按 tracker 触发 PostFlush -``` - -PostEnqueue 释放的是现有表路径的背压，不是 durable 确认。正常磁盘压力优先通过 spool、高低水位、强制小批 MERGE、延迟 PostEnqueue 吸收；不能只因普通水位升高就报 fatal。 - -入口按保守字节与元素数限制，并预留一个现有交付组可能带来的有限突发空间。共享 chunk 按其被 sink 引用的实际容量保守计费；可多计不能漏计。Detach 回调后逐步解除源数据引用；必须证明后台任务没通过闭包继续捕获整个 chunk。 - -现有交付组超过可接纳范围、单行超支持上限、磁盘/IO 严重错误属于明确异常策略：失败停机而非无限分配或无限等待。现有源 batch 没有足够硬上限时，不承诺永不触发该策略；用压测记录支持规模，不为此默认改 collector。 - -### 4.2 transport reset 与 owner/lifetime 分离 - -- 同 manager/dispatcher 的 transport session reset 不取消或丢弃 sink 已接管的数据，不因 collector epoch 改变清空入口/spool。 -- 重放可能与已接管事件重复，按源身份收敛；每个现有 progress 登记仍由自己的真实完成回调完成，不能因为逻辑事件重复就丢其 callback。 -- 未接管发生致命失败后，不在同 dispatcher 中尝试自定义 delivery 重绑/清 blocker；结束旧生命周期，重建后从安全 checkpoint 重放。 -- 真正 owner/lifetime 结束时，停止该旧 sink 新写入；这不证明已经发出的远端 SQL 已终止，见§9。 - -### 4.3 接管失败不是只设 IsNormal=false - -必须以首次 fatal 原因驱动单调状态： - -```text -RUNNING → FAILED（不可在本生命周期内复活） - → 立即关闭本地 admission 和 mutation gate - → Run 返回非context.Canceled的首因；上报控制面 - → 旧 manager/dispatcher 最终关闭，未完成事件不flush - → 核验旧远端 SQL；新生命周期从安全 checkpoint 重建 -``` - -固定基线中 Run 错误经 handleError/collectErrors 上报并重发，**并不立即销毁 dispatcher**。因此： - -1. sink 自己立即拒绝后续 Add/DDL/新 SQL；所有 worker 检查相同 failed gate。后续调用不能继续排队；失败回调只记首因，不忙循环。 -2. 不为清理旧 blocker 伪造 PostFlush。旧生命周期保留未完成 blocker 是安全停机状态；不能把它当健康任务继续 session reset。 -3. 验证控制面收到错误后关闭旧 manager/dispatcher，恢复时创建新对象。若既有路径不保证关闭，增加 Databend fatal 专属小接入，复用已有关闭/LocalFence能力；不能在运行 goroutine 里等待自身退出，不能取消上报上下文后把首因吞成 Canceled。 -4. 关闭/上报链未实测前，受控 PoC 需要人工停掉旧进程并确认 manager 不再运行，不承诺自动恢复。不得只调用一个 Close/LocalFence 就宣称整个控制面路径已验证。 - -## 5. spool、批处理与事件完成 - -### 5.1 缓冲策略 - -内存热数据→限额 spool segment；spool 是临时缓冲，不是恢复 WAL。预算包含入口引用、编码临时值、payload、回读批次、tracker/callback 元数据。超大行无内存直通例外，sync.Pool 不等于配额。 - -磁盘按实际占用/预留计费，未删成功的 segment 继续占额；启动只清理确认不活跃的自身目录。共享本地 BudgetManager 若可复用则复用，不能因多个 feed 各有10GiB就忘记节点空间；不把预算扩成分布式服务。 - -压力触发唤醒所有钉住受压 segment 的表，不只热点申请者；独立读回/应用资源保证消费者能排空。首版先验证强制排空和物理回收，不默认新增复杂搬迁系统；不能排空则告警/停止新接管。ENOSPC 仍可能来自共盘其他组件，要明确处理，不承诺绝不发生。 - -### 5.2 表内串行与攒批 - -每表/代际/raw_generation 一条 lane，按行数、编码字节、最长等待、DDL marker、压力触发。计时从最老待处理数据起算。不同表有限并发，单表 raw/核验/MERGE/DDL/compaction 不交错。 - -raw batch 可包含多个事件分片；大事件可跨多批。通过有界窗口边编码边应用，不能把累计事件总量作为必须同时驻留的大小。 - -### 5.3 SEALED tracker 和独立回收 - -```text -事件：OPEN → SEALED → COMPLETED（或 FAILED） -分片：REGISTERED → STAGED → RAW_VERIFIED → MERGING → APPLIED → RELEASED -``` - -- 分片先登记、增加 pending，再发布给消费者；生产者遍历完整事件后才 Seal。 -- 只有 `SEALED && pending==0 && !failed && CAS(completed)` 才调用 flush 一次。 -- APPLIED 分片立即可释放 payload/entry、回收空间，不等整事件完成；只保留有界未完成描述符和小 tracker。 -- Detached flush 闭包自身没有 once 保护，由 tracker 保证；PostEnqueue 与 flush 不混用。 -- 停止后未完成分片不确认；Close 可回收临时文件，恢复靠源 checkpoint 重放，不能靠文件删除当作已完成。 - -## 6. manifest、K/H/S 与直接 MERGE - -### 6.1 本批 manifest - -每批在 sink 内有只读 manifest，含：batch_id、binding/incarnation/raw_generation/schema、分片到成员映射，以及每个成员的稳定 event_id、完整 row_key、commit_ts、净动作、payload_hash。 - -manifest 是有界 batch 元数据，可随 spool 暂存；不要求普通 DML durable applied ledger。不能只保存预期行数或摘要：需要能精确判断成员存在、内容一致以及属于哪个事件分片。源恢复可重建新批次/manifest。 - -### 6.2 raw 完整性核验 - -1. 参数化批量写 raw，可拆多个有限 SQL；失败重试同批成员，不改变业务版本。 -2. 读取/核验该 batch 的 raw 逻辑成员集合，核对完整事件身份和内容；重复物理行按相同身份去重，相同身份异内容为错误。 -3. 缺任何 manifest 成员则继续补写/核验，不 MERGE、不确认；额外未知成员说明 batch 绑定错误，停止。 -4. 核验结果若分页需读取完整，不能 LIMIT一页、COUNT相等或比较一个无成员能力的digest就当全集证明。核验本身也在有界批次/内存预算内。 - -### 6.3 集合定义与应用算法 - -```text -M = 已验证的本批 manifest 成员 -K = DISTINCT row_key FROM M -H = 当前 binding/incarnation/raw_generation 合法范围内、row_key∈K 的 retained raw -S = H 中每键最高 commit_ts 的确定净动作（含 DELETE tombstone） -``` - -顺序：冻结该表其他 raw/DDL/compaction → 验证 M 全到齐 → 验证 H 的同 key/TSO 冲突 → 形成 S → MERGE target USING S → 成功终态 → 仅完成 M 对应分片。 - -**不是直接 MERGE 本批旧行，也不是看到 higher raw 就跳过旧行。** 例如 M 有 K@150 而 H 有 K@200：本次实际将 K@200 的净动作写入目标，即使@200之前只是raw写入尚未应用，也能支配@150。不能仅过滤掉@150后声称它已完成。 - -概念 SQL 结构（不是已验证可执行 SQL）： - -```sql --- 在实际实现中使用转义标识符和绑定参数；先核验 manifest/同版本冲突。 -WITH batch_keys AS ( - SELECT DISTINCT row_key FROM RAW_TABLE - WHERE batch_id = BATCH_ID AND raw_generation = RAW_GENERATION -), latest AS ( - SELECT r.* FROM RAW_TABLE r JOIN batch_keys k ON r.row_key = k.row_key - WHERE r.raw_generation = RAW_GENERATION - QUALIFY ROW_NUMBER() OVER ( - PARTITION BY r.row_key ORDER BY r.commit_ts DESC - ) = 1 -) -MERGE INTO TARGET_TABLE t USING latest s ON TYPED_PK_MATCH -WHEN MATCHED AND s.action = 'DELETE' THEN DELETE -WHEN MATCHED AND s.action = 'UPSERT' THEN UPDATE SET EXPLICIT_COLUMNS -WHEN NOT MATCHED AND s.action = 'UPSERT' THEN INSERT EXPLICIT_COLUMNS; -``` - -ROW_NUMBER只在同版本净动作/内容已验证一致后用于消除物理重复，不能拿它解决冲突。真实SQL必须包含完整binding/代际约束、精确payload投影/NULL规则，不能跨schema误读。 - -### 6.4 覆盖条件 - -本批完成需同时证明： - -- M完整进入raw；K准确对应M，没有未解释缺失/额外成员。 -- 对每个M成员，S包含同key、版本不低于它的合法净动作；其更高版本被**实际纳入本次MERGE**。 -- SQL无额外LIMIT/漏JOIN/过滤，目标PK匹配/投影覆盖完整S；同表冻结期间核验到MERGE使用一致有效数据，快照/查询计划行为需真实验证。 -- MERGE服务端提交终态明确成功；零业务行修改也可以完成，如删除不存在的键，但不能仅凭返回计数判断成功。 - -S可能顺便应用H中其他批次的更高事件，但只完成M对应callback；其他批次仍走自己的核验/幂等应用，不猜其进度。若只覆盖部分分片，未覆盖部分保持pending，不一次确认整事件。 - -没有stream，stream hint/cursor/no-op消费witness不参与正确性；但MERGE源集合与终态证明仍不可省略。若无法证明上述条件，停止，不加一张写有“计划coverage”的表假装解决。 - -## 7. 提交、重试与恢复：不引入普通 DML applied ledger - -| 情况 | 行为 | -|---|---| -| 部分 raw INSERT 成功 | 不确认，补写缺少成员；同事件物理重复允许但逻辑去重 | -| 明确未提交/回滚 | 在旧请求不再运行且所有权有效后重试 | -| MERGE成功、callback之前崩溃 | 从checkpoint重放，重建manifest并按K/H/S重应用，不要求恢复旧batch_id | -| MERGE成功、响应丢失 | COMMIT_UNKNOWN，不确认、不发下一批；先核验旧请求终态 | -| 业务提交已确认，仅操作统计解析失败 | 保留成功，统计unavailable，不为补统计重做SQL | -| 查询记录缺失、断连、取消返回nil | 不能推导未提交/已终止，进入人工恢复 | - -最小 TargetClient 需能保留当前操作/query ID、读取完整结果和服务端执行终态，并区分执行错误、传输错误、值解码错误与统计解析失败。QueryContext返回Rows不是通用提交证明，驱动Kill/Close成功也不是服务端终止证明。 - -不要求把每次普通DML attempt写入durable applied ledger。代价是：重启丢query关联、旧查询查不到时，**不承诺自动恢复**；人工确认所有旧mutation已终结且目标独占，再从安全checkpoint幂等重应用。进程退出/固定sleep不能替代确认。业务幂等不意味着可以允许旧SQL晚于新批次提交。 - -DDL仍有必要durable journal，因为重复TRUNCATE会破坏新代际；不能将“普通DML无需ledger”扩展为“所有持久化恢复元数据都不要”。 - -## 8. 建表、DDL、schema fence 与 TRUNCATE - -### 8.1 建表与历史数据范围 - -首次DML按权威TableInfo建表/核验，接收到CREATE TABLE等受支持DDL时也按事件建表；不查询最新schema解释旧事件。目标已存在须验证结构、任务绑定与PK唯一性，不盲目IF NOT EXISTS接管。没有DML/DDL的历史空表不要求出现在目标，不做全库发现/预建，也不借此引入全量工具。 - -同一raw_generation只接受可由同一projection解释的数据。已完成会改变行投影的DDL后，优先切换新的raw_generation；旧raw隔离，不把旧payload直接CAST成新schema。此前目标行的回填/默认值由已验证DDL处理，后续完整after-image重新建立新generation的raw记录。 - -旧schema事件的跳过依据是durable applied-DDL fence，必须证明该边界前数据已排空或被TRUNCATE合法清除；不是当前schema的更新时间。只有对源顺序前缀/fence契约成立时才可安全退役旧raw墓碑，否则停止而非清理。 - -### 8.2 普通 DDL - -```text -现有 FlushDMLBeforeBlock / 全局 DDL 协调 - → 强制旧分片全部MERGE完成 - → sink核验DDL身份/owner/当前schema - → durable journal PREPARED（含before/after fingerprint及drain fence） - → 执行白名单DDL，取得终态并校验 - → 切换/验证projection和必要raw_generation - → journal APPLIED持久化 - → DDL PostFlush，放行新schema -``` - -支持CREATE、有限非键ADD/DROP/RENAME/MODIFY；默认值/NULL/生成列须明确。复杂多操作、PK定义更改、DROP/RENAME表等未定义项停止。二级索引DDL只有不改变行身份/投影时才可no-op，raw-change契约持续有效。 - -### 8.3 DDL/TRUNCATE最小恢复顺序 - -先关闭新写→人工/受控确认旧SQL终结和目标独占→核验binding/journal→恢复pending DDL/reset→安装已应用fence→协调旧DML重放→开放当前代际。 - -| durable状态 | 恢复行为 | -|---|---| -| PREPARED，尚未发送或明确未提交 | 核验before条件，按同DDL身份执行；未知旧SQL先协调 | -| 执行结果不明 | 查终态/校验可唯一证明的after状态；不能只见列不存在就猜成功 | -| APPLIED但上游checkpoint尚未保存 | 相同DDL只跳过，不重复破坏性操作；旧DML按已证fence处理 | -| metadata冲突/丢失 | 停止、人工核验或重新初始化，不用重新TRUNCATE“修复” | - -TRUNCATE记录源DDL身份、旧/新表血缘和代际，不只用表名/全局MAX(TSO)： - -```text -DRAINED → PREPARED(new incarnation CLOSED) - → TARGET_RESET已确认 - → 新raw_generation/binding校验完成 - → APPLIED持久化 - → 才可开放新代际DML -``` - -PREPARED中重做reset仅在旧SQL明确终结、目标独占且新代际从未开放/无新写入证据充分时允许；APPLIED之后绝不重做。journal写入本身不确定就停在该步骤，不越过。不存在假定的强制唯一约束/跨DDL事务万能锁。 - -重点验收：TRUNCATE@500完成，新代际@510写入后从@490重放，@510不得被再次清空。此顺序属于sink必要恢复，不因此重写调度框架。 - -## 9. 受控 PoC 的排他与生命周期边界 - -首版只在隔离单capture、静态独占目标环境验证；禁止正常运行时跨owner漂移，自动替换/重启后也不能未经核验直接开放写入。`enable-table-across-nodes=false`仅限制拆表分布，不等于禁止整表HA，不能将其当唯一保护。 - -失租/fatal立即关本地gate，禁止新raw/MERGE/DDL/compaction；已发送的远端SQL仍需查终态。旧MERGE晚到也可能覆盖新值，旧TRUNCATE晚到可清空新代际，去掉stream并不解决这些问题。 - -若目标版本/接口不能可靠定位并终结旧SQL，保持停机，人工核验或重初始化；不发布自动HA。自动HA若需要目标侧fence或更多控制面工作，单独RFC，不夹进新增sink的首版合入。 - -## 10. 清理、容量与成本 - -- raw保留每键最新版本及删除墓碑，删除严格被支配历史；最新物理重复需安全去重策略，不因为SQL成功就认为空间不再增长。 -- raw compaction与同表写入/MERGE串行；不能删manifest仍需要的成员/恢复证据。未处理完的批次受本地预算约束，不能清raw后才发现还需要核验成员。 -- 持续新键INSERT→DELETE会让target很小但raw增长，必须监控raw/墓碑/retained快照、历史键及扫描成本，达到阈值停新raw；旧SQL/清理在途也计入告警判断。 -- 首版是**监控型停写阈值**，有测量延迟、物理放大和多capture超调风险；不承诺全局原子feed硬额度。受控PoC不引入跨capture预算服务；将来需要严格额度时单独评估静态切分或已有外部配额。 -- 不删最新墓碑解决超额。恢复方式是显式扩额、安全去重或新一致快照重初始化；物理文件/快照释放可能延迟，不把逻辑删除量当已回收字节。 -- 无stream不等于便宜：K关联H可能扫很多raw。固定事件率、行宽、历史键数、热点、小表数量、可见延迟SLO，实测Databend扫描/credit、TiCDC CPU/RSS/spool IO，再与旧链比较。 - -## 11. 配置草案与增量接入边界 - -所有Databend新字段均未实现；以下是可读的压测起点，不是可运行配置。认证通过受保护配置/运行时凭据注入，不把真实TOKEN写入URI/日志。 - -连接标识：`databend://HOST:PORT/DB_NAME?sslmode=require`。 - -```toml -[filter] -rules = ["DB_NAME.*"] - -[scheduler] -enable-table-across-nodes = false # 不是禁用HA；PoC还需隔离部署和恢复准入 - -[sink.databend-config] -auto-create-table = true -batch-max-rows = 5000 -batch-max-bytes = 8388608 -flush-interval = "1s" -worker-count = 4 # 跨表，同表串行 -entry-queue-max-events = 1024 -sink-memory-quota = 268435456 # 含入口引用/编码/回读/metadata，不等于进程RSS -spool-memory-quota = 67108864 # 包含在sink-memory-quota内 -spool-disk-quota = 10737418240 -raw-observe-interval = "30s" -raw-stop-threshold-bytes = 1099511627776 # 示例告警/停写阈值，非硬计费quota -``` - -镜像模式固定处理DELETE，内部固定raw-change=true；不开放allow-delete=false、stream消费hint、DML ledger模式、自动HA、分布式预算开关。单行/交付规模、SQL请求开销和内存分池必须校验兼容，不能让queue count乘以巨大chunk绕过字节预算。配置减少到用量以下时停新输入继续安全排空，不直接删文件。 - -sink从TiCDC交付的增量事件开始处理，遵守已配置start-ts与现有checkpoint，不负责搬运该边界之前的存量行。用户若要求目标成为全量镜像，需要在本功能之外准备可衔接的目标基线；本设计不指定工具、不实现或编排该流程，也不强制所有增量用例先做全量导入。没有基线且历史行一直不变时，这些行不会自动出现在目标，这是增量范围边界而非自动补数能力。源保留窗口不足仍应失败，不承诺无限期背压恢复。 - -## 12. 具体验收：先实现最小路径，不继续扩通用协议 - -以下均为待执行测试，未因为本文写出而通过。 - -| ID | 场景 | 必须证明 | -|---|---|---| -| A01 | PK固定仅UK更新；raw=true初建/重注册；合法真实PK变化对照 | 普通UPDATE无人工DELETE/INSERT冲突；未验证变键不虚报支持 | -| A02 | 一批多个raw子写，部分成功、重复物理行、缺成员/同身份异payload | manifest逐成员核验，缺失不MERGE、不提前回调；跨batch重复收敛 | -| A03 | UPSERT@100→DELETE@200→迟到UPSERT@150；higher raw已入但未应用 | K/H/S实际应用最高净动作，删除不复活，不因仅raw存在就确认 | -| A04 | 实际MERGE SQL、不同schema/代际、查询投影失败、零目标修改 | 完整S应用与终态成立；无stream hint依赖；不跨schema解释旧payload | -| A05 | F1完成时F2未登记；事件总量>spool；Seal/Apply/取消竞争 | OPEN不确认、分片可独立回收、全集完成恰回调一次、驻留有界 | -| A06 | 正常磁盘压力、冷表pin、删除失败、入口无法接管 | 常规压力由背压/排空吸收；异常非阻塞fatal，无假释放/无限队列 | -| A07 | 已接管后transport reset；Add接管失败；Run错误上报/控制面关闭 | reset不丢数据；fatal后无新接管/SQL；旧blocker不伪flush；新manager从安全checkpoint恢复而非旧session复活 | -| A08 | MERGE提交后响应丢失、callback前崩溃、统计解析失败、旧query查不到 | 无未知结果确认；终态排他后幂等重应用；统计失败不重做；查不到时人工停机 | -| A09 | DDL各journal中断、schema generation切换、旧DML/DDL重放 | 正确fence/投影，APPLIED前不开新schema；不靠最新版schema猜跳过 | -| A10 | TRUNCATE每步骤崩溃、新数据写入后回放旧TRUNCATE、晚到旧SQL | 新代际不再被清空；旧SQL终态无法证明则不放行 | -| A11 | target小但历史键churn；同SLO旧链/无stream SQL成本对照 | 成功清理下仍能告警停写；量化延迟/超调，不声称严格quota；扫描/费用实测 | -| A12 | start-ts/checkpoint接入、首条DML/CREATE建表、类型边界/复合键/collation | 只验增量交付边界和数据转换，不要求全量工具或历史空表预建；既有bend-cdc增量核心案例逐项对照 | - -### 12.1 当前仅保留七个落地门槛 - -| ID | 待关闭事项 | 处理边界 | -|---|---|---| -| Q1 | manifest成员核验、K/H/S SQL、同版本净事件与重放 | sink内实现+A01–A04，不能用新ledger替代集合证明 | -| Q2 | 非阻塞接管、正常背压与fatal生命周期闭环 | sink为主+A06/A07；仅证明确需时加manager小接入 | -| Q3 | SEALED tracker、spool字节预算和回收活性 | sink内实现+A05/A06，不能承诺框架全RSS硬界 | -| Q4 | 服务端终态与旧SQL排他 | sink客户端+A08/A10；未证明则人工恢复PoC，不扩自动HA承诺 | -| Q5 | DDL journal、schema fence、TRUNCATE与恢复分支 | sink为主、schema/recovery小接入+A09/A10 | -| Q6 | raw保留/去重、监控停写、扫描与机器总成本 | sink内+A11，严格跨capture额度不在首版 | -| Q7 | 事件元数据建表、类型矩阵、真实源契约和兼容接入 | 增量mapper/建表及小注册分支+A01/A12；不包含全量工作包 | - -Q1–Q7均未以实现/实验关闭。功能或规模缩减必须可执行；若最小路径在既有框架不能正确停止/恢复，先提出精确小补丁，不继续扩写通用信用/progress/调度系统。 - -## 13. 交付顺序、灰度和回退 - -1. P0：固定server/driver，验证K/H/S SQL、raw完整核验、提交终态、原生事件及现有fatal关闭调用链；不创建生产资源。 -2. P1：受控单capture、PK表DML纵切，完成buffer/spool、manifest、tracker、raw/MERGE、错误停机；覆盖A01–A08。 -3. P2：事件驱动建表/类型、有限DDL与TRUNCATE恢复，对齐bend-cdc已支持的增量核心行为，完成A09–A12；不引入全量导出/导入工具。 -4. P3：独立影子目标、对齐快照/水位比对，测容量与成本后决定是否继续生产化；自动HA另议，不因PoC通过默认开放。 - -旧链写旧目标，新sink写独立影子目标，不共享raw/journal；受控drain/对齐边界比较，不能任意时刻比两个异步目标行数。切读前保留旧链回放窗口，回退先追平旧链再切读。内部raw/journal格式版本化，不直接复用bend-cdc文件游标或不兼容旧schema。 - -## 14. 源码证据与验证状态 - -以下是固定源码证据，不代表新sink已经运行。路径基于Git对象，无需当前checkout存在同文件。 - -- [Sink接口/factory](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/sink.go)。 -- [raw-change初始化与Run错误传递](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go)；[session注册/重注册传播](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/eventcollector/dispatcher_session.go)。 -- [UK拆分](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/eventservice/event_scanner.go#L970-L1014)；[IsUKChanged](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/codec.go#L219-L259)。 -- [同步Add与progress先登记](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatcher/basic_dispatcher.go#L293-L335)；[progress按flush移除](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatcher/table_progress.go#L97-L125)。 -- [Run错误进入handleError，L393–404](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go#L393-L404)；[错误队列/持续上报，L699–749](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go#L699-L749)；[TryClose/LocalFence/stopWritePath，L1057–1173](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go#L1057-L1173)。 -- [flush/enqueue及Detach闭包](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/dml_event.go#L660-L711)；[spool实现](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/sink/spool/spool.go)。 -- [TableSchemaStore类型分支](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/table_schema_store.go)；[dispatcher恢复专属分支](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/task.go)。 -- bend-cdc：[实际MERGE适配入口](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go)、[raw/MERGE/compaction](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go)、[DDL epoch](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/schema_epoch.go)、[TRUNCATE](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/truncate.go)、[现有MERGE结果标记](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/merge_result.go)。 - -验证状态： - -- 前序已运行bend-cdc既有两包本地测试通过，真实Databend连接未配置、相关集成用例跳过；不能当作新sink证据。 -- 本轮只统一文档和评审入口，保留v0.2历史快照；未改业务Go代码、未切分支、未构建/部署、未创建PR。 -- A01–A12、K/H/S真实SQL、错误控制面关闭链、终态/恢复、扫描和费用实验均未执行。 -- 文档结构/TOML/Git对象链接检查与上述运行验证分开报告。去掉旧协议是范围裁决，不是正确性已被证明。 diff --git a/docs/design/databend-native-sink-design-review.md b/docs/design/databend-native-sink-design-review.md deleted file mode 100644 index 95780135e9..0000000000 --- a/docs/design/databend-native-sink-design-review.md +++ /dev/null @@ -1,395 +0,0 @@ -# 定稿 v1.1：TiCDC 原生 Databend Sink——按 bend-cdc 既有语义迁移 - -| 项目 | 当前裁决 | -|---|---| -| 版本 / 日期 | v1.1 / 2026-10-25；v1.0定稿后按用户要求增加conflict-key及指定DDL跳过 | -| 状态 | 设计已接受，首版实现及受控真实链路验收通过；不表示生产HA/成本已验收 | -| 实现分支 | `feat/databend-native-sink`，从固定新架构commit `2bf6dacd0aec9ac2fad464600e47341f9aa951f2` 创建 | -| 正确性基线 | 对齐固定版本 bend-cdc 的 TiDB 增量路径，不为迁移另造更强的通用同步系统 | -| 实施主线 | 迁移现有 Databend raw/MERGE、mapper、schema epoch 核心；新增原生事件、spool、TiCDC回调适配 | -| 输入与来源默认值 | 与固定版本CloudStorage默认配置保持拆分模式 raw-change=false；首版固定raw source，stream仅保留后续验收入口 | -| 不新增 | batch manifest持久化/逐成员读回、K/H/S新SQL算法、event_id/payload_hash体系、AppliedCycle/DML ledger、raw_generation协议、collector信用/progress迁移、分布式锁/预算服务 | -| 缓冲 | 保留有界内存 + 限额本地spool + 现有PostEnqueue背压 | -| 全量范围 | 不支持/编排Dumpling、CSV、COPY INTO，不实现历史空表发现/预建 | -| 首版边界 | 沿用单逻辑目标表单writer、目标静态独占前提；故障恢复先验证，不承诺新建自动HA体系 | -| 已验证 | 本地/race/lint/build及真实TiUP→CDC→staging Lake：9组PASS、32列、磁盘spool/排空重启；见§8.2–8.3，未验收生产HA/成本 | - -**本文是唯一当前定稿设计。** [v0.3](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review-v0.3.md)和[v0.2](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review-v0.2.md)仅保留历史分析；其中manifest、K/H/S、raw generation等不是并行实施要求。设计评审不再以新增通用协议扩大范围；实现中若发现不可规避的范围变更，单独说明并复审。 - -## 0. 固定源码与原则 - -- TiCDC：`/Users/jiangjianyuan/work/git/pingcap/ticdc`，commit `2bf6dacd0aec9ac2fad464600e47341f9aa951f2`，origin `git@github.com:sdojjy/tiflow.git`。 -- bend-cdc：`/Users/jiangjianyuan/work/git/pingcap/webapi` 下 `bend-cdc`，commit `b6481621021e82aee659ad88e8e6c00f6dd11906`。 -- 设计期间曾在旧架构master / `03ed2432bd7782198742d3aa20e07edcc47097e6` 下按Git对象核对；用户批准实现后已创建并切到上述新架构基线的实现分支，没有在旧master开发。 - -原则：**替换输入和运行时外壳，不先重设计已经存在的目标应用算法。** 新增约束必须归入三类之一：既有行为迁移、移除S3后必须补的接入、可延期的增强；不能只因理论上可加强就进入首版。 - -“同样消费TiCDC”不意味着故障语义完全相同：原链有S3持久缓冲、文件/epoch顺序与独立消费进程；直接sink必须把必要的顺序/完成语义接到现有dispatcher/callback。也不能把旧代码注释/假设当成全部合法输入都已正确的证明。 - -本轮只补六个局部缺口：拆分输入兼容、规范化一次、raw source默认值、内层UNKNOWN截断、spool无成功回调清理、落库精度排序。没有重新引入manifest/ledger/witness或collector/progress协议。默认值来自固定源码，不代表已核实任何线上changefeed配置；旧链若显式启用raw=true，其PK更新行为需另列既有缺口。 - -## 1. 实际代码基线：已有的保证、假设与限制 - -| 主题 | bend-cdc实际代码 | 迁移处理 | -|---|---|---| -| 核心输入 | CanalJSONMessage转common.Event，再经normalizeCDCEvent交CDCManager | 原生DMLEvent经NativeNormalizer进入规范行入口，不二次做Canal解码 | -| 版本 | 优先 `_tidb.commitTs` → TiDBCommitTS → event_pos；无扩展时兼容退回毫秒TS | 直接用原生CommitTs，复用排序规则；不继承“TSO每行唯一”的错误注释 | -| 主键 | 完整PK集合canonical化、复合conflictIdentity与MERGE JOIN | 直接从TableInfo获取相同键语义，保留复合键测试 | -| raw记录 | `id, table_name, raw_data, add_time, event_pos, action` | 优先保持形状，不为迁移增加manifest/event_id表或新一套版本身份 | -| UPDATE/DELETE | UPDATE使用After，DELETE使用Before；默认拆分模式提供PK/UK变更的DELETE/INSERT | 保持默认拆分和输出顺序；不把raw=true变键UPDATE直接交After-only核心 | -| 顺序与重放 | 严格模式按(event_pos,add_time)选候选，并查询retained raw抑制旧值 | 先迁移既有SQL和清理语义；同TSO合法净动作须做源事件对照，不用到达时间掩盖新增差异 | -| 清理 | 删除被更高版本支配历史，保留每键最新watermark/tombstone | 保留，不引入TTL删最新墓碑；不承诺历史键集合有界 | -| stream | 创建append-only stream；失败时用raw作为同一个MERGE builder的source | 首版显式固定raw；stream代码可保留，但未验收不创建/消费，不用静默fallback当完成证明 | -| 完成进度 | raw成功后writeGen递增；mergeTable捕获startGen，成功后mergedGen推进；Flush排空 | 复用批次/代数级完成，映射TiCDC callback，不默认读回每行 | -| DDL顺序 | 文件schema版本边界先Flush，ApplySchema保存pending/applied epoch | 使用现有TiCDC屏障代替文件队列，迁移epoch/reconcile | -| TRUNCATE | Flush→pending epoch→drop stream/raw→truncate target→applied | 迁移现有reset顺序，测试重启重放，不增加多套代际状态机 | -| 不支持DDL | 某些DDL只warning并推进epoch，并没有执行目标DDL | 不把warning跳过算“已支持”；只对齐已正确投影的能力，不承诺任意DDL | - -代码事实不等于运行证明。例如writeGen/mergedGen假定MERGE覆盖了已记账raw批次；QueryContext返回Rows被现有代码视作executed；这些前提接入callback前需要局部验证，不能靠新增一套架构掩盖，也不能不经验证就宣布安全。 - -## 2. 最小实现：三个主体，而非新框架 - -```text -现有TiCDC dispatcher / TableProgress / DDL协调 - → NativeAdapter - TableInfo、CommitTs、before/after → 等价内部事件/原生类型转换 - → Buffer + Spool + TableWorker - 有界接管、按表攒批、内存/本地磁盘、有限并发、callback记账 - → 迁移的DatabendCore - 建表、参数化raw INSERT、已有MERGE/compaction、schema epoch/TRUNCATE -``` - -对每张表，第一版由一个worker串行控制： - -```text -取一个有界批次 - → 全部raw子INSERT成功 - → 调用等价的FlushTable（复用现有Flush/mergeTable逻辑） - → 确认本轮排空成功 - → 完成覆盖的分片、释放spool - → 源事件全部完成时flush callback一次 -``` - -这样无需在单表内并发raw写入与MERGE，也无需manifest逐成员核验。普通INSERT成功按完整SQL执行结果记账；如果分多条SQL，全部成功才使该批具备完成资格。失败/结果未知不确认，局部重试或停止后从checkpoint重放；不把“写了一部分raw”当作一批成功。 - -实现可保留writeGen/mergedGen或等价本地批次序号，属于进程内callback记账，不是持久化应用日志。移植核心算法，不照搬每表连接池、Background context、无界goroutine/重复后台循环，也不让TiCDC依赖整个webapi服务模块。 - -## 3. raw/stream/MERGE如何复用，哪些不能盲抄 - -### 3.1 不再重设计MERGE来源：首版固定raw - -保留现有 `buildMergeSQL(target, source, raw, schema)`、复合键JOIN、列投影和compaction；首版调用显式传 `source=raw`，初始化不创建stream，也不在后台自动切换为stream。复用现有raw分支，不是新K/H/S算法。 - -- 首版使用现有full-raw路径：单writer停止新写，完整MERGE当前epoch的raw，再确认该批。无stream cursor、消费hint或witness依赖。 -- stream作为后续增量读取优化保留，不以“删除stream代码”作为简化目标。启用时必须证明当前server/每个连接的有效设置下Flush覆盖所记批次，且验证no-op消费；如果只处理一部分，不能按startGen确认全部，也不能循环现有Flush冒充排空，因为pending可能已经被清零。 -- 不强制新增batch_id/manifest、K/H/S或读回所有raw成员；也不因为不用stream就保证更省钱。full-raw可能扫描更多，性能决定是否需要启用已有stream优化。 - -验收门槛很具体：两次raw INSERT、两个不同键，Flush必须将两键应用后才完成两个callback；加入no-op、旧事件重放。stream消费限额是后续启用测试，不能“与旧代码一样”就提前确认；不新增消费witness或逐成员读回。 - -### 3.2 同TSO和类型：先比较等价输入 - -**本版撤销强制raw-change=true。** 固定CloudStorageConfig未设置该开关时返回false，manager对新增SinkType也默认false；首版沿用拆分模式，不需要新增“强制true”分支。启动/重注册测试确认传播值为false，用户不能把它当调优项切成true。该模式下PK/UK更新由既有scanner拆分；adapter/worker必须保留其DELETE/延后INSERT输出顺序，不能重新逐行交错或仅按TSO排序。 - -例如非聚簇PK从1变2，兼容路径应收到并应用DELETE(1)、INSERT(2)；不能将带oldPK!=newPK的raw UPDATE直接送After-only核心。若意外收到这种UPDATE，首版显式拒绝/停写，不静默仅写After。将来需要raw=true时，须补oldPK删除和同事务顺序适配后再开放，不因此新建事务DAG。 - -source事务TSO不是每行唯一；同key/TSO合法相反动作可能依赖上述既有源顺序。分片/重启后同样要通过最终键集合测试，不用本机时钟创造源业务版本。 - -**规范化恰一次**：迁移核心提供两个边界清晰的入口，汇合后共享raw构造/INSERT/MERGE： - -```text -Canal原始事件 → normalizeCDCEvent（原ordinal/Latin-1等规则） ┐ -Native datum → NativeNormalizer（按FieldType取label/bytes） ┴→ NormalizedRow → raw构造/INSERT -``` - -NormalizedRow携带规范PK/列值/动作和CommitTs；Native路径不再进入normalizeCDCEvent。Source仍是TiDB，不通过伪装其他Source绕过解码；旧getTiDBSequence仍依赖它，或在规范行接口中显式传递同一CommitTs。 - -比较的是规范化后的键、payload、动作、TSO，而非native datum与Canal原始字符串。数字label `ENUM('2','x')`：Canal ordinal `"1"`和native label `"2"`均只能最终得到`"2"`，不能二次解码成`"x"`。还需SET高位、全字节二进制、精确decimal、大整数JSON、SQL NULL/JSON null、时间精度；raw JSON序列化不再次做类型解码。 - -### 3.2.1 add_time按落库精度单调，而非仅Go时间单调 - -既有核心按1ns递增，但raw参数只输出6位小数，可能落成相同时间。依赖同key/TSO顺序的兼容路径需在raw构造中局部修正：先将candidate量化到实际存储/参数精度（当前格式为微秒），若不大于该表lastStored，则取lastStored+1个存储精度单位，再格式化。不要先做纳秒比较再截断。 - -单writer按源顺序调用该分配器；测试必须检查最终SQL参数及真实raw时间/目标状态，而不只检查内存Time.After。add_time仍是既有同TSO决胜辅助，不成为新的业务版本体系，也不解决任意源冲突/失租并发；同payload纯重复不作为该错误反例。 - -### 3.3 完成结果与旧缺口 - -**必须在迁移的客户端/核心最内层分类，不能只在外层worker判断UNKNOWN。** 旧executeMergeInto在QueryContext返回Rows后设executed，后续Rows.Err也携带true；旧mergeTable会把executed错误改成成功，其他临时错误则可能已在内部重发。原分支不能透明照搬。 - -迁移后使用最小提交状态 `COMMITTED / NOT_COMMITTED / UNKNOWN`，统计解析错误单独携带；raw写、MERGE和显式COMMIT同样适用。每次结果进入核心后，**先判断UNKNOWN，再判断观测失败或可重试错误**： - -- COMMITTED且仅统计失败：业务成功、统计unavailable，不为补统计重做。 -- 明确NOT_COMMITTED且可重试：才进入既有有界重试；不能把所有网络错误都归入该类。 -- UNKNOWN：立即退出内层循环，上层不得按transient再重试；不推进mergedGen/pending，不callback，不进下一批/DDL。先核验旧SQL终态，取得排他后用retained raw幂等重应用；无法证明就停写、人工恢复。 - -mock检查错误分支与物理请求次数；真实实验检查提交响应丢失后的恢复。mock构造的“首页后失败”不是声称真实MERGE已复现。只改Databend客户端/核心，不引入DML ledger、AppliedCycle、查询注册中心或分布式fence。 - -## 4. 哪些仍是直接sink必须新增的适配 - -### 4.1 MERGE后才确认TiCDC - -bend-cdc已有MergeProgressReporter/applied watermark区分“文件处理/raw写入”与“目标已MERGE”。原链的文件游标、S3持久数据和lake applied进度不是同一件事。 - -新sink用现有PostFlush表达目标应用完成，不能在raw INSERT成功时就推进TiCDC checkpoint。否则去掉S3后，可能没有一个可靠消费者负责把已确认raw继续应用。这不是追求比bend-cdc更强的exactly-once，而是把已有目标完成语义接到不同的上游接口。 - -同一事件的回调由本地completion对象持有；分片先登记后发布，编码完成Seal，`SEALED && pending==0`时once回调。已应用分片可独立回收，避免大事件等整事件确认才释放空间。只保留有界计数/未完成描述符，不增持久化tracker。 - -### 4.2 保留spool和现有流控，不改collector - -- 有界入口、内存热数据、限额磁盘、字节/元素数记账；单行超上限明确失败，不无限内存直通。 -- 正常压力延迟PostEnqueue、强制小批Flush；磁盘物理未回收继续计费，压力唤醒钉住segment的表。 -- 已接管事件不因transport session reset丢弃。失败不伪造PostFlush，不清迁旧TableProgress。 -- AddDMLEvent不无限等待共享线程；真正无法接管/IO致命错误关闭sink本地admission和写gate，Run报首因；旧manager/dispatcher需最终结束，不只reset旧session继续跑。 -- 先验证现有关闭链，确有缺口仅补Databend fatal小分支。上游chunk已在sink前分配，不承诺全进程RSS硬上限；不为此改collector/dynstream信用协议。 - -**spool包装有两个不可透明继承的行为**：固定spool的Discard会先跑entry postFlush再Release；TryEnqueue对超过disk quota的entry有oversized内存接受分支。native包装必须隔离： - -1. 源事件成功callback只由SEALED tracker持有；spool message/entry的postFlush列表保持为空，也不能间接挂上会递减tracker成功计数的回调。取消/fatal/Close仅走无成功回调的资源Release；不调用语义不匹配的Discard来确认未应用数据。PostEnqueue只用于背压，不转成成功确认。 -2. enqueue前按包含metadata/编码开销的保守字节上界拆分entry；单行仍超限就拒绝，不进入oversized旁路。若缺少一致大小估算接口，仅补spool小型计量/包装，不改collector；“接受后发现oversized再失败”不能算预防超额。 -3. 正常MERGE成功时由worker显式完成tracker并释放分片；重入/迟到Release不得再次完成源事件。分别在raw前、raw后MERGE前注入fatal/Close，源flush次数必须为0；普通压力仍可提前Flush恢复，而非频繁fatal。 - -## 5. DDL：迁移现有epoch，而不是新建DDL协调系统 - -沿用 `ApplySchema`、`SchemaChanges`、`SchemaEpoch/SaveSchemaEpoch`、reconcile和TruncateCDCTable已有语义： - -```text -TiCDC现有屏障排空旧DML - → 读取目标epoch；已应用的同DDL跳过 - → Flush该表 - → 保存pending epoch - → 翻译/执行已支持DDL或TRUNCATE reset - → 保存applied epoch - → DDL PostFlush，再放新DML -``` - -固定TiCDC的SchemaFile.Build明确使用event.FinishedTs填TableVersion，因而迁移字段映射直接固定： - -| 字段 | native取值/用途 | -|---|---| -| raw event_pos | DMLEvent.CommitTs | -| 实际DDL epoch | DDLEvent.FinishedTs | -| TableInfoVersion/更新时间 | 解释列元数据，不自动成为applied fence | - -还必须迁移旧DatabendMergeSink.writeBatch的外层保护，不只复制CDCManager：写前读epoch；CommitTs<=epoch的旧事件仅在其排空/DDL恢复前提已成立时按旧规则跳过；pending时新数据禁止写入，先reconcile到applied。启动加载schema只读/核验，不能凭空SaveSchemaEpoch标成已处理历史。epoch==pending也不等于DDL已经执行，实际DDL仍按原恢复分支处理。 - -TRUNCATE继续清理旧raw/stream并清目标，applied前不得写新代际。测试执行后checkpoint未保存时重放，不重复清空新数据。若旧epoch记录不足以识别特定中断，只补必要字段/检查，不默认增加before/after全量快照、独立raw_generation和多份journal。 - -普通增量自动建表仍使用权威事件元数据；类型映射/有限列DDL按bend-cdc现有支持范围。warning跳过但未执行的DDL保留明确限制，不能当镜像一致性保证。复杂/无证明操作可停止，不为追平“任意DDL”扩大首版。 - -## 6. 外部改动边界与非目标 - -| 范围 | 当前处理 | -|---|---| -| 新Databend包/测试 | 主要工作量：提取迁移核心、native adapter、buffer/spool、worker/callback | -| scheme/type/config/factory/metrics | 常规sink注册小改动 | -| raw-change注册/重注册 | 首版沿用默认false；复用并测试现有传播，不新增强制true分支 | -| TableSchemaStore/recovery | 增加必要类型/恢复分支；不伪装MySQL，不改进度算法 | -| manager fatal关闭 | 仅验证不足时的有限小补丁 | -| collector/dynstream/TableProgress/调度核心 | 不做通用协议重构 | - -不实现Dumpling/CSV/COPY INTO、全量编排、历史空表发现/预建；不实现新的全库schema snapshot服务；不要求跨表事务原子性、分布式预算、目标分布式锁或通用自动HA系统。 - -原有单目标writer、无外部写入的前提必须在TiCDC部署中成立，不能因为来源都是TiCDC就自动假定。across-nodes=false只禁止拆表分布；旧owner在途SQL不是客户端cancel就结束。首版在受控单capture/目标独占下做迁移与恢复实验，不为新增sink顺带重做HA。无法确认旧SQL结束时停写，这不是已交付生产HA。 - -远端raw增长按现有compaction和监控告警处理，可加本地阈值停新raw；不叫严格跨capture quota，不新增容量账本。历史PK新增后删除仍会保留墓碑，这项成本按旧链同样负载比较，不因它存在就要求另建watermark存储系统。 - -## 7. 迁移验收：证明不退化，另测接入差异 - -以相同输入和相同能力前提运行旧Canal→bend-cdc核心与新native→迁移核心，比较最终目标状态、错误与DDL处理。不是只看“两个都返回nil”，也不把旧实现的已知错误当金标准。 - -| ID | 迁移测试 | 核查重点 | -|---|---|---| -| T1 | 两路径规范化结果，数字label ENUM、SET高位、全字节binary、decimal/JSON/NULL/时间 | 规范化恰一次；Source/CommitTs不改丢；键、payload、动作、TSO等价 | -| T2 | 默认拆分、非聚簇PK单行变键、UK更新、合法同TSO键复用、同微秒时间、跨分片/重启 | 旧键消失新键唯一；保留源DELETE/INSERT顺序；检查SQL时间参数及raw落库精度，墓碑阻止旧值复活 | -| T3 | 首版raw source、低流量/no-op；后续stream强制部分消费 | 目标真正覆盖记账工作才确认；stream不得只凭pending/mergedGen自证排空，未消费B不得回调B | -| T4 | raw子SQL失败/UNKNOWN、MERGE首响应/提交响应丢失、callback前重启 | 最内层UNKNOWN前置截断，检查请求次数不重发；不提前确认，终态排他后重放收敛 | -| T5 | FinishedTs映射、外层epoch过滤、pending阻新DML、TRUNCATE中断/重放 | bootstrap不写applied fence；旧schema不污染新结构，旧TRUNCATE不清新数据 | -| T6 | OPEN/分片回收、spool超限、raw前/raw后MERGE前fatal/Close、reset、压力 | 源flush不得由Discard/清理触发，未应用为0；无oversized旁路，正常压力不变异常，旧生命周期结束 | -| T7 | 同吞吐/行宽/历史键/热点/延迟SLO比较旧链和新sink | 记录扫描及full-raw重复更新保留键造成的目标重写、CPU/RSS/IO/credit与总费用；不预先承诺比例 | - -T1–T7是完整迁移验收门槛：当前已有其中的本地单测/HTTP模拟覆盖，真实TiDB→Databend、原生/Canal差分及成本实验仍待执行。完成它们所需的最小Databend局部修正是范围内工作；把它们替换成新调度/交付/事务协议不是默认答案。 - -## 8. 本轮已执行与未执行 - -源码静态核对覆盖：DatabendMergeSink输入/PK/配置、CDCManager raw/Flush/mergeTable/SQL/compaction、MergeProgressReporter/applied watermark、ApplySchema/schema epoch/TRUNCATE、现有相关单测与TiCDC回调接入。 - -设计阶段v0.4在bend-cdc执行（v0.4.1评审时未重复运行，也尚未实现新sink）： - -```text -go test ./pkg/sink/databend ./pkg/source/tidb -count=1 -ok databendcloud/bend-cdc/pkg/sink/databend 20.922s -ok databendcloud/bend-cdc/pkg/source/tidb 8.784s -``` - -未配置BENDCDC_TEST_DSN/TIDB_DDL_INTEGRATION_DSN，相应真实warehouse测试跳过。这证明既有本地单测/模拟测试通过，不证明stream全量消费、driver终态或新sink恢复已实测。虽然source包包含全量工具单测，运行它们不意味着迁移那些工具。 - -v0.4.1新增静态核对：CloudStorage raw-change默认false/manager默认值、normalizeCDCEvent的Canal专属解码/getTiDBSequence、spool Discard/oversized分支、raw时间格式与单调分配、SchemaFile.Build的FinishedTs映射、writeBatch外层epoch过滤。在v0.4.1评审时，六项修正仅在设计中明确，未标记为代码已修复；定稿后的首批实现状态见下文。 - -上述记录属于设计阶段。用户批准后已创建实现分支，现已完成sink运行时接入与主程序本地构建，并在2026-10-25通过受控真实TiDB/Lake链路（§8.3）；未部署共享CDC、未做在途真实故障或成本实验，未提交或推送。历史版本不作当前规范。 - -### 8.1 首批实现记录（2026-10-24，历史进度） - -首批代码集中在 `/Users/jiangjianyuan/work/git/pingcap/ticdc/downstreamadapter/sink/databend/`：规范行入口、复合键身份、参数化raw/既有MERGE/compaction SQL、微秒时间分配、提交状态前置截断、SEALED completion及对应单测。错误码位于 `/Users/jiangjianyuan/work/git/pingcap/ticdc/pkg/errors/databend.go`。 - -当时仅为核心实现切片，不是已可创建changefeed的完整sink（后续接入见§8.2）：真实HTTP/driver终态客户端、DMLEvent datum适配、配置/factory注册、spool worker、自动建表/DDL epoch恢复尚待后续接入。当前writer通过执行函数边界要求明确提交结果，不能将测试注入的结果当真实服务端终态协议已完成。完整T1–T7仍待后续集成验收。 - -本批5个生产文件、4个测试文件（15个测试函数），没有修改collector、TableProgress、dynstream或scheduler；未注册不可用的scheme/factory。验证记录： - -```text -GOMAXPROCS=4 go test -p 2 ./downstreamadapter/sink/databend ./pkg/errors -count=1 -PASS（两个包） -GOMAXPROCS=4 go test -p 2 -race ./downstreamadapter/sink/databend ./pkg/errors -count=1 -PASS（两个包） -GOMAXPROCS=4 go vet -p 2 ./downstreamadapter/sink/databend -PASS -``` - -gci/gofumports、当前文档本地链接/代码围栏/空白检查通过。单测覆盖规范值保持、实际raw参数的微秒排序、SQL构造、子INSERT/MERGE未知结果不重发、已提交后统计失败不重试，以及OPEN/SEALED/失败/并发重入回调。SQL仅检查构造和调用顺序，未在真实Databend执行；尚无完整sink运行入口，因此未运行sink集成、真实故障与性能测试，也未执行全仓测试/主程序构建。 - -### 8.2 运行时接入记录（2026-10-25） - -实现分支仍为 `feat/databend-native-sink`，基线未变；没有提交、推送或更改webapi。 - -- 原生接入：`native.go`直接读取TableInfo/typed chunk；ENUM/SET取label、binary保留字节、decimal/大整数/JSON保持精确文本，SQL NULL与JSON null分开。TIMESTAMP按TiCDC配置的源时区转UTC；DATETIME保留wall clock，HTTP会话固定UTC。PK变键的未拆分UPDATE显式拒绝。 -- 运行时：`sink.go`/`lane.go`固定数量lane按逻辑目标表hash，保持单表串行；源事件池按字节/元素有界，Add不阻塞共享handler。正常spool压力提前排空/延迟PostEnqueue，容量异常停止整个旧生命周期；不操作collector/progress条目。 -- 攒批：支持行数/编码字节/定时/DDL屏障/配额压力触发；大事件分片边应用边回收，封口前不flush源事件。复用共享spool；新增的通用接口仅有`AccountingBytes`，用于接纳前阻止oversized旁路。源成功回调始终不放入spool消息。 -- 客户端：直接使用Databend HTTP Query API的`params`与终态，绕开驱动内层POST重试。仅匹配query ID的Succeeded确认mutation；Running首页含结果不算成功。UNKNOWN带请求ID报错、停止lane/整个sink，不自动重发、不根据取消调用返回值宣布旧SQL结束。统计不可用不撤销已确认提交。URI继续请求限制同origin，禁止重定向和错误中泄漏SQL/参数/凭据。 -- SQL：参数化raw INSERT→首版固定full-raw MERGE→已有strict compaction；排序时间按微秒递增，并在重启时读取retained raw的MAX(add_time)，避免本机时钟回退导致同TSO决胜顺序倒退。源列名`action`不再与内部动作列冲突；真实验收后投影改为`get(raw_data, 字面键)`，避免引号列名被错误解释为JSONPath。 -- schema：按权威类型创建目标/raw；复用每表schema epoch，补充一份有界的post-DDL定义/翻译步骤供pending恢复。它不是DML ledger、manifest或新generation协议。pending恢复先于新DML，applied epoch按FinishedTs过滤旧输入；bootstrap不保存applied。TRUNCATE先清raw/target再applied，applied重放不重复清新数据。 -- 接入点：scheme/type/factory、TableSchemaStore类型分支、错误分类和metrics；没有改collector、dynstream、TableProgress、调度或manager算法。Run/DDL错误使用既有unretryable路径结束旧生命周期；真实UNKNOWN故障下的控制面关闭/人工恢复仍待验收。 -- 本地验证包含：原生精确类型、原始UPDATE拒绝、实际raw时间参数、OPEN/SEALED、并发/重入回调、部分raw结果未知、MERGE未知、低流量/transport epoch变化、配额压力、多表排空、Close/接纳竞争、pending列DDL与TRUNCATE恢复、applied重放、远端监控阈值、factory注册。 - -HTTP fixture只模拟协议与journal持久化，不执行SQL，不能证明真实Databend提交原子性。真实入口为`TestDatabendNativeLiveIntegration`，仅在显式配置`DATABEND_NATIVE_TEST_URI`时运行，创建/删除独立随机测试database；运行时接入阶段曾跳过，随后获授权连接staging Lake通过（§8.3）。没有启动或修改生产资源。 - -本轮验证命令与最终结果见§10.4；全量`pkg/common/event`测试曾在TiDB测试依赖中出现`duplicate metrics collector registration attempted` panic。未改无关依赖；缩小到本次改动相关的TableSchemaStore测试后通过。全仓测试未运行。 - -### 8.3 真实 TiUP→CDC→staging Lake 验收（2026-10-25） - -获用户授权，使用其staging Lake和本地TiUP v8.5.1，当前分支构建新架构CDC；只操作随机独立库。修复后从新库完整重跑，9组全部PASS：DML/PK/UK、复合键、特殊标识符、32列类型和值、有限列DDL、TRUNCATE、批处理/spool、受控进程重启、DROP TABLE。 - -- 实测发现并修复DESCRIBE unsigned类型别名不兼容、含双引号列名的JSONPath投影；补回归测试并重跑完整链路。增加`lake://` HTTPS别名以支持用户实际URI。 -- 1200行事务使用64 KiB磁盘spool、极小内存热集，实际41次磁盘读回；证明本负载能边应用边回收，不等于性能/RSS硬上限证明。 -- 排空/暂停并终止旧CDC，停机期间积压DML/DDL，同data-dir重启后从原checkpoint恢复并逐行比对；不是在途UNKNOWN或自动HA实验。 -- 真实核心测试PASS（13.36s）：旧UPSERT不复活删除键、applied过滤和重复TRUNCATE。6包race、sink lint、主程序构建通过。 -- 源/目标测试库、feed按具体名字确认已清理，本次本地CDC/TiUP已停止；未提交、推送或部署共享集群。 - -详见[真实验收报告](./databend-native-sink-e2e-report.md)及[可重复测试说明](../../tests/integration_tests/databend_native/README.md)。真实UNKNOWN故障、跨capture/HA、长期墓碑增长及成本/SLO未验收，T1–T7不据此整体关闭。 - -### 8.4 bend-cdc 对照、DDL矩阵与真实响应丢失（2026-10-25） - -用户要求补齐的测试已执行，详见[补充验收报告](./databend-native-sink-parity-fault-report.md)。这里纠正之前过宽的范围表述：TiCDC的`GetPrimaryKeyColumnNames`包含非空唯一键fallback；无显式PK但有NOT NULL UNIQUE的原生DML/变键已实测通过，不需要新增framework协议。 - -固定bend-cdc真实core对照确认：RENAME TABLE和DROP DATABASE不执行目标操作，只跳过/推进epoch，不能算已支持；真正无可用PK/非空UK时，旧版可用显式conflictKey，native尚未迁移；旧core能扁平投影分区元数据及普通DML，native仍拒绝分区dispatcher。多列组合DDL旧版拒绝，native支持的ADD/DROP/RENAME组合已实测。 - -本轮还复现并局部修复INT→VARCHAR(200)被过度拒绝：只开放完整容纳有符号INT32的VARCHAR(n>=11)，不开放同字符类型未知宽度/长度缩小。原failed feed更换本地二进制后从原checkpoint恢复成功；生产改动仅在sink/schema.go。 - -真实Lake故障注入覆盖部分raw已提交/第二条未发、MERGE成功但成功响应被丢弃，以及CREATE/ALTER/TRUNCATE/DROP共19个关键提交边界。客户端未取得证据时无成功callback、旧sink停止；在测试方已独立取得匹配终态的前提下重建manager，恢复journal及重放验证通过。该测试不伪造SQL执行，但不是每个OS指令点SIGKILL、服务端执行中崩溃或自动HA证明。具体最终矩阵与清理证据以补充报告为准。 - -### 8.5 显式conflict-key与跳过DDL（2026-10-25） - -按用户最新要求：增加URI `conflict-key`，对应bend-cdc持久化配置`conflict_key`/运行时`ConflictKey`，不是新建SQL主键。与旧版相同，单列override可以替代无键/单列源键，但不允许覆盖复合键；按可见列元数据正名，NULL身份或未拆分的变键UPDATE拒绝。操作者保证业务键唯一、非空、稳定；真正无索引键的源表还需启用既有force-replicate。DDL journal保存实际选取的键，DML和DDL共用这一配置，不改通用框架。 - -RENAME TABLE/RENAME TABLES与DROP DATABASE在旧DML排空后warning跳过、正常回调，不更改目标DDL或伪造applied epoch。历史目标保留；重命名后的DML按新表名进入新投影，不会搬迁历史数据。旧DML结果未知仍必须停止，不能靠跳过后续DDL绕过。 - -真实TiUP→CDC→staging Lake补测：无键CRUD+ADD COLUMN及单PK override、重命名后进度/后续DML、DROP DATABASE后进度/保留目标/新库表DML，3组均通过。详细证据见[补充报告§8](./databend-native-sink-parity-fault-report.md)。分区保护未放开：lane只约束单进程，跨capture同一逻辑表排他未证明；不是Lake无法存储分区数据。 - -### 8.6 本地 Databend 自动后台 E2E（2026-10-26） - -新增 `make integration_test_databend`，由 [本地 runner](/Users/jiangjianyuan/work/git/pingcap/ticdc/tests/integration_tests/databend_native/local.py) 自动管理独立 Docker Databend（meta/query、fs存储）、TiUP PD/TiKV/TiDB及本分支TiCDC，复用已有真实行值、DDL和故障断言。无需外部Lake凭据、S3或Dumpling。仅修改测试、Make入口及文档，没有修改TiCDC框架或新增生产运行时协议。 - -macOS arm64 + Docker Desktop + TiDB v8.5.1 + Databend `v1.2.925-patch-13` 固定多架构digest实跑：9组全部PASS，包含27个Python场景及3个真实后端Go测试；DDL响应丢失覆盖19个提交边界。磁盘spool观测41次实际读回，排空后的进程重启及后续DML/DDL恢复通过。7个不依赖服务的runner测试通过。各suite对象清理及runner进程/容器清理PASS；不会停止共享Docker daemon。Linux/amd64 CI尚未运行，不能把镜像多架构等同于跨平台实测。 - -旧 `v1.2.852-nightly` 镜像参数绑定预检失败，不列为支持版本，不引入字符串插值降级。新入口还修复了测试固定重启端口、owner早于coordinator就绪、TIME_WAIT导致连续重跑失败、macOS路径别名导致清理误判的问题。受控本地fs后端测试不替代云网关、S3存储、成本/容量或自动HA验收。使用方法及测试边界见 [E2E说明](/Users/jiangjianyuan/work/git/pingcap/ticdc/tests/integration_tests/databend_native/README.md)。 - -## 9. 可复核源码 - -bend-cdc链接按固定commit的本地源码核对；远端reviewer按相同Git对象路径读取。 - -- [Canal事件转换与现有目标配置，L376–559](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go#L376-L559)。 -- [PK/schema与ApplySchema，L93–163](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go#L93-L163)。 -- [raw写入与代数记账，L640–840](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L640-L840)；[MergeProgress，L399–416](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L399-L416)。 -- [Flush排空，L503–578](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L503-L578)；[MERGE/代数推进/清理，L1230–1416](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L1230-L1416)。 -- [现有MERGE SQL，L1628–1705](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L1628-L1705)；[stream创建及fallback，L889–942](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L889-L942)。 -- [applied watermark与MergeProgress接口](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/applied_watermark.go)；[schema epoch](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/schema_epoch.go)；[TRUNCATE](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/truncate.go)。 -- [schema队列的flush/警告范围](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/schema_queue.go)；[Flush错误传播测试](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/flush_permanent_test.go)。 -- [TiCDC Sink接口](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/sink.go)；[原生DML回调](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/dml_event.go#L660-L711)。 -- [raw-change与sink Run/error/close接入](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go)；[原生DDL字段](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/ddl_event.go)。 -- [CloudStorage raw-change默认值，L774–784](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/config/sink.go#L774-L784)；[scanner拆分与延后INSERT，L970–1014](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/eventservice/event_scanner.go#L970-L1014)。 -- [规范化入口，L14–39](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/tidb_values.go#L14-L39)；[外层epoch保护，L215–235](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go#L215-L235)。 -- [spool oversized接纳，L503–510](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/sink/spool/spool.go#L503-L510)；[Discard执行postFlush，L734–746](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/sink/spool/spool.go#L734-L746)。 -- [raw微秒格式化，L995–1003](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L995-L1003)；[纳秒单调时间，L1755–1763](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L1755-L1763)；[SchemaFile使用FinishedTs，L208–215](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/cloudstorage/schema_file.go#L208-L215)。 - - -## 10. 首版使用方式与实际支持边界 - -### 10.1 配置入口 - -所有Databend专属参数放在sink URI，不引入一套通用框架配置。`lake://`与`databend://`均为同一原生HTTPS sink（已通过真实Lake验收），不是MySQL协议。HTTPS： - -```text -databend://DB_USER:TOKEN@HOST/DB_NAME?exclusive=true&warehouse=WAREHOUSE&workers=4&batch-rows=2048&batch-bytes=1048576&flush-interval=1s&spool-bytes=10737418240&memory-ratio=0.02&source-bytes=268435456&source-events=128&max-entries=4096&query-timeout=2m -``` - -`DB_USER`/`TOKEN`必须URL编码；不应把实际密码粘贴到review或日志。`databend+http://DB_USER:TOKEN@HOST:PORT?...`仅用于明确允许明文的隔离环境。留空database路径时使用源schema名；指定database时同名源表不能汇入同一目标表，不实现任意表路由。 - -| 参数 | 默认 | 意义 | -|---|---|---| -| exclusive | 无，必须为true | 操作者确认目标静态独占；不是锁或自动HA证明 | -| conflict-key | 未指定 | 可选feed级单列override，替代无/单列源键；不能覆盖复合键，必须唯一、非NULL且稳定，不能在原目标上随意更换 | -| workers | 4，最大64 | 固定lane数；表hash到lane，单表串行 | -| batch-rows / batch-bytes | 2048 / 1 MiB | 攒批/分片阈值；单行编码超限立即停止，不无界直通 | -| flush-interval | 1s | 低流量定时排空 | -| spool-bytes | 10 GiB | 共享本地磁盘预算；已释放但物理未回收仍由spool计费 | -| memory-ratio | 0.02 | spool内存热数据额度占磁盘预算比例；独立于source池 | -| source-bytes / source-events | 256 MiB / 128 | 整个sink已接纳原生事件池；共享chunk保守按每个event完整计费 | -| max-entries | 4096 | 每lane待应用分片/缓存表元数据上限，达到分片上限主动排空 | -| spool-dir | TiCDC data-dir下 | spool根目录；按capture/feed隔离，不是恢复日志 | -| query-timeout | 2m | 单SQL及分页总超时；到期不代表服务端已停止 | -| warehouse | 未指定 | HTTP warehouse头 | -| remote-bytes-limit | 未开启 | 基于system.tables中目标database raw压缩字节测量的停写阈值；不是跨capture硬额度 | - -源chunk在进入sink之前已经分配，source池、编码、读回、HTTP和元数据分别有界不等于全进程RSS严格上限。batch/spool参数必须联合满足最小额度，未知/重复参数拒绝。普通压力用背压和提前排空吸收；事件池无法接管属于明确fatal容量异常，不伪造成功回调。 - -### 10.2 已实现的首版能力 - -- 具备TiCDC可选冲突键（显式PRIMARY KEY或全列NOT NULL的UNIQUE键），或显式配置conflict-key的普通非分区表；INSERT/UPDATE/DELETE，PK/UK变化沿用raw-change=false的scanner拆分顺序。 -- 增量自动建表，复合键与保留墓碑，参数化批量raw INSERT、full-raw MERGE、strict历史清理。 -- 主要TiDB标量/文本/binary/ENUM/SET/bit/decimal/JSON/date/time/datetime/timestamp映射，生成列只投影CDC可见的STORED值，不生成目标表达式。 -- CREATE DATABASE、CREATE TABLE/RECOVER、DROP TABLE、TRUNCATE；有限ADD/DROP/RENAME/CHANGE/MODIFY列，保守类型widening、字面默认值与nullability；二级索引DDL不创建Databend索引但推进已应用schema fence。 -- 实际列DDL遇到缺失目标时，可依据该事件定义创建空投影；不扫描或预建没有事件的历史空表。 -- pending schema步骤重入恢复、applied DDL重放跳过、旧DML fence过滤；无成功回调的fatal/Close清理。 - -### 10.3 必须保留的限制 - -- 受控单capture、目标静态独占；旧SQL终态无法确认时人工排除后才能resume。`exclusive=true`是运行前提声明，不自动检测/获得下游租约。跨capture自动故障接管没有验收，不承诺生产HA。 -- 首版拒绝分区dispatcher、跨节点拆表、syncpoint、active-active、redo、low-latency模式、列选择、任意路由和table级事务原子性。分区调度可能使同逻辑目标出现多writer，因此在证明隔离前不静默接纳。 -- 拒绝删除/重命名冲突键、修改复合冲突键、不安全type conversion、除INT32→VARCHAR(n>=11)之外的参数化字符MODIFY/CHANGE及表达式默认值DDL；复杂分区等不做隐式有损翻译。按用户最新裁决，表重命名（含多表）及DROP DATABASE只warning后跳过，目标对象不变、进度继续；不宣称镜像执行了这些DDL。其他unsupported操作仍fail-closed。 -- 不支持Dumpling/CSV/COPY INTO或全量编排；首次创建空目标不补历史存量。用户必须已独立准备一致的起始状态与start-ts。 -- 不创建stream、DML ledger、manifest/witness或新owner协议。retained raw最新墓碑不TTL删除；不同历史PK持续churn仍会增长。 -- HTTP端点必须实际支持Query API参数绑定与终态。Verify以绑定参数SELECT做预检；不会自动降级到字符串插值。本轮staging端点的参数绑定、终态、支持DDL/类型与排空重启已实测；不同端点/版本仍需预检，不推广到所有版本/许可或在途故障恢复。 -- 当前journal是新sink专属的`epoch/applied/plan`形状，不自动接管正在运行的bend-cdc目标或其旧两列journal。迁移必须停旧链并独立验证起点；不绕过DDL恢复证据。 - -### 10.4 验证与发布状态 - -代码已接入factory并可构建；用户随后提供staging Lake，真实核心及TiUP全链路通过（§8.3），不再仅有HTTP fixture。用户随后要求提交并推送当前实现分支，提交/远端状态以该分支Git记录为准；下文实验记录描述测试当时的dirty工作区。未建PR、未部署共享CDC。成本/SLO与在途故障/自动HA未验收，T1–T7不能标为全部关闭。 - - -2026-10-25 本地最终验证记录（最后一批代码）： - -```text -GOMAXPROCS=4 go test -p 2 -race ./downstreamadapter/sink/databend ./downstreamadapter/sink ./pkg/sink/spool ./pkg/errors ./pkg/config ./downstreamadapter/sink/metrics -count=1 -timeout=180s -PASS：6个包 -GOMAXPROCS=4 go test -p 2 -race ./pkg/common/event -run 'TestTableSchemaStore|TestNeedTableNameStore' -count=1 -PASS -GOMAXPROCS=4 go test -p 2 ./pkg/metrics -run TestDownstreamTypeFromSinkURI -count=1 -PASS -GOMAXPROCS=4 tools/bin/golangci-lint run --timeout 5m ./downstreamadapter/sink/databend/... ./downstreamadapter/sink/metrics/... -0 issues -GOMAXPROCS=4 go vet -p 2 ./downstreamadapter/sink/databend ./downstreamadapter/sink ./pkg/sink/spool ./pkg/common/event ./pkg/errors ./downstreamadapter/sink/metrics -PASS -GOMAXPROCS=4 go test -p 2 ./downstreamadapter/sink/databend -run TestDatabendNativeLiveIntegration -v -count=1 -PASS：显式配置用户授权staging URI，13.36s；独立随机库自动清理（URI不写命令/报告） -GOMAXPROCS=4 go build -p 2 -o /tmp/ticdc-databend-native ./cmd/cdc -PASS，临时产物不进入仓库；version子命令可执行，未注入发布版本元数据 -``` - -新增Databend包为11个生产文件、11个测试文件、54个测试函数（包含3个显式opt-in live测试）；另有错误定义和sink metrics文件。factory/类型/schema store/配置scheme/错误分类/metrics/spool计量等现有文件仅有限接入改动。HTTP fixture仍仅是进程内协议测试；真实SQL/TiDB链路另见§8.3，生产故障恢复不据此宣称通过。 diff --git a/docs/design/databend-native-sink-e2e-report.md b/docs/design/databend-native-sink-e2e-report.md deleted file mode 100644 index a8c00dd116..0000000000 --- a/docs/design/databend-native-sink-e2e-report.md +++ /dev/null @@ -1,131 +0,0 @@ -# TiCDC 原生 Lake Sink：本地 TiUP → staging Lake 实测报告 - -日期：2026-10-25（Asia/Shanghai）。状态：**本轮受控端到端验收通过；未提交、未推送、未部署到共享 CDC 集群。** - -后续追加的bend-cdc对照、DDL默认值/nullability矩阵和真实响应丢失实验见[补充报告](./databend-native-sink-parity-fault-report.md)。下述9组是首轮记录，不替代后续实验；“无PK不支持”应精确为“无可用PK/非空唯一键且依赖显式conflictKey的表”。 - -## 1. 结论与证据边界 - -使用本地 TiUP TiDB/PD/TiKV、当前分支新架构 CDC，以及用户授权的真实 staging Lake，修复兼容性问题后，**从全新随机库重新执行的 9 组测试全部通过**。不是仅运行 mock，也没有使用 S3/Canal JSON/bend-cdc 代写。 - -- DML、有限列 DDL、TRUNCATE、DROP TABLE、32 个目标列的类型/值比对通过。 -- 1,200 行事务超过 64 KiB 磁盘 spool 配额，仍能边应用边回收；实际发生 41 次 spool 磁盘读回。 -- 排空、暂停、终止旧 CDC 后，制造源端 DML/DDL 积压，重启同一 data-dir 并从原 checkpoint 恢复，最终数据一致。 -- 独立真实 Lake 核心测试验证旧 UPSERT 不复活已删除键、applied fence 与重复 TRUNCATE。 -- **不代表**任意 DDL/列类型均已支持、在途 UNKNOWN 请求故障接管已通过、多 capture HA 已通过，或成本/吞吐已达到生产要求。 - -当前设计与限制见 [定稿设计 §10](./databend-native-sink-design-review.md)。可重复运行的测试见 [runner](../../tests/integration_tests/databend_native/run.py) 和 [操作说明](../../tests/integration_tests/databend_native/README.md)。 - -## 2. 实际环境与版本 - -| 组件 | 本次使用 | -|---|---| -| 源 TiDB / PD / TiKV | TiUP playground v8.5.1，各 1 个；TiDB `SELECT VERSION()` 返回 `8.0.11-TiDB-v8.5.1` | -| CDC 源码 | 分支 `feat/databend-native-sink`；基线 `2bf6dacd0aec9ac2fad464600e47341f9aa951f2` 加当前未提交实现 | -| CDC 运行 | 当前工作区构建，`--newarch --tz UTC`；单 capture，独占本次目标 | -| CDC 版本标签 | 本地注入 `v8.5.1` / `2bf6dacd-dirty-e2e`，仅用于源版本兼容检查，**不是官方 v8.5.1 发布包** | -| Lake | `tnww1vflk.gw.aliyun-ap-northeast-1.default.staging.lake.tidbcloud.com:443`，warehouse `default`，HTTPS Query API | -| Lake 实际版本 | `Lake Query v1.2.931-nightly-0ca7e6af79(rust-1.94.0-nightly-2026-08-02T22:24:51.279542089Z)` | -| 测试库 | `native_e2e_20261024_171016_bbf3e64f`（源、目标相同的独立随机库，测试后删除） | -| Changefeed | `native-e2e-20261024-171016-bbf3e64f`（测试后删除） | - -用户原 URI 的 database 被替换为本次随机库；没有修改 `default` 内现有表。凭据仅从仓库外 0600 文件读取，不记录到本文或测试代码。 - -本地 TiUP SQL/PD/TiKV/CDC API 分别为 `127.0.0.1:14000/12379/30160/18300`。本机空间不足触发 TiDB 快速 ADD INDEX 的磁盘保护，故仅对本次独立 TiUP 关闭 `tidb_ddl_enable_fast_reorg` 和 `tidb_enable_dist_task` 后运行索引 DDL。这是**源端测试环境调整**，不属于 sink 修复，不能据此更改共享源配置。 - -## 3. 最终全新运行：9 / 9 PASS - -耗时是该组 wall time，含远端 SQL 和等待收敛，不是吞吐基准。 - -| 测试组 | 具体断言 | 结果 / 秒 | -|---|---|---| -| DML | INSERT/UPDATE/DELETE；固定 PK 改二级 UK；非聚簇 PK 变键；同事务 DELETE+重插后最终键集合一致 | PASS / 16.571 | -| 复合键 | 包含分隔符/转义字符的复合主键不碰撞，源目标逐键一致 | PASS / 10.622 | -| 特殊标识符 | 表/列包含双引号；源列 `action`、`__ticdc_operation` 不覆盖内部动作列 | PASS / 10.246 | -| 类型/值 | 32 个投影列，初始、NULL、极值及更新后比对；VIRTUAL 不投影 | PASS / 12.801 | -| 列 DDL | ADD NOT NULL 字面默认值及旧行回填、RENAME、INT→BIGINT 后写大整数、DROP、TEXT NOT NULL 隐式空默认；二级 ADD/DROP INDEX | PASS / 42.352 | -| TRUNCATE | 旧行清空、旧 raw 代际清理后仅保留新 DML | PASS / 18.195 | -| 批处理/spool | 单事务 1,200 行×256 字符；随后 UPDATE 一行、DELETE 100 行；分别比对 1,200/1,100 行；低流量尾批排空 | PASS / 40.592 | -| 进程重启 | checkpoint 排空后暂停并终止旧 CDC；停机期间 UPDATE/INSERT/ADD COLUMN；同 data-dir 重启/恢复，完整比对积压结果 | PASS / 22.769 | -| DROP TABLE | 源 DROP 后目标表及对应 raw 表均不存在 | PASS / 12.387 | - -### 类型覆盖与比较方法 - -| 源类型/边界 | 真实目标观察/比较 | -|---|---| -| TINY/SMALL/MEDIUM/INT/BIGINT，含 UNSIGNED | 覆盖有符号正负边界、`BIGINT` 最小值、`UINT64` 最大值；精确十进制文本比较，无 float64 中转 | -| DECIMAL(65,30) | 大正数、大负数、30 位小数和最小小数位更新；精确比较 | -| FLOAT/DOUBLE | 源端以足够精度读回后按浮点容差比对，不能用 MySQL CLI 默认短显示当精确值 | -| CHAR/TEXT/BOOLEAN | Unicode、emoji、引号、反斜线、换行、NULL；BOOLEAN 保留 TiDB TINYINT 表示 | -| BLOB/BINARY/VARBINARY | 0–255 全字节、定长补零、变长及更新；canonical hex 精确比较 | -| BIT(64) | 全 64 位值映射到 unsigned BIGINT，精确比较 | -| ENUM/SET | 数字 label `ENUM('2','x')` 不二次解码；64 项 SET 的 bit63 及多 label 更新 | -| JSON | 超过有符号范围的大整数、数组/布尔；显式 `is_null_value` 区分 JSON null 与 SQL NULL | -| DATETIME(6)/TIMESTAMP(6)/DATE | 微秒与日期保持；本轮源会话/CDC 为 UTC，不据此宣称任意时区组合已实测 | -| TIME(6)/YEAR | 负且超过 24h 的 TIME 精确文本、YEAR 数值 | -| 生成列 | STORED 实际值同步，含 BIGINT 最大值；VIRTUAL 列不在目标 DESCRIBE 中 | - -按主键 canonical 化后比较，不依赖两端数值/VARCHAR 的不同排序规则。最终 Lake `DESCRIBE types_test` 为 **32 列**；此数字是测试表覆盖列数，不是“所有 MySQL 类型的数量”。 - -### spool 与恢复的实际证据 - -测试配置：`workers=1`、`batch-rows=64`、`batch-bytes=32768`、`flush-interval=200ms`、`spool-bytes=65536`、`memory-ratio=0.001`。真实 `ticdc_databend_spool_load_bytes_count` 累计 **41**，不是只根据阈值推测使用了磁盘。该测试证明本负载可在小配额下推进，不构成全进程 RSS/物理磁盘硬封顶或性能报告。 - -重启时旧 PID `41562` 已终止，新 PID `41917`；正数源事务读 TSO 为 `469988010726522887`，恢复后 checkpoint 为 `469988011722670087`。未覆盖 checkpoint，未跳过积压。**这是受控排空重启，不是在 MERGE/TRUNCATE 结果未知时直接启动新 owner。** - -## 4. 本轮发现并修复的真实兼容问题 - -### 4.1 DESCRIBE 的 unsigned 类型别名 - -Lake 返回 `TINYINT/SMALLINT/INT/BIGINT UNSIGNED`，原校验只认内部 UINT8/16/32/64，导致合法目标被安全拒绝。`canonicalType` 统一别名及 Nullable 包装，不放宽 Decimal 精度检查；增加回归测试。真实极值类型测试已重跑通过。 - -### 4.2 双引号列名不是 JSONPath - -原 SQL 的 `s.raw_data:"a""b"` 虽经过 SQL 标识符转义，仍触发真实 Lake `1006 InvalidKeyPath`。改为字面对象键 `get(s.raw_data, 'a"b')`，统一 SQL 字面量转义，并测试点号、引号、反斜线。没有替换 raw/MERGE 主算法，也没有引入 stream/manifest。 - -先核验旧错误请求为终态后，用 checkpoint 和 retained raw 恢复该失败案例，目标正确；随后使用修复后的完整代码，从新库完整重跑，9 组均通过。不是把多个失败运行的局部结果拼成一次全绿。 - -### 4.3 接入用户实际 URI - -增加 `lake://` 作为同一 HTTPS native sink 的 scheme 别名，接通 factory/config/downstream metrics；`exclusive=true` 等首版限制保持不变。用户名/密码继续按 URI URL 编码，不降级明文。 - -测试脚本本身还修正了 reserved identifier、源 FLOAT 读出精度、hex 大小写、JSON null 观察、`/tmp` realpath 和 coordinator 启动等待等环境/fixture 问题，未将其计为数据复制缺陷。 - -## 5. 独立真实核心与本地回归 - -真实 Lake 核心测试 `TestDatabendNativeLiveIntegration`：**PASS，13.36s**（package 15.127s）。另外创建/清理独立随机库，验证 DELETE 后旧 UPSERT 不复活、applied epoch 过滤旧输入、相同 FinishedTs 的 TRUNCATE 重放不清空新数据。它不是 TiUP 源端完整链路的替代证明。 - -最终代码验证： - -```sh -GOMAXPROCS=4 go test -p 2 -race ./downstreamadapter/sink/databend ./downstreamadapter/sink ./pkg/sink/spool ./pkg/errors ./pkg/config ./downstreamadapter/sink/metrics -count=1 -timeout=180s -# PASS，6 个包 -GOMAXPROCS=4 tools/bin/golangci-lint run --timeout 5m ./downstreamadapter/sink/databend/... ./downstreamadapter/sink/metrics/... -# 0 issues -``` - -当前 Databend 包包含 45 个测试函数（含 1 个 opt-in 真实 Lake 测试）。历史本地回归还通过 TableSchemaStore 针对性测试、scheme metrics 测试、go vet 和主程序构建。未运行全仓测试；之前全量 common/event 曾因既有 TiDB 测试依赖重复注册 metrics panic，未改无关依赖，针对相关测试通过。 - -## 6. 清理与证据位置 - -- 最终 run 状态和 cleanup 均为 PASS。 -- 对 4 次运行的**具体自有库名/feed ID**查询确认：Lake 测试库、源测试库、changefeed 均无残留；不是只看 DELETE 返回成功。 -- 本轮启动的 CDC 与 TiUP 已停止；本地 14000/12379/30160/18300 端口均已关闭。 -- 凭据不入 Git；临时 credential 文件在完成清理核验后删除。日志/result 保留在本机私有目录用于复核，不保证跨机器可访问。 - -证据根目录： - -```text -/var/folders/lq/3n41d_w97w55hrx2mzmd42w80000gn/T/ticdc-native-e2e-0t2q351n -``` - -其中 `acceptance-4/result.json` / `acceptance-4.log` 为最终全新运行；`live-core-test-final.log` 为真实核心测试；`cleanup-verification.json` 为逐名字清理核验；`acceptance-3/quoted-recovery.json` 为特殊列名修复后恢复案例。最终测试 runner SHA-256：`81c753f3f38e471f7c6bb17a83d342215ea6352fb4147fbfca2c7056a7f25226`。 - -## 7. 尚未验收，不应扩大结论 - -- 在途 HTTP 断连/首页后失败、服务端仍执行的 UNKNOWN SQL、旧 owner 与新 owner 并发、进程任意中断点恢复：本地 fixture 覆盖部分分支，**本轮没有真实故障注入证明**。 -- 多 capture、自动 HA、跨表事务原子性、分区表、无可用PK/非空唯一键、复杂/不安全 DDL、任意路由、非 UTC 时区矩阵不在本轮验收范围。 -- 长期 retained raw 墓碑增长、多表高并发、成本/扫描量/SLO、长期满盘/IO 故障未做 benchmark/soak。 -- 从既有 bend-cdc journal/目标在线无缝接管、历史全量初始化不在范围；不能把首次自动建空表当存量已同步。 - -因此本轮可以确认“首版支持范围内，真实 TiDB→当前 CDC→该 staging Lake 的功能纵切通过”，不能把 T1–T7 全部风险或生产就绪标为关闭。 diff --git a/docs/design/databend-native-sink-parity-fault-report.md b/docs/design/databend-native-sink-parity-fault-report.md deleted file mode 100644 index 73ac537b89..0000000000 --- a/docs/design/databend-native-sink-parity-fault-report.md +++ /dev/null @@ -1,198 +0,0 @@ -# bend-cdc 能力对照与 native sink DDL / 故障补充验收 - -日期：2026-10-25。结论：**已完成真实 Lake 能力对照、11 组真实源端矩阵、2 类 DML 故障及 19 个 DDL 提交边界恢复测试；发现并修复一个局部迁移退化。仍有明确未迁移能力，不宣称全部替代 bend-cdc。** - -**后续更新**：下述对照矩阵是上一轮状态；按用户新要求，`conflict-key`现已实现，RENAME/DROP DATABASE改为跳过不阻塞。最新实现/真实测试见§8；分区保护仍保留。 - -## 1. 先纠正能力判断 - -固定基线为 bend-cdc `b6481621021e82aee659ad88e8e6c00f6dd11906`，不是另一个 HEAD。旧库保持干净，通过 Go overlay 添加临时测试，不改其源码。Native 为 TiCDC `2bf6dacd0aec9ac2fad464600e47341f9aa951f2` 加当前分支未提交实现；不改 collector/dispatcher/progress/调度框架。 - -| 场景 | bend-cdc 固定版实际证据 | native 当前结果 | 判定 | -|---|---|---|---| -| 分区表普通 DML | 真实 core 接受分区 CREATE 元数据，创建扁平目标，Canal 行 INSERT/Flush 后值存在 | 真实源 CREATE 分区表触发 unretryable，未建新目标 | **仍有迁移缺口**；旧 core 验证不等于已测完整 S3 分区文件/分区 DDL | -| 无显式 PK，但有 NOT NULL UNIQUE | native 与旧路径可使用 TiCDC 提供的隐式冲突键，不等于真正无键 | 真实 TiDB→CDC→Lake 的 INSERT、唯一键变更、DELETE 全通过，未开 force_replicate | **已支持**；之前笼统说“无 PK 都不支持”不准确 | -| 无可用 PK/非空唯一键 | 无 pkNames 且无 override 时拒绝；设置 `conflictKey=id` 后真实 ApplySchema、WriteBatch、Flush 通过 | force_replicate 后实际 INSERT，native 拒绝且 checkpoint 停止 | **仍缺显式 conflictKey override**；旧版依赖操作者保证所选键有效，不是任意无键表都安全 | -| RENAME TABLE | 真实旧 ApplySchema 返回成功，但旧表仍在、新表不存在；代码按 unsupported 跳过并推进 epoch | 拒绝，旧目标保留 | **旧版没有实际重命名支持**，不应把 nil error 当已支持 | -| DROP DATABASE | 旧 SchemaChanges 判 unsupported；给旧 ApplySchema 带表元数据探针后仍保留目标库/表 | 真实源 DROP DATABASE 后 native 拒绝，目标库表保留 | **旧版没有实际 DROP DATABASE 投影**；旧探针不是完整 S3 DROP DATABASE 测试 | -| 默认值/nullability | 真实旧 MODIFY 默认值、NOT NULL→NULL 通过 | 真实源多种组合通过，见§2 | 对应支持场景已补证据 | -| INT→VARCHAR(200) | 真实旧 core 通过，原值转文本保留 | 修复前真实 failed；局部修复后通过且原 feed 重放恢复 | **已修复迁移退化** | -| 多操作列 DDL | 旧 parser 对两列 ADD 等组合明确报 multi-operation 错误 | ADD+DROP+RENAME、CHANGE 完整定义等受测组合通过 | 不属于旧版全支持、新版遗漏；仍不承诺任意组合 | -| 缩窄/表达式默认值 | 旧版有明确限制，不是任意 DDL 都执行 | 本轮真实 source 探针验证 fail-closed | 拒绝测试通过≠功能支持 | - -非空唯一键的源码依据是 [GetPrimaryKey 隐式键选择](../../pkg/common/table_info_helper.go#L593-L637)，native 复用该 helper，**不是本轮新增框架机制**。旧 unsupported 分类及处理见 [SchemaChanges](../../../webapi/bend-cdc/pkg/source/tidb/schema_ddl.go) 和 [ApplySchema](../../../webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go)。 - -## 2. 修复后全新库：11 / 11 PASS - -真实环境：本地 TiUP TiDB/PD/TiKV v8.5.1、当前分支新架构单 capture CDC、用户授权 staging Lake。TiUP 仅用本轮独立实例；源为 UTC。CDC 注入的 `v8.5.1` 是本地兼容标签，不是官方 CDC 发布包；最终 git_hash 标签 `2bf6dacd-dirty-extended-e2e`。 - -最终随机库 `native_e2e_20261025_012759_8da770af`，结果及 cleanup 均 PASS。下面时间包括网络与收敛等待，不是性能指标。 - -| 测试 | 秒 / 结果 | -|---|---| -| 无显式 PK 的 NOT NULL UNIQUE 表，包含键变更/删除 | 11.240 / PASS | -| INT→VARCHAR(200)、INT 最小值保留、后续 `'0012'` 文本及默认值 | 16.797 / PASS | -| 同类型 MODIFY 字面默认值，已有行不被回填改写 | 8.122 / PASS | -| NOT NULL→NULL，显式 DEFAULT NULL | 7.830 / PASS | -| 清除源 NULL 后 NULL→NOT NULL，后续默认值写入 | 9.606 / PASS | -| MODIFY 完整定义省略 NOT NULL，使目标恢复可空 | 9.225 / PASS | -| MODIFY 省略 DEFAULT，删除旧默认值 | 8.022 / PASS | -| TINYINT→INT→BIGINT，VARCHAR→TEXT | 12.184 / PASS | -| ADD Decimal/文本/NULL 字面默认值，旧行回填及转义字符串 | 16.237 / PASS | -| CHANGE 重命名及完整默认值/nullability 定义 | 8.363 / PASS | -| 同一 DDL 的 ADD+DROP+RENAME，随后 DML | 9.161 / PASS | - -不只比行数：按键比较实际值，并用真实目标 DESCRIBE 检查默认值和可空性。`VARCHAR(200)` 只存在于源定义，目标按既有 mapper 使用不定长 VARCHAR。 - -另有 **6 个独立真实拒绝探针**通过：分区表、真正无键表（显式 force_replicate 并写行）、表重命名、DROP DATABASE、INT→TINYINT 缩窄、CURRENT_TIMESTAMP 表达式默认值。断言为稳定 failed、checkpoint 不再推进、旧目标值未变/新目标未创建；不能将这些 PASS 计为支持了相应操作。 - -初始无键探针若不 force_replicate 会在上游被 eligibility 过滤，feed 可保持 normal，不能当 sink 拒绝证据。初始误把 NOT NULL UNIQUE 当“不支持无 PK”也导致测试预期失败；读取真实目标、追踪 helper 后已改为正向测试。保留这些原始结果，未把失败记录改成通过。 - -## 3. 两类真实 DML 故障：PASS - -[实现与复现入口](../../downstreamadapter/sink/databend/live_fault_test.go)。SQL 都由真实 Lake 执行，不使用 HTTP fixture 伪造数据库成功。 - -### 3.1 部分 raw INSERT 成功 - -四行批次拆为两条 INSERT：第一条真实提交两行，第二条在发送前注入传输失败。 - -- 观察真实 raw=2、target=0，applyResult 未应用。 -- writer 未重发未知请求、未继续 MERGE（匹配 raw POST 总数为2，第二条未送服务器）。 -- 测试已知旧第一条成功、第二条未发，替换所有本地 manager/writer 状态，再完整重放四行。 -- 最终真实 target=4、strict compaction 后 raw=4。 - -这是“部分子 SQL 成功”，不是声称同一条 INSERT 内部半提交。 - -### 3.2 MERGE 已提交，但成功响应丢失 - -委托真实 HTTP transport 执行，独立取得**相同 query_id 的 Succeeded**后，在 sink 看到该响应之前丢弃它。 - -- 真实 target 已有两行，但旧 sink 因 UNKNOWN 终止，源 flush callback=0。 -- MERGE POST 只有1次，无内层/外层暗中重发。 -- 测试方已独立取得旧 query 的终态且关闭旧 sink 后，重建 manager/writer、重放同一输入。 -- 最终 target=2，compaction 后 raw=2，没有重复目标行。 - -原任意 UNKNOWN 请求在实际运维中仍必须取得终态及目标排他后才能恢复；测试夹具掌握的证据不等于新建了自动终态发现/HA 服务。 - -## 4. DDL 真实提交边界：19 / 19 PASS - -每个断点均先在真实服务器提交该语句，再丢弃客户端响应；经实际 `Sink.WriteBlockEvent` 路径观察错误、Run 停止、DDL callback=0。随后在旧请求终态已知的前提下，用新 schemaManager 只读取真实 Lake 的 journal/表状态恢复，不继承旧内存状态。 - -| DDL | 丢失响应的关键提交边界 | 数量 | -|---|---|---| -| CREATE | pending journal、目标建表、raw建表、applied journal | 4 | -| ALTER | pending、ADD、RENAME、MODIFY、raw确保存在、applied | 6 | -| TRUNCATE | pending、DROP raw、TRUNCATE target、重建raw、applied | 5 | -| DROP | pending、DROP target、DROP raw、applied | 4 | - -每例均验证恢复至 epoch=150/applied=true；非 DROP 场景再写 ts=160 的新数据并核对完整值，重放相同 DDL 不清新数据，ts=120 旧 DML 被 fence 跳过；DROP 后 target/raw 均不存在。ALTER 同时核对旧行 `label='old',extra=NULL` 和新行 `label='new',extra=7`，不只核对行数。 - -最终命令带 `-race`，2个父测试及21个故障子场景通过，package **199.112s**。19条匹配 query_id 的真实终态证据保留在 `fault-summary.json` 和原始日志。 - -**边界**：这是客户端真实响应丢失、旧 sink 生命周期终止和本地 manager 状态重建，不是服务端执行中崩溃、OS 任意指令点 SIGKILL、跨 capture 故障接管。没有声称“所有 DDL 的所有可能中断点”已覆盖，也没有对旧 S3/bend-cdc 整条链做同样故障注入。 - -## 5. 本轮修复与真实升级恢复 - -生产行为仅修改 [schema.go](../../downstreamadapter/sink/databend/schema.go) 的参数化字符转换判定：此前将所有参数化 VARCHAR MODIFY/CHANGE 一律拒绝，误挡住 bend-cdc 的 INT→VARCHAR(200)。现在仅放行 **INT32→VARCHAR(n>=11)**（足够完整容纳包括符号的 INT32 十进制域）；同字符类型未知原宽度、VARCHAR(10)、CHAR(200)、BIGINT→VARCHAR 等仍按保守规则拒绝。增加边界单测，不扩展通用 casting/框架协议。 - -真实复现先保留 failed feed，再停止旧 CDC、换本轮构建、同 data-dir 重启，从**原 checkpoint**手工 resume，不覆盖时间戳： - -- 原 PID `67290` → 新 PID `68998`。 -- checkpoint `469995752722530332` → `469995795137953793`。 -- 原行 `-2147483648` 转文本保留；积压新行 `'0012'` 不丢前导零。 -- 原错误 DDL 重放恢复成功，独立记录 `upgrade-recovery.json`；随后另起新库完整跑§2矩阵。 - -## 6. 验证、清理和可重复运行 - -新增/扩展测试入口： - -- [extended.py](../../tests/integration_tests/databend_native/extended.py)：源端矩阵与拒绝探针。 -- [baseline.py](../../tests/integration_tests/databend_native/baseline.py) / [固定版 Go overlay](../../tests/integration_tests/databend_native/baseline_parity_test.go.txt)：旧 core 对照。 -- [live_fault_test.go](../../downstreamadapter/sink/databend/live_fault_test.go)：真实 Query API 故障注入。 -- [README](../../tests/integration_tests/databend_native/README.md)：私有凭据、启动与命令。 - -固定 bend-cdc 的7个真实对照子测试及原有 IndexWarning/TRUNCATE 集成测试通过（package **38.659s**）；选定的原有分类/默认值/冲突键单测也通过。旧 checkout 未修改。 - -最终修改后，6包 race、sink lint（0 issues）、go vet、当前分支主程序构建通过。Databend包现49个测试函数，其中3个 opt-in live 父测试。全仓测试、生产部署、成本/长期负载没有执行。 - -本轮按**具体自有名字**确认16个源命名空间、26个远端命名空间（含baseline/fault随机库）均无残留，对应changefeed均已删除；没有操作原 `default` 中的已有表。自有本地CDC/TiUP停止，临时凭据清除；没有提交、推送或建PR。 - -私有证据根目录（仅本机可访问）： - -```text -/var/folders/lq/3n41d_w97w55hrx2mzmd42w80000gn/T/ticdc-native-extended-il8v40pb -``` - -关键记录：`ddl-matrix-fixed-final/result.json`、`baseline-fixed-final/bend-live-parity.log`、`live-fault-final.log`、`fault-summary.json`、`upgrade-recovery.json`、`cleanup-verification.json`。首轮9组DML/32列/spool/排空重启仍见[首轮报告](./databend-native-sink-e2e-report.md)，与本轮不同运行分开记账。 - -## 7. 当前仍未闭合的事项 - -1. **分区普通 DML**与**显式 conflictKey override**仍是具体迁移差异，不能写成已覆盖；本轮没有用跳过错误或删除安全检查冒充支持。 -2. 任意复杂/不安全DDL、所有类型/时区组合不是11组测试能够穷举；RENAME和DROP DATABASE也不能因旧版“成功返回”就宣称正确支持。 -3. 服务端仍在执行时的 UNKNOWN 终态取得、自动HA、长期资源与性能成本继续保留未验收状态。 - -所以本轮关闭的是具体功能/恢复测试缺口及一个局部转换退化，不是宣称 bend-cdc 全功能、全部故障或生产就绪已验收。 - -## 8. 最新用户要求：conflict-key 配置、RENAME/DROP DATABASE 跳过 - -本节取代§1/§7中相应的“尚未迁移/拒绝”状态；不是改写上一轮实验记录。分区保护仍未开放。 - -### 8.1 配置及语义 - -bend-cdc 在 `DatabendConfig` 中保存单列 `conflict_key`，TiDB wrapper 的 `ConflictKey` 将其交给 raw identity / MERGE 作为业务键；**没有给源表自动添加 PRIMARY KEY 或唯一索引**。旧 `resolveConflictKeys` 允许覆盖无键或单列源键，拒绝覆盖复合源键。 - -新 sink 已按该规则增加 URI 参数：在原 URI 查询串追加： - -```text -&exclusive=true&conflict-key=id -``` - -`exclusive=true`只应出现一次，已有时只追加`conflict-key`。真正没有可用源索引键的表，还需既有TiCDC选项： - -```toml -force-replicate = true -``` - -API对应 `replica_config.force_replicate=true`。无需改TiCDC通用配置结构。 - -规则： - -- 未配置时仍自动选择显式 PK / NOT NULL UNIQUE；配置后是feed级**单列override**，列名按可见元数据大小写归一。 -- 不能覆盖复合源键；缺失/不可见列、NULL身份拒绝。此配置适合独立feed，不能随意把复合键表混入同一override任务。 -- 业务方保证值唯一、非NULL且稳定。重复业务键会按同键更新处理，不会自动获得源端唯一约束。 -- 任意未索引逻辑键不一定触发TiCDC拆分；未拆分UPDATE改变该键时native拒绝，不静默留下旧行。本轮没有新造通用变键协议。 -- DML与DDL journal均使用相同的实际选取键；不要在原target/raw上直接更换键，须独立重新初始化。 - -生产改动集中于 sink 包的 `config.go`、`native.go`、`lane.go`、`schema.go`；不改collector、dispatcher、progress或调度算法。 - -### 8.2 跳过语义 - -`RENAME TABLE`（含多表类型）和 `DROP DATABASE` 在旧DML成功排空后 warning 并正常完成DDL回调；不发目标DDL，不创建虚假的applied epoch。不再因这些指定操作停任务；其他不安全DDL仍拒绝。 - -重要影响：目标库/旧表/旧数据保留；**重命名不会搬迁历史行**。后续DML使用新表名，按需创建新投影。DROP DATABASE不删除目标库；源库重建后的新表仍能同步。同名库表重建不会自动清理旧目标数据，需要空目标时应独立重新初始化。这是用户要求的跳过策略，不是完整DDL镜像。 - -如果前序DML的MERGE结果未知，屏障仍会失败并停止；不能以“跳过后续DDL”绕过旧写入的安全问题。 - -### 8.3 真实验收：3组PASS - -使用本地TiUP v8.5.1、当前新架构CDC及同一授权staging Lake；CDC本地标签`2bf6dacd-dirty-options-e2e`，不是官方发布包。入口为[options.py](../../tests/integration_tests/databend_native/options.py)。 - -| 场景 | 结果 | -|---|---| -| 无任何PK/UK的表：force-replicate + `conflict-key=ID`，INSERT/UPDATE/DELETE/ADD COLUMN，journal键为`id`；另测覆盖不同的单列源PK | PASS / 32.927s | -| 真实RENAME后checkpoint跨过DDL、旧目标数据保留；对新表名继续UPDATE/INSERT，新投影值正确且旧表不变 | PASS / 16.873s | -| 真实DROP DATABASE后checkpoint继续、目标库和数据保留；重建源库、新建表的DML继续成功 | PASS / 16.143s | - -单测/race同时验证：配置格式、大小写、空值、NULL身份、复合键拒绝、未拆分变键拒绝、配置贯穿DML/journal、跳过三种DDL无目标SQL/无虚构epoch、回调顺序、后续DML推进，以及旧UNKNOWN写不能被跳过掩盖。4包race、lint（0 issues）、go vet、主程序构建通过；全仓测试/HA/成本未跑。 - -私有证据根目录： - -```text -/var/folders/lq/3n41d_w97w55hrx2mzmd42w80000gn/T/ticdc-native-options-gdj9c83o -``` - -确切路径由本机`/tmp/ticdc-native-options-current`指针记录；`manual-key-1/result.json`、`skip-ddl-1/result.json`和`cdc.log`分别保存结果及warning。两个run的status/cleanup均PASS；最终按具体名字核验源/目标库及feed清理，自有CDC/TiUP停止，临时凭据移除。未提交、推送或部署共享CDC。 - -### 8.4 为什么仍拒绝分区dispatcher - -限制来自首版主动保护，不是Lake不能写分区数据。一个capture内，同一逻辑目标已经hash到同一lane串行；但不同物理分区可由不同capture承担，多个进程可能同时MERGE同一target。`enable_table_across_nodes=false`并不等于所有物理分区都在同一capture，`exclusive=true`也不是跨capture锁。当前仍拒绝分区输入，避免把“单进程串行”误当跨进程排他；这轮没有顺手删除该保护或引入框架锁协议。 diff --git a/docs/design/databend-native-sink-plan.md b/docs/design/databend-native-sink-plan.md deleted file mode 100644 index 60dc756710..0000000000 --- a/docs/design/databend-native-sink-plan.md +++ /dev/null @@ -1,278 +0,0 @@ -# TiCDC 原生 Databend / Lake Warehouse Sink：代码分析与迁移计划 - -日期：2026-10-24。状态：历史分析摘要；当前设计已评审定稿，首版实现及受控真实链路验收通过，未提交、未部署共享CDC。 - -当前规范见 [定稿设计 v1.1](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review.md)，配套 [实现评审入口](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-review-prompt.md)。本文件保留为初始分析历史摘要，不是当前实施规范；唯一主路径与范围以v1.0为准。实现分支为 `feat/databend-native-sink`；v0.2/v0.3已另存历史版本，不同时实施。 - -v0.4.1局部修正：首版raw-change=false保持默认拆分和源顺序，首版固定raw source；Native只规范化一次；UNKNOWN在核心内层截断；spool清理不挂成功回调且不进oversized旁路；add_time按落库精度单调；DDL epoch明确取FinishedTs并迁移外层过滤。上述为已接受的范围；首批核心、运行时接入与待验收状态见定稿§8.1–8.2/§10，没有扩大框架范围。 - -最新裁决：对齐bend-cdc既有正确性，迁移raw/stream或raw source/MERGE、mapper和schema epoch核心，只新增native adapter、spool与TiCDC callback适配。删除manifest/K-H-S新算法、DML ledger与新raw_generation，不因review无限增设框架协议。保留MERGE后确认、SEALED/分片独立回收和必要失败关闭；stream全批覆盖、终态、原生事件与旧路径差异要局部验证。当前有T1–T6的部分本地覆盖及[9组真实TiUP/Lake功能验收](./databend-native-sink-e2e-report.md)，在途故障/HA与成本仍未验收，T1–T7不整体关闭；无全量工具工作包。 - -需求修订（2026-10-24）：第一版必须采用新架构 S3 sink 类似的 **内存缓冲 + 有配额本地磁盘 spool + 背压**，不再采用仅内存攒批的初版建议。磁盘 spool 是临时缓冲，不是恢复日志；确认点仍是 MERGE 成功。 - -## 1. 结论与边界 - -可行，建议新增独立的 Databend sink，直接消费 TiCDC 的原生 DML/DDL 事件，迁移 bend-cdc 的 **schema/type mapping + raw staging + stream + MERGE + ordering/tombstone + schema epoch**，不搬运它的对象存储消费者。 - -本文的 lake 指 bend-cdc 当前实际连接的 Databend SQL warehouse，不是泛化的 Iceberg/Delta/Hudi 文件 sink。如果目标接口不同，需要另做 writer/事务语义设计。 - -现状： - -```text -TiDB → TiCDC → Canal JSON 编码 → S3/OSS - → bend-cdc LIST/GET/SQS → Canal JSON 解码 - → schema/类型转换 → Databend raw table → stream → MERGE → target -``` - -建议： - -```text -TiDB → TiCDC DMLEvent / DDLEvent - → Databend sink [原生类型适配 + schema barrier + 有界批处理] - → 有界内存 / 限额本地磁盘 spool（超限背压） - → Databend 持久化 raw table → append-only stream → MERGE → target - ↓ - 成功后触发 PostFlush -``` - -去掉的是增量链路中的 Canal JSON envelope、文件生产/扫描/下载、独立 bend-cdc consumer，以及相应 CPU、内存和网络开销。**不去掉** Databend 自身存储、raw 写入、MERGE、必要的重放水位，也不承诺完全没有 JSON：第一版仍可用 raw_data JSON 承载行内容。 - -代码分析基线：webapi `b6481621021e82aee659ad88e8e6c00f6dd11906`，ticdc `2bf6dacd0aec9ac2fad464600e47341f9aa951f2`，调查开始时两边 working tree 干净。均为本地源码证据，不代表生产版本。 - -## 2. 现有实现真正要迁移的内容 - -### 2.1 路径与职责 - -| 模块 | 已确认行为 | 迁移处理 | -|---|---|---| -| `service/tidb_runner.go` | TiDB 专用 runner，明确绕过通用 channel pipeline，保存对象游标 | 不迁 runner；让 TiCDC 管生命周期/checkpoint | -| `source/tidb/canal_json.go` | 读取 pkNames/mysqlType/_tidb.commitTs，文件分类与 schema version | 不迁解码；从 DMLEvent/TableInfo 取信息 | -| `source/tidb/manager.go`、`schema_queue.go` | 表级队列、存储进度、schema 对象排在新 epoch DML 前 | 改为 TiCDC dispatcher + sink barrier | -| `source/tidb/databend_merge_sink.go` | 每目标表一个 DatabendSink；PK 推导、库表映射、schema epoch、事件转换 | 迁移语义，重写入口及资源管理 | -| `sink/databend/databend.go` | 连接、建表、DESCRIBE cache、类型推导、加列 | 迁移；用权威 TableInfo 替代首行猜类型 | -| `sink/databend/databend_cdc.go` | raw 批量参数化 INSERT、append-only stream、MERGE、去重、重试、清理 | 核心迁移；重写调度/确认/所有权 | -| `sink/databend/schema_change.go`、`schema_epoch.go`、`truncate.go` | 列变更白名单，pending/applied epoch，清空 target/raw/stream | 迁移并接入 TiCDC DDL 恢复流程 | -| `sink/databend/tidb_values.go`、`enum_set.go` | Canal 字符串格式的 binary/ENUM/SET/JSON 还原 | 迁移语义和测试，不照搬 Canal 专用表示 | -| `source/tidb/applied_watermark.go` | 区分对象消费进度与 MERGE 可见进度 | 保留“可见才确认”的契约，不迁对象轮询算法 | -| `source/tidb/dumpling*.go` | 全量 schema、CSV、COPY INTO、快照元数据 | 独立保留，不能用增量 sink 自动替代 | - -**不要选错入口**：`source/tidb/databend_sink.go` 是简单 INSERT/DELETE 路径（DELETE 固定取 id），实际 TiDB runner 创建的是 `DatabendMergeSink`。只迁前者会丢失现有核心能力。 - -### 2.2 “临时表”实际上是持久化状态 - -当前建的是普通 `
_cdc_raw`，不是 TEMP TABLE。包含 `id/table_name/raw_data/add_time/event_pos/action`；另外创建 `
_cdc_stream`，使用 `APPEND_ONLY = true`。 - -`buildMergeSQL` 使用 ROW_NUMBER 按冲突键选最新事件；严格顺序模式还查 retained raw history，拒绝被更大 event_pos 覆盖的旧事件。DELETE 同样保留最新 raw 记录作为墓碑。`buildRawCompactionSQL` 删除旧版本而保留每键最新水位，避免旧事件重放后复活已删除数据。 - -所以迁移目标应是 **持久化 staging + 安全 compaction**，不能在 MERGE 后直接清空全部 raw，不能只按 24h TTL 删除最新墓碑。保留每键水位意味着空间通常随历史键集合增长，删除过的键也会占空间；它不是固定大小缓存。 - -## 3. TiCDC 已有接入能力，以及不能漏掉的改动 - -`downstreamadapter/sink/sink.go` 已有统一接口： - -- `AddDMLEvent` 接受含 TableInfo、CommitTs、Row/PreRow 的原生事件。 -- `FlushDMLBeforeBlock` 是 DDL/syncpoint 前的 flush/barrier 钩子。 -- `WriteBlockEvent` 成功后需要调用事件 `PostFlush`。 -- `Run/Close/IsNormal` 管理生命周期及错误。 -- `SetWriteGate` 要求在真实下游写边界再次检查写许可。 -- `AddCheckpointTs` 接收已经计算出的 changefeed checkpoint，不是写入完成确认接口，不能反过来用它触发必须先完成的 MERGE，形成进度依赖环。 - -建议新目录： - -```text -downstreamadapter/sink/databend/ # 事件适配、队列、barrier、callback、生命周期 -pkg/sink/databend/ # 客户端、配置、schema、类型、raw、merge、恢复 -tests/integration_tests/databend_sink/ # 真实 TiDB/TiCDC/Databend 故障与正确性测试 -``` - -配置建议 `databend://...` + 独立 `DatabendConfig`；不是 MySQL scheme，也不是 Canal JSON protocol。下面是设计项，当前均未实现： - -1. sink `New/Verify` 分支、scheme 常量、独立 `DatabendSinkType`。 -2. `pkg/config/sink.go`、changefeed config 的默认值、校验、克隆/序列化/更新、`RmUnusedFields`，避免新增字段被清掉。 -3. `pkg/common/event/table_schema_store.go` 新 sink 的需求分支；现有默认分支会 panic。DB/All 级 DDL 所需表集合要明确维护。 -4. dispatcher/dispatchermanager 中 sink-type 分支、恢复、routing、调度兼容性；不能只改 factory。 -5. 不能返回 `MysqlSinkType` 偷用流程：manager 和 task 中会将 sink 强转成 `*mysql.Sink` 调 `GetTableRecoveryInfo`。 -6. 将必要恢复能力抽成窄接口，或新增 Databend 专属恢复分支；不借此重构所有 sink。 -7. URI/DSN/token 脱敏、TLS、token 刷新、权限预检、metrics、示例配置、mock/集成测试。 - -生产实现不建议直接 import 整个 bend-cdc module：会耦合多源捕获、应用配置和日志体系，还会影响 TiDB parser/驱动/Arrow 依赖选择。先做有来源记录的最小代码迁移，保留适用版权与 NOTICE；只有确有两边持续共享需求时再抽公共库。 - -## 4. 第一版必须确定的正确性契约 - -### 4.1 完成确认：MERGE 成功后再推进 checkpoint - -当前 bend-cdc `Write()` 返回只代表 raw INSERT 成功；MERGE 是异步的。TiCDC 不能把它直接当作“目标已写完”。 - -第一版建议： - -1. 原生事件转换成自有、不可变记录，按目标表/epoch 进入内存/磁盘 spool，保存事务 flush callback；内存与磁盘预算均显式受限，不能无限接收。 -2. 数据转交后的 enqueue 与最终 flush 分开；可参考 storage sink `DetachPostCallbacks` 的做法，不能提前释放仍被 worker 引用的 chunk。 -3. 达到行数、字节数或 flush interval 任一上限就批写 raw；低流量表也必须被定时 flush。 -4. 同一目标表只运行一个 MERGE；明确它覆盖了哪些已提交 raw 批次。 -5. MERGE 确认提交后，释放其覆盖的 spool entry，并只完成已覆盖事件的 callback，每个事务 callback 恰好一次；拆批事务必须等全部片段完成。不能因本地落盘或 raw INSERT 成功提前调用 PostFlush。 -6. MERGE 已提交但 callback/上游 checkpoint 未保存：允许重放，依靠稳定身份和版本去重收敛。 -7. MERGE 失败：不确认；临时错误有限重试，永久错误由 `Run` 返回并使 changefeed 可见地失败。 - -目标保证是 **at-least-once 传输 + 幂等最终状态**，不是未经证明的 exactly-once，也不默认保证跨表事务原子可见。 - -raw 落盘即确认也可以设计，但那需要独立持久化的待 MERGE 任务、重启扫描、接管协议及另一套 applied watermark；不建议第一版为此引入额外复杂度。bend-cdc 的 pendingEvents/writeGen/mergedGen 是内存状态，不能直接充当接管后的恢复日志。 - -### 4.2 事件身份与顺序 - -- 直接使用 `DMLEvent.CommitTs`，不用毫秒时间或本机 add_time 定义业务顺序。 -- **commit TSO 是事务版本，不是每行唯一 ID**。同一事务可以更新多行，现有 Canal 代码“每行唯一”的注释不能成为新实现假设。 -- 用源集群/命名空间、changefeed、逻辑表身份/建表代际、schema epoch、规范化复合键定义隔离域；事件版本另保存 commitTs，必要的同事务子操作顺序必须来自稳定的归约规则，不能使用重启后改变的 dispatcher seq/本机时钟。 -- 同一键、同一版本重放必须内容一致；不一致要检测并报错，不能通过新的 add_time 随机覆盖。 -- PK 值变更要处理 before/after 身份：旧键 tombstone + 新键 upsert。现有 CDCManager 的 UPDATE 只使用 After，不能直接宣称覆盖主键值更新。还要验证 TiCDC 入口是否已拆成 DELETE/INSERT，覆盖同事务换键/交换键的净结果。 -- 第一版仅接受已验证的非空单列/复合主键，不把 TiCDC handle key 一概当作用户主键；无 PK、nullable unique key、force-replicate 表拒绝或单独设计 append-only 模式。 -- 字符串键需测试大小写、collation、尾空格与 Unicode；复合键编码避免分隔符歧义。目标库表映射必须检测多源碰撞。 - -### 4.3 多节点与接管 - -bend-cdc 是单任务管理目标表，map/mutex 只保护进程内；TiCDC 每节点都有 sink，不能原样复制后同时消费同一个 stream、MERGE 同一目标表。 - -第一版限制为：**每个逻辑目标表一个明确 owner**，禁用 table-across-nodes，拒绝多 changefeed 写同一目标表，并验证调度/迁移时确实满足限制。分区表可能存在多个 physical-table dispatcher 汇入同一个逻辑表，禁用 table-across-nodes 不自动解决这个问题；未实现 owner 汇聚前第一版拒绝分区表。 - -接管必须包括停旧 writer、取消/等待其在途 SQL、fence、恢复 pending DDL、识别未完成 raw、继续 MERGE；验证 SQL 结果不确定时的重放。`SetWriteGate` 要覆盖 raw INSERT、MERGE、建表/ALTER、epoch 写、cleanup，不只是 AddDMLEvent 入口。capture lease 只是必要条件，还要证明调度转移期间旧请求不会晚于新 owner 落库；不能声称进程 mutex 或入口 gate 就提供分布式 fencing。 - -更大规模阶段再选择：逻辑表单 owner 汇聚多个 dispatcher，或分 writer staging + 单 merge owner。不能让每个节点自行操作共享 stream cursor。 - -### 4.4 DDL 与恢复 - -继续采用：旧 epoch DML 全部 MERGE → durable pending DDL journal → ALTER/TRUNCATE → applied journal → DDL PostFlush → 新 epoch DML。 - -需要复用 TiCDC 的 block 协调：本地 `FlushDMLBeforeBlock` 完成本节点相关队列屏障，配合 maintainer 对受影响 dispatcher 的协调，不把一个节点的 flush 当成整个 changefeed 的全局屏障。 - -仅保存 applied epoch 不够：还应保存可恢复的 DDL 标识/类型、目标身份、schema fingerprint/操作内容与 pending 状态。重启从 TiCDC checkpoint 较早位置重放时，必须识别已执行 DDL、跳过已覆盖的旧 schema DML，并禁止旧 DML 用新表结构解释。当前只有 MySQL-class 有显式 `GetTableRecoveryInfo`，这是新 sink 的正式开发项，不是自动继承能力。 - -支持策略： - -| DDL | 第一版建议 | -|---|---| -| 建库/建表 | 支持，依据权威 TableInfo，保留列顺序/精度/NULL 属性 | -| ADD/DROP/RENAME COLUMN | 非键列、明确默认值语义下支持，保留 epoch 恢复 | -| MODIFY COLUMN | 沿用明确的安全转换白名单，不做任意 CAST | -| 二级索引 DDL | 明确 no-op，并推进必要 schema 状态 | -| TRUNCATE | 支持前需完成 table ID 代际、raw/stream 重置和中断恢复测试 | -| PK 定义变更、复杂多操作 ALTER | 第一版拒绝 | -| RENAME/DROP TABLE/SCHEMA、分区 DDL | 明确阻塞/拒绝，第二阶段扩展，不静默忽略 | - -现有 bend-cdc 对某些不支持 DDL 会告警并推进 epoch，新 sink 不应无条件照搬；只有已证明不影响数据投影/身份的操作才可跳过。DDLEvent 的 `NotSync` 与过滤语义也要遵守。 - -**既有空表建表是独立问题**：新 changefeed 不保证为所有历史表重放 CREATE TABLE。首个 DML 的 TableInfo 可用于懒建表，但空表永远没有首行；需从 start-ts 对应 schema snapshot 获取元数据做 bootstrap，或由全量初始化任务预建表。不能把 schema store 里的名称清单当成完整列定义。 - -## 5. 类型改写：语义保留，表示重写 - -以 TableInfo/ColumnInfo 的 FieldType、flags、Flen/Decimal、Elems 和源时区为权威；同一套 mapper 服务建表、DML 编码及 DDL，消除 bend-cdc 中多个字符串映射函数的漂移。 - -| 源类型 | 目标与编码原则 | -|---|---| -| signed/unsigned integer | 精确宽度映射；BIGINT UNSIGNED → UINT64，禁止 float64 中转 | -| DECIMAL(p,s) | 精确保留 p/s；不沿用缺 metadata 时 DECIMAL(76,30) 的妥协，更不能转 DOUBLE | -| BIT(n) | UINT64，验证 BIT(64) 最大值 | -| BINARY/VARBINARY/BLOB | 目标 binary；直接从 datum 原始 bytes 转 HEX/二进制参数，禁止套 Canal 的 Latin-1 字符串还原 | -| ENUM/SET | 目标字符串，依据 typed datum/Elems 还原；不能把原生 label 再当 Canal 数字解析 | -| JSON | VARIANT/JSON；保留数值文本精度，区分 SQL NULL 与 JSON null；只解析一次 | -| DATE/DATETIME/TIMESTAMP | 明确时间范围、时区、精度和零日期策略；DATETIME 不应意外做时区平移 | -| TIME/YEAR | 延续 TIME 字符串、YEAR 小整数语义，验证负 TIME 和超 24h 值 | -| CHAR/VARCHAR/TEXT | 转 string；测试字符集、大小写、引号/反斜杠/控制字符 | -| generated column | 沿用 TiCDC 可见列策略：VIRTUAL 不写，STORED 作为物化值写入，不复制 TiDB 生成表达式到 Databend | -| 其他/不支持类型 | 明确报错或显式配置映射，不静默推成字符串 | - -首版可继续使用 parameterized INSERT 写 raw_data，再用受控 projection 转目标类型。后续评估 typed staging 或驱动批量上传路径；不为追求“完全零 JSON”阻塞正确性迁移。 - -官方 [MERGE 文档](https://docs.databend.com/sql/sql-commands/dml/dml-merge) 确认支持基于 source/subquery 的 INSERT/UPDATE/DELETE。目标 warehouse 的 stream、事务、DDL、session sticky state、结果不确定恢复与性能，仍须用实际版本验证；公开语法支持不能替代真实兼容性测试。 - -## 6. 性能与资源设计 - -1. 不沿用“每张表一个连接池 + 一组后台循环”：建议每 changefeed/目标 endpoint 共享有界连接池和 worker pool，表状态按需维护，设置空闲回收。 -2. 同时配置 max batch rows、max bytes、flush interval、max in-flight bytes 和 merge concurrency。按目标表串行、多表有限并行，避免大量小表各自频繁小 MERGE。 -3. backpressure 要贯通到 TiCDC dispatcher；不能无限保存未确认事件，也不能用无限 goroutine 批触发 MERGE。 -4. raw INSERT、MERGE、compaction 分别计时/计费；cleanup 失败不能把已成功业务写入误报为未提交并导致无界重试。compaction 不能删除尚未安全应用的更新或最新墓碑。 -5. 所有后台任务使用 sink 生命周期和超时；不把现有 context.Background() merge 循环直接带入 TiCDC，避免 Close/失租后继续写。 -6. 监控区分 accepted、raw durable、target applied、checkpoint，以及 raw backlog/bytes、MERGE SQL 时间、重试、schema 阻塞、写许可阻塞。吞吐不能只统计进入队列的行数。 - -### 6.1 第一版必需：有界内存 + 限额磁盘 spool - -正常在内存中攒批；达到内存预算后，新增记录溢写本地 segment 文件。按批大小、等待时间或 DDL barrier 刷到 Databend raw，再执行 MERGE。第一版将 spool entry 保留至对应 MERGE 确认提交后释放，使未完成工作受同一套预算约束。 - -采用基线 `2bf6dacd0` 的 `spool.TryEnqueue`、segment 管理和 flush marker 思路，但不能直接照搬全部默认策略：该实现有“单 entry 超过磁盘配额时在内存接受并立即刷出”的特殊路径。新 sink 要对超大记录/事务单独定义限制，不能声称复用后内存天然存在严格硬上限。 - -必须具备的空间与背压约束： - -- **单 changefeed 配额 + capture 共享总预算**：多张表、多 writer 共享所属配额；所有 changefeed 还受节点总预算约束，不能每表/每 worker 各自分配一份上限后将磁盘累加写满。 -- **磁盘实际占用计费**：包括 segment 文件头、未回收空间和在途写入预留；删除/截断失败时继续计费，不能只扣逻辑消息大小就宣称空间已释放。 -- **最低剩余空间保护**：写入前同时检查 spool 配额及文件系统可用空间，为日志、其他 TiCDC 数据及其他进程留余量；多个 spool 实例原子预留共享预算。其他进程仍可能抢占空间，因此 ENOSPC/IO 错误必须可诊断、停止接收且不提前确认。 -- **达到限制立即背压**：先调度已有 raw flush/MERGE，释放空间后再接收；不删除未完成数据腾空间，不无限重试落盘，也不通过无界内存队列绕过配额。 -- **防止调度死锁**：MERGE、回收、DDL barrier 等控制路径不与等待配额的生产者互相占用唯一 worker/锁；下游不可用时允许进度停住，恢复后应能自动排空积压。 -- **读回同样有界**：spool 回读、编码缓冲和 SQL batch 都有预算，不能一次将所有磁盘数据加载回内存。源事件/chunk 的生命周期及队列元数据也必须计入内存设计，不能把 spool payload 配额当成进程 RSS 上限。 -- **超大事务/单行**：事务可以按预算拆分持久化/写入，回调等所有片段 MERGE 后完成；单行超过支持的最大大小时明确报错，不无限加内存或磁盘预算。 -- **重启恢复**：spool 不作为 WAL。重启/节点迁移仍由已保存 checkpoint 重放，依赖 raw/MERGE 幂等；清理 spool 只能操作当前实例拥有的目录，不能删除其他活跃实例数据。 -- **监控与验收**：增加 spool 内存/实际磁盘字节、共享预算使用率、可用空间、配额等待、segment 回收失败；覆盖多 changefeed、磁盘满、回收失败、超大行、慢 MERGE、失租、重启和背压解除测试。 - -配置接口建议（待实现，不是当前可运行配置；容量仅为示例，需按节点磁盘和 changefeed 数量定额）： - -```toml -[sink.databend-config] -spool-memory-quota = 67108864 # 64 MiB，单 changefeed 热数据预算 -spool-disk-quota = 10737418240 # 10 GiB，单 changefeed 磁盘上限 -``` - -节点级还必须有跨 changefeed 的 spool 总预算和最低剩余空间阈值；这两项属于 capture/server 级配置，不应复制成每表配置。具体字段名及与 TiCDC 现有 quota 的整合方式在实现前确定。上述预算不限制 Databend 端 raw 的历史水位/墓碑容量，后者仍按安全 compaction 单独管理。 - -成本比较固定同样输入/目标延迟/SLO：原链 TiCDC 编码成本 + 存储请求与容量 + consumer LIST/GET/解码/调度成本，对比新链新增的 TiCDC writer 成本；另单独计算 Databend raw/MERGE/compaction 与网络费用。直接 sink 会失去对象存储的长时间缓冲，warehouse 故障将更快反馈到 TiCDC lag 和上游保留窗口，需约定最长可恢复停机时间。 - -不提前承诺节省百分比；如果 Databend MERGE 计算是主成本，链路缩短不一定等比例降低总费用。 - -## 7. 全量初始化、上线与回退 - -增量 sink 不负责导出现存 TiDB 全量。保留 Dumpling + COPY INTO 作为独立初始化任务：选一致快照 S → 从 S 建 changefeed 并确保增量可保留 → 按 S 的 schema/数据导入 → 校验 → 放行大于 S 的增量。实施前验证 start-ts 边界和 GC/日志保留；全量期间不能直接让增量覆盖后又被旧快照写回。 - -切换建议使用 **独立影子目标库表**：旧链继续写原目标，新 sink 写影子目标；对齐快照/TSO 后核对 count/hash/关键业务查询及 schema，再切读并停旧链。禁止两个链路同时消费同一 stream、共用 raw 或写同一目标表。 - -回退需预留旧链原始数据和消费进度，在连续覆盖窗口内追平到已验证水位再切读，不能只替换二进制。新 sink 目标的 target/raw/stream/epoch/owner 状态按一组管理；不自动复用旧 bend-cdc 的内部表，也不自动转换其文件游标。 - -## 8. 分阶段工作包与验收 - -| 阶段 | 工作与产物 | 退出标准 | -|---|---|---| -| P0：语义验证 | 锁定 warehouse/驱动版本；验证 raw/stream/MERGE、重复批次、提交结果不确定、sticky session、fencing/接管、DDL 中断恢复 | 核心恢复模型明确；无法证明时先不扩展功能 | -| P1：增量纵切 PoC | 独立 scheme/type/config；原生事件适配；PK/复合 PK；内存/磁盘 spool；节点共享配额与背压；raw/MERGE；MERGE 后 ack；单 owner | insert/update/delete、重放、低流量 flush、超大事务、磁盘满/回收失败/多 changefeed 配额、cancel/Close、失租不丢数据 | -| P2：核心能力对齐 | 权威建表/空表 bootstrap；类型矩阵；列 DDL/epoch 恢复；TRUNCATE；库表映射；错误与监控 | 达到现有已支持核心行为，补齐 PK 值更新及故障案例；不支持项可诊断地拒绝 | -| P3：灰度与优化 | 快照接续、影子流量、全链故障注入、成本基准、多表并发；再评估分区/跨节点 owner 汇聚及 typed staging | 正确性与目标 SLO 达标，成本实测下降，可演练回退 | - -粗估：一名熟悉两端代码的工程师，受限 PoC 约 1–2 周；DDL/恢复/完整类型和灰度再预留数周。该估算不是交付承诺，跨节点/fencing 和真实 warehouse 兼容性可能成为关键路径，应在 P0 后重新排期。 - -必须覆盖的验收用例： - -- 复合键、NULL/极值、大整数与 DECIMAL、二进制全字节、ENUM/SET、JSON 大数、时区和生成列。 -- 同毫秒不同事务；同事务多行；同键旧版本延迟；DELETE 后旧 INSERT 重放；PK 值更新和换键。 -- raw INSERT 前后、MERGE 提交前后、callback 前后强杀；响应丢失；重启/调度接管；重复数据不复活/不回退。 -- pending/applied DDL 各中断点，rename/drop/re-add column、TRUNCATE 代际、不带后续 DML 的 DDL、历史空表。 -- 多节点失租/恢复、旧节点在途请求、目标撞名、分区/跨节点不支持时的创建拒绝。 -- raw cleanup 与 MERGE 并发、墓碑保留、warehouse 长时间不可用、超限背压、token 刷新、权限变化。 -- 内存溢写磁盘、节点总预算、最低剩余空间、ENOSPC、segment 回收失败、超大 entry 限制、配额等待时 MERGE 仍能推进并释放空间。 -- 快照 S 接续与影子切换，不把 source checkpoint lag 当作 target applied lag。 - -## 9. 本次实际验证与证据索引 - -已执行：`go test ./pkg/sink/databend ./pkg/source/tidb -count=1`（在 bend-cdc module），两包通过。未设置 BENDCDC_TEST_DSN/TIDB_DDL_INTEGRATION_DSN，真实 Databend 集成测试按现有条件跳过，因此这些结果只验证现有本地单元/模拟测试，不证明原生 sink 已可运行。 - -未改生产代码，未跑新 sink E2E、故障注入、成本 benchmark 或部署。TiCDC 尚未新增 sink，且本机默认 Go 1.25.5 与该 repo go.mod 要求 1.26.8 不同；本次不构建 TiCDC。 - -主要源码证据（绝对路径便于本机复核）： - -- [实际 TiDB runner，L66–138](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/service/tidb_runner.go#L66-L138) -- [schema epoch 与写入适配，L93–260](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go#L93-L260) -- [复合主键与有序模式，L305–474](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/databend_merge_sink.go#L305-L474) -- [raw 写入与异步调度，L642–855](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L642-L855) -- [持久化 raw/stream 与参数化写入，L888–1016](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L888-L1016) -- [墓碑 compaction，L1188–1203](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L1188-L1203) -- [严格顺序 MERGE，L1625–1704](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/sink/databend/databend_cdc.go#L1625-L1704) -- [现有列 DDL 白名单，L80–188](/Users/jiangjianyuan/work/git/pingcap/webapi/bend-cdc/pkg/source/tidb/schema_ddl.go#L80-L188) -- [TiCDC sink 接口和工厂，L36–105](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/sink.go#L36-L105) -- [原生 DML 元数据，L375–430](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/dml_event.go#L375-L430) -- [DML flush/enqueue callback，L660–705](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/dml_event.go#L660-L705) -- [TableSchemaStore sink-type 分支，L65–109](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/pkg/common/event/table_schema_store.go#L65-L109) -- [MySQL 专属恢复分支，L551–570](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/dispatchermanager/dispatcher_manager.go#L551-L570) -- [storage sink callback 解耦参考，L51–68](https://github.com/sdojjy/tiflow/blob/2bf6dacd0aec9ac2fad464600e47341f9aa951f2/downstreamadapter/sink/cloudstorage/task.go#L51-L68) diff --git a/docs/design/databend-native-sink-review-prompt.md b/docs/design/databend-native-sink-review-prompt.md deleted file mode 100644 index 67dab2a07e..0000000000 --- a/docs/design/databend-native-sink-review-prompt.md +++ /dev/null @@ -1,74 +0,0 @@ -# 定稿后的实现评审入口：按 bend-cdc 既有语义迁移 - -用户已确认设计评审通过，[定稿v1.1](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review.md)按v0.4.1范围进入 `feat/databend-native-sink` 分支实现。后续优先评审实际代码与该定稿的偏差，不重新扩展通用协议；实现状态见设计§8.1–8.3及§10；[真实验收报告](./databend-native-sink-e2e-report.md)记录本地TiUP→当前CDC→staging Lake的9组PASS、32列及排空重启。单测与受控链路通过不等于在途故障/自动HA验收。 - -最新补充：[bend-cdc能力对照与真实故障验收](./databend-native-sink-parity-fault-report.md)。重点看已验证的迁移差异，不把旧版“warning跳过”算已支持；非空唯一键fallback已验证，参数化字符widening局部修复见源码。 - -最新用户裁决已实现：URI `conflict-key`、指定RENAME/DROP DATABASE warning跳过；真实3组通过，见补充报告§8。此前对应“仍拒绝/未迁移”的结论是历史状态；分区保护未开放。 - -## 基线与范围 - -- 新架构TiCDC：`/Users/jiangjianyuan/work/git/pingcap/ticdc`，固定commit `2bf6dacd0aec9ac2fad464600e47341f9aa951f2`。 -- bend-cdc：`/Users/jiangjianyuan/work/git/pingcap/webapi` 下 `bend-cdc`，固定commit `b6481621021e82aee659ad88e8e6c00f6dd11906`。 -- 设计基线按固定Git对象核验；实现位于上述新分支，以实际分支/工作区代码为准，不切分支或覆盖开发中的改动。 -- 用户要求：只对齐bend-cdc当前TiDB增量路径已正确支持的语义，核心改动集中sink，不为迁移重做TiCDC框架。 -- [v0.3](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review-v0.3.md)、[v0.2](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review-v0.2.md)以及旧review comment都是历史讨论；manifest/K-H-S/AppliedCycle/新raw_generation不是当前要求。 - -## 唯一主线 - -```text -原生事件adapter（不经Canal编解码） - → 有界内存/限额spool与按表worker - → 迁移现有Databend raw写入、Flush/MERGE、compaction - → 目标完成后回调；SEALED分片计数独立回收 -``` - -复用现有raw字段、复合PK、TSO排序、墓碑、stream/raw source、schema epoch和TRUNCATE顺序，不默认增加per-row manifest读回、event_id/payload_hash、DML ledger、分布式锁/预算、collector信用/progress迁移。 - -普通DML无需逐成员回读，但多条raw SQL必须全部成功且Flush覆盖对应工作后才确认。**首版显式固定raw source，不自动创建/消费stream**；stream代码可保留，启用前验证batch limit/no-op与代数记账覆盖，不能循环pending已清零的旧Flush冒充排空。full-raw扫描和大量保留键重复更新成本仍要测。 - -## 本轮六项明确决策 - -1. 首版沿用固定CloudStorage默认拆分模式`raw-change=false`，撤销强制true的接入；保留scanner的DELETE/延后INSERT顺序。非聚簇PK变键要真实验证；若意外收到oldPK!=newPK的原始UPDATE则拒绝，不直接交After-only核心。默认源码不等于实际旧任务配置，后者须另核验。 -2. Canal原始值入口与NativeNormalizer规范行入口分离，汇合后共享raw写入/MERGE；规范化恰一次，不伪造Source绕过解码/丢失TSO。数字label ENUM('2','x')必须测试。 -3. 默认raw source解决首版部分stream消费前提；writeGen/mergedGen仍不是独立覆盖证明，不因“沿用”就跳过T3。 -4. 在客户端/核心**内层重试之前**传播COMMITTED/NOT_COMMITTED/UNKNOWN；UNKNOWN不得被executed分支吞成成功或transient分支重发，外层也不能再重试。 -5. spool成功回调列表为空，源callback只在SEALED tracker；取消/Discard/Release不能间接完成源事件。enqueue前拆分/拒绝oversized，不接受后再声称有界。 -6. add_time先按真实存储精度量化再保证单调；检验SQL参数/落库，而非只测Go内存递增；不把它提升为任意源冲突的业务版本。 - -schema映射固定：raw event_pos=CommitTs，实际DDL epoch=FinishedTs，TableInfoVersion只解释schema。迁移外层writeBatch的旧事件过滤与pending阻新数据；bootstrap不得凭空保存applied epoch。 - -不做Dumpling/CSV/COPY INTO、全量编排或历史空表预建。单writer/无外部写的运行前提必须成立；不因来源相同就假设旧SQL、原生session/callback与S3文件恢复完全相同。 - -## 当前实现与证据边界 - -运行时代码集中在 `/Users/jiangjianyuan/work/git/pingcap/ticdc/downstreamadapter/sink/databend/`。当前已经有native mapper、HTTP终态/参数绑定、按表lane、共享spool与回调、schema epoch恢复，以及factory接入；不是只有接口草图。 - -先读定稿§8.2–8.3/§10：受控PoC明确拒绝分区dispatcher/复杂DDL，未验证自动HA；新journal附带必要的post-DDL定义与步骤，不自动接管旧bend-cdc目标。local/race/lint/build通过不代表真实Databend SQL和完整控制面恢复通过。请特别核对源码已有的scope限制、边界输入与SQL实际语义，避免把HTTP fixture当成数据库实现。 - -## 请按“是否退化”评审，不默认增加理想化要求 - -每个发现请标注： - -1. **迁移退化**：bend-cdc已正确处理，而当前native/spool/callback适配会破坏；必须修。 -2. **接入差异**：去掉S3或使用TiCDC原生生命周期后新增的必要责任；给最小sink/接入补丁。 -3. **既有缺口**：旧代码也存在假设或潜在bug；明确证据及是否在支持场景下会造成错误，不能盲抄，也不能默认为迁移要重建整个系统。 -4. **增强需求**：比基线更强的HA/跨表原子性/全局硬预算等，默认后移，不作为新增sink首版合入要求。 - -优先检查： - -- native与Canal转换后的内部行差分：复合键、二进制、ENUM/SET、decimal/JSON/时间、同TSO合法变化、UK更新。 -- 现有WriteToRawTable/Flush/mergeTable/writeGen/mergedGen是否足以绑定完成的callback；stream部分消费时是否可能错误确认，最小修正是什么。 -- raw子SQL部分成功、MERGE提交未知、callback前重启后重放；不新增ledger能否在受控排他前提下收敛。 -- 现有schema epoch的数值含义与原生FinishedTs/表版本是否正确映射，pending/applied与TRUNCATE重放是否会退化；不能用新raw_generation替代必要源码审查。 -- spool正常压力/封口/回收、transport reset不丢已接管数据、fatal结束旧生命周期；既有框架接入若不够，给确切最小修改，不恢复整套delivery协议。 -- 现有warning跳过的DDL不算完整支持；已知问题不能因“等价迁移”被宣称正确，支持范围之外不无限扩展。 -- 对相同SLO/负载比较raw source与stream、旧链与新sink的扫描和总成本；不预设任一方案更便宜。 - -## 输出 - -总体：接受 / 修改后接受 / 拒绝。 - -每项：证据commit/定位、基线与新方案差异、最小触发时序、影响、最小修复、验证断言；标注上述四种类型以及“sink内 / 小接入 / 超出范围”。按正文T1–T7提出具体验收，不把旧本地测试通过当新sink通过。 - -本轮只读评审；不要修改文件/代码、切分支、创建任务或执行生产操作。 diff --git a/docs/design/databend-native-sink-scope-review-comment.md b/docs/design/databend-native-sink-scope-review-comment.md deleted file mode 100644 index 54f68e0150..0000000000 --- a/docs/design/databend-native-sink-scope-review-comment.md +++ /dev/null @@ -1,94 +0,0 @@ -# Review comment：请先评审“sink 内聚、最小框架改动”的收敛方案 - -> 状态更新：设计已接受，当前规范为[定稿v1.0](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-design-review.md)，以bend-cdc既有正确性为基线。以下是历史review comment，其中manifest/K-H-S候选及raw-change=true已由当前基线迁移决策替代，不是第二套实施要求；继续评审请使用[当前任务说明](/Users/jiangjianyuan/work/git/pingcap/ticdc/docs/design/databend-native-sink-review-prompt.md)。方向裁决不代表实现/实验通过。 - -## 结论与评审约束 - -v0.2 对数据正确性的讨论有价值，但新增 collector delivery/信用协议、progress 迁移和跨 capture 账本，会把“新增一个 Databend sink”扩大成 TiCDC 框架改造，不利于合入。本轮请优先评审下面的收敛方案，**不要默认通过增加框架协议来修复每个反例**。 - -这是针对 v0.2 的架构调整提议，不是已实现或已验证的 v0.3。原报告尚未整体替换为本文；有冲突处作为本轮待裁决事项，不能把两套协议同时当实施要求。 - -固定源码基线： - -- TiCDC：`2bf6dacd0aec9ac2fad464600e47341f9aa951f2`，仓库 `/Users/jiangjianyuan/work/git/pingcap/ticdc`。 -- bend-cdc：`b6481621021e82aee659ad88e8e6c00f6dd11906`，仓库 `/Users/jiangjianyuan/work/git/pingcap/webapi` 下的 `bend-cdc`。 -- 当前 checkout 不代表上述新架构版本，请用固定 Git 对象复核，不切分支。 - -## 1. 建议保留的工作流程 - -```text -现有 dispatcher 交付原生 DML/DDL - → sink 内取 TableInfo、主键、commit_ts、before/after - → 统一类型转换与内部行记录 - → 有界内存缓冲 + 限额本地 spool - → 同表/同 schema 攒批，不同表有限并行 - → 批量 INSERT 持久化 raw staging（带 batch_id / 事件身份） - → MERGE INTO 目标表 - → 分片应用成功后独立释放本地资源 - → 事件 SEALED 且全部分片完成后，现有 flush callback 恰执行一次 - → TiCDC 原有机制推进 checkpoint -``` - -- 保留内存+磁盘 spool，不改回纯内存；raw 写成功不能确认源事件。 -- 同表 raw INSERT、MERGE、DDL、compaction 首版串行，避免先引入高并发覆盖协议。 -- 保留 `OPEN → SEALED → COMPLETED` 和已应用分片独立回收，不为等事件全集确认而钉住全部 spool。 -- Databend 内部固定 `raw-change=true`，只增加必要的初始化/注册接入；普通二级 UK 更新必须正确。真实 PK/handle 变更仍需源事件实验，不把这个开关当万能解决方案。 - -## 2. 首版考虑不以 stream 作为应用/确认依据 - -建议评审一个更小的替代：**按明确 batch_id 从 raw 读取本批数据进行 MERGE，不依赖 stream 的消费 cursor 判断完成。** Stream 不作为首版必要依赖，后续若成本实测证明有必要，再单独引入。 - -这样可以移除本轮“stream hint 只消费了 F1，却给 F1/F2 写了 witness”的故障路径；也不再需要为 stream 构造跨层消费证明系统。但必须注意： - -1. batch_id 是下游写入批次身份，不是源业务版本；本次逻辑批次的重试复用身份，重启重新分批仍需按事件身份/commit_ts 幂等收敛。 -2. 每个批次写 raw 可拆多个 SQL；全部待应用记录明确写入成功后才执行 MERGE。不能用 INSERT/MERGE 返回的操作行数充当事件成员覆盖证明。 -3. 不能简单把本批旧值直接覆盖目标。候选思路是：取本批涉及主键，结合同代际、合法 schema 范围的 raw 历史，重算各键最高有效版本并实际应用，包含 DELETE 墓碑。 -4. 不能只因 raw 有更高版本就跳过并确认旧事件；更高版本也可能尚未应用。若采用上述重算，应真的将最高版本纳入当前 MERGE，而非仅用它过滤旧行。 -5. 相同 key/TSO 的冲突、跨 schema 历史、同键乱序/重放、未完成批次、compaction 都必须有明确行为；SQL 与快照一致性仍待验证。 -6. 若某批只部分可应用，未覆盖分片不能确认。`batch_id + MERGE 成功`不是自动正确性证明，必须说明 SQL 来源集合完整、无意外过滤/限制，以及对应 callback 的成员集合。 - -请比较这个直接批次方案与 stream 方案的正确性、SQL复杂度和扫描成本。**去掉 stream 可能增加 raw 扫描，不能预先认定一定更便宜；本提议也不视为关闭所有覆盖风险。** - -## 3. 不改 collector/dynstream/TableProgress 的入口处理 - -删除 v0.2 新增的 delivery 重投、transport epoch 重绑、progress blocker 迁移等定制协议，优先沿用现有 AddDMLEvent / PostEnqueue / PostFlush 生命周期。 - -- 已被 sink 接管的数据由 sink 生命周期管理，不因 collector 的 transport session reset 直接丢弃。源重放由现有框架处理，sink 负责幂等；不能伪造 PostFlush 清 blocker。 -- sink 的入口队列、编码窗口、spool 和元数据必须有界。PostEnqueue 按真实接管/现有表路径背压语义触发，不提前虚报。 -- 不允许在 AddDMLEvent 中无限等待磁盘并占住共享处理线程。无法在有界入口内接管时，首版可通过既有 sink 错误路径停止任务，不无限排队、静默丢弃或自行搬迁 progress；需要复核现有错误路径确实会完成必要的生命周期清理。 -- 上述取舍不承诺“只靠 sink 就能硬限制全 TiCDC RSS”，也不承诺资源耗尽时其他表完全不受影响。原生大 chunk 已在 sink 前分配的问题要明确限制/测量，不靠文档声称消失。 - -请给出“不新增 delivery 协议”的最小调用时序，核查 session reset、接管失败、任务停止/重启时是否仍会遗留 blocker 或漏事件。如果原框架存在无法规避的问题，先给具体反例和最小补丁位置，不能直接恢复整套信用协议。 - -## 4. DDL、恢复及容量控制的简化边界 - -- 建表、类型映射、raw/MERGE、白名单 DDL、DDL journal 与 TRUNCATE 恢复保留在 sink。使用已有 DDL 屏障，不重做全局调度；历史空表可由独立初始化预建，不为此新增框架 schema 服务。 -- 保留 TRUNCATE 的持久化身份/代际和“APPLIED 前不开新代际”的恢复顺序；不能为了减少代码去掉必要的破坏性操作幂等保护。 -- 提交未知时不确认，不继续同表下一批；先核验旧 SQL 终态。驱动取消返回 nil、进程退出、固定等待时间均不是终态证明。 -- 正常迁移/异常接管仍需排除旧在途 SQL；across-nodes=false 不等于禁用自动 HA。若只能依赖大规模框架变更才可生产安全，首版应明确缩为隔离、受控、需人工恢复的 PoC，不虚报自动 HA。 -- 远端 raw/tombstone 增长仍需监控、阈值停写和扩容/重初始化流程。但首版不为此默认新增跨 capture 强一致预算服务；严格 feed 总预算应另评估静态配额切分/已有外部配额。无法保证时，明确是监控型停止阈值而非全局原子硬额度，不能对外承诺严格封顶。 - -## 5. 允许与不允许的修改范围 - -| 范围 | 本轮要求 | -|---|---| -| 新 Databend sink 包 | 事件适配、类型/SQL、buffer/spool、batch worker、tracker、DDL、恢复、错误与指标 | -| scheme/type/config/factory/metrics | 必要注册与参数校验 | -| raw-change模式、schema store、recovery接入 | 允许有限分支/小接口，需列出确切触点及原因，不能伪装成MySQL sink | -| collector/dynstream/TableProgress/调度核心 | 不做通用协议改造；如发现确实阻塞，先提交最小必要变更论证 | - -“主要改在 sink”不等于强求外部零改动；但新增 sink 不应变成 TiCDC 全链路流控/调度重构。 - -## 6. 请 reviewer 给出明确裁决 - -请重点回答： - -1. 不新增 delivery/信用协议，复用原有接口能否做到不丢数据、正常完成、资源失败可诊断？列出最小必要的外部改动。 -2. 直接 batch_id + raw MERGE 能否比 stream/witness 方案明显简单？请给出候选 SQL/成员映射及会击穿它的最小反例，不只说“需验证”。 -3. 旧事件重放、不同 batch 重复、最新墓碑、同TSO合法更新、跨schema恢复是否仍正确？ -4. raw扫描/历史保留成本是否会抵消去掉中间链路的收益，最小实验怎么设计？ -5. 哪些风险必须在sink中解决，哪些需要小接入补丁，哪些只能通过明确缩减首版范围处理？ - -输出：**接受 / 修改后接受 / 拒绝**；每个发现写证据、触发时序、影响、最小修正、验证方法，并标注“sink内修改 / 小接入修改 / 超出本轮范围”。不要为了给通过而补隐含前提，也不要只为追求理想化全局保障无限扩大框架范围。 - -本轮仅评审，未实现该收敛方案，未运行真实 Databend/新 sink 故障实验。不要修改代码、切分支、创建任务或执行生产操作。 diff --git a/docs/design/lake-sink-design.md b/docs/design/lake-sink-design.md new file mode 100644 index 0000000000..3e482408df --- /dev/null +++ b/docs/design/lake-sink-design.md @@ -0,0 +1,508 @@ +# Native Lake Sink: Final Design and E2E Results + +Last updated: 2026-10-26. + +Status: experimental incremental sink; controlled single-capture operation with +an exclusively owned target. This is the only design and acceptance document for +this feature. Historical proposals and separate review/test reports are removed. + +## 1. Purpose and scope + +Replace the intermediate `TiCDC -> Canal JSON -> object storage -> bend-cdc -> +Lake` path with a native sink: + +```text +Existing TiCDC dispatcher and DDL barriers + -> native typed-row adapter + -> shared source admission + bounded memory/disk spool + -> fixed table lanes + -> retained raw INSERT -> target MERGE -> raw compaction + -> source completion callback +``` + +The correctness reference is bend-cdc's TiDB incremental merge path at +`b6481621021e82aee659ad88e8e6c00f6dd11906`. Migrate its key, raw/MERGE, type and +schema-epoch semantics rather than introduce another general CDC framework. +The upstream integration base is TiCDC `b9179182f`. These are source comparison +identifiers, not deployment versions. + +### Supported first-version behavior + +- Incremental INSERT/UPDATE/DELETE for ordinary non-partitioned tables. +- Explicit primary keys, TiCDC-selected non-null unique keys, composite source + keys, and an explicit single-column `conflict-key` override. +- Lazy target/raw table creation, parameter-bound batched writes, retained + tombstones, strict version ordering, target MERGE and historical compaction. +- CREATE DATABASE, CREATE/RECOVER TABLE, DROP TABLE, TRUNCATE and a conservative + subset of ADD/DROP/RENAME/CHANGE/MODIFY COLUMN. +- Literal defaults, nullability changes and supported widening conversions. +- Durable pending/applied DDL recovery and old-event/DDL replay fences. + +### Explicit boundaries + +- One controlled capture and a statically exclusive target. `exclusive=true` is + an operator assertion, **not** a distributed lock or an automatic HA protocol. +- Reject partition dispatchers, across-node table splitting, syncpoints, + active-active replication, redo, low-latency mode, column selectors, target + routing and table-level transaction atomicity. +- RENAME TABLE, including multi-table rename, and DROP DATABASE are explicitly + skipped with warnings after draining old DML. They preserve target objects; + this is not successful target-side DDL translation. Subsequent DML uses its + current source name and may create a new projection without moving old rows. +- No Dumpling, CSV, COPY INTO, full snapshot orchestration or historical empty + table discovery. Users independently prepare a consistent starting state. +- No stream, persistent DML ledger, batch manifest, consumption witness, new + collector credit protocol, progress migration or distributed quota service. + +## 2. Integration and implementation map + +Core logic is owned by [the Lake sink package](../../downstreamadapter/sink/lake). +Existing-code changes are limited to scheme/factory/type registration, +schema-store requirements, error classification, metrics and shared spool byte +accounting. The dispatcher/collector framework is not redesigned. + +| Component | Responsibility | +|---|---| +| [sink.go](../../downstreamadapter/sink/lake/sink.go) | Lifecycle, shared source admission, DDL barriers and callbacks | +| [native.go](../../downstreamadapter/sink/lake/native.go), [row.go](../../downstreamadapter/sink/lake/row.go) | Authoritative source schema and typed-value normalization | +| [lane.go](../../downstreamadapter/sink/lake/lane.go), [completion.go](../../downstreamadapter/sink/lake/completion.go) | Incremental fragments, bounded spool, batching and event completion | +| [writer.go](../../downstreamadapter/sink/lake/writer.go), [sql.go](../../downstreamadapter/sink/lake/sql.go) | Serial raw/MERGE/compaction and version ordering | +| [client.go](../../downstreamadapter/sink/lake/client.go) | Bound Query API requests and matching terminal-state evidence | +| [schema.go](../../downstreamadapter/sink/lake/schema.go) | Durable schema plans, reconciliation and recovery | +| [local.py](../../tests/integration_tests/lake_native/local.py) | Owned local backend/source/capture E2E lifecycle | + +Public names are Lake: `lake://`, `lake+http://`, `LakeSinkType`, Lake errors, +`ticdc_lake_*` metrics and `integration_test_lake`. Backend wire headers and the +compatible backend image keep their actual identifiers. The optional fixed +bend-cdc baseline overlay also retains that module's real API/driver names; +those identifiers are not new public sink names. + +## 3. Native events, identities and values + +Keep `outputRawChangeEvent=false`, matching the fixed cloud-storage input path. +The existing scanner splits relevant PK/UK changes into DELETE and delayed +INSERT. Preserve that order, including opposite actions with the same key and +commit TSO. Commit TSO is not a unique row/event identity. An unexpected unsplit +UPDATE changing the effective conflict identity fails closed instead of leaving +an old key behind. + +Use authoritative `TableInfo` and native `CommitTs`. Native rows do not pass +through Canal ordinal/Latin-1 decoding a second time. Normalize once into an +owned immutable payload, action and canonical key representation: + +- Resolve a source PK or non-null unique key using TiCDC's existing helper. +- Composite keys use a canonical named key/value representation; SQL joins use + every effective key column. Null/missing identities are rejected. +- `conflict-key` is a feed-wide single-column override, not a source index. It + can replace no key or a single native key, but not a composite key. The operator + guarantees unique, non-null, stable values and must not change it on an + existing projection without reinitialization. +- **A truly keyless source also requires `force-replicate = true` in the TiCDC + replica configuration** (`force_replicate` in the API). `conflict-key` alone + cannot make the upstream emit an otherwise ineligible table. + +Type mapping preserves signed/unsigned integer widths, exact decimals up to the +supported precision, numeric-label ENUM and SET labels, BIT(64), binary bytes, +JSON documents, SQL NULL versus JSON null and microsecond temporal values. +Binary is hex-encoded once and projected using `unhex`. JSON uses a separate +projection so SQL NULL is not confused with a JSON document containing `null`. +TIMESTAMP converts the configured source timezone to UTC; DATETIME retains wall +clock values; TIME/duration uses text. Unsupported or invalid values fail closed. +Visible STORED generated values become ordinary nullable columns. Invisible +VIRTUAL generated columns are not materialized or recreated as expressions. + +## 4. Admission, batching and bounded buffering + +### Shared admission + +`source-bytes` and `source-events` are **sink-wide**, not per-dispatcher limits. +Charge shared source chunks conservatively in full for each admitted event, +including bounded metadata overhead. Dispatcher batch hints are not a global +memory reservation. + +`AddDMLEvent` waits on shared quota or lane-queue pressure instead of marking the +changefeed unretryable. It publishes only after reserving bytes/counts under the +admission mutex. A lane dequeue or source release wakes waiters. Waiting does +not hold the mutex, allocate another event queue or start per-event goroutines. +Cancellation, fatal failure and Close wake/terminate waiting calls without a +success callback. An individual event larger than the entire configured source +byte pool is an unretryable configuration/data-size error, not ordinary pressure. + +Waiting occurs inside the existing synchronous sink call. A shared handler can +therefore temporarily wait with other paths on that handler; there is no promise +of per-table latency isolation. Independent lane workers, timer flushes and +quota-pressure draining do not require new upstream DML to make progress. +Waiting callers still own already-allocated chunks under upstream accounting; +the sink source limit bounds **admitted** retention, not total process RSS or +all framework-delivered memory. No collector/dynstream API is changed. + +### Encoding and spool + +Tables hash to a fixed number of serial lanes. Each lane incrementally encodes +bounded immutable fragments rather than copy an entire large transaction into a +second in-memory buffer. The source event is released after encoding. Low +traffic flushes on a timer; row/byte/entry thresholds and spool pressure trigger +earlier flushes. Quota pressure drains pending tables and reclaims applied +fragments, allowing an event larger than the available spool quota to progress. + +The spool has a bounded memory hot set plus a physical disk quota. Released but +not physically reclaimed segments remain charged by the shared spool. Entries +are checked with `spool.AccountingBytes` before enqueue; a single oversized row +or entry is rejected rather than use the shared spool's oversized-memory bypass. +Source retention, encoded fragments, readback, HTTP results and schema metadata +have separate limits; their sum is not an absolute RSS/physical-volume cap. + +### Completion and cleanup + +A source event owns an `OPEN -> SEALED -> COMPLETED` tracker. Register a fragment +before publishing it. Seal only after enumerating the complete event. Complete +once only when sealed, all registered fragments are applied, and no failure +occurred. Applied fragments release storage independently of whole-event ACK. + +`PostEnqueue` is a flow-control signal, not target application. `PostFlush` is +allowed only after confirmed target application or a proven applied-DDL fence. +Spool entries never own source-success callbacks. Fatal/Close cleanup uses +resource release, not `Discard`, whose callback semantics could falsely ACK. +Transport session reset does not discard data already transferred to the sink. +Fatal errors end the old lifecycle; do not clear progress by fabricating flushes. + +## 5. Raw writes, MERGE and retention + +Each table has a target projection and a `
_cdc_raw` table containing: + +```text +id, table_name, raw_data, add_time, event_pos, action +``` + +For a serial table batch: + +1. Validate the complete batch and filter rows fenced by an applied DDL epoch. +2. Execute all parameter-bound raw INSERT sub-statements in source order. +3. MERGE the target from retained raw rows newer than that epoch. Select each + key's maximum `(event_pos, add_time)` action and suppress dominated versions. +4. Obtain matching server terminal success before completing covered fragments. +5. Compact strictly dominated historical raw versions. + +The source is deliberately **raw**, not a stream cursor. There is no stream +consume hint or planned-coverage witness to interpret. Full-raw MERGE can scan +and rewrite more than an incremental stream path; benchmark it before making a +cost or throughput claim. + +`event_pos` is the native commit TSO. `add_time` is a compatibility tie-breaker, +not a new source business version: quantize to stored microseconds, then advance +by one microsecond when needed. Seed the clock from retained `MAX(add_time)` on +bootstrap so a clock rollback/restart cannot reorder a later same-TSO action. +This relies on the exclusive serial writer and existing source ordering; it is +not a solution for conflicting concurrent owners. + +Compaction keeps each key's latest watermark/tombstone. Do not TTL-delete the +latest tombstone: an older replayed UPSERT could otherwise resurrect a deletion. +Distinct historical keys can grow raw even if the target stays small and all +maintenance succeeds. `remote-bytes-limit` is a measured raw-storage stop +threshold with batch/statistics lag, not a strict cross-capture budget or a +physical billing cap. Reaching it requires diagnosis and capacity planning or +controlled reinitialization, not unsafe tombstone deletion. + +Raw INSERTs, target MERGE and compaction are separate mutations, not one atomic +transaction. A successful MERGE followed by a maintenance failure remains +applied; do not replay business SQL merely to repair maintenance statistics. + +## 6. Query outcomes and write fencing + +The client sends bound SQL parameters and a client query ID. It follows only +same-origin query continuations, carries required routing hints, disallows +redirects and checks capture write admission again immediately before mutation +POST. Errors must not expose credentials, SQL payloads or row values. + +| Result | Required handling | +|---|---| +| COMMITTED | Matching query ID and terminal `Succeeded`; complete application even if optional statistics are unavailable | +| NOT_COMMITTED | Definitive server execution failure; only explicitly proven retryable failures may enter bounded retry | +| UNKNOWN | Missing/contradictory terminal evidence, transport loss or timeout; stop immediately, no hidden retry, next batch, DDL or source ACK | + +A result page containing rows is not a commit certificate. Poll through terminal +execution; distinguish execution/transport failures from observation failures. +The HTTP adapter is conservative and does not label arbitrary network errors as +safe retries. Query timeout or a successful cancellation function is not evidence +that the server has stopped. An arbitrary UNKNOWN needs independent old-query +terminal/quiescence evidence and target exclusivity before manual recovery. +The sink does not implement automatic discovery or distributed fencing of old +MERGE/TRUNCATE requests across captures. + +## 7. DDL journal, barriers and recovery + +Use native DDL `FinishedTs` as the application epoch, not `TableInfoVersion`. +Existing TiCDC block-event handling drains old DML. Each table has a sink-owned +`
_cdc_schema_epoch` journal with `epoch`, `applied` and a serialized schema +plan. Plans contain the intended projection and explicit reconciliation steps. + +```text +drain old DML -> load journal -> skip an already-applied DDL + -> persist pending plan -> reconcile target/raw + -> verify projection -> persist applied epoch -> DDL callback +``` + +On restart, pending work is reconciled before accepting new-schema DML. Use the +durable applied schema to handle old events; rows at/before the applied epoch +are skipped. Bootstrap alone does not invent an applied epoch. Schema state is +bounded by `max-entries`; exceeding a schema-cache limit is a metadata-capacity +error rather than unbounded table retention. + +TRUNCATE recovery order is pending journal, drop raw, truncate target, recreate +raw, applied journal. Reenter pending steps before opening writes. Applied DDL +replay skips the reset so newly written rows survive. DROP removes target/raw +while preserving the journal needed for replay handling. + +Column reconciliation checks actual target metadata. Supported cases include +literal-default backfill, nullability changes, TINYINT -> INT -> BIGINT widening, +VARCHAR -> TEXT, and signed INT32 -> VARCHAR(n >= 11). Reject unknown narrowing, +unsafe casts, expression-default changes and conflict-key removal/rename or +unsupported composite-key modification. Secondary index DDL does not create a +Lake index but can advance the compatible schema fence. These rules do not +promise arbitrary multi-operation SQL or source/target transaction equivalence. + +## 8. Configuration and operational boundaries + +All sink-specific settings live in the URI. Credentials must be URL-encoded and +kept out of command history, logs and shared reports. + +```text +lake://DB_USER:TOKEN@HOST/DB_NAME?exclusive=true&workers=4&batch-rows=2048&batch-bytes=1048576&flush-interval=1s&spool-bytes=10737418240&memory-ratio=0.02&source-bytes=268435456&source-events=128&max-entries=4096&query-timeout=2m +``` + +`lake://` uses HTTPS. `lake+http://` is restricted to literal loopback IPs or +`localhost` for isolated local testing; other hostnames/IPs are rejected before +sending Basic authentication. There are no alternative branded public scheme +aliases in this initial unreleased feature. An explicit database path overrides +the source schema name. Do not collapse different source tables onto one target; +arbitrary schema/table routing is not supported. + +| Option | Default | Contract | +|---|---|---| +| `exclusive` | required `true` | Operator-owned static target exclusivity, not a lock | +| `conflict-key` | unset | Feed-wide stable unique non-null single-column override; truly keyless tables also need `force-replicate = true` | +| `workers` | 4 | Fixed lanes, maximum 64; one table hashes to one lane | +| `batch-rows` / `batch-bytes` | 2048 / 1 MiB | Fragment and write thresholds; maximum 100000 rows / 64 MiB, minimum 1 KiB | +| `flush-interval` | 1s | Low-traffic flush bound under a healthy backend | +| `spool-bytes` | 10 GiB | Shared local physical spool budget, at least twice batch bytes | +| `memory-ratio` | 0.02 | Spool hot-memory fraction of spool budget, strictly between 0 and 1 | +| `source-bytes` / `source-events` | 256 MiB / 128 | Shared admitted native-event pool; bytes must cover at least one batch | +| `max-entries` | 4096 | Per-lane fragment pressure and schema-cache limit | +| `spool-dir` | data-directory based | Capture/feed-separated temporary spool, not a replay log | +| `query-timeout` | 2m | Total single-query/pagination deadline, not proof of server termination | +| `warehouse` | unset | Backend warehouse routing header | +| `remote-bytes-limit` | disabled | Measured raw-storage stop threshold; not a global hard quota | + +Unknown/duplicate options and unsupported TiCDC modes are rejected. Source-event +and metadata counts are positive and at most 65536. Disk quota is at most 1 PiB. +The target must support parameter binding, matching query IDs, terminal query +states and the required SQL. `Verify` checks a bound SELECT; there is no string +interpolation fallback for incompatible endpoints. + +### Migration and recovery procedure + +1. Independently prepare consistent target contents and a compatible start-ts. +2. Stop the old writer. Establish that its in-flight SQL cannot mutate the target + later. An S3 file read cursor alone is not proof of applied target progress. +3. Do not transparently reuse the old two-column bend-cdc journal: the new sink + has its own pending/applied schema-plan format. Plan target/journal ownership + and reinitialization explicitly before cutover. +4. Start one capture with the target-exclusive assertion and verify actual data, + checkpoints, spool pressure and raw growth. +5. If an outcome becomes UNKNOWN, keep writes closed until old-query terminal + evidence and exclusivity are independently established, then replay from a + safe checkpoint with a fresh lifecycle. + +Removing the intermediate consumer/storage hop can reduce infrastructure and +serialization work. Type conversion still runs in TiCDC; local spool, raw/target +storage and warehouse computation remain. Full-raw scans and compaction can add +cost. No cost-saving percentage or production HA/SLO has been measured here. + +## 9. Reproducible tests + +### Fully managed local E2E + +From the checkout root: + +```sh +make integration_test_lake +# Or select/repeat suites: +python3 tests/integration_tests/lake_native/local.py --suite acceptance +# Reuse an explicitly built current-tree binary: +E2E_CDC_BINARY="$PWD/bin/cdc" make integration_test_lake +``` + +Prerequisites: a running Docker engine, TiUP, mysql CLI, Python **3.10+** and the +repository's Go/build dependencies. Allow enough disk for Go's build cache and +the source cluster, and approximately 8 GiB Docker memory. The runner checks the +Python minimum before starting a build and hashes binaries incrementally rather +than depend on Python 3.11-only APIs. It does not install tools or start Docker. + +The runner builds current-tree TiCDC and starts a unique backend meta/query +container using the pinned multi-platform image: + +```text +datafuselabs/databend:v1.2.925-patch-13@sha256:7f93f005aa0d626ebc7554d142b0a6b6ebb90e085b0d5bd0d97470084c42d41b +``` + +The actual image identifier is required to reproduce the compatible server; it +is not the public sink name. The container uses filesystem storage, publishes +HTTP only on a random loopback port, and uses random temporary credentials. No +cloud bucket, MinIO or external Lake credentials are needed. Readiness includes +real SQL parameter binding. An older tested image at v1.2.852-nightly failed +binding and is not listed as supported; no unsafe interpolation workaround was +introduced. + +Start an owned TiUP v8.5.1 source with one PD/TiKV/TiDB, no TiFlash/monitor, and an +owned `--newarch --tz UTC` capture. Probe source ports by connection and bind; +refuse reachable loopback/wildcard listeners rather than run SQL against an +existing cluster. Permit TIME_WAIT for immediate reruns. Owner election is not +sufficient readiness: wait for the changefeed read API after coordinator bootstrap. +Only on this owned source, disable fast/distributed index construction to avoid +low-disk laptop failures unrelated to CDC. + +Run acceptance, DDL, explicit-key, skipped-DDL, four expected-rejection probes +and real-backend Go tests. On success, failure or interruption, stop only owned +processes/container and clean the unique TiUP tag. Restart cleanup verifies the +exact canonical binary and data-directory paths. Do not use global `pkill` or +`tiup clean --all`; the shared Docker daemon is left running. Target container +data is ephemeral even on failure; private logs/results remain and the temporary +URI file is removed. + +The runner prints its artifacts directory. `summary.json` records build/server +versions, source commit/dirty status, image identity, suite exits and cleanup. +Each Python suite has a `result.json`; real-backend assertions are in `fault.log`. +Artifacts contain no credentials intended for publication. No personal machine +paths, generated database IDs or PIDs are required in this document. + +Optional environment: `LAKE_E2E_IMAGE`, `TIDB_E2E_VERSION`, `E2E_PORT_OFFSET` +(default 10000), `E2E_CDC_BINARY`, and an existing `OUT_DIR` parent. The managed +runner does not adopt inherited `LAKE_URI_FILE`, `LAKE_NATIVE_TEST_URI`, capture +address or source-port settings. A source/capture already running outside this +runner is not its test environment. + +### External authorized warehouse and baseline probes + +The [acceptance runner](../../tests/integration_tests/lake_native/run.py) can also +use an explicitly authorized test warehouse and already-owned local TiUP/capture. +Set `LAKE_URI_FILE` to a private mode-0600 file, `CDC_SERVER` (default local port +18300), `MYSQL_PORT` (default 14000) and a fresh `E2E_RUN_DIR` per invocation. +Never pass real credentials on argv or point these tests at production. + +```sh +python3 tests/integration_tests/lake_native/run.py +python3 tests/integration_tests/lake_native/extended.py +python3 tests/integration_tests/lake_native/options.py --manual-key +python3 tests/integration_tests/lake_native/options.py +``` + +External-mode failures retain the isolated database/feed for inspection. After +confirming old requests have terminated, use the same environment/run directory +with `run.py --cleanup`. This mode does not stop borrowed TiUP/capture services; +that remains the operator's responsibility. Its optional drained restart needs +`E2E_CDC_BINARY`, `E2E_CDC_DATA_DIR` and `E2E_PD` identifying owned processes. + +The optional [baseline overlay](../../tests/integration_tests/lake_native/baseline.py) +requires `BEND_CDC_DIR` to identify a clean, fixed `b648162` checkout and the same +private URI/run-directory setup. It modifies no baseline files. It exercises the +old core on an authorized HTTPS endpoint, not the entire S3 listing/decoding +pipeline. Its legacy driver/API identifiers must remain unchanged. + +### Focused local checks + +```sh +python3 -m unittest discover -s tests/integration_tests/lake_native -p 'test_*.py' +go test -p 2 -race ./downstreamadapter/sink/lake ./downstreamadapter/sink ./pkg/sink/spool ./pkg/errors ./pkg/config ./downstreamadapter/sink/metrics -count=1 -timeout=180s +go test -p 2 -race ./pkg/common/event -run 'TestTableSchemaStore|TestNeedTableNameStore' -count=1 +go test -p 2 ./pkg/metrics -run TestDownstreamTypeFromSinkURI -count=1 +tools/bin/golangci-lint run --timeout 5m ./downstreamadapter/sink/lake/... ./downstreamadapter/sink/metrics/... +``` + +The local managed suite supplies `LAKE_NATIVE_TEST_URI` to the Go live tests via +the environment. Without explicit opt-in these tests skip; an offline HTTP +fixture is not equivalent to real SQL execution. + +## 10. E2E results and evidence boundaries + +### Completed acceptance record + +On 2026-10-26 the Lake-named implementation with the review fixes passed all nine +suites on macOS arm64, Go 1.26.9, TiDB v8.5.1 and the pinned compatible backend +above. Validation used the review working tree on top of `a3fb6259a`, based on +upstream `b9179182f`. These are local results, not a claim about current CI. +The acceptance configuration deliberately limits the shared source pool to one +event while exercising multiple source tables; normal contention does not stop +the task. + +| Suite | Assertions | Result | +|---|---|---| +| Core acceptance | 9: CRUD; PK/UK changes; composite/quoted keys; type fidelity; column DDL; TRUNCATE; large transaction; drained restart; DROP cleanup | PASS | +| Extended DDL | 11: non-null unique handle; INT-to-wide-VARCHAR; defaults/nullability; widening; literal backfill; CHANGE; multi-operation projection | PASS | +| Explicit conflict key | 1: forced keyless CRUD/ADD COLUMN and single-native-key override | PASS | +| Skipped DDL | 2: RENAME and DROP DATABASE preserve target objects while later DML/checkpoints progress | PASS | +| Expected rejections | 4 separate probes: partition dispatcher, genuinely keyless/no override, narrowing, expression default | PASS as rejection, not feature support | +| Real-backend Go tests | 3 parent tests: replay/fences, partial raw/MERGE response loss, DDL interruption recovery | PASS | + +There are 27 Python scenarios plus three live Go parent tests. The acceptance +run observed **41 actual disk-spool readbacks**, with a 1200-row transaction and +64 KiB spool quota. Comparisons are key-based and check actual values, not merely +row counts. They include 32-column scalar/binary/decimal/JSON/temporal fidelity; +FLOAT uses tolerance while exact values, null distinctions and schema defaults +are explicitly checked. The drained restart creates a source DML/DDL backlog +and resumes from the original safe checkpoint, without overriding its timestamp. + +Real response-loss injection delegates SQL to the actual server, independently +observes a matching terminal success, then drops that response before the sink +sees it. Partial raw testing commits the first INSERT and blocks the second +before send. Unknown paths issue no early flush or blind retry. Recovery is +allowed because the test knows the old terminal outcome and has ended the old +sink lifecycle. This is not a general automatic UNKNOWN recovery service. + +| DDL | Lost-response boundaries | Count | +|---|---|---| +| CREATE | pending journal, target create, raw create, applied journal | 4 | +| ALTER | pending, ADD, RENAME, MODIFY, raw ensure, applied | 6 | +| TRUNCATE | pending, raw drop, target truncate, raw recreate, applied | 5 | +| DROP | pending, target drop, raw drop, applied | 4 | + +All **19 DDL boundaries** passed. Non-DROP recovery verifies new rows and rejects +old DML/applied-DDL replay without clearing new data. DROP verifies target/raw +absence. These are commit-boundary response-loss tests, not every OS instruction +point, server failure during execution or cross-capture failover. + +All suite-owned database/feed cleanup passed. Independent checks found no old +or restarted capture PIDs, no owned backend containers and no URI file. The +current binary was built with `make cdc GOBUILD='CGO_ENABLED=1 GO111MODULE=on go build -p 2'` to reuse the local Go cache, then supplied through `E2E_CDC_BINARY`. +Production build defaults were not changed. + +A separate 2026-10-25 authorized staging run also passed the core/DDL matrix and +response-loss probes. Fixed-version baseline core probes identified the original +scope boundaries: non-null unique keys are valid handles, table rename/database +drop are skipped rather than projected, and INT32-to-wide-VARCHAR must remain +supported. They do not prove complete old-S3-chain equivalence or seamless live +journal takeover. + +### Review regressions and remaining limits + +Review regression validation passed: + +- Event-count, byte-budget and control-filled lane-queue pressure wait without + stopping the sink or bypassing the shared quota; independent workers drain + them. Close/fatal failure terminates waiters with zero false flush callbacks. +- Admission/Close race scenarios passed 30 repeated race-enabled runs. +- Plaintext configuration accepts only loopback IPs/localhost; remote, wildcard + and misleading hostname cases are rejected before authentication. +- Real loopback/wildcard listeners are rejected before source startup/SQL; a + closed listener permits a new run. +- Nine service-free Python lifecycle tests passed, including streaming hashing + and early minimum-version rejection. All runners parse with Python 3.10 + grammar; an older installed interpreter exits before building services. +- Six focused Go race-test packages, schema-store/metrics tests, scoped vet and + fresh-cache lint passed (zero lint issues). Go/import/Python/shell formatting + and repository-relative document-link checks passed. + +No Linux/amd64 CI execution, production deployment, general HA/fencing proof, +long-running churn/capacity bound, cloud gateway/S3 qualification, cost benchmark +or performance/SLO acceptance is implied by these tests. Pinning a multi-platform +image is not evidence that a second platform was tested. Latest tombstones and +old-query exclusivity remain operational responsibilities, not closed risks. diff --git a/downstreamadapter/sink/databend/client.go b/downstreamadapter/sink/lake/client.go similarity index 78% rename from downstreamadapter/sink/databend/client.go rename to downstreamadapter/sink/lake/client.go index 909ed51027..3b0cdcf979 100644 --- a/downstreamadapter/sink/databend/client.go +++ b/downstreamadapter/sink/lake/client.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "bytes" @@ -69,7 +69,7 @@ func (c *httpClient) selectRows(ctx context.Context, sql string, args []any) ([] return nil, err } if state.state != committed { - return nil, errors.ErrDatabendExecution.GenWithStackByArgs() + return nil, errors.ErrLakeExecution.GenWithStackByArgs() } return rows, nil } @@ -78,7 +78,7 @@ func (c *httpClient) query(ctx context.Context, sql string, args []any, readData ctx, cancel := context.WithTimeout(ctx, c.opts.queryTimeout) defer cancel() if err := writelease.WaitForWrite(ctx, c.gate.Load()); err != nil { - return nil, mutationResult{state: notCommitted}, errors.WrapError(errors.ErrDatabendExecution, err) + return nil, mutationResult{state: notCommitted}, errors.WrapError(errors.ErrLakeExecution, err) } id := uuid.NewString() payload := struct { @@ -89,7 +89,7 @@ func (c *httpClient) query(ctx context.Context, sql string, args []any, readData }{sql, args, map[string]any{"database": "default", "settings": map[string]string{"timezone": "UTC"}}, map[string]int{"wait_time_secs": 1, "max_rows_per_page": 1024, "max_rows_in_buffer": 1024}} body, err := json.Marshal(payload) if err != nil { - return nil, mutationResult{state: notCommitted}, errors.WrapError(errors.ErrDatabendExecution, err) + return nil, mutationResult{state: notCommitted}, errors.WrapError(errors.ErrLakeExecution, err) } method, path := http.MethodPost, "/v1/query" node, route := "", "" @@ -105,7 +105,7 @@ func (c *httpClient) query(ctx context.Context, sql string, args []any, readData // queries, especially successful mutations, must echo our identity. preStartFailure := method == http.MethodPost && page.State == "Failed" && page.ID == "" if page.ID != id && !preStartFailure { - return nil, mutationResult{state: commitUnknown}, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "query identity mismatch") + return nil, mutationResult{state: commitUnknown}, errors.ErrLakeProtocol.GenWithStackByArgs(id, "query identity mismatch") } if page.NodeID != "" { node = page.NodeID @@ -118,10 +118,10 @@ func (c *httpClient) query(ctx context.Context, sql string, args []any, readData if page.Error != nil { code = page.Error.Code } - return nil, mutationResult{state: notCommitted}, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "server terminal failure code "+jsonInt(code)) + return nil, mutationResult{state: notCommitted}, errors.ErrLakeProtocol.GenWithStackByArgs(id, "server terminal failure code "+jsonInt(code)) } if page.Error != nil || (page.State != "Succeeded" && page.State != "Starting" && page.State != "Running") { - return nil, mutationResult{state: commitUnknown}, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "missing or inconsistent execution state") + return nil, mutationResult{state: commitUnknown}, errors.ErrLakeProtocol.GenWithStackByArgs(id, "missing or inconsistent execution state") } if page.State == "Succeeded" { state.state = committed @@ -133,18 +133,18 @@ func (c *httpClient) query(ctx context.Context, sql string, args []any, readData if readData && len(page.Data) > 0 { var data [][]*string if err := json.Unmarshal(page.Data, &data); err != nil { - return nil, state, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "invalid result data") + return nil, state, errors.ErrLakeProtocol.GenWithStackByArgs(id, "invalid result data") } // Administrative reads are intentionally bounded, independently of paging. total += len(page.Data) if total > maxResponseBytes { - return nil, state, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "result exceeds metadata budget") + return nil, state, errors.ErrLakeProtocol.GenWithStackByArgs(id, "result exceeds metadata budget") } rows = append(rows, data...) } if page.NextURI == "" || strings.HasSuffix(page.NextURI, "/final") { if state.state != committed { - return nil, state, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "no terminal execution evidence") + return nil, state, errors.ErrLakeProtocol.GenWithStackByArgs(id, "no terminal execution evidence") } c.finalize(ctx, page.FinalURI, id, node, route) return rows, state, nil @@ -152,7 +152,7 @@ func (c *httpClient) query(ctx context.Context, sql string, args []any, readData method, path, body = http.MethodGet, page.NextURI, nil select { case <-ctx.Done(): - return nil, state, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "query polling interrupted; verify server termination before resuming") + return nil, state, errors.ErrLakeProtocol.GenWithStackByArgs(id, "query polling interrupted; verify server termination before resuming") case <-time.After(10 * time.Millisecond): } } @@ -164,11 +164,11 @@ func (c *httpClient) request(ctx context.Context, method, path string, body []by // Never forward credentials to an absolute or network-path URI supplied by a server. u, err := url.Parse(path) if err != nil || u.IsAbs() || u.Host != "" || !strings.HasPrefix(path, "/v1/query") { - return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "invalid query continuation URI") + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "invalid query continuation URI") } req, err := http.NewRequestWithContext(ctx, method, c.opts.endpoint+path, bytes.NewReader(body)) // #nosec G704 -- endpoint is operator configured; continuation above is restricted to the same origin. if err != nil { - return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "invalid request") + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "invalid request") } req.SetBasicAuth(c.opts.username, c.opts.password) req.Header.Set("Content-Type", "application/json") @@ -185,24 +185,24 @@ func (c *httpClient) request(ctx context.Context, method, path string, body []by } if method == http.MethodPost { if err := writelease.WaitForWrite(ctx, c.gate.Load()); err != nil { - return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "write admission closed before request") + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "write admission closed before request") } } resp, err := c.transport.Do(req) // #nosec G704 -- intentional configured sink endpoint, same origin only and redirects disabled. if err != nil { - return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "transport failed; verify server termination before resuming") + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "transport failed; verify server termination before resuming") } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "HTTP status "+jsonInt(resp.StatusCode)) + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "HTTP status "+jsonInt(resp.StatusCode)) } data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) if err != nil || len(data) > maxResponseBytes { - return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "truncated or oversized response") + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "truncated or oversized response") } var page queryPage if err := json.Unmarshal(data, &page); err != nil { - return queryPage{}, nil, errors.ErrDatabendProtocol.GenWithStackByArgs(id, "invalid execution response") + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "invalid execution response") } return page, resp.Header, nil } diff --git a/downstreamadapter/sink/databend/client_test.go b/downstreamadapter/sink/lake/client_test.go similarity index 99% rename from downstreamadapter/sink/databend/client_test.go rename to downstreamadapter/sink/lake/client_test.go index 3a35f4ed83..54abb4c216 100644 --- a/downstreamadapter/sink/databend/client_test.go +++ b/downstreamadapter/sink/lake/client_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "context" diff --git a/downstreamadapter/sink/databend/completion.go b/downstreamadapter/sink/lake/completion.go similarity index 99% rename from downstreamadapter/sink/databend/completion.go rename to downstreamadapter/sink/lake/completion.go index 7c3f70721c..d899ba6c57 100644 --- a/downstreamadapter/sink/databend/completion.go +++ b/downstreamadapter/sink/lake/completion.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "sync" diff --git a/downstreamadapter/sink/databend/completion_test.go b/downstreamadapter/sink/lake/completion_test.go similarity index 99% rename from downstreamadapter/sink/databend/completion_test.go rename to downstreamadapter/sink/lake/completion_test.go index f159cebd1d..92d9eef5c7 100644 --- a/downstreamadapter/sink/databend/completion_test.go +++ b/downstreamadapter/sink/lake/completion_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "sync" diff --git a/downstreamadapter/sink/databend/config.go b/downstreamadapter/sink/lake/config.go similarity index 91% rename from downstreamadapter/sink/databend/config.go rename to downstreamadapter/sink/lake/config.go index 20f2319db7..eecdb057a8 100644 --- a/downstreamadapter/sink/databend/config.go +++ b/downstreamadapter/sink/lake/config.go @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( + "net" "net/url" "strconv" "strings" @@ -41,9 +42,9 @@ type options struct { func parseOptions(uri *url.URL, cfg *config.ChangefeedConfig) (options, error) { o := options{workers: 4, batchRows: 2048, batchBytes: 1 << 20, spoolBytes: 10 << 30, sourceBytes: 256 << 20, sourceEvents: 128, maxEntries: 4096, memoryRatio: 0.02, flushInterval: time.Second, queryTimeout: 2 * time.Minute} bad := func() (options, error) { - return options{}, errors.ErrSinkURIInvalid.GenWithStackByArgs("invalid Databend options; require exclusive=true, bounded positive quotas and supported incremental settings") + return options{}, errors.ErrSinkURIInvalid.GenWithStackByArgs("invalid Lake options; require exclusive=true, bounded positive quotas and supported incremental settings") } - if uri == nil || (config.GetScheme(uri) != config.DatabendScheme && config.GetScheme(uri) != config.DatabendHTTPScheme && config.GetScheme(uri) != config.LakeScheme) || uri.Host == "" || uri.Fragment != "" { + if uri == nil || (config.GetScheme(uri) != config.LakeScheme && config.GetScheme(uri) != config.LakeHTTPScheme) || uri.Host == "" || uri.Fragment != "" { return bad() } q, queryErr := url.ParseQuery(uri.RawQuery) @@ -146,7 +147,12 @@ func parseOptions(uri *url.URL, cfg *config.ChangefeedConfig) (options, error) { endpoint.RawQuery = "" endpoint.RawPath = "" endpoint.Scheme = "https" - if config.GetScheme(uri) == config.DatabendHTTPScheme { + if config.GetScheme(uri) == config.LakeHTTPScheme { + host := uri.Hostname() + ip := net.ParseIP(host) + if !strings.EqualFold(host, "localhost") && (ip == nil || !ip.IsLoopback()) { + return bad() + } endpoint.Scheme = "http" } o.endpoint = endpoint.String() diff --git a/downstreamadapter/sink/databend/config_test.go b/downstreamadapter/sink/lake/config_test.go similarity index 63% rename from downstreamadapter/sink/databend/config_test.go rename to downstreamadapter/sink/lake/config_test.go index fc197c2743..9844b1d0c0 100644 --- a/downstreamadapter/sink/databend/config_test.go +++ b/downstreamadapter/sink/lake/config_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "net/url" @@ -22,8 +22,8 @@ import ( "github.com/stretchr/testify/require" ) -func TestDatabendOptions(t *testing.T) { - uri, err := url.Parse("databend://user:pass@warehouse/lake?exclusive=true&workers=2&batch-bytes=4096&spool-bytes=16384") +func TestLakeOptions(t *testing.T) { + uri, err := url.Parse("lake://user:pass@warehouse/lake?exclusive=true&workers=2&batch-bytes=4096&spool-bytes=16384") require.NoError(t, err) o, err := parseOptions(uri, nil) require.NoError(t, err) @@ -37,7 +37,7 @@ func TestDatabendOptions(t *testing.T) { require.NoError(t, err) require.Equal(t, "https://warehouse:443", lake.endpoint) for _, raw := range []string{ - "databend://host", "databend://host?exclusive=false", "databend://host?exclusive=true&workers=0", "databend://host?exclusive=true&memory-ratio=NaN", "databend://host?exclusive=true&batch-bytes=4096&spool-bytes=4096", "databend://host?exclusive=true&raw-change=true", "databend://host?exclusive=true&workers=1&workers=2", "databend://host?exclusive=true&query-timeout=0s", + "lake://host", "lake://host?exclusive=false", "lake://host?exclusive=true&workers=0", "lake://host?exclusive=true&memory-ratio=NaN", "lake://host?exclusive=true&batch-bytes=4096&spool-bytes=4096", "lake://host?exclusive=true&raw-change=true", "lake://host?exclusive=true&workers=1&workers=2", "lake://host?exclusive=true&query-timeout=0s", } { u, err := url.Parse(raw) require.NoError(t, err) @@ -65,3 +65,19 @@ func TestConflictKeyOption(t *testing.T) { require.Error(t, err) } } + +func TestLakePlaintextRequiresLoopback(t *testing.T) { + for _, host := range []string{"localhost", "LOCALHOST:8000", "127.0.0.1:8000", "127.1.2.3", "[::1]:8000", "[::ffff:127.0.0.1]:8000"} { + u, err := url.Parse("lake+http://user:pass@" + host + "/default?exclusive=true") + require.NoError(t, err) + o, err := parseOptions(u, nil) + require.NoError(t, err, host) + require.Equal(t, "http://"+host, o.endpoint) + } + for _, host := range []string{"warehouse.example:8000", "localhost.example", "127.0.0.1.example", "10.0.0.1", "0.0.0.0", "[::]", "[2001:db8::1]"} { + u, err := url.Parse("lake+http://user:pass@" + host + "?exclusive=true") + require.NoError(t, err) + _, err = parseOptions(u, nil) + require.Error(t, err, host) + } +} diff --git a/downstreamadapter/sink/databend/integration_test.go b/downstreamadapter/sink/lake/integration_test.go similarity index 94% rename from downstreamadapter/sink/databend/integration_test.go rename to downstreamadapter/sink/lake/integration_test.go index c3b6206016..690c852694 100644 --- a/downstreamadapter/sink/databend/integration_test.go +++ b/downstreamadapter/sink/lake/integration_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "context" @@ -33,10 +33,10 @@ import ( // Explicit opt-in only. This test creates and drops its own unique database; // never use a production warehouse or the running bend-cdc target for it. -func TestDatabendNativeLiveIntegration(t *testing.T) { - raw := os.Getenv("DATABEND_NATIVE_TEST_URI") +func TestLakeNativeLiveIntegration(t *testing.T) { + raw := os.Getenv("LAKE_NATIVE_TEST_URI") if raw == "" { - t.Skip("DATABEND_NATIVE_TEST_URI not configured; no live Databend evidence") + t.Skip("LAKE_NATIVE_TEST_URI not configured; no live Lake evidence") } uri, err := url.Parse(raw) require.NoError(t, err) diff --git a/downstreamadapter/sink/databend/lane.go b/downstreamadapter/sink/lake/lane.go similarity index 86% rename from downstreamadapter/sink/databend/lane.go rename to downstreamadapter/sink/lake/lane.go index 0905c9a2be..25d43adc71 100644 --- a/downstreamadapter/sink/databend/lane.go +++ b/downstreamadapter/sink/lake/lane.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "context" @@ -62,12 +62,15 @@ func (l *tableLane) run(ctx context.Context) error { return err } case item := <-l.input: + l.sink.mu.Lock() + l.sink.notifyAdmissionLocked() + l.sink.mu.Unlock() var err error switch { case item.event != nil: err = l.encode(ctx, &item) case item.ddl != nil: - err = l.sink.statistics.RecordDDLExecution(func() (string, error) { return "databend", l.schemas.applyDDL(ctx, item.ddl) }) + err = l.sink.statistics.RecordDDLExecution(func() (string, error) { return "lake", l.schemas.applyDDL(ctx, item.ddl) }) default: err = l.flushAll(ctx) } @@ -130,7 +133,7 @@ func (l *tableLane) encode(ctx context.Context, item *workItem) (err error) { blob = append(blob, ']') fragment, ok := tracker.register() if !ok { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("fragment registered after completion") + return errors.ErrLakeInvalidData.GenWithStackByArgs("fragment registered after completion") } var callback func() if last { @@ -156,10 +159,10 @@ func (l *tableLane) encode(ctx context.Context, item *workItem) (err error) { } encoded, err := json.Marshal(spoolRow{row.id, row.payload, row.commitTs, row.action}) if err != nil { - return errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot encode spool row") + return errors.WrapError(errors.ErrLakeInvalidData, err, "cannot encode spool row") } if int64(len(encoded)+2) > l.sink.opts.batchBytes { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("single encoded row exceeds batch byte limit") + return errors.ErrLakeInvalidData.GenWithStackByArgs("single encoded row exceeds batch byte limit") } if rows > 0 && (int64(len(blob)+len(encoded)+2) > l.sink.opts.batchBytes || rows >= l.sink.opts.batchRows) { if err := publish(false); err != nil { @@ -176,7 +179,7 @@ func (l *tableLane) encode(ctx context.Context, item *workItem) (err error) { count++ } if count != event.Len() { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("incomplete native row iteration") + return errors.ErrLakeInvalidData.GenWithStackByArgs("incomplete native row iteration") } event = nil release() // Only callback closures and immutable bounded fragments remain. @@ -191,7 +194,7 @@ func (l *tableLane) enqueue(ctx context.Context, state *tableState, blob []byte, msg := &codec.Message{Value: blob} // Never attach source success to spool.Callback. msgs := []*codec.Message{msg} if spool.AccountingBytes(msgs) > l.sink.opts.spoolBytes { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("encoded fragment exceeds spool quota") + return errors.ErrLakeInvalidData.GenWithStackByArgs("encoded fragment exceeds spool quota") } for { action, entry, err := l.sink.spool.TryEnqueue(msgs, enqueue) @@ -199,7 +202,7 @@ func (l *tableLane) enqueue(ctx context.Context, state *tableState, blob []byte, return err } if action == spool.EnqueueActionAcceptedOversized { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("spool size invariant violated") + return errors.ErrLakeInvalidData.GenWithStackByArgs("spool size invariant violated") } if action == spool.EnqueueActionWaitDiskQuota { // Drain every local table, not just the large event. Other lanes keep @@ -258,7 +261,7 @@ func (l *tableLane) flush(ctx context.Context, key string) error { return err } if n >= l.sink.opts.remoteLimit { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("remote raw monitoring threshold reached; expand budget or reinitialize manually") + return errors.ErrLakeInvalidData.GenWithStackByArgs("remote raw monitoring threshold reached; expand budget or reinitialize manually") } } var rows []normalizedRow @@ -272,14 +275,14 @@ func (l *tableLane) flush(ctx context.Context, key string) error { return err } if !ok { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("empty spool fragment") + return errors.ErrLakeInvalidData.GenWithStackByArgs("empty spool fragment") } var decoded []spoolRow if err := json.Unmarshal(data, &decoded); err != nil { - return errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot decode spool fragment") + return errors.WrapError(errors.ErrLakeInvalidData, err, "cannot decode spool fragment") } if len(decoded) != f.count { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("incomplete spool fragment") + return errors.ErrLakeInvalidData.GenWithStackByArgs("incomplete spool fragment") } for _, r := range decoded { rows = append(rows, normalizedRow{r.ID, r.Payload, r.CommitTs, r.Action}) @@ -327,11 +330,11 @@ func (l *tableLane) cleanup() { // parseRemoteBytes rejects malformed measurements instead of treating them as 0. func parseRemoteBytes(rows [][]*string) (int64, error) { if len(rows) != 1 || len(rows[0]) != 1 || rows[0][0] == nil { - return 0, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid remote raw measurement") + return 0, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid remote raw measurement") } var n int64 if err := json.Unmarshal([]byte(*rows[0][0]), &n); err != nil || n < 0 { - return 0, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid remote raw byte count") + return 0, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid remote raw byte count") } return n, nil } diff --git a/downstreamadapter/sink/databend/live_fault_test.go b/downstreamadapter/sink/lake/live_fault_test.go similarity index 97% rename from downstreamadapter/sink/databend/live_fault_test.go rename to downstreamadapter/sink/lake/live_fault_test.go index a6352be9dc..1568489aab 100644 --- a/downstreamadapter/sink/databend/live_fault_test.go +++ b/downstreamadapter/sink/lake/live_fault_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "bytes" @@ -116,9 +116,9 @@ func (f *liveResponseCut) RoundTrip(req *http.Request) (*http.Response, error) { func liveFaultEnvironment(t *testing.T) (context.Context, options, *httpClient) { t.Helper() - raw := os.Getenv("DATABEND_NATIVE_TEST_URI") + raw := os.Getenv("LAKE_NATIVE_TEST_URI") if raw == "" { - t.Skip("explicit DATABEND_NATIVE_TEST_URI is required; no real warehouse evidence") + t.Skip("explicit LAKE_NATIVE_TEST_URI is required; no real warehouse evidence") } u, err := url.Parse(raw) require.NoError(t, err) @@ -152,7 +152,7 @@ func liveFaultEnvironment(t *testing.T) (context.Context, options, *httpClient) func liveRuntime(ctx context.Context, t *testing.T, o options, name string) *Sink { t.Helper() - u, err := url.Parse(os.Getenv("DATABEND_NATIVE_TEST_URI")) + u, err := url.Parse(os.Getenv("LAKE_NATIVE_TEST_URI")) require.NoError(t, err) u.Path, u.RawPath = "/"+o.database, "" q := u.Query() @@ -179,7 +179,7 @@ func liveCount(ctx context.Context, t *testing.T, c *httpClient, name string, ex require.Equal(t, expected, *rows[0][0]) } -func TestDatabendNativeLiveRawPartialAndMergeResponseLoss(t *testing.T) { +func TestLakeNativeLiveRawPartialAndMergeResponseLoss(t *testing.T) { ctx, o, admin := liveFaultEnvironment(t) t.Run("partial_raw_before_second_insert", func(t *testing.T) { info := liveNamedInfo("partial_raw") @@ -260,7 +260,7 @@ func TestDatabendNativeLiveRawPartialAndMergeResponseLoss(t *testing.T) { }) } -func TestDatabendNativeLiveDDLInterruptionRecovery(t *testing.T) { +func TestLakeNativeLiveDDLInterruptionRecovery(t *testing.T) { ctx, o, admin := liveFaultEnvironment(t) for _, kind := range []string{"create", "alter", "truncate", "drop"} { cuts := []string{"pending", "target", "raw", "applied"} diff --git a/downstreamadapter/sink/databend/metrics.go b/downstreamadapter/sink/lake/metrics.go similarity index 51% rename from downstreamadapter/sink/databend/metrics.go rename to downstreamadapter/sink/lake/metrics.go index 1a0ae3e66b..a95347cdb8 100644 --- a/downstreamadapter/sink/databend/metrics.go +++ b/downstreamadapter/sink/lake/metrics.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( sinkmetrics "github.com/pingcap/ticdc/downstreamadapter/sink/metrics" @@ -23,24 +23,24 @@ import ( func newSpoolMetrics(id common.ChangeFeedID) *spool.Metrics { k, f := id.Keyspace(), id.Name() return &spool.Metrics{ - MemoryBytes: sinkmetrics.DatabendBufferBytes.WithLabelValues(k, f, "memory"), - DiskBytes: sinkmetrics.DatabendBufferBytes.WithLabelValues(k, f, "disk"), - PendingPostEnqueue: sinkmetrics.DatabendBufferEntries.WithLabelValues(k, f, "enqueue"), - DiskQuotaWaiters: sinkmetrics.DatabendBufferEntries.WithLabelValues(k, f, "waiters"), - SegmentCount: sinkmetrics.DatabendBufferEntries.WithLabelValues(k, f, "segments"), - DiskQuotaWait: sinkmetrics.DatabendSpoolDuration.WithLabelValues(k, f), - LoadedBytes: sinkmetrics.DatabendSpoolLoad.WithLabelValues(k, f), - RotatedCount: sinkmetrics.DatabendSpoolRotations.WithLabelValues(k, f), + MemoryBytes: sinkmetrics.LakeBufferBytes.WithLabelValues(k, f, "memory"), + DiskBytes: sinkmetrics.LakeBufferBytes.WithLabelValues(k, f, "disk"), + PendingPostEnqueue: sinkmetrics.LakeBufferEntries.WithLabelValues(k, f, "enqueue"), + DiskQuotaWaiters: sinkmetrics.LakeBufferEntries.WithLabelValues(k, f, "waiters"), + SegmentCount: sinkmetrics.LakeBufferEntries.WithLabelValues(k, f, "segments"), + DiskQuotaWait: sinkmetrics.LakeSpoolDuration.WithLabelValues(k, f), + LoadedBytes: sinkmetrics.LakeSpoolLoad.WithLabelValues(k, f), + RotatedCount: sinkmetrics.LakeSpoolRotations.WithLabelValues(k, f), Close: func() { for _, kind := range []string{"memory", "disk"} { - sinkmetrics.DatabendBufferBytes.DeleteLabelValues(k, f, kind) + sinkmetrics.LakeBufferBytes.DeleteLabelValues(k, f, kind) } for _, kind := range []string{"enqueue", "waiters", "segments"} { - sinkmetrics.DatabendBufferEntries.DeleteLabelValues(k, f, kind) + sinkmetrics.LakeBufferEntries.DeleteLabelValues(k, f, kind) } - sinkmetrics.DatabendSpoolDuration.DeleteLabelValues(k, f) - sinkmetrics.DatabendSpoolLoad.DeleteLabelValues(k, f) - sinkmetrics.DatabendSpoolRotations.DeleteLabelValues(k, f) + sinkmetrics.LakeSpoolDuration.DeleteLabelValues(k, f) + sinkmetrics.LakeSpoolLoad.DeleteLabelValues(k, f) + sinkmetrics.LakeSpoolRotations.DeleteLabelValues(k, f) }, } } diff --git a/downstreamadapter/sink/databend/native.go b/downstreamadapter/sink/lake/native.go similarity index 81% rename from downstreamadapter/sink/databend/native.go rename to downstreamadapter/sink/lake/native.go index 3b2a9179e5..dac524b6d9 100644 --- a/downstreamadapter/sink/databend/native.go +++ b/downstreamadapter/sink/lake/native.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "encoding/hex" @@ -33,17 +33,17 @@ import ( func schemaFromNative(info *common.TableInfo, database, conflictKey string) (tableSchema, error) { if info == nil { - return tableSchema{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("missing authoritative table metadata") + return tableSchema{}, errors.ErrLakeInvalidData.GenWithStackByArgs("missing authoritative table metadata") } if info.IsPartitionTable() { - return tableSchema{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("partition dispatchers are outside the single-table writer proof; partition DDL is unsupported") + return tableSchema{}, errors.ErrLakeInvalidData.GenWithStackByArgs("partition dispatchers are outside the single-table writer proof; partition DDL is unsupported") } s := tableSchema{database: info.GetSchemaName(), table: info.GetTableName(), primaryKey: info.GetPrimaryKeyColumnNames(), source: info.GetSchemaName() + "." + info.GetTableName()} if database != "" { s.database = database } if strings.HasSuffix(s.table, "_cdc_raw") || strings.HasSuffix(s.table, "_cdc_schema_epoch") { - return tableSchema{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("source table uses a reserved CDC suffix") + return tableSchema{}, errors.ErrLakeInvalidData.GenWithStackByArgs("source table uses a reserved CDC suffix") } for _, c := range info.GetColumns() { if !common.IsColCDCVisible(c) { @@ -63,7 +63,7 @@ func schemaFromNative(info *common.TableInfo, database, conflictKey string) (tab // Match bend-cdc's persisted single-column override. Never silently // replace a composite key with only one of its members. if len(s.primaryKey) > 1 { - return tableSchema{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("conflict-key cannot override a composite source key") + return tableSchema{}, errors.ErrLakeInvalidData.GenWithStackByArgs("conflict-key cannot override a composite source key") } actual := "" for _, col := range s.columns { @@ -73,7 +73,7 @@ func schemaFromNative(info *common.TableInfo, database, conflictKey string) (tab } } if actual == "" { - return tableSchema{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("conflict-key must name a visible source column") + return tableSchema{}, errors.ErrLakeInvalidData.GenWithStackByArgs("conflict-key must name a visible source column") } s.primaryKey = []string{actual} } @@ -105,7 +105,7 @@ func nativeColumnType(c *model.ColumnInfo) (string, columnEncoding, error) { case mysql.TypeNewDecimal: p, scale := c.GetFlen(), c.GetDecimal() if p <= 0 || p > 76 || scale < 0 || scale > p { - return "", 0, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid decimal metadata") + return "", 0, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid decimal metadata") } return fmt.Sprintf("DECIMAL(%d,%d)", p, scale), scalarColumn, nil case mysql.TypeDate, mysql.TypeNewDate: @@ -124,7 +124,7 @@ func nativeColumnType(c *model.ColumnInfo) (string, columnEncoding, error) { } return "VARCHAR", scalarColumn, nil default: - return "", 0, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported native column type") + return "", 0, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported native column type") } } @@ -139,7 +139,7 @@ func nativeValues(info *common.TableInfo, row chunk.Row, location *time.Location } offset, ok := info.GetRowColumnsOffset()[col.ID] if !ok || offset >= row.Len() { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("row does not match authoritative schema") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("row does not match authoritative schema") } d := row.GetDatum(offset, &col.FieldType) var v any @@ -152,7 +152,7 @@ func nativeValues(info *common.TableInfo, row chunk.Row, location *time.Location case mysql.TypeBit: n, err := d.GetMysqlBit().ToInt(types.DefaultStmtNoWarningContext) if err != nil { - return nil, errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid bit datum") + return nil, errors.WrapError(errors.ErrLakeInvalidData, err, "invalid bit datum") } v = strconv.FormatUint(n, 10) case mysql.TypeJSON: @@ -165,7 +165,7 @@ func nativeValues(info *common.TableInfo, row chunk.Row, location *time.Location stamp := d.GetMysqlTime() if !stamp.IsZero() { if err := stamp.ConvertTimeZone(location, time.UTC); err != nil { - return nil, errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot normalize timestamp timezone") + return nil, errors.WrapError(errors.ErrLakeInvalidData, err, "cannot normalize timestamp timezone") } } v = stamp.String() @@ -176,7 +176,7 @@ func nativeValues(info *common.TableInfo, row chunk.Row, location *time.Location default: text, err := d.ToString() if err != nil { - return nil, errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot format native datum") + return nil, errors.WrapError(errors.ErrLakeInvalidData, err, "cannot format native datum") } v = text } @@ -210,7 +210,7 @@ func normalizeChange(info *common.TableInfo, schema tableSchema, change commonEv return normalizedRow{}, err } if old.id != row.id { - return normalizedRow{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("conflict key changed in an unsplit UPDATE; use a stable conflict-key or split change mode") + return normalizedRow{}, errors.ErrLakeInvalidData.GenWithStackByArgs("conflict key changed in an unsplit UPDATE; use a stable conflict-key or split change mode") } } return row, nil @@ -234,7 +234,7 @@ func nativeDefault(c *model.ColumnInfo, encoding columnEncoding) (string, bool, case bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: text = fmt.Sprint(v) default: - return "", false, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported column default representation") + return "", false, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported column default representation") } expr := c.DefaultIsExpr if (c.GetType() == mysql.TypeTimestamp || c.GetType() == mysql.TypeDatetime) && strings.HasPrefix(strings.ToUpper(text), "CURRENT_TIMESTAMP") { @@ -246,7 +246,7 @@ func nativeDefault(c *model.ColumnInfo, encoding columnEncoding) (string, bool, if c.GetType() == mysql.TypeBit { n, err := types.BinaryLiteral([]byte(text)).ToInt(types.DefaultStmtNoWarningContext) if err != nil { - return "", false, errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid bit default") + return "", false, errors.WrapError(errors.ErrLakeInvalidData, err, "invalid bit default") } return strconv.FormatUint(n, 10), false, nil } @@ -255,7 +255,7 @@ func nativeDefault(c *model.ColumnInfo, encoding columnEncoding) (string, bool, } if encoding == jsonColumn { if !json.Valid([]byte(text)) { - return "", false, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid JSON default") + return "", false, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid JSON default") } return "parse_json(" + sqlString(text) + ")", false, nil } diff --git a/downstreamadapter/sink/databend/native_test.go b/downstreamadapter/sink/lake/native_test.go similarity index 99% rename from downstreamadapter/sink/databend/native_test.go rename to downstreamadapter/sink/lake/native_test.go index fc83420538..ed262eec63 100644 --- a/downstreamadapter/sink/databend/native_test.go +++ b/downstreamadapter/sink/lake/native_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "encoding/hex" diff --git a/downstreamadapter/sink/databend/row.go b/downstreamadapter/sink/lake/row.go similarity index 74% rename from downstreamadapter/sink/databend/row.go rename to downstreamadapter/sink/lake/row.go index 3991c3452c..ab5f3fe92d 100644 --- a/downstreamadapter/sink/databend/row.go +++ b/downstreamadapter/sink/lake/row.go @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package databend owns the native Databend sink implementation. Its row and SQL +// Package lake owns the native Lake sink implementation. Its row and SQL // layer is ported from bend-cdc's TiDB path, without the Canal JSON decoder. -package databend +package lake import ( "encoding/hex" @@ -58,17 +58,17 @@ type tableSchema struct { func (s tableSchema) validate() error { validName := func(name string) bool { return name != "" && !strings.ContainsRune(name, 0) } if !validName(s.database) || !validName(s.table) || len(s.columns) == 0 || len(s.primaryKey) == 0 { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("table metadata and a primary key are required") + return errors.ErrLakeInvalidData.GenWithStackByArgs("table metadata and a primary key are required") } columns := make(map[string]struct{}, len(s.columns)) folded := make(map[string]struct{}, len(s.columns)) for _, col := range s.columns { if !validName(col.name) || col.encoding > jsonColumn { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid column metadata") + return errors.ErrLakeInvalidData.GenWithStackByArgs("invalid column metadata") } lower := strings.ToLower(col.name) if _, ok := folded[lower]; ok { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("duplicate column name") + return errors.ErrLakeInvalidData.GenWithStackByArgs("duplicate column name") } folded[lower] = struct{}{} columns[col.name] = struct{}{} @@ -76,10 +76,10 @@ func (s tableSchema) validate() error { keys := make(map[string]struct{}, len(s.primaryKey)) for _, key := range s.primaryKey { if _, ok := columns[key]; !ok { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("primary key is missing from columns") + return errors.ErrLakeInvalidData.GenWithStackByArgs("primary key is missing from columns") } if _, ok := keys[key]; ok { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("duplicate primary key column") + return errors.ErrLakeInvalidData.GenWithStackByArgs("duplicate primary key column") } keys[key] = struct{}{} } @@ -117,10 +117,10 @@ func normalizeNativeRow(schema tableSchema, action rowAction, commitTs uint64, v return normalizedRow{}, err } if commitTs == 0 || commitTs > math.MaxInt64 { - return normalizedRow{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("commit TSO cannot be represented in raw BIGINT") + return normalizedRow{}, errors.ErrLakeInvalidData.GenWithStackByArgs("commit TSO cannot be represented in raw BIGINT") } if action != insertAction && action != updateAction && action != deleteAction { - return normalizedRow{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported row action") + return normalizedRow{}, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported row action") } data := make(map[string]any, len(values)) for _, col := range schema.columns { @@ -129,7 +129,7 @@ func normalizeNativeRow(schema tableSchema, action rowAction, commitTs uint64, v if action == deleteAction { continue // A tombstone needs the old primary key, not an after image. } - return normalizedRow{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("incomplete after image") + return normalizedRow{}, errors.ErrLakeInvalidData.GenWithStackByArgs("incomplete after image") } normalized, err := normalizeNativeValue(col.encoding, value) if err != nil { @@ -139,7 +139,7 @@ func normalizeNativeRow(schema tableSchema, action rowAction, commitTs uint64, v } for name := range values { if _, exists := data[name]; !exists { - return normalizedRow{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("value has no column metadata") + return normalizedRow{}, errors.ErrLakeInvalidData.GenWithStackByArgs("value has no column metadata") } } id, err := conflictIdentity(data, schema.canonicalKeys()) @@ -148,7 +148,7 @@ func normalizeNativeRow(schema tableSchema, action rowAction, commitTs uint64, v } payload, err := json.Marshal(data) if err != nil { - return normalizedRow{}, errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot encode normalized row") + return normalizedRow{}, errors.WrapError(errors.ErrLakeInvalidData, err, "cannot encode normalized row") } return normalizedRow{id: id, payload: string(payload), commitTs: commitTs, action: action}, nil } @@ -164,7 +164,7 @@ func normalizeNativeValue(encoding columnEncoding, value any) (any, error) { case string: return hex.EncodeToString([]byte(v)), nil // Original bytes, not Canal Latin-1. default: - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("binary datum must contain bytes") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("binary datum must contain bytes") } } if encoding == jsonColumn { @@ -177,22 +177,22 @@ func normalizeNativeValue(encoding columnEncoding, value any) (any, error) { case json.RawMessage: document = v default: - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("JSON datum must contain an exact document") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("JSON datum must contain an exact document") } if !json.Valid(document) { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid JSON document") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid JSON document") } return string(document), nil // Includes JSON null, distinct from SQL NULL. } switch v := value.(type) { case string: if !utf8.ValidString(v) { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid text encoding") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid text encoding") } return strings.Clone(v), nil case []byte: if !utf8.Valid(v) { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid text encoding") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid text encoding") } return string(v), nil case bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: @@ -200,7 +200,7 @@ func normalizeNativeValue(encoding columnEncoding, value any) (any, error) { case json.Number: text := string(v) if len(text) == 0 || !json.Valid([]byte(text)) || (text[0] != '-' && (text[0] < '0' || text[0] > '9')) { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid exact JSON number") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid exact JSON number") } return json.Number(strings.Clone(string(v))), nil case float32: @@ -212,7 +212,7 @@ func normalizeNativeValue(encoding columnEncoding, value any) (any, error) { return v, nil } } - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported native scalar datum") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported native scalar datum") } // conflictIdentity preserves bend-cdc's single-key text and canonical composite @@ -226,12 +226,12 @@ func conflictIdentity(values map[string]any, keys []string) (string, error) { for _, key := range keys { value, ok := values[key] if !ok || value == nil { - return "", errors.ErrDatabendInvalidData.GenWithStackByArgs("primary key is missing or null") + return "", errors.ErrLakeInvalidData.GenWithStackByArgs("primary key is missing or null") } text := fmt.Sprint(value) if len(keys) == 1 { if strings.TrimSpace(text) == "" { - return "", errors.ErrDatabendInvalidData.GenWithStackByArgs("empty primary key") + return "", errors.ErrLakeInvalidData.GenWithStackByArgs("empty primary key") } return text, nil } @@ -239,7 +239,7 @@ func conflictIdentity(values map[string]any, keys []string) (string, error) { } encoded, err := json.Marshal(parts) if err != nil { - return "", errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot encode primary key") + return "", errors.WrapError(errors.ErrLakeInvalidData, err, "cannot encode primary key") } return string(encoded), nil } diff --git a/downstreamadapter/sink/databend/row_test.go b/downstreamadapter/sink/lake/row_test.go similarity index 99% rename from downstreamadapter/sink/databend/row_test.go rename to downstreamadapter/sink/lake/row_test.go index 2e64d59905..41d9e7ac69 100644 --- a/downstreamadapter/sink/databend/row_test.go +++ b/downstreamadapter/sink/lake/row_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "encoding/hex" diff --git a/downstreamadapter/sink/databend/runtime_test.go b/downstreamadapter/sink/lake/runtime_test.go similarity index 83% rename from downstreamadapter/sink/databend/runtime_test.go rename to downstreamadapter/sink/lake/runtime_test.go index 0d1bd44edb..d7290b058c 100644 --- a/downstreamadapter/sink/databend/runtime_test.go +++ b/downstreamadapter/sink/lake/runtime_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "context" @@ -37,7 +37,7 @@ import ( ) // apiFixture models only the HTTP boundary and journal persistence. It does not -// execute Databend SQL and must not be cited as proof of server atomicity. +// execute Lake SQL and must not be cited as proof of server atomicity. type apiFixture struct { mu sync.Mutex columns []columnDef @@ -154,9 +154,9 @@ func newAPIFixture(t *testing.T) (*apiFixture, *httptest.Server) { func runtimeSink(t *testing.T, server *httptest.Server, extra string) *Sink { t.Helper() - uri, err := url.Parse(strings.Replace(server.URL, "http://", "databend+http://", 1) + "?exclusive=true&workers=1&flush-interval=1h&query-timeout=1s&spool-dir=" + url.QueryEscape(t.TempDir()) + extra) + uri, err := url.Parse(strings.Replace(server.URL, "http://", "lake+http://", 1) + "?exclusive=true&workers=1&flush-interval=1h&query-timeout=1s&spool-dir=" + url.QueryEscape(t.TempDir()) + extra) require.NoError(t, err) - s, err := New(context.Background(), common.NewChangeFeedIDWithName("databend-test", "default"), &config.ChangefeedConfig{}, uri, 0) + s, err := New(context.Background(), common.NewChangeFeedIDWithName("lake-test", "default"), &config.ChangefeedConfig{}, uri, 0) require.NoError(t, err) t.Cleanup(s.Close) return s @@ -262,7 +262,7 @@ func TestRuntimeLargeEventStreamsAndReclaimsBeforeSeal(t *testing.T) { func TestRuntimeFatalAdmissionAndCloseNeverFlush(t *testing.T) { _, server := newAPIFixture(t) - t.Run("capacity", func(t *testing.T) { + t.Run("oversized event", func(t *testing.T) { s := runtimeSink(t, server, "&source-bytes=1048576") e := nativeTestEvent(nativeTestInfo(), 100, 100000) var flushed atomic.Int64 @@ -338,7 +338,7 @@ func TestConcurrentAdmissionAndCloseReleaseAllSources(t *testing.T) { func TestInternalTimeoutRemainsFatalNotLifecycleCancel(t *testing.T) { _, server := newAPIFixture(t) s := runtimeSink(t, server, "") - err := errors.WrapError(errors.ErrDatabendExecution, context.DeadlineExceeded) + err := errors.WrapError(errors.ErrLakeExecution, context.DeadlineExceeded) s.stop(err) require.True(t, errors.Is(context.Cause(s.ctx), context.DeadlineExceeded)) err = s.Run(context.Background()) @@ -495,3 +495,92 @@ func TestSkippedDDLDoesNotBypassUnknownDML(t *testing.T) { require.Error(t, <-done) s.Close() } + +func TestRuntimeSharedAdmissionBackpressure(t *testing.T) { + for _, budget := range []string{"events", "bytes", "lane queue"} { + t.Run(budget, func(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "&source-events=2&batch-bytes=1024&spool-bytes=4096") + var flushed atomic.Int64 + first := nativeTestEvent(nativeTestInfo(), 100, 1) + second := nativeTestEvent(nativeTestInfo(), 120, 1) + first.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + second.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + wantFlushed := int64(2) + charge := sourceEventBytes(first) + require.Equal(t, charge, sourceEventBytes(second)) + switch budget { + case "events": + s.opts.sourceEvents = 1 + s.AddDMLEvent(first) + case "bytes": + s.opts.sourceBytes = charge + charge/2 + s.AddDMLEvent(first) + case "lane queue": + // Existing barrier items are not part of the DML source pool. + for range cap(s.lanes[0].input) { + s.lanes[0].input <- workItem{done: make(chan error, 1)} + } + wantFlushed = 1 + } + admitted := make(chan struct{}) + go func() { + s.AddDMLEvent(second) + close(admitted) + }() + select { + case <-admitted: + t.Fatal("full shared admission must wait, not fail or bypass quota") + case <-time.After(50 * time.Millisecond): + } + require.True(t, s.IsNormal(), "ordinary pressure must not stop the feed") + require.Zero(t, flushed.Load()) + s.mu.Lock() + bytes, events := s.sourceBytes, s.sourceEvents + s.mu.Unlock() + require.LessOrEqual(t, bytes, s.opts.sourceBytes) + require.LessOrEqual(t, events, s.opts.sourceEvents) + done := runRuntime(t, s) + waitEnqueued(t, admitted) + require.NoError(t, s.FlushDMLBeforeBlock(nil)) + require.Equal(t, wantFlushed, flushed.Load()) + s.Close() + require.ErrorIs(t, <-done, context.Canceled) + }) + } +} + +func TestRuntimeAdmissionWaitStopsWithoutFlush(t *testing.T) { + for _, fatal := range []bool{false, true} { + t.Run(strconv.FormatBool(fatal), func(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "&source-events=1") + var flushed atomic.Int64 + for i := range 2 { + e := nativeTestEvent(nativeTestInfo(), uint64(100+i), 1) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + if i == 0 { + s.AddDMLEvent(e) + continue + } + returned := make(chan struct{}) + go func() { s.AddDMLEvent(e); close(returned) }() + select { + case <-returned: + t.Fatal("source pool must apply backpressure") + case <-time.After(50 * time.Millisecond): + } + if fatal { + s.stop(errors.ErrLakeExecution.GenWithStackByArgs()) + } + s.Close() + waitEnqueued(t, returned) + } + require.Zero(t, flushed.Load()) + s.mu.Lock() + require.Zero(t, s.sourceBytes) + require.Zero(t, s.sourceEvents) + s.mu.Unlock() + }) + } +} diff --git a/downstreamadapter/sink/databend/schema.go b/downstreamadapter/sink/lake/schema.go similarity index 80% rename from downstreamadapter/sink/databend/schema.go rename to downstreamadapter/sink/lake/schema.go index c15a9ed023..927255f768 100644 --- a/downstreamadapter/sink/databend/schema.go +++ b/downstreamadapter/sink/lake/schema.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "context" @@ -131,9 +131,9 @@ func (m *schemaManager) exec(ctx context.Context, sql string, args ...any) error result, err := m.client.mutate(ctx, sql, args) if result.state == commitUnknown { if err == nil { - err = errors.ErrDatabendCommitUnknown.GenWithStackByArgs() + err = errors.ErrLakeCommitUnknown.GenWithStackByArgs() } else { - err = errors.WrapError(errors.ErrDatabendCommitUnknown, err) + err = errors.WrapError(errors.ErrLakeCommitUnknown, err) } return errors.WrapChangefeedUnretryableErr(err) } @@ -141,7 +141,7 @@ func (m *schemaManager) exec(ctx context.Context, sql string, args ...any) error return err } if result.state != committed { - return errors.ErrDatabendExecution.GenWithStackByArgs() + return errors.ErrLakeExecution.GenWithStackByArgs() } return nil } @@ -161,26 +161,26 @@ func (m *schemaManager) loadEpoch(ctx context.Context, s tableSchema) (epochReco return epochRecord{}, nil } if len(rows) != 1 || len(rows[0]) != 3 || rows[0][0] == nil || rows[0][1] == nil || rows[0][2] == nil { - return epochRecord{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid schema epoch journal") + return epochRecord{}, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid schema epoch journal") } e, err := strconv.ParseUint(*rows[0][0], 10, 64) if err != nil || e == 0 || e > math.MaxInt64 { - return epochRecord{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid applied schema epoch") + return epochRecord{}, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid applied schema epoch") } applied, err := strconv.ParseBool(*rows[0][1]) if err != nil { - return epochRecord{}, errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid epoch status") + return epochRecord{}, errors.WrapError(errors.ErrLakeInvalidData, err, "invalid epoch status") } r := epochRecord{epoch: e, applied: applied} if err := json.Unmarshal([]byte(*rows[0][2]), &r.plan); err != nil { - return epochRecord{}, errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid epoch plan") + return epochRecord{}, errors.WrapError(errors.ErrLakeInvalidData, err, "invalid epoch plan") } saved := r.plan.Schema.schema() if err := saved.validate(); err != nil { return epochRecord{}, err } if saved.database != s.database || saved.table != s.table || saved.source != s.source { - return epochRecord{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("target belongs to another source or schema journal") + return epochRecord{}, errors.ErrLakeInvalidData.GenWithStackByArgs("target belongs to another source or schema journal") } return r, nil } @@ -188,7 +188,7 @@ func (m *schemaManager) loadEpoch(ctx context.Context, s tableSchema) (epochReco func (m *schemaManager) saveEpoch(ctx context.Context, s tableSchema, r epochRecord) error { plan, err := json.Marshal(r.plan) if err != nil { - return errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid schema plan") + return errors.WrapError(errors.ErrLakeInvalidData, err, "invalid schema plan") } return m.exec(ctx, "MERGE INTO "+s.journalName()+" t USING (SELECT ?::BIGINT AS epoch, ?::BOOLEAN AS applied, ?::VARCHAR AS plan) s ON true WHEN MATCHED THEN UPDATE * WHEN NOT MATCHED THEN INSERT *", int64(r.epoch), r.applied, string(plan)) } @@ -237,7 +237,7 @@ func (m *schemaManager) targetExists(ctx context.Context, s tableSchema) (bool, return false, err } if count > 1 { - return false, errors.ErrDatabendInvalidData.GenWithStackByArgs("ambiguous target table metadata") + return false, errors.ErrLakeInvalidData.GenWithStackByArgs("ambiguous target table metadata") } return count == 1, nil } @@ -250,7 +250,7 @@ func (m *schemaManager) describe(ctx context.Context, s tableSchema) (map[string cols := make(map[string]columnDef, len(rows)) for _, r := range rows { if len(r) < 3 || r[0] == nil || r[1] == nil || r[2] == nil { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("incomplete target column metadata") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("incomplete target column metadata") } name := *r[0] cols[name] = columnDef{Name: name, Type: canonicalType(*r[1]), Nullable: strings.EqualFold(*r[2], "YES") || strings.HasPrefix(strings.ToUpper(*r[1]), "NULLABLE(")} @@ -264,12 +264,12 @@ func (m *schemaManager) verifyTarget(ctx context.Context, s tableSchema) error { return err } if len(cols) != len(s.columns) { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("target column set differs from source metadata") + return errors.ErrLakeInvalidData.GenWithStackByArgs("target column set differs from source metadata") } for _, c := range s.columns { actual, ok := cols[c.name] if !ok || canonicalType(actual.Type) != canonicalType(c.typ) || actual.Nullable != c.nullable { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("target type or nullability differs from source projection") + return errors.ErrLakeInvalidData.GenWithStackByArgs("target type or nullability differs from source projection") } } return nil @@ -280,7 +280,7 @@ func (m *schemaManager) initialize(ctx context.Context, s tableSchema) (*tableSt return state, nil } if len(m.tables) >= m.opts.maxEntries { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("table metadata capacity exhausted") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("table metadata capacity exhausted") } r, err := m.loadEpoch(ctx, s) if err != nil { @@ -319,12 +319,12 @@ func (m *schemaManager) initialize(ctx context.Context, s tableSchema) (*tableSt return nil, err } if len(rows) != 1 || len(rows[0]) != 1 || rows[0][0] == nil { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid retained raw time watermark") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid retained raw time watermark") } if text := *rows[0][0]; text != "" { stamp, err := time.ParseInLocation("2006-01-02 15:04:05.999999", text, time.UTC) if err != nil { - return nil, errors.WrapError(errors.ErrDatabendInvalidData, err, "invalid retained raw timestamp") + return nil, errors.WrapError(errors.ErrLakeInvalidData, err, "invalid retained raw timestamp") } writer.clock.last = stamp } @@ -343,7 +343,7 @@ func (m *schemaManager) prepareDML(ctx context.Context, s tableSchema, ts uint64 return state, true, nil } if state.dropped || !reflect.DeepEqual(toSchemaDef(state.schema), toSchemaDef(s)) { - return nil, false, errors.ErrDatabendInvalidData.GenWithStackByArgs("DML schema is not the applied target schema; replay its DDL first") + return nil, false, errors.ErrLakeInvalidData.GenWithStackByArgs("DML schema is not the applied target schema; replay its DDL first") } return state, false, nil } @@ -358,11 +358,11 @@ func allowedModify(from, to columnDef) bool { func makeColumnSteps(query string, s tableSchema, current map[string]columnDef) ([]schemaStep, error) { stmt, err := parser.New().ParseOneStmt(query, "", "") if err != nil { - return nil, errors.WrapError(errors.ErrDatabendInvalidData, err, "cannot parse incremental DDL") + return nil, errors.WrapError(errors.ErrLakeInvalidData, err, "cannot parse incremental DDL") } alter, ok := stmt.(*ast.AlterTableStmt) if !ok { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported column DDL") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported column DDL") } desired := make(map[string]columnDef) for _, c := range toSchemaDef(s).Columns { @@ -397,15 +397,15 @@ func makeColumnSteps(query string, s tableSchema, current map[string]columnDef) oldKey, nameKey := strings.ToLower(old), strings.ToLower(name) c := desired[nameKey] if (kind == "add" || kind == "modify") && c.DefaultExpr { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("expression default DDL is unsupported") + return errors.ErrLakeInvalidData.GenWithStackByArgs("expression default DDL is unsupported") } if kind == "drop" { c = columnDef{Name: name} } else if c.Name == "" { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("DDL references an invisible or missing column") + return errors.ErrLakeInvalidData.GenWithStackByArgs("DDL references an invisible or missing column") } if (kind == "drop" || kind == "rename") && (isKey(old) || isKey(name)) { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("changing a conflict key is unsupported") + return errors.ErrLakeInvalidData.GenWithStackByArgs("changing a conflict key is unsupported") } prev, present := current[oldKey] if present { @@ -417,22 +417,22 @@ func makeColumnSteps(query string, s tableSchema, current map[string]columnDef) switch kind { case "add": if _, ok := current[nameKey]; ok { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("column already exists before new DDL") + return errors.ErrLakeInvalidData.GenWithStackByArgs("column already exists before new DDL") } if !c.Nullable && c.Default == "" { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("ADD NOT NULL requires a literal source default") + return errors.ErrLakeInvalidData.GenWithStackByArgs("ADD NOT NULL requires a literal source default") } case "drop", "rename", "modify": if !present { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("old column is missing before new DDL") + return errors.ErrLakeInvalidData.GenWithStackByArgs("old column is missing before new DDL") } } if kind == "modify" && (!allowedModify(prev, c) || (isKey(old) && len(s.primaryKey) > 1)) { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported type transition or composite key modification") + return errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported type transition or composite key modification") } if kind == "rename" { if _, ok := current[nameKey]; ok { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("rename target already exists") + return errors.ErrLakeInvalidData.GenWithStackByArgs("rename target already exists") } delete(current, oldKey) } @@ -476,11 +476,11 @@ func makeColumnSteps(query string, s tableSchema, current map[string]columnDef) } case ast.AlterTableModifyColumn: if len(spec.NewColumns) != 1 { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid MODIFY") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid MODIFY") } def := spec.NewColumns[0] if !parameterizedCharacterSafe(def.Name.Name.O, def) { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("parameterized character MODIFY may narrow source data") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("parameterized character MODIFY may narrow source data") } name := spec.NewColumns[0].Name.Name.O if err := add("modify", name, name); err != nil { @@ -488,11 +488,11 @@ func makeColumnSteps(query string, s tableSchema, current map[string]columnDef) } case ast.AlterTableChangeColumn: if len(spec.NewColumns) != 1 { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid CHANGE") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid CHANGE") } old, name := spec.OldColumnName.Name.O, spec.NewColumns[0].Name.Name.O if !parameterizedCharacterSafe(old, spec.NewColumns[0]) { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("parameterized character CHANGE may narrow source data") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("parameterized character CHANGE may narrow source data") } if old != name { if err := add("rename", old, name); err != nil { @@ -504,11 +504,11 @@ func makeColumnSteps(query string, s tableSchema, current map[string]columnDef) } case ast.AlterTableAddConstraint: if spec.Constraint == nil || spec.Constraint.Tp == ast.ConstraintPrimaryKey { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("changing primary keys is unsupported") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("changing primary keys is unsupported") } case ast.AlterTableDropIndex, ast.AlterTableRenameIndex, ast.AlterTableIndexInvisible: // Secondary indexes have no target projection. default: - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported DDL; target has not been changed") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported DDL; target has not been changed") } } return steps, nil @@ -548,7 +548,7 @@ func (m *schemaManager) finishPlan(ctx context.Context, r epochRecord) error { case "add": if renamed { if canonicalType(newCol.Type) != canonicalType(step.Column.Type) || newCol.Nullable != step.Column.Nullable { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("pending ADD has conflicting target representation") + return errors.ErrLakeInvalidData.GenWithStackByArgs("pending ADD has conflicting target representation") } continue } @@ -563,16 +563,16 @@ func (m *schemaManager) finishPlan(ctx context.Context, r epochRecord) error { continue } if !present || renamed { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("pending RENAME is ambiguous") + return errors.ErrLakeInvalidData.GenWithStackByArgs("pending RENAME is ambiguous") } suffix = "RENAME COLUMN " + quoteIdentifier(step.OldName) + " TO " + quoteIdentifier(step.Column.Name) case "modify": if !present || !allowedModify(old, step.Column) { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("pending MODIFY has conflicting target representation") + return errors.ErrLakeInvalidData.GenWithStackByArgs("pending MODIFY has conflicting target representation") } suffix = "MODIFY COLUMN " + step.Column.definition() default: - return errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid durable DDL operation") + return errors.ErrLakeInvalidData.GenWithStackByArgs("invalid durable DDL operation") } if err := m.exec(ctx, "ALTER TABLE "+s.targetName()+" "+suffix); err != nil { return err @@ -585,7 +585,7 @@ func (m *schemaManager) finishPlan(ctx context.Context, r epochRecord) error { return err } default: - return errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported durable DDL plan") + return errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported durable DDL plan") } r.applied = true return m.saveEpoch(ctx, s, r) @@ -599,14 +599,14 @@ func (m *schemaManager) applyDDL(ctx context.Context, e *commonEvent.DDLEvent) e return nil } if e.FinishedTs == 0 || e.FinishedTs > math.MaxInt64 { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid DDL FinishedTs") + return errors.ErrLakeInvalidData.GenWithStackByArgs("invalid DDL FinishedTs") } switch model.ActionType(e.Type) { case model.ActionRenameTable, model.ActionRenameTables, model.ActionDropSchema: // Compatibility with bend-cdc's non-projecting path. The outer sink // has already drained prior DML; only these explicitly opted-out DDLs // succeed without a target mutation or an invented applied epoch. - log.Warn("Databend sink skipped source DDL and kept target objects unchanged", + log.Warn("Lake sink skipped source DDL and kept target objects unchanged", zap.String("ddlType", model.ActionType(e.Type).String()), zap.String("schema", e.SchemaName), zap.String("table", e.TableName), zap.Uint64("finishedTs", e.FinishedTs)) @@ -618,12 +618,12 @@ func (m *schemaManager) applyDDL(ctx context.Context, e *commonEvent.DDLEvent) e db = m.opts.database } if db == "" || strings.ContainsRune(db, 0) { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid schema name") + return errors.ErrLakeInvalidData.GenWithStackByArgs("invalid schema name") } return m.exec(ctx, "CREATE DATABASE IF NOT EXISTS "+quoteIdentifier(db)) } if e.TableInfo == nil { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("DDL without table metadata is outside the incremental projection") + return errors.ErrLakeInvalidData.GenWithStackByArgs("DDL without table metadata is outside the incremental projection") } s, err := schemaFromNative(e.TableInfo, m.opts.database, m.opts.conflictKey) if err != nil { @@ -666,17 +666,17 @@ func (m *schemaManager) applyDDL(ctx context.Context, e *commonEvent.DDLEvent) e // empty projection; this is not historical-table discovery. stmt, parseErr := parser.New().ParseOneStmt(e.Query, "", "") if parseErr != nil { - return errors.WrapError(errors.ErrDatabendInvalidData, parseErr, "cannot parse incremental DDL") + return errors.WrapError(errors.ErrLakeInvalidData, parseErr, "cannot parse incremental DDL") } alter, ok := stmt.(*ast.AlterTableStmt) if !ok { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported incremental column DDL") + return errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported incremental column DDL") } for _, spec := range alter.Specs { switch spec.Tp { case ast.AlterTableAddColumns, ast.AlterTableDropColumn, ast.AlterTableModifyColumn, ast.AlterTableChangeColumn, ast.AlterTableRenameColumn: default: - return errors.ErrDatabendInvalidData.GenWithStackByArgs("unsupported DDL on missing projection") + return errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported DDL on missing projection") } } plan.Kind = "create" @@ -691,7 +691,7 @@ func (m *schemaManager) applyDDL(ctx context.Context, e *commonEvent.DDLEvent) e return err } default: - return errors.ErrDatabendInvalidData.GenWithStackByArgs(fmt.Sprintf("unsupported DDL action %d", e.Type)) + return errors.ErrLakeInvalidData.GenWithStackByArgs(fmt.Sprintf("unsupported DDL action %d", e.Type)) } // No new mutation is issued for rejected DDLs. Callers have already drained // every affected lane; all old SQL must be terminated before a manual resume. diff --git a/downstreamadapter/sink/databend/schema_test.go b/downstreamadapter/sink/lake/schema_test.go similarity index 99% rename from downstreamadapter/sink/databend/schema_test.go rename to downstreamadapter/sink/lake/schema_test.go index 8ab1f4ae68..5202c8ac9c 100644 --- a/downstreamadapter/sink/databend/schema_test.go +++ b/downstreamadapter/sink/lake/schema_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "context" diff --git a/downstreamadapter/sink/databend/sink.go b/downstreamadapter/sink/lake/sink.go similarity index 77% rename from downstreamadapter/sink/databend/sink.go rename to downstreamadapter/sink/lake/sink.go index 7796c711bc..747568b9a3 100644 --- a/downstreamadapter/sink/databend/sink.go +++ b/downstreamadapter/sink/lake/sink.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "context" @@ -53,6 +53,7 @@ type Sink struct { mu sync.Mutex sourceBytes int64 sourceEvents int + admissionChanged chan struct{} started bool closed bool done chan struct{} @@ -71,7 +72,7 @@ func Verify(ctx context.Context, cfg *config.ChangefeedConfig, uri *url.URL) err return err } if len(rows) != 1 || len(rows[0]) != 1 || rows[0][0] == nil || *rows[0][0] != "1" { - return errors.ErrDatabendInvalidData.GenWithStackByArgs("server must support HTTP bound parameters and terminal query states") + return errors.ErrLakeInvalidData.GenWithStackByArgs("server must support HTTP bound parameters and terminal query states") } return nil } @@ -81,19 +82,19 @@ func New(ctx context.Context, id common.ChangeFeedID, cfg *config.ChangefeedConf if err != nil { return nil, err } - buffer, err := spool.New(id, spool.WithRootDir(o.spoolDir), spool.WithDirectoryNamespace("databend-sink-spool", config.GetGlobalServerConfig().AdvertiseAddr), spool.WithMetrics(newSpoolMetrics(id)), spool.WithDiskQuotaBytes(o.spoolBytes), spool.WithSegmentBytes(min(64<<20, o.spoolBytes/4)), spool.WithMemoryRatio(o.memoryRatio)) + buffer, err := spool.New(id, spool.WithRootDir(o.spoolDir), spool.WithDirectoryNamespace("lake-sink-spool", config.GetGlobalServerConfig().AdvertiseAddr), spool.WithMetrics(newSpoolMetrics(id)), spool.WithDiskQuotaBytes(o.spoolBytes), spool.WithSegmentBytes(min(64<<20, o.spoolBytes/4)), spool.WithMemoryRatio(o.memoryRatio)) if err != nil { return nil, err } ctx, cancel := context.WithCancelCause(ctx) - s := &Sink{opts: o, client: newHTTPClient(o), spool: buffer, ctx: ctx, cancel: cancel, done: make(chan struct{}), statistics: metrics.NewStatistics(id, keyspaceID, "databend")} + s := &Sink{opts: o, client: newHTTPClient(o), spool: buffer, ctx: ctx, cancel: cancel, done: make(chan struct{}), admissionChanged: make(chan struct{}), statistics: metrics.NewStatistics(id, keyspaceID, "lake")} s.normal.Store(true) for i := 0; i < o.workers; i++ { s.lanes = append(s.lanes, &tableLane{sink: s, input: make(chan workItem, o.sourceEvents), schemas: newSchemaManager(s.client, o), batches: make(map[string]*pendingTable)}) } return s, nil } -func (s *Sink) SinkType() common.SinkType { return common.DatabendSinkType } +func (s *Sink) SinkType() common.SinkType { return common.LakeSinkType } func (s *Sink) IsNormal() bool { return s.normal.Load() && s.ctx.Err() == nil } func (s *Sink) BatchCount() int { return min(s.opts.sourceEvents, s.opts.batchRows) } func (s *Sink) BatchBytes() int { return int(min(s.opts.sourceBytes/2, s.opts.batchBytes)) } @@ -121,44 +122,72 @@ func (s *Sink) releaseSource(charge int64) { s.mu.Lock() s.sourceBytes -= charge s.sourceEvents-- + s.notifyAdmissionLocked() s.mu.Unlock() } +// notifyAdmissionLocked broadcasts a quota/queue change without allocating +// goroutines or retaining a second waiting-event queue. The caller holds mu. +func (s *Sink) notifyAdmissionLocked() { + close(s.admissionChanged) + s.admissionChanged = make(chan struct{}) +} + +func sourceEventBytes(e *commonEvent.DMLEvent) int64 { + charge := e.Rows.MemoryUsage() + max(e.GetSize(), int64(0)) + int64(len(e.RowTypes))*8 + int64(len(e.RowKeys))*24 + int64(len(e.Checksum))*64 + 1024 + for _, key := range e.RowKeys { + charge += int64(len(key)) + } + return charge +} + func (s *Sink) AddDMLEvent(e *commonEvent.DMLEvent) { if !s.IsNormal() { return } if e == nil || e.TableInfo == nil || e.Rows == nil { - s.stop(errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid native DML event")) + s.stop(errors.ErrLakeInvalidData.GenWithStackByArgs("invalid native DML event")) return } // Shared chunks are conservatively charged in full for each admitted event. - // This can reject early but cannot hide source retention outside the pool. - charge := e.Rows.MemoryUsage() + max(e.GetSize(), int64(0)) + int64(len(e.RowTypes))*8 + int64(len(e.RowKeys))*24 + int64(len(e.Checksum))*64 + 1024 - for _, key := range e.RowKeys { - charge += int64(len(key)) - } - s.mu.Lock() - if !s.IsNormal() || charge > s.opts.sourceBytes-s.sourceBytes || s.sourceEvents >= s.opts.sourceEvents { - s.mu.Unlock() - s.stop(errors.ErrDatabendInvalidData.GenWithStackByArgs("source admission capacity exhausted; old sink lifecycle must terminate")) + // Waiting callers still own their event under the upstream memory budget. + charge := sourceEventBytes(e) + if charge > s.opts.sourceBytes { + s.stop(errors.ErrLakeInvalidData.GenWithStackByArgs("single source event exceeds admission byte limit")) return } - s.sourceBytes += charge - s.sourceEvents++ db := e.TableInfo.GetSchemaName() if s.opts.database != "" { db = s.opts.database } lane := s.lane(db, e.TableInfo.GetTableName()) - select { - case lane.input <- workItem{event: e, charge: charge}: - s.mu.Unlock() - default: - s.sourceBytes -= charge - s.sourceEvents-- + for { + s.mu.Lock() + if !s.IsNormal() || s.closed { + s.mu.Unlock() + return + } + if charge <= s.opts.sourceBytes-s.sourceBytes && s.sourceEvents < s.opts.sourceEvents { + s.sourceBytes += charge + s.sourceEvents++ + select { + case lane.input <- workItem{event: e, charge: charge}: + s.mu.Unlock() + return + default: + // A control/barrier item may occupy this lane. Roll back the + // reservation and wait for the independent lane worker. + s.sourceBytes -= charge + s.sourceEvents-- + } + } + changed := s.admissionChanged s.mu.Unlock() - s.stop(errors.ErrDatabendInvalidData.GenWithStackByArgs("source admission queue exhausted")) + select { + case <-s.ctx.Done(): + return + case <-changed: + } } } @@ -197,7 +226,7 @@ func (s *Sink) WriteBlockEvent(event commonEvent.BlockEvent) error { } e, ok := event.(*commonEvent.DDLEvent) if !ok { - err := errors.ErrDatabendInvalidData.GenWithStackByArgs("syncpoints are not supported") + err := errors.ErrLakeInvalidData.GenWithStackByArgs("syncpoints are not supported") s.stop(err) return err } @@ -239,7 +268,7 @@ func (s *Sink) Run(ctx context.Context) error { } if s.started { s.mu.Unlock() - return errors.ErrDatabendInvalidData.GenWithStackByArgs("sink lifecycle cannot be reused") + return errors.ErrLakeInvalidData.GenWithStackByArgs("sink lifecycle cannot be reused") } s.started = true s.mu.Unlock() diff --git a/downstreamadapter/sink/databend/sql.go b/downstreamadapter/sink/lake/sql.go similarity index 97% rename from downstreamadapter/sink/databend/sql.go rename to downstreamadapter/sink/lake/sql.go index 3cf729e7cd..66ccaabf2a 100644 --- a/downstreamadapter/sink/databend/sql.go +++ b/downstreamadapter/sink/lake/sql.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "fmt" @@ -38,7 +38,7 @@ id VARCHAR(255), table_name VARCHAR(255), raw_data JSON, add_time TIMESTAMP, event_pos BIGINT, action VARCHAR(20))`, s.rawName()) } -// addTimeClock assigns timestamps in the precision actually sent to Databend. +// addTimeClock assigns timestamps in the precision actually sent to Lake. // Comparing nanoseconds before formatting microseconds would lose tie ordering. type addTimeClock struct { last time.Time @@ -70,7 +70,7 @@ func (s tableSchema) mergeSQL(epoch uint64) string { for _, col := range s.columns { // A source name is a literal JSON object key, not a JSONPath. Quoting // it as an SQL identifier still leaves embedded quotes unescaped in - // Databend's get_by_keypath conversion. GET uses the exact string key. + // Lake's get_by_keypath conversion. GET uses the exact string key. path := "get(s.raw_data, " + sqlString(col.name) + ")" alias := quoteIdentifier(col.name) switch col.encoding { diff --git a/downstreamadapter/sink/databend/sql_test.go b/downstreamadapter/sink/lake/sql_test.go similarity index 99% rename from downstreamadapter/sink/databend/sql_test.go rename to downstreamadapter/sink/lake/sql_test.go index 39c6932b3e..2d48f8ac97 100644 --- a/downstreamadapter/sink/databend/sql_test.go +++ b/downstreamadapter/sink/lake/sql_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "strings" diff --git a/downstreamadapter/sink/databend/writer.go b/downstreamadapter/sink/lake/writer.go similarity index 88% rename from downstreamadapter/sink/databend/writer.go rename to downstreamadapter/sink/lake/writer.go index 068149a3e8..f2cdd635ce 100644 --- a/downstreamadapter/sink/databend/writer.go +++ b/downstreamadapter/sink/lake/writer.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "context" @@ -71,7 +71,7 @@ func newTableWriter(schema tableSchema, execute executeMutation, batchRows, maxA return nil, err } if execute == nil || batchRows <= 0 || maxAttempts <= 0 || epoch > math.MaxInt64 { - return nil, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid table writer configuration") + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid table writer configuration") } // Keep metadata owned even if the caller reuses its slices for a later DDL. schema.columns = append([]column(nil), schema.columns...) @@ -92,7 +92,7 @@ func (w *tableWriter) applyBatch(ctx context.Context, rows []normalizedRow, now for _, row := range rows { if row.id == "" || !json.Valid([]byte(row.payload)) || row.commitTs == 0 || row.commitTs > math.MaxInt64 || (row.action != insertAction && row.action != updateAction && row.action != deleteAction) { - return applyResult{}, errors.ErrDatabendInvalidData.GenWithStackByArgs("invalid normalized row") + return applyResult{}, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid normalized row") } } pending := make([]normalizedRow, 0, len(rows)) @@ -130,31 +130,31 @@ func (w *tableWriter) applyBatch(ctx context.Context, rows []normalizedRow, now func (w *tableWriter) runMutation(ctx context.Context, statement string, args []any) error { for attempt := 0; attempt < w.maxAttempts; attempt++ { if err := ctx.Err(); err != nil { - return errors.WrapError(errors.ErrDatabendExecution, err) + return errors.WrapError(errors.ErrLakeExecution, err) } result, err := w.execute(ctx, statement, args) switch result.state { case committed: if err != nil { // Contradictory execution evidence is not an observation error. - return errors.WrapError(errors.ErrDatabendCommitUnknown, err) + return errors.WrapError(errors.ErrLakeCommitUnknown, err) } // A confirmed commit is not undone by result statistics parsing. return nil case notCommitted: if !result.retryable || attempt+1 == w.maxAttempts { if err != nil { - return errors.WrapError(errors.ErrDatabendExecution, err) + return errors.WrapError(errors.ErrLakeExecution, err) } - return errors.ErrDatabendExecution.GenWithStackByArgs() + return errors.ErrLakeExecution.GenWithStackByArgs() } default: // Check before transient-error handling: no unknown request is resent. if err != nil { - return errors.WrapError(errors.ErrDatabendCommitUnknown, err) + return errors.WrapError(errors.ErrLakeCommitUnknown, err) } - return errors.ErrDatabendCommitUnknown.GenWithStackByArgs() + return errors.ErrLakeCommitUnknown.GenWithStackByArgs() } } - return errors.ErrDatabendExecution.GenWithStackByArgs() + return errors.ErrLakeExecution.GenWithStackByArgs() } diff --git a/downstreamadapter/sink/databend/writer_test.go b/downstreamadapter/sink/lake/writer_test.go similarity index 91% rename from downstreamadapter/sink/databend/writer_test.go rename to downstreamadapter/sink/lake/writer_test.go index b843941550..38af6384cf 100644 --- a/downstreamadapter/sink/databend/writer_test.go +++ b/downstreamadapter/sink/lake/writer_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package databend +package lake import ( "context" @@ -67,7 +67,7 @@ func TestTableWriterStopsUnknownBeforeInnerRetry(t *testing.T) { require.Error(t, err) code, ok := errors.RFCCode(err) require.True(t, ok) - require.Equal(t, errors.ErrDatabendCommitUnknown.RFCCode(), code) + require.Equal(t, errors.ErrLakeCommitUnknown.RFCCode(), code) require.False(t, result.applied) require.Equal(t, failAt, calls, "unknown must not reach an inner retry, MERGE, or compaction") require.Zero(t, w.merged) @@ -83,9 +83,9 @@ func TestTableWriterRetriesOnlyConfirmedNotCommitted(t *testing.T) { execute := func(context.Context, string, []any) (mutationResult, error) { calls++ if calls == 1 { - return mutationResult{state: notCommitted, retryable: true}, errors.ErrDatabendInvalidData.GenWithStackByArgs("confirmed rollback") + return mutationResult{state: notCommitted, retryable: true}, errors.ErrLakeInvalidData.GenWithStackByArgs("confirmed rollback") } - return mutationResult{state: committed, observationErr: errors.ErrDatabendInvalidData.GenWithStackByArgs("statistics unavailable")}, nil + return mutationResult{state: committed, observationErr: errors.ErrLakeInvalidData.GenWithStackByArgs("statistics unavailable")}, nil } w, err := newTableWriter(testSchema(), execute, 1, 2, 0) require.NoError(t, err) @@ -97,7 +97,7 @@ func TestTableWriterRetriesOnlyConfirmedNotCommitted(t *testing.T) { calls = 0 w, err = newTableWriter(testSchema(), func(context.Context, string, []any) (mutationResult, error) { calls++ - return mutationResult{state: notCommitted, retryable: true}, errors.ErrDatabendInvalidData.GenWithStackByArgs("confirmed rollback") + return mutationResult{state: notCommitted, retryable: true}, errors.ErrLakeInvalidData.GenWithStackByArgs("confirmed rollback") }, 1, 2, 0) require.NoError(t, err) result, err = w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 100, 1)}, time.Now()) @@ -111,7 +111,7 @@ func TestTableWriterDoesNotRevokeAppliedOnCompactionFailure(t *testing.T) { execute := func(context.Context, string, []any) (mutationResult, error) { calls++ if calls == 3 { - return mutationResult{state: notCommitted}, errors.ErrDatabendInvalidData.GenWithStackByArgs("compaction denied") + return mutationResult{state: notCommitted}, errors.ErrLakeInvalidData.GenWithStackByArgs("compaction denied") } return mutationResult{state: committed}, nil } @@ -133,7 +133,7 @@ func TestTableWriterRejectsMissingOrContradictoryCommitEvidence(t *testing.T) { err error }{ {name: "missing terminal evidence"}, - {name: "contradictory evidence", result: mutationResult{state: committed}, err: errors.ErrDatabendInvalidData.GenWithStackByArgs("execution failed")}, + {name: "contradictory evidence", result: mutationResult{state: committed}, err: errors.ErrLakeInvalidData.GenWithStackByArgs("execution failed")}, } { t.Run(tc.name, func(t *testing.T) { calls := 0 @@ -146,7 +146,7 @@ func TestTableWriterRejectsMissingOrContradictoryCommitEvidence(t *testing.T) { require.Error(t, err) code, ok := errors.RFCCode(err) require.True(t, ok) - require.Equal(t, errors.ErrDatabendCommitUnknown.RFCCode(), code) + require.Equal(t, errors.ErrLakeCommitUnknown.RFCCode(), code) require.False(t, result.applied) require.Equal(t, 1, calls) }) diff --git a/downstreamadapter/sink/metrics/databend.go b/downstreamadapter/sink/metrics/databend.go deleted file mode 100644 index 3f446893ef..0000000000 --- a/downstreamadapter/sink/metrics/databend.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package metrics - -import "github.com/prometheus/client_golang/prometheus" - -var ( - DatabendBufferBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: "ticdc", Subsystem: "databend", Name: "buffer_bytes", Help: "Local Databend memory and physical spool bytes."}, []string{"namespace", "changefeed", "kind"}) - DatabendBufferEntries = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: "ticdc", Subsystem: "databend", Name: "buffer_entries", Help: "Local Databend pending enqueue, segment and quota waiter counts."}, []string{"namespace", "changefeed", "kind"}) - DatabendSpoolDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{Namespace: "ticdc", Subsystem: "databend", Name: "spool_wait_seconds", Help: "Databend spool quota wait time.", Buckets: prometheus.ExponentialBuckets(0.001, 2, 20)}, []string{"namespace", "changefeed"}) - DatabendSpoolLoad = prometheus.NewHistogramVec(prometheus.HistogramOpts{Namespace: "ticdc", Subsystem: "databend", Name: "spool_load_bytes", Help: "Databend encoded bytes loaded from local spool.", Buckets: prometheus.ExponentialBuckets(1024, 4, 9)}, []string{"namespace", "changefeed"}) - DatabendSpoolRotations = prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: "ticdc", Subsystem: "databend", Name: "spool_rotations_total", Help: "Databend local spool segment rotations."}, []string{"namespace", "changefeed"}) -) - -func InitDatabendMetrics(registry *prometheus.Registry) { - registry.MustRegister(DatabendBufferBytes, DatabendBufferEntries, DatabendSpoolDuration, DatabendSpoolLoad, DatabendSpoolRotations) -} diff --git a/downstreamadapter/sink/metrics/lake.go b/downstreamadapter/sink/metrics/lake.go new file mode 100644 index 0000000000..cbc594c851 --- /dev/null +++ b/downstreamadapter/sink/metrics/lake.go @@ -0,0 +1,29 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +var ( + LakeBufferBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: "ticdc", Subsystem: "lake", Name: "buffer_bytes", Help: "Local Lake memory and physical spool bytes."}, []string{"namespace", "changefeed", "kind"}) + LakeBufferEntries = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: "ticdc", Subsystem: "lake", Name: "buffer_entries", Help: "Local Lake pending enqueue, segment and quota waiter counts."}, []string{"namespace", "changefeed", "kind"}) + LakeSpoolDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{Namespace: "ticdc", Subsystem: "lake", Name: "spool_wait_seconds", Help: "Lake spool quota wait time.", Buckets: prometheus.ExponentialBuckets(0.001, 2, 20)}, []string{"namespace", "changefeed"}) + LakeSpoolLoad = prometheus.NewHistogramVec(prometheus.HistogramOpts{Namespace: "ticdc", Subsystem: "lake", Name: "spool_load_bytes", Help: "Lake encoded bytes loaded from local spool.", Buckets: prometheus.ExponentialBuckets(1024, 4, 9)}, []string{"namespace", "changefeed"}) + LakeSpoolRotations = prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: "ticdc", Subsystem: "lake", Name: "spool_rotations_total", Help: "Lake local spool segment rotations."}, []string{"namespace", "changefeed"}) +) + +func InitLakeMetrics(registry *prometheus.Registry) { + registry.MustRegister(LakeBufferBytes, LakeBufferEntries, LakeSpoolDuration, LakeSpoolLoad, LakeSpoolRotations) +} diff --git a/downstreamadapter/sink/sink.go b/downstreamadapter/sink/sink.go index d1d04b6c44..c51c36c827 100644 --- a/downstreamadapter/sink/sink.go +++ b/downstreamadapter/sink/sink.go @@ -19,8 +19,8 @@ import ( "github.com/pingcap/ticdc/downstreamadapter/sink/blackhole" "github.com/pingcap/ticdc/downstreamadapter/sink/cloudstorage" - "github.com/pingcap/ticdc/downstreamadapter/sink/databend" "github.com/pingcap/ticdc/downstreamadapter/sink/kafka" + "github.com/pingcap/ticdc/downstreamadapter/sink/lake" "github.com/pingcap/ticdc/downstreamadapter/sink/mysql" "github.com/pingcap/ticdc/downstreamadapter/sink/pulsar" "github.com/pingcap/ticdc/pkg/common" @@ -73,8 +73,8 @@ func New(ctx context.Context, cfg *config.ChangefeedConfig, changefeedID common. return pulsar.New(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive, keyspaceID) case config.S3Scheme, config.FileScheme, config.GCSScheme, config.GSScheme, config.AzblobScheme, config.AzureScheme, config.CloudStorageNoopScheme: return cloudstorage.New(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive, cfg.EnableTableAcrossNodes, nil, keyspaceID) - case config.DatabendScheme, config.DatabendHTTPScheme, config.LakeScheme: - return databend.New(ctx, changefeedID, cfg, sinkURI, keyspaceID) + case config.LakeScheme, config.LakeHTTPScheme: + return lake.New(ctx, changefeedID, cfg, sinkURI, keyspaceID) case config.BlackHoleScheme: return blackhole.New(changefeedID, keyspaceID) } @@ -100,8 +100,8 @@ func Verify(ctx context.Context, cfg *config.ChangefeedConfig, changefeedID comm return pulsar.Verify(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive) case config.S3Scheme, config.FileScheme, config.GCSScheme, config.GSScheme, config.AzblobScheme, config.AzureScheme, config.CloudStorageNoopScheme: return cloudstorage.Verify(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive, cfg.EnableTableAcrossNodes) - case config.DatabendScheme, config.DatabendHTTPScheme, config.LakeScheme: - return databend.Verify(ctx, cfg, sinkURI) + case config.LakeScheme, config.LakeHTTPScheme: + return lake.Verify(ctx, cfg, sinkURI) case config.BlackHoleScheme: return nil } diff --git a/downstreamadapter/sink/sink_test.go b/downstreamadapter/sink/sink_test.go index 4a2244f10f..88bfc4bcb8 100644 --- a/downstreamadapter/sink/sink_test.go +++ b/downstreamadapter/sink/sink_test.go @@ -78,17 +78,17 @@ func requireInvalidSinkURIError(t *testing.T, err error) { require.Contains(t, errMsg, "invalid URL escape") } -func TestDatabendFactoryRegistration(t *testing.T) { - cfg := &config.ChangefeedConfig{SinkURI: "databend+http://127.0.0.1:1?exclusive=true&spool-dir=" + url.QueryEscape(t.TempDir())} +func TestLakeFactoryRegistration(t *testing.T) { + cfg := &config.ChangefeedConfig{SinkURI: "lake+http://127.0.0.1:1?exclusive=true&spool-dir=" + url.QueryEscape(t.TempDir())} s, err := New(context.Background(), cfg, common.NewChangefeedID(common.DefaultKeyspaceName), common.DefaultKeyspaceID) require.NoError(t, err) defer s.Close() - require.Equal(t, common.DatabendSinkType, s.SinkType()) + require.Equal(t, common.LakeSinkType, s.SinkType()) store := commonEvent.NewTableSchemaStore(nil, s.SinkType(), false) s.SetTableSchemaStore(store) - require.False(t, config.IsStorageScheme("databend")) - require.False(t, config.IsMySQLCompatibleScheme("databend")) - cfg.SinkURI = "databend://user:verysecure@localhost?exclusive=false" + require.False(t, config.IsStorageScheme("lake")) + require.False(t, config.IsMySQLCompatibleScheme("lake")) + cfg.SinkURI = "lake://user:verysecure@localhost?exclusive=false" _, err = New(context.Background(), cfg, common.NewChangefeedID(common.DefaultKeyspaceName), common.DefaultKeyspaceID) require.Error(t, err) require.NotContains(t, err.Error(), "verysecure") diff --git a/pkg/common/event/table_schema_store.go b/pkg/common/event/table_schema_store.go index 0cfe0b701c..20277e1c6f 100644 --- a/pkg/common/event/table_schema_store.go +++ b/pkg/common/event/table_schema_store.go @@ -91,7 +91,7 @@ func newTableSchemaStoreRequirements( updateTableIDs: false, needTableNames: true, } - case commonType.CloudStorageSinkType, commonType.BlackHoleSinkType, commonType.DatabendSinkType: + case commonType.CloudStorageSinkType, commonType.BlackHoleSinkType, commonType.LakeSinkType: return tableSchemaStoreRequirements{ needTableIDs: false, updateTableIDs: false, diff --git a/pkg/common/event/table_schema_store_test.go b/pkg/common/event/table_schema_store_test.go index 2aa18f407e..716da0cc8a 100644 --- a/pkg/common/event/table_schema_store_test.go +++ b/pkg/common/event/table_schema_store_test.go @@ -254,8 +254,8 @@ func TestTableSchemaStoreNonMysqlTableIDsBootstrapOnly(t *testing.T) { require.ElementsMatch(t, []int64{10, 20}, store.GetAllNormalTableIds()) } -func TestTableSchemaStoreWhenDatabendSink(t *testing.T) { - store := NewTableSchemaStore(nil, common.DatabendSinkType, false) +func TestTableSchemaStoreWhenLakeSink(t *testing.T) { + store := NewTableSchemaStore(nil, common.LakeSinkType, false) require.NotNil(t, store) require.Nil(t, store.TableIDStore) require.Nil(t, store.tableNameStore) diff --git a/pkg/common/types.go b/pkg/common/types.go index 82e718786e..acae4d5b36 100644 --- a/pkg/common/types.go +++ b/pkg/common/types.go @@ -344,7 +344,7 @@ const ( CloudStorageSinkType BlackHoleSinkType RedoSinkType - DatabendSinkType + LakeSinkType ) type RowType byte diff --git a/pkg/config/sink_protocol.go b/pkg/config/sink_protocol.go index ed76ed65ea..bd24688c73 100644 --- a/pkg/config/sink_protocol.go +++ b/pkg/config/sink_protocol.go @@ -114,12 +114,10 @@ func (p Protocol) String() string { const ( // KafkaScheme indicates the scheme is kafka. KafkaScheme = "kafka" - // DatabendScheme is the native HTTPS warehouse sink. - DatabendScheme = "databend" - // LakeScheme is the TiDB Cloud Lake HTTPS endpoint alias. + // LakeScheme is the native Lake HTTPS warehouse sink. LakeScheme = "lake" - // DatabendHTTPScheme is the explicit plaintext local/test warehouse sink. - DatabendHTTPScheme = "databend+http" + // LakeHTTPScheme is the explicit plaintext local/test warehouse sink. + LakeHTTPScheme = "lake+http" // KafkaSSLScheme indicates the scheme is kafka+ssl. KafkaSSLScheme = "kafka+ssl" // BlackHoleScheme indicates the scheme is blackhole. diff --git a/pkg/errors/helper.go b/pkg/errors/helper.go index 8a76b559f3..b5e6188bfe 100644 --- a/pkg/errors/helper.go +++ b/pkg/errors/helper.go @@ -102,8 +102,8 @@ func RFCCode(err error) (errors.RFCErrorCode, bool) { } var changefeedUnRetryableErrors = []*errors.Error{ - ErrDatabendInvalidData, - ErrDatabendCommitUnknown, + ErrLakeInvalidData, + ErrLakeCommitUnknown, ErrExpressionColumnNotFound, ErrExpressionParseFailed, ErrSchemaSnapshotNotFound, diff --git a/pkg/errors/databend.go b/pkg/errors/lake.go similarity index 55% rename from pkg/errors/databend.go rename to pkg/errors/lake.go index 41d7d88ffb..d9962b72a4 100644 --- a/pkg/errors/databend.go +++ b/pkg/errors/lake.go @@ -17,11 +17,11 @@ package errors import "github.com/pingcap/errors" var ( - ErrDatabendProtocol = errors.Normalize("Databend request %s: %s", errors.RFCCodeText("CDC:ErrDatabendProtocol")) - ErrDatabendInvalidData = errors.Normalize( - "invalid Databend sink data: %s", errors.RFCCodeText("CDC:ErrDatabendInvalidData")) - ErrDatabendExecution = errors.Normalize( - "Databend mutation failed", errors.RFCCodeText("CDC:ErrDatabendExecution")) - ErrDatabendCommitUnknown = errors.Normalize( - "Databend mutation commit state is unknown", errors.RFCCodeText("CDC:ErrDatabendCommitUnknown")) + ErrLakeProtocol = errors.Normalize("Lake request %s: %s", errors.RFCCodeText("CDC:ErrLakeProtocol")) + ErrLakeInvalidData = errors.Normalize( + "invalid Lake sink data: %s", errors.RFCCodeText("CDC:ErrLakeInvalidData")) + ErrLakeExecution = errors.Normalize( + "Lake mutation failed", errors.RFCCodeText("CDC:ErrLakeExecution")) + ErrLakeCommitUnknown = errors.Normalize( + "Lake mutation commit state is unknown", errors.RFCCodeText("CDC:ErrLakeCommitUnknown")) ) diff --git a/pkg/metrics/downstream_type.go b/pkg/metrics/downstream_type.go index c960036802..9afc5dcfed 100644 --- a/pkg/metrics/downstream_type.go +++ b/pkg/metrics/downstream_type.go @@ -59,8 +59,8 @@ func downstreamTypeFromScheme(scheme string) string { return config.AzblobScheme case config.CloudStorageNoopScheme: return config.CloudStorageNoopScheme - case config.DatabendScheme, config.DatabendHTTPScheme, config.LakeScheme: - return config.DatabendScheme + case config.LakeScheme, config.LakeHTTPScheme: + return config.LakeScheme case config.BlackHoleScheme: return config.BlackHoleScheme case "": diff --git a/pkg/metrics/downstream_type_test.go b/pkg/metrics/downstream_type_test.go index d3ac34ac43..f6fe517e72 100644 --- a/pkg/metrics/downstream_type_test.go +++ b/pkg/metrics/downstream_type_test.go @@ -45,9 +45,9 @@ func TestDownstreamTypeFromSinkURI(t *testing.T) { {sinkURI: "azblob://bucket/prefix", expectedLabel: "azblob"}, {sinkURI: "noop://bucket/prefix", expectedLabel: "noop"}, {sinkURI: "blackhole:///", expectedLabel: "blackhole"}, - {sinkURI: "databend://warehouse", expectedLabel: "databend"}, - {sinkURI: "lake://warehouse:443/default", expectedLabel: "databend"}, - {sinkURI: "databend+http://127.0.0.1:8000", expectedLabel: "databend"}, + {sinkURI: "lake://warehouse", expectedLabel: "lake"}, + {sinkURI: "lake://warehouse:443/default", expectedLabel: "lake"}, + {sinkURI: "lake+http://127.0.0.1:8000", expectedLabel: "lake"}, {sinkURI: "127.0.0.1:3306", expectedLabel: "unknown"}, {sinkURI: "mysql://[::1", expectedLabel: "unknown"}, {sinkURI: "", expectedLabel: "unknown"}, diff --git a/pkg/metrics/init.go b/pkg/metrics/init.go index a6b0133aaa..ef5db0b60e 100644 --- a/pkg/metrics/init.go +++ b/pkg/metrics/init.go @@ -44,7 +44,7 @@ func InitMetrics(registry *prometheus.Registry) { kafka.InitMetrics(registry) gc.InitMetrics(registry) metrics.InitCloudStorageMetrics(registry) - metrics.InitDatabendMetrics(registry) + metrics.InitLakeMetrics(registry) initRedoMetrics(registry) initDDLMetrics(registry) } diff --git a/tests/integration_tests/databend_native/README.md b/tests/integration_tests/databend_native/README.md deleted file mode 100644 index 4847c4776e..0000000000 --- a/tests/integration_tests/databend_native/README.md +++ /dev/null @@ -1,301 +0,0 @@ -# Native Databend / Lake acceptance test - -## One-command local E2E (no external Lake required) - -```sh -make integration_test_databend -# Or use the integration-case entry point: -bash tests/integration_tests/databend_native/run.sh databend -# Faster, selected scenario (repeat --suite to select more): -python3 tests/integration_tests/databend_native/local.py --suite acceptance -``` - -Prerequisites: a running Docker engine (Linux or Docker Desktop on macOS), TiUP, -the `mysql` CLI, Python 3 and the repository's Go/build dependencies. No cloud -credentials, manually running database or S3/MinIO service is required. Allow -enough free disk for the Go build and source cluster; 8 GiB Docker memory is -recommended. The runner does not install tools or start the Docker daemon. - -`local.py` follows the existing integration tests' owned-background-service -pattern, but reuses these Python assertions rather than a MySQL downstream diff: - -1. Build current-branch TiCDC using `make cdc` (or use `E2E_CDC_BINARY`). -2. Start a unique Databend **meta + query** all-in-one background container, - pinned to `v1.2.925-patch-13` and its multi-platform image digest. Use local - filesystem storage, not cloud object storage; publish HTTP only on loopback - with a random port and temporary random credentials. -3. Start an owned TiUP v8.5.1 source (one PD/TiKV/TiDB, no TiFlash/monitor) and - a new-architecture TiCDC capture. Wait for actual SQL and owner readiness. -4. Run the existing acceptance, DDL, explicit-key, skipped-DDL, four rejection - probes and real-Databend response-loss tests. The acceptance suite includes - a drained capture restart, disk-spool readback and full value comparisons. -5. On success, failure or interruption, stop only this run's processes/container - and clean its unique TiUP tag. Preserve private logs and per-suite results; - remove the temporary URI file. The container's target data is ephemeral and - is removed even on failure, unlike external-warehouse mode below. - -The summary path is printed at startup and exit. `summary.json` records source -commit/dirty status, build/server versions, image identity, suite exits and -cleanup. Detailed case assertions remain in each suite's `result.json`; fault -assertions are in `fault.log`. A rejected-scope PASS proves fail-closed behavior, -not support. Local fs-backed Databend proves SQL/adapter behavior, not cloud S3, -cloud gateway routing, cost, distributed HA or production capacity. - -Optional environment: `DATABEND_E2E_IMAGE` to explicitly test another image, -`TIDB_E2E_VERSION` for the source version, `E2E_PORT_OFFSET` (default 10000) for -source ports, `E2E_CDC_BINARY` for an already-built capture, and `OUT_DIR` for an -existing artifacts parent directory. Occupied source ports fail rather than -attaching to someone else's cluster. The runner intentionally ignores inherited -`LAKE_URI_FILE`, `DATABEND_NATIVE_TEST_URI`, capture address and source-port -settings, so it cannot accidentally use an external target. - -Service-free lifecycle checks: - -```sh -python3 -m unittest discover -s tests/integration_tests/databend_native -p 'test_*.py' -``` - -### Local execution evidence (2026-10-26) - -Executed on macOS arm64 with Docker Desktop, TiDB v8.5.1 and the pinned Databend -v1.2.925-patch-13 image; TiCDC was built from `457c2f9bd` plus this test-only dirty -tree. `make integration_test_databend` (reusing that current-branch binary via -`E2E_CDC_BINARY`) passed **all nine suites**: 27 Python case assertions and three -real-backend Go tests, including 19 DDL response-loss boundaries. Acceptance -observed 41 disk-spool readbacks and a successful drained capture restart. All -suite-owned database/feed cleanup and runner process/container cleanup passed. -After fixing canonical-path cleanup, the default build path was rerun separately -with `local.py --suite acceptance` (no `E2E_CDC_BINARY`): build, all nine acceptance -cases and automatic cleanup passed again. Independent checks found neither old -nor restarted capture PIDs, no owned Databend containers, and no URI file. -The service-free lifecycle suite passed seven tests. Linux/amd64 CI execution was -not performed in this run; the image is multi-platform, not evidence of a tested -second platform. - -The initial v1.2.852-nightly probe failed real parameter binding; it is **not** a -supported backend established by this test. No SQL string-interpolation fallback -was introduced. Startup now checks binding before creating the source/capture. -The managed tests also exposed and fixed fixture issues: hard-coded -restart port, owner election preceding coordinator readiness, and TIME_WAIT -blocking immediate reruns, plus canonical-path comparison during cleanup on -macOS (`/var` versus `/private/var`). These are test-lifecycle fixes, not TiCDC -framework changes. - -## External authorized warehouse mode - -Opt-in integration test for **a local TiUP TiDB + current-branch new-architecture -TiCDC + a real, explicitly authorized Lake test warehouse**. It uses only the -Python standard library and a local `mysql` CLI; no Dumpling or S3 is involved. - -The runner creates a random `native_e2e_*` database and a matching changefeed, -compares actual TiDB rows against actual Lake rows, and removes only those owned -resources after success. On failure, it keeps its isolated database/feed and a -private result file for diagnosis. Never point it at a production source/target. - -## Prepare - -1. Build `cmd/cdc` from this branch. Use `--newarch` and UTC for the test capture. - Any compatibility release label injected into this local build is **not** an - official upstream release; record the source commit and dirty-tree status. -2. Start TiUP with one TiDB/PD/TiKV and no TiFlash/monitor. For example: - - ```sh - tiup playground v8.5.1 --tag native-lake-test \ - --db 1 --pd 1 --kv 1 --tiflash 0 --ticdc 0 \ - --without-monitor --port-offset 10000 - ``` - -3. Start the built CDC with `--newarch --tz UTC`, local PD - `http://127.0.0.1:12379`, API `127.0.0.1:18300`, and a private test data dir. - The source is always `127.0.0.1`; `MYSQL_PORT` defaults to `14000`. -4. Put the authorized Lake URI in a **0600** file outside the repository. The - runner reads `LAKE_URI_FILE`; never put the URI/password on a command line, - into a shared report, or into Git. `lake://` is the HTTPS sink alias. -5. On low-free-space development laptops, TiDB v8.5.1's fast/distributed ADD - INDEX can reject a tiny test table before CDC sees any DDL. For this **owned - local TiUP only**, use legacy index construction if necessary: - - ```sql - SET GLOBAL tidb_ddl_enable_fast_reorg=OFF; - SET GLOBAL tidb_enable_dist_task=OFF; - ``` - - Record this source-environment adjustment. Do not change these settings on - a shared/production source just to make this test pass. - -## Run - -```sh -umask 077 -export LAKE_URI_FILE=/absolute/path/to/private/lake-uri -export E2E_RUN_DIR=/absolute/path/to/new/private/run-directory -export CDC_SERVER=http://127.0.0.1:18300 -export MYSQL_PORT=14000 -# Optional process-restart scenario: only the exact owned local binary is stopped. -export E2E_CDC_BINARY=/absolute/path/to/current-branch-cdc -export E2E_CDC_DATA_DIR=/absolute/path/to/the/cdc/data-dir -export E2E_PD=http://127.0.0.1:12379 -python3 tests/integration_tests/databend_native/run.py -``` - -Use a fresh run directory each time. Dedicated source and target database names -are chosen automatically; the URI's original database (e.g. `default`) is not -used as the test database. Existing tables there are never altered or dropped. - -The current suite covers: - -- DML insert/update/delete, nonclustered PK updates, secondary UK updates, - same-transaction delete/reinsert and composite keys. -- Quoted table/column names, `action` and internal-discriminator name collisions. -- Signed/unsigned integer boundaries, DECIMAL(65,30), FLOAT/DOUBLE, CHAR/TEXT, - binary/BLOB/VARBINARY padding/bytes, BIT(64), numeric-label ENUM, 64-element SET, - exact JSON integers, SQL NULL versus JSON null, dates, microsecond - datetime/timestamp/time, YEAR, and visible STORED generated columns. - VIRTUAL generated columns must not appear in the target. -- ADD/default/backfill, RENAME, integer widening, DROP column, secondary indexes, - TRUNCATE/new DML and DROP TABLE/raw cleanup. -- A 1,200-row transaction larger than the **64 KiB** disk quota, with a small - memory hot set. A live spool read histogram must show disk readback; this is - not inferred solely from configured thresholds. -- Optional drained process restart: wait for a positive source transaction TSO - checkpoint, pause, verify the owned CDC process, terminate it, create a source - DML/DDL backlog, restart, wait through coordinator bootstrap, resume without - overriding checkpoint, then compare the recovered state. This is **not** - a claim of in-flight ambiguous-query failover or multi-capture HA. - -Comparison is key-based, not dependent on differing numeric/VARCHAR ORDER BY -semantics. JSON null is observed explicitly with `is_null_value`; FLOAT is read -at sufficient source precision and compared with tolerance. All other integer, -Decimal, text and binary comparisons are exact (binary is canonicalized as hex). - -## Failure inspection and cleanup - -`result.json` records individual cases, versions, target schema and restart/spool -evidence. It excludes credentials. Logs and temporary credentials stay in the -private run directory, not the repository. - -First inspect a failure and ensure the old CDC queries are terminated. To remove -just the owned failed run: - -```sh -python3 tests/integration_tests/databend_native/run.py --cleanup -``` - -Use the same environment/run directory. Cleanup validates the generated namespace -and matching feed, pauses a healthy feed before deletion, and never drops the -URI's original database. The runner does not stop an externally launched TiUP or -CDC after the test: stop your owned local processes separately. Do not confuse a -successful source/target test with fault-injection, cost/SLO, unsupported-DDL or -production-readiness validation. - -## Extended DDL and baseline parity probes - -`extended.py` uses the same local services/private environment and a fresh -`E2E_RUN_DIR`. It verifies defaults and nullability with actual target DESCRIBE, -not only row counts. Its positive suite also covers a NOT NULL UNIQUE handle -without a declared PK and sufficiently wide `INT -> VARCHAR(200)`. - -```sh -python3 tests/integration_tests/databend_native/extended.py -# Separate fresh run directories for each intentionally unsupported probe: -python3 tests/integration_tests/databend_native/extended.py --reject partition -python3 tests/integration_tests/databend_native/extended.py --reject no-pk -python3 tests/integration_tests/databend_native/extended.py --reject narrow -python3 tests/integration_tests/databend_native/extended.py --reject expression-default -``` - -A rejection PASS means **stable failed feed + nonadvancing checkpoint + unchanged -existing target (or no new projection)**, NOT feature support. The `no-pk` probe -forces replication and writes a source row, so upstream eligibility filtering -cannot masquerade as sink rejection. A NOT NULL UNIQUE table is a positive test, -not a `no-pk` rejection. Optional `--unique-only` / `--varchar-only` isolate those -positive cases for diagnosis. - -`baseline.py` runs an ephemeral Go overlay against the **clean, fixed b648162** -independent bend-cdc module. It changes no baseline files. Set `BEND_CDC_DIR` to -that checkout, retain the same private URI-file environment, and use a fresh run -directory. `baseline_parity_test.go.txt` is the reviewable overlay fixture. - -```sh -export BEND_CDC_DIR=/absolute/path/to/webapi/bend-cdc -python3 tests/integration_tests/databend_native/baseline.py -``` - -These baseline probes exercise its actual Canal/schema-to-Lake core, not the -entire S3 listing/decoding path. In particular they assert RENAME / DROP DATABASE -are **not projected**, rather than treating a nil error as feature support. - -## Real warehouse response-loss injection - -`live_fault_test.go` is opt-in. All SQL executes on the real authorized warehouse. -A delegating HTTP RoundTripper drops a matching real Succeeded response before -the sink sees it; it does NOT mock SQL success. For partial raw writes it blocks -the second INSERT before sending, after the first real INSERT committed. - -```sh -python3 - <<'PY' -import os, pathlib, subprocess -p = pathlib.Path(os.environ['LAKE_URI_FILE']) -assert not p.stat().st_mode & 0o077 - -env = os.environ.copy() -env['DATABEND_NATIVE_TEST_URI'] = p.read_text().strip() -env.setdefault('GOMAXPROCS', '4') -subprocess.run([ - 'go', 'test', '-p', '2', '-race', './downstreamadapter/sink/databend', - '-run', 'TestDatabendNativeLive(RawPartialAndMergeResponseLoss|DDLInterruptionRecovery)$', - '-v', '-count=1', '-timeout=20m', -], env=env, check=True) -PY -``` - -Coverage: partial raw recovery; committed MERGE with lost terminal response and -zero source flush; 19 DDL commit boundaries across CREATE/ALTER/TRUNCATE/DROP, -zero early DDL callbacks, durable pending/applied recovery, full recovered values, -and old-DML/applied-DDL replay fences. Each test creates/drops a random -`ticdc_fault_test_*` database. - -Recovery is allowed because the test independently knows the old matching query -reached terminal success (or was never sent), and has ended the old sink -lifecycle. Schema-manager state is rebuilt inside the test process; this is not -a SIGKILL-at-every-instruction test, mid-execution server failure, or automatic -HA proof. A real arbitrary UNKNOWN still requires manual terminal/exclusivity -verification before resume. - - -## Explicit conflict key and skipped DDL - -These are current positive acceptance scenarios (not the historical rejection -probes). Use the same local setup and a new E2E_RUN_DIR for each command: - -```sh -python3 tests/integration_tests/databend_native/options.py --manual-key -python3 tests/integration_tests/databend_native/options.py -``` - -The first adds `conflict-key=ID` to the feed URI and enables API -`replica_config.force_replicate=true`. It verifies a genuinely keyless table's -INSERT/UPDATE/DELETE/ADD COLUMN, persisted effective identity, and an override of -a different single native PK. Source columns resolve case insensitively. - -This is a feed-wide **single-column override**, matching bend-cdc, not automatic -uniqueness discovery: the operator must guarantee unique, non-NULL, stable values. -Do not change the key on an existing target without reinitialization. Composite -source keys cannot be replaced; absent/virtual columns and NULL row identities -are rejected. An unsplit UPDATE that changes the configured identity fails -closed; this version does not synthesize a new delete/insert protocol for arbitrary -mutable logical keys. A truly ineligible source table still needs TiCDC's existing -`force-replicate = true` (TOML) / `force_replicate` (API) setting. - -The second proves RENAME TABLE / DROP DATABASE are explicitly skipped with a -warning, the checkpoint advances, previous target objects/data remain, and later -source DML/DDL still applies. Rename does not move historical rows: later DML -uses the new target name and may lazily create that projection. Dropping the -source database does not delete the target database; recreating the same names -does not automatically clear retained target rows. Reinitialize separately if -an empty projection is required. Only these requested DDLs -are skipped; unsafe column DDL and prior UNKNOWN writes still stop the task. - -Partition dispatchers remain rejected: the existing lane only serializes within -one process, not between captures owning different physical partitions. These -options do not claim automatic multi-capture ownership/fencing support. diff --git a/tests/integration_tests/databend_native/.gitignore b/tests/integration_tests/lake_native/.gitignore similarity index 100% rename from tests/integration_tests/databend_native/.gitignore rename to tests/integration_tests/lake_native/.gitignore diff --git a/tests/integration_tests/databend_native/baseline.py b/tests/integration_tests/lake_native/baseline.py similarity index 100% rename from tests/integration_tests/databend_native/baseline.py rename to tests/integration_tests/lake_native/baseline.py diff --git a/tests/integration_tests/databend_native/baseline_parity_test.go.txt b/tests/integration_tests/lake_native/baseline_parity_test.go.txt similarity index 100% rename from tests/integration_tests/databend_native/baseline_parity_test.go.txt rename to tests/integration_tests/lake_native/baseline_parity_test.go.txt diff --git a/tests/integration_tests/databend_native/extended.py b/tests/integration_tests/lake_native/extended.py similarity index 100% rename from tests/integration_tests/databend_native/extended.py rename to tests/integration_tests/lake_native/extended.py diff --git a/tests/integration_tests/databend_native/local.py b/tests/integration_tests/lake_native/local.py similarity index 87% rename from tests/integration_tests/databend_native/local.py rename to tests/integration_tests/lake_native/local.py index 4366bf7564..c0e2ffdb67 100755 --- a/tests/integration_tests/databend_native/local.py +++ b/tests/integration_tests/lake_native/local.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -"""Owned local TiUP + TiCDC + Docker Databend lifecycle for native sink E2E. +"""Owned local TiUP + TiCDC + Docker Lake lifecycle for native sink E2E. Unlike run.py's external-warehouse mode, this runner always creates its own -loopback-only Databend, credentials, source and capture. No cloud URI is read. +loopback-only Lake, credentials, source and capture. No cloud URI is read. """ import argparse @@ -63,20 +63,47 @@ def free_port(): return sock.getsockname()[1] +def require_python(): + if sys.version_info < (3, 10): + raise SystemExit("Lake E2E requires Python 3.10 or newer") + + +def check_source_port(port): + # On BSD/macOS a specific-address SO_REUSEADDR bind can coexist with a + # wildcard listener. Detect a live loopback/wildcard service before binding. + with socket.socket() as probe: + probe.settimeout(1) + if probe.connect_ex(("127.0.0.1", port)) == 0: + raise RuntimeError("source port %d is already in use" % port) + with socket.socket() as sock: + # TIME_WAIT is not a live listener; allow immediate reruns. + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", port)) + + +def file_sha256(path): + # file_digest is only available in Python 3.11+. + digest = hashlib.sha256() + with path.open("rb") as binary: + for chunk in iter(lambda: binary.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + class Local: def __init__(self, root, image, version): self.root = root self.image = image self.version = version self.processes = [] - self.container = "ticdc-databend-e2e-" + uuid.uuid4().hex[:12] + self.container = "ticdc-lake-e2e-" + uuid.uuid4().hex[:12] self.container_created = False self.tag = "native-e2e-" + uuid.uuid4().hex[:12] self.env = os.environ.copy() # Never inherit a cloud target or another runner's capture/restart options. for key in ( "LAKE_URI_FILE", - "DATABEND_NATIVE_TEST_URI", + "LAKE_NATIVE_TEST_URI", "E2E_CDC_BINARY", "E2E_CDC_DATA_DIR", "E2E_PD", @@ -121,11 +148,7 @@ def prepare(self): ) # Fixed source ports must not silently attach to an existing playground. for base in (4000, 10080, 2379, 2380, 20160, 20180): - with socket.socket() as sock: - # Permit immediate reruns after our previous cluster closes; - # TIME_WAIT is not a live listener. No SO_REUSEPORT is enabled. - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(("127.0.0.1", base + self.offset)) + check_source_port(base + self.offset) self.summary["source_commit"] = self.command( ["git", "-C", str(REPO), "rev-parse", "HEAD"], capture_output=True, @@ -153,23 +176,20 @@ def prepare(self): self.summary["cdc_version"] = self.command( [str(self.binary), "version"], capture_output=True, text=True ).stdout.strip() - with self.binary.open("rb") as binary: - self.summary["cdc_sha256"] = hashlib.file_digest( - binary, "sha256" - ).hexdigest() - print("Starting owned local Databend background container", flush=True) + self.summary["cdc_sha256"] = file_sha256(self.binary) + print("Starting owned local Lake background container", flush=True) with (self.root / "pull.log").open("w") as log: self.command( ["docker", "pull", self.image], stdout=log, stderr=subprocess.STDOUT ) - self.summary["databend_image"] = json.loads( + self.summary["lake_image"] = json.loads( self.command( ["docker", "image", "inspect", self.image], capture_output=True, text=True, ).stdout )[0]["Id"] - self.summary["databend_image_tag"] = self.image + self.summary["lake_image_tag"] = self.image # No host network, no cloud storage, no MinIO download, no exposed password. password = uuid.uuid4().hex self.env["QUERY_DEFAULT_PASSWORD"] = password @@ -200,12 +220,12 @@ def prepare(self): capture_output=True, text=True, ).stdout.strip() - uri = f"databend+http://ticdc_e2e:{password}@{address}/default?exclusive=true" + uri = f"lake+http://ticdc_e2e:{password}@{address}/default?exclusive=true" credentials = self.root / "local-uri" credentials.write_text(uri) credentials.chmod(0o600) self.env["LAKE_URI_FILE"] = str(credentials) - self.env["DATABEND_NATIVE_TEST_URI"] = uri + self.env["LAKE_NATIVE_TEST_URI"] = uri self.env.pop("QUERY_DEFAULT_PASSWORD") self.env["MYSQL_PORT"] = str(4000 + self.offset) self.env["E2E_RUN_DIR"] = str(self.root / "readiness") @@ -219,10 +239,10 @@ def prepare(self): finally: os.environ.clear() os.environ.update(old_env) - wait_ready(lambda: probe.lake("SELECT 1") == [["1"]], None, 90, "Databend") - self.summary["databend_version"] = probe.lake("SELECT version()")[0][0] + wait_ready(lambda: probe.lake("SELECT 1") == [["1"]], None, 90, "Lake") + self.summary["lake_version"] = probe.lake("SELECT version()")[0][0] if probe.lake("SELECT ?::BIGINT", [1]) != [["1"]]: - raise RuntimeError("Databend must support Query API bound parameters") + raise RuntimeError("Lake must support Query API bound parameters") print("Starting owned local TiDB/PD/TiKV", flush=True) source = self.start( [ @@ -311,9 +331,9 @@ def suites(self, names): "-p", "2", "-race", - "./downstreamadapter/sink/databend", + "./downstreamadapter/sink/lake", "-run", - "TestDatabendNativeLive", + "TestLakeNativeLive", "-v", "-count=1", "-timeout=20m", @@ -412,7 +432,7 @@ def cleanup(self): errors.append("TiUP cleanup: " + type(error).__name__) if self.container_created: try: - with (self.root / "databend.log").open("wb") as log: + with (self.root / "lake.log").open("wb") as log: subprocess.run( ["docker", "logs", self.container], stdout=log, @@ -420,7 +440,7 @@ def cleanup(self): timeout=30, ) except Exception as error: - errors.append("Databend logs: " + type(error).__name__) + errors.append("Lake logs: " + type(error).__name__) try: result = subprocess.run( ["docker", "rm", "--force", "--volumes", self.container], @@ -429,9 +449,9 @@ def cleanup(self): timeout=45, ) if result.returncode: - errors.append("Databend container cleanup failed") + errors.append("Lake container cleanup failed") except Exception as error: - errors.append("Databend cleanup: " + type(error).__name__) + errors.append("Lake cleanup: " + type(error).__name__) (self.root / "local-uri").unlink(missing_ok=True) self.summary["cleanup"] = "PASS" if not errors else errors if errors: @@ -440,6 +460,7 @@ def cleanup(self): def main(): + require_python() parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--suite", @@ -450,12 +471,12 @@ def main(): args = parser.parse_args() os.umask(0o077) root = pathlib.Path( - tempfile.mkdtemp(prefix="ticdc-databend-local-", dir=os.environ.get("OUT_DIR")) + tempfile.mkdtemp(prefix="ticdc-lake-local-", dir=os.environ.get("OUT_DIR")) ) print("Local E2E artifacts: " + str(root), flush=True) runner = Local( root, - os.environ.get("DATABEND_E2E_IMAGE", IMAGE), + os.environ.get("LAKE_E2E_IMAGE", IMAGE), os.environ.get("TIDB_E2E_VERSION", "v8.5.1"), ) diff --git a/tests/integration_tests/databend_native/options.py b/tests/integration_tests/lake_native/options.py similarity index 100% rename from tests/integration_tests/databend_native/options.py rename to tests/integration_tests/lake_native/options.py diff --git a/tests/integration_tests/databend_native/run.py b/tests/integration_tests/lake_native/run.py similarity index 99% rename from tests/integration_tests/databend_native/run.py rename to tests/integration_tests/lake_native/run.py index abcce2b12a..bdd8a66bb0 100755 --- a/tests/integration_tests/databend_native/run.py +++ b/tests/integration_tests/lake_native/run.py @@ -47,7 +47,7 @@ def __init__(self, existing=False): ) self.feed = self.database.replace("_", "-") # Plain HTTP is only enabled by the sink's explicit test scheme. - scheme = "http" if self.parsed.scheme == "databend+http" else "https" + scheme = "http" if self.parsed.scheme == "lake+http" else "https" self.base = scheme + "://" + self.parsed.netloc.rsplit("@", 1)[-1] self.results = { "database": self.database, @@ -268,7 +268,7 @@ def compare(self, table, columns, numeric=(), json_columns=(), timeout=75): data = self.lake(target) actual = [dict(zip([c[0] for c in columns], row)) for row in data] - # Databend may bind ORDER BY id to the projected VARCHAR alias. + # Lake may bind ORDER BY id to the projected VARCHAR alias. # Compare the key-indexed set, never numeric-vs-text sort order. def key(row): return str(row["id"]), str(row.get("tenant", "")) @@ -332,7 +332,7 @@ def setup(self): "spool-bytes": ["65536"], "memory-ratio": ["0.001"], "source-bytes": ["268435456"], - "source-events": ["256"], + "source-events": ["1"], "spool-dir": [str(self.root / "spool")], } ) @@ -589,7 +589,7 @@ def test_batch(self, db): count = 0 for line in text.splitlines(): if ( - line.startswith("ticdc_databend_spool_load_bytes_count{") + line.startswith("ticdc_lake_spool_load_bytes_count{") and ('changefeed="' + self.feed + '"') in line ): count += float(line.rsplit(" ", 1)[-1]) diff --git a/tests/integration_tests/databend_native/run.sh b/tests/integration_tests/lake_native/run.sh similarity index 66% rename from tests/integration_tests/databend_native/run.sh rename to tests/integration_tests/lake_native/run.sh index 797c77d77e..9bbbb47444 100755 --- a/tests/integration_tests/databend_native/run.sh +++ b/tests/integration_tests/lake_native/run.sh @@ -1,8 +1,8 @@ #!/bin/bash # The generic integration runner also discovers this case for other sink types. set -euo pipefail -if [ "${1:-databend}" != databend ]; then - echo "databend_native: requires the databend sink suite; skipping" +if [ "${1:-lake}" != lake ]; then + echo "lake_native: requires the lake sink suite; skipping" exit 0 fi CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/integration_tests/databend_native/test_local.py b/tests/integration_tests/lake_native/test_local.py similarity index 78% rename from tests/integration_tests/databend_native/test_local.py rename to tests/integration_tests/lake_native/test_local.py index 082687bb44..274a1241c4 100644 --- a/tests/integration_tests/databend_native/test_local.py +++ b/tests/integration_tests/lake_native/test_local.py @@ -2,20 +2,46 @@ """Service-free checks for the local E2E lifecycle and target selection.""" import json +import hashlib import os import pathlib import shlex +import socket import subprocess import sys import tempfile import unittest from unittest.mock import patch -from local import Local, wait_ready +from local import Local, check_source_port, file_sha256, require_python, wait_ready from run import Run class LocalRunnerTest(unittest.TestCase): + def test_source_ports_reject_live_loopback_and_wildcard_listeners(self): + for address in ("127.0.0.1", "0.0.0.0"): + with self.subTest(address=address), socket.socket() as listener: + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind((address, 0)) + listener.listen() + port = listener.getsockname()[1] + with self.assertRaisesRegex(RuntimeError, "already in use"): + check_source_port(port) + # A previous connection's TIME_WAIT does not forbid a new run. + check_source_port(port) + + def test_python_requirement_and_chunked_digest(self): + with patch("local.sys.version_info", (3, 9)): + with self.assertRaisesRegex(SystemExit, "Python 3.10"): + require_python() + with patch("local.sys.version_info", (3, 10)): + require_python() + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "binary" + data = b"lake-test" * (1 << 19) + path.write_bytes(data) + self.assertEqual(file_sha256(path), hashlib.sha256(data).hexdigest()) + def test_owned_background_process_is_stopped(self): with tempfile.TemporaryDirectory() as directory: runner = Local(pathlib.Path(directory), "test-image", "v8.5.1") @@ -38,7 +64,7 @@ def test_never_inherits_cloud_or_other_capture(self): os.environ, { "LAKE_URI_FILE": "/not/read/cloud-secret", - "DATABEND_NATIVE_TEST_URI": "must-not-use", + "LAKE_NATIVE_TEST_URI": "must-not-use", "CDC_SERVER": "https://not-local.invalid", "E2E_CDC_BINARY": "/unowned/cdc", "MYSQL_PORT": "1234", @@ -48,7 +74,7 @@ def test_never_inherits_cloud_or_other_capture(self): runner = Local(pathlib.Path(directory), "test-image", "v8.5.1") for name in ( "LAKE_URI_FILE", - "DATABEND_NATIVE_TEST_URI", + "LAKE_NATIVE_TEST_URI", "CDC_SERVER", "E2E_CDC_BINARY", "MYSQL_PORT", @@ -65,9 +91,7 @@ def test_explicit_http_scheme_only(self): with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) credential = root / "uri" - credential.write_text( - "databend+http://user:password@127.0.0.1:18000/default" - ) + credential.write_text("lake+http://user:password@127.0.0.1:18000/default") credential.chmod(0o600) with patch.dict( os.environ, @@ -153,7 +177,7 @@ def execute(args, **kwargs): runner.cleanup() self.assertEqual(run.call_args_list[-1].args[0][:2], ["docker", "rm"]) self.assertEqual(runner.summary["status"], "FAIL") - self.assertIn("Databend logs", runner.summary["cleanup"][0]) + self.assertIn("Lake logs", runner.summary["cleanup"][0]) if __name__ == "__main__":