diff --git a/Makefile b/Makefile index 215d263f8f..ec6a3106b8 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ generate-protobuf generate_mock \ cdc kafka_consumer storage_consumer pulsar_consumer filter_helper kafka_auth_server \ prepare_test_binaries \ - unit_test_in_verify_ci integration_test_build integration_test_build_fast integration_test_mysql integration_test_kafka integration_test_storage integration_test_pulsar \ + unit_test_in_verify_ci integration_test_build integration_test_build_fast integration_test_mysql integration_test_kafka integration_test_storage integration_test_pulsar integration_test_lake \ generate-next-gen-grafana check-next-gen-grafana @@ -255,6 +255,11 @@ check_failpoint_ctl: tools/bin/failpoint-ctl integration_test: integration_test_mysql +# Self-contained local source/capture/Lake; does not require unrelated Kafka, +# Pulsar, MinIO, diff-inspector or downstream TiDB test binaries. +integration_test_lake: + python3 tests/integration_tests/lake_native/local.py + integration_test_mysql: check_third_party_binary tests/integration_tests/run.sh mysql "$(CASE)" "$(START_AT)" diff --git a/docs/design/lake-sink-design.md b/docs/design/lake-sink-design.md new file mode 100644 index 0000000000..3e482408df --- /dev/null +++ b/docs/design/lake-sink-design.md @@ -0,0 +1,508 @@ +# Native Lake Sink: Final Design and E2E Results + +Last updated: 2026-10-26. + +Status: experimental incremental sink; controlled single-capture operation with +an exclusively owned target. This is the only design and acceptance document for +this feature. Historical proposals and separate review/test reports are removed. + +## 1. Purpose and scope + +Replace the intermediate `TiCDC -> Canal JSON -> object storage -> bend-cdc -> +Lake` path with a native sink: + +```text +Existing TiCDC dispatcher and DDL barriers + -> native typed-row adapter + -> shared source admission + bounded memory/disk spool + -> fixed table lanes + -> retained raw INSERT -> target MERGE -> raw compaction + -> source completion callback +``` + +The correctness reference is bend-cdc's TiDB incremental merge path at +`b6481621021e82aee659ad88e8e6c00f6dd11906`. Migrate its key, raw/MERGE, type and +schema-epoch semantics rather than introduce another general CDC framework. +The upstream integration base is TiCDC `b9179182f`. These are source comparison +identifiers, not deployment versions. + +### Supported first-version behavior + +- Incremental INSERT/UPDATE/DELETE for ordinary non-partitioned tables. +- Explicit primary keys, TiCDC-selected non-null unique keys, composite source + keys, and an explicit single-column `conflict-key` override. +- Lazy target/raw table creation, parameter-bound batched writes, retained + tombstones, strict version ordering, target MERGE and historical compaction. +- CREATE DATABASE, CREATE/RECOVER TABLE, DROP TABLE, TRUNCATE and a conservative + subset of ADD/DROP/RENAME/CHANGE/MODIFY COLUMN. +- Literal defaults, nullability changes and supported widening conversions. +- Durable pending/applied DDL recovery and old-event/DDL replay fences. + +### Explicit boundaries + +- One controlled capture and a statically exclusive target. `exclusive=true` is + an operator assertion, **not** a distributed lock or an automatic HA protocol. +- Reject partition dispatchers, across-node table splitting, syncpoints, + active-active replication, redo, low-latency mode, column selectors, target + routing and table-level transaction atomicity. +- RENAME TABLE, including multi-table rename, and DROP DATABASE are explicitly + skipped with warnings after draining old DML. They preserve target objects; + this is not successful target-side DDL translation. Subsequent DML uses its + current source name and may create a new projection without moving old rows. +- No Dumpling, CSV, COPY INTO, full snapshot orchestration or historical empty + table discovery. Users independently prepare a consistent starting state. +- No stream, persistent DML ledger, batch manifest, consumption witness, new + collector credit protocol, progress migration or distributed quota service. + +## 2. Integration and implementation map + +Core logic is owned by [the Lake sink package](../../downstreamadapter/sink/lake). +Existing-code changes are limited to scheme/factory/type registration, +schema-store requirements, error classification, metrics and shared spool byte +accounting. The dispatcher/collector framework is not redesigned. + +| Component | Responsibility | +|---|---| +| [sink.go](../../downstreamadapter/sink/lake/sink.go) | Lifecycle, shared source admission, DDL barriers and callbacks | +| [native.go](../../downstreamadapter/sink/lake/native.go), [row.go](../../downstreamadapter/sink/lake/row.go) | Authoritative source schema and typed-value normalization | +| [lane.go](../../downstreamadapter/sink/lake/lane.go), [completion.go](../../downstreamadapter/sink/lake/completion.go) | Incremental fragments, bounded spool, batching and event completion | +| [writer.go](../../downstreamadapter/sink/lake/writer.go), [sql.go](../../downstreamadapter/sink/lake/sql.go) | Serial raw/MERGE/compaction and version ordering | +| [client.go](../../downstreamadapter/sink/lake/client.go) | Bound Query API requests and matching terminal-state evidence | +| [schema.go](../../downstreamadapter/sink/lake/schema.go) | Durable schema plans, reconciliation and recovery | +| [local.py](../../tests/integration_tests/lake_native/local.py) | Owned local backend/source/capture E2E lifecycle | + +Public names are Lake: `lake://`, `lake+http://`, `LakeSinkType`, Lake errors, +`ticdc_lake_*` metrics and `integration_test_lake`. Backend wire headers and the +compatible backend image keep their actual identifiers. The optional fixed +bend-cdc baseline overlay also retains that module's real API/driver names; +those identifiers are not new public sink names. + +## 3. Native events, identities and values + +Keep `outputRawChangeEvent=false`, matching the fixed cloud-storage input path. +The existing scanner splits relevant PK/UK changes into DELETE and delayed +INSERT. Preserve that order, including opposite actions with the same key and +commit TSO. Commit TSO is not a unique row/event identity. An unexpected unsplit +UPDATE changing the effective conflict identity fails closed instead of leaving +an old key behind. + +Use authoritative `TableInfo` and native `CommitTs`. Native rows do not pass +through Canal ordinal/Latin-1 decoding a second time. Normalize once into an +owned immutable payload, action and canonical key representation: + +- Resolve a source PK or non-null unique key using TiCDC's existing helper. +- Composite keys use a canonical named key/value representation; SQL joins use + every effective key column. Null/missing identities are rejected. +- `conflict-key` is a feed-wide single-column override, not a source index. It + can replace no key or a single native key, but not a composite key. The operator + guarantees unique, non-null, stable values and must not change it on an + existing projection without reinitialization. +- **A truly keyless source also requires `force-replicate = true` in the TiCDC + replica configuration** (`force_replicate` in the API). `conflict-key` alone + cannot make the upstream emit an otherwise ineligible table. + +Type mapping preserves signed/unsigned integer widths, exact decimals up to the +supported precision, numeric-label ENUM and SET labels, BIT(64), binary bytes, +JSON documents, SQL NULL versus JSON null and microsecond temporal values. +Binary is hex-encoded once and projected using `unhex`. JSON uses a separate +projection so SQL NULL is not confused with a JSON document containing `null`. +TIMESTAMP converts the configured source timezone to UTC; DATETIME retains wall +clock values; TIME/duration uses text. Unsupported or invalid values fail closed. +Visible STORED generated values become ordinary nullable columns. Invisible +VIRTUAL generated columns are not materialized or recreated as expressions. + +## 4. Admission, batching and bounded buffering + +### Shared admission + +`source-bytes` and `source-events` are **sink-wide**, not per-dispatcher limits. +Charge shared source chunks conservatively in full for each admitted event, +including bounded metadata overhead. Dispatcher batch hints are not a global +memory reservation. + +`AddDMLEvent` waits on shared quota or lane-queue pressure instead of marking the +changefeed unretryable. It publishes only after reserving bytes/counts under the +admission mutex. A lane dequeue or source release wakes waiters. Waiting does +not hold the mutex, allocate another event queue or start per-event goroutines. +Cancellation, fatal failure and Close wake/terminate waiting calls without a +success callback. An individual event larger than the entire configured source +byte pool is an unretryable configuration/data-size error, not ordinary pressure. + +Waiting occurs inside the existing synchronous sink call. A shared handler can +therefore temporarily wait with other paths on that handler; there is no promise +of per-table latency isolation. Independent lane workers, timer flushes and +quota-pressure draining do not require new upstream DML to make progress. +Waiting callers still own already-allocated chunks under upstream accounting; +the sink source limit bounds **admitted** retention, not total process RSS or +all framework-delivered memory. No collector/dynstream API is changed. + +### Encoding and spool + +Tables hash to a fixed number of serial lanes. Each lane incrementally encodes +bounded immutable fragments rather than copy an entire large transaction into a +second in-memory buffer. The source event is released after encoding. Low +traffic flushes on a timer; row/byte/entry thresholds and spool pressure trigger +earlier flushes. Quota pressure drains pending tables and reclaims applied +fragments, allowing an event larger than the available spool quota to progress. + +The spool has a bounded memory hot set plus a physical disk quota. Released but +not physically reclaimed segments remain charged by the shared spool. Entries +are checked with `spool.AccountingBytes` before enqueue; a single oversized row +or entry is rejected rather than use the shared spool's oversized-memory bypass. +Source retention, encoded fragments, readback, HTTP results and schema metadata +have separate limits; their sum is not an absolute RSS/physical-volume cap. + +### Completion and cleanup + +A source event owns an `OPEN -> SEALED -> COMPLETED` tracker. Register a fragment +before publishing it. Seal only after enumerating the complete event. Complete +once only when sealed, all registered fragments are applied, and no failure +occurred. Applied fragments release storage independently of whole-event ACK. + +`PostEnqueue` is a flow-control signal, not target application. `PostFlush` is +allowed only after confirmed target application or a proven applied-DDL fence. +Spool entries never own source-success callbacks. Fatal/Close cleanup uses +resource release, not `Discard`, whose callback semantics could falsely ACK. +Transport session reset does not discard data already transferred to the sink. +Fatal errors end the old lifecycle; do not clear progress by fabricating flushes. + +## 5. Raw writes, MERGE and retention + +Each table has a target projection and a `_cdc_raw` table containing: + +```text +id, table_name, raw_data, add_time, event_pos, action +``` + +For a serial table batch: + +1. Validate the complete batch and filter rows fenced by an applied DDL epoch. +2. Execute all parameter-bound raw INSERT sub-statements in source order. +3. MERGE the target from retained raw rows newer than that epoch. Select each + key's maximum `(event_pos, add_time)` action and suppress dominated versions. +4. Obtain matching server terminal success before completing covered fragments. +5. Compact strictly dominated historical raw versions. + +The source is deliberately **raw**, not a stream cursor. There is no stream +consume hint or planned-coverage witness to interpret. Full-raw MERGE can scan +and rewrite more than an incremental stream path; benchmark it before making a +cost or throughput claim. + +`event_pos` is the native commit TSO. `add_time` is a compatibility tie-breaker, +not a new source business version: quantize to stored microseconds, then advance +by one microsecond when needed. Seed the clock from retained `MAX(add_time)` on +bootstrap so a clock rollback/restart cannot reorder a later same-TSO action. +This relies on the exclusive serial writer and existing source ordering; it is +not a solution for conflicting concurrent owners. + +Compaction keeps each key's latest watermark/tombstone. Do not TTL-delete the +latest tombstone: an older replayed UPSERT could otherwise resurrect a deletion. +Distinct historical keys can grow raw even if the target stays small and all +maintenance succeeds. `remote-bytes-limit` is a measured raw-storage stop +threshold with batch/statistics lag, not a strict cross-capture budget or a +physical billing cap. Reaching it requires diagnosis and capacity planning or +controlled reinitialization, not unsafe tombstone deletion. + +Raw INSERTs, target MERGE and compaction are separate mutations, not one atomic +transaction. A successful MERGE followed by a maintenance failure remains +applied; do not replay business SQL merely to repair maintenance statistics. + +## 6. Query outcomes and write fencing + +The client sends bound SQL parameters and a client query ID. It follows only +same-origin query continuations, carries required routing hints, disallows +redirects and checks capture write admission again immediately before mutation +POST. Errors must not expose credentials, SQL payloads or row values. + +| Result | Required handling | +|---|---| +| COMMITTED | Matching query ID and terminal `Succeeded`; complete application even if optional statistics are unavailable | +| NOT_COMMITTED | Definitive server execution failure; only explicitly proven retryable failures may enter bounded retry | +| UNKNOWN | Missing/contradictory terminal evidence, transport loss or timeout; stop immediately, no hidden retry, next batch, DDL or source ACK | + +A result page containing rows is not a commit certificate. Poll through terminal +execution; distinguish execution/transport failures from observation failures. +The HTTP adapter is conservative and does not label arbitrary network errors as +safe retries. Query timeout or a successful cancellation function is not evidence +that the server has stopped. An arbitrary UNKNOWN needs independent old-query +terminal/quiescence evidence and target exclusivity before manual recovery. +The sink does not implement automatic discovery or distributed fencing of old +MERGE/TRUNCATE requests across captures. + +## 7. DDL journal, barriers and recovery + +Use native DDL `FinishedTs` as the application epoch, not `TableInfoVersion`. +Existing TiCDC block-event handling drains old DML. Each table has a sink-owned +`
_cdc_schema_epoch` journal with `epoch`, `applied` and a serialized schema +plan. Plans contain the intended projection and explicit reconciliation steps. + +```text +drain old DML -> load journal -> skip an already-applied DDL + -> persist pending plan -> reconcile target/raw + -> verify projection -> persist applied epoch -> DDL callback +``` + +On restart, pending work is reconciled before accepting new-schema DML. Use the +durable applied schema to handle old events; rows at/before the applied epoch +are skipped. Bootstrap alone does not invent an applied epoch. Schema state is +bounded by `max-entries`; exceeding a schema-cache limit is a metadata-capacity +error rather than unbounded table retention. + +TRUNCATE recovery order is pending journal, drop raw, truncate target, recreate +raw, applied journal. Reenter pending steps before opening writes. Applied DDL +replay skips the reset so newly written rows survive. DROP removes target/raw +while preserving the journal needed for replay handling. + +Column reconciliation checks actual target metadata. Supported cases include +literal-default backfill, nullability changes, TINYINT -> INT -> BIGINT widening, +VARCHAR -> TEXT, and signed INT32 -> VARCHAR(n >= 11). Reject unknown narrowing, +unsafe casts, expression-default changes and conflict-key removal/rename or +unsupported composite-key modification. Secondary index DDL does not create a +Lake index but can advance the compatible schema fence. These rules do not +promise arbitrary multi-operation SQL or source/target transaction equivalence. + +## 8. Configuration and operational boundaries + +All sink-specific settings live in the URI. Credentials must be URL-encoded and +kept out of command history, logs and shared reports. + +```text +lake://DB_USER:TOKEN@HOST/DB_NAME?exclusive=true&workers=4&batch-rows=2048&batch-bytes=1048576&flush-interval=1s&spool-bytes=10737418240&memory-ratio=0.02&source-bytes=268435456&source-events=128&max-entries=4096&query-timeout=2m +``` + +`lake://` uses HTTPS. `lake+http://` is restricted to literal loopback IPs or +`localhost` for isolated local testing; other hostnames/IPs are rejected before +sending Basic authentication. There are no alternative branded public scheme +aliases in this initial unreleased feature. An explicit database path overrides +the source schema name. Do not collapse different source tables onto one target; +arbitrary schema/table routing is not supported. + +| Option | Default | Contract | +|---|---|---| +| `exclusive` | required `true` | Operator-owned static target exclusivity, not a lock | +| `conflict-key` | unset | Feed-wide stable unique non-null single-column override; truly keyless tables also need `force-replicate = true` | +| `workers` | 4 | Fixed lanes, maximum 64; one table hashes to one lane | +| `batch-rows` / `batch-bytes` | 2048 / 1 MiB | Fragment and write thresholds; maximum 100000 rows / 64 MiB, minimum 1 KiB | +| `flush-interval` | 1s | Low-traffic flush bound under a healthy backend | +| `spool-bytes` | 10 GiB | Shared local physical spool budget, at least twice batch bytes | +| `memory-ratio` | 0.02 | Spool hot-memory fraction of spool budget, strictly between 0 and 1 | +| `source-bytes` / `source-events` | 256 MiB / 128 | Shared admitted native-event pool; bytes must cover at least one batch | +| `max-entries` | 4096 | Per-lane fragment pressure and schema-cache limit | +| `spool-dir` | data-directory based | Capture/feed-separated temporary spool, not a replay log | +| `query-timeout` | 2m | Total single-query/pagination deadline, not proof of server termination | +| `warehouse` | unset | Backend warehouse routing header | +| `remote-bytes-limit` | disabled | Measured raw-storage stop threshold; not a global hard quota | + +Unknown/duplicate options and unsupported TiCDC modes are rejected. Source-event +and metadata counts are positive and at most 65536. Disk quota is at most 1 PiB. +The target must support parameter binding, matching query IDs, terminal query +states and the required SQL. `Verify` checks a bound SELECT; there is no string +interpolation fallback for incompatible endpoints. + +### Migration and recovery procedure + +1. Independently prepare consistent target contents and a compatible start-ts. +2. Stop the old writer. Establish that its in-flight SQL cannot mutate the target + later. An S3 file read cursor alone is not proof of applied target progress. +3. Do not transparently reuse the old two-column bend-cdc journal: the new sink + has its own pending/applied schema-plan format. Plan target/journal ownership + and reinitialization explicitly before cutover. +4. Start one capture with the target-exclusive assertion and verify actual data, + checkpoints, spool pressure and raw growth. +5. If an outcome becomes UNKNOWN, keep writes closed until old-query terminal + evidence and exclusivity are independently established, then replay from a + safe checkpoint with a fresh lifecycle. + +Removing the intermediate consumer/storage hop can reduce infrastructure and +serialization work. Type conversion still runs in TiCDC; local spool, raw/target +storage and warehouse computation remain. Full-raw scans and compaction can add +cost. No cost-saving percentage or production HA/SLO has been measured here. + +## 9. Reproducible tests + +### Fully managed local E2E + +From the checkout root: + +```sh +make integration_test_lake +# Or select/repeat suites: +python3 tests/integration_tests/lake_native/local.py --suite acceptance +# Reuse an explicitly built current-tree binary: +E2E_CDC_BINARY="$PWD/bin/cdc" make integration_test_lake +``` + +Prerequisites: a running Docker engine, TiUP, mysql CLI, Python **3.10+** and the +repository's Go/build dependencies. Allow enough disk for Go's build cache and +the source cluster, and approximately 8 GiB Docker memory. The runner checks the +Python minimum before starting a build and hashes binaries incrementally rather +than depend on Python 3.11-only APIs. It does not install tools or start Docker. + +The runner builds current-tree TiCDC and starts a unique backend meta/query +container using the pinned multi-platform image: + +```text +datafuselabs/databend:v1.2.925-patch-13@sha256:7f93f005aa0d626ebc7554d142b0a6b6ebb90e085b0d5bd0d97470084c42d41b +``` + +The actual image identifier is required to reproduce the compatible server; it +is not the public sink name. The container uses filesystem storage, publishes +HTTP only on a random loopback port, and uses random temporary credentials. No +cloud bucket, MinIO or external Lake credentials are needed. Readiness includes +real SQL parameter binding. An older tested image at v1.2.852-nightly failed +binding and is not listed as supported; no unsafe interpolation workaround was +introduced. + +Start an owned TiUP v8.5.1 source with one PD/TiKV/TiDB, no TiFlash/monitor, and an +owned `--newarch --tz UTC` capture. Probe source ports by connection and bind; +refuse reachable loopback/wildcard listeners rather than run SQL against an +existing cluster. Permit TIME_WAIT for immediate reruns. Owner election is not +sufficient readiness: wait for the changefeed read API after coordinator bootstrap. +Only on this owned source, disable fast/distributed index construction to avoid +low-disk laptop failures unrelated to CDC. + +Run acceptance, DDL, explicit-key, skipped-DDL, four expected-rejection probes +and real-backend Go tests. On success, failure or interruption, stop only owned +processes/container and clean the unique TiUP tag. Restart cleanup verifies the +exact canonical binary and data-directory paths. Do not use global `pkill` or +`tiup clean --all`; the shared Docker daemon is left running. Target container +data is ephemeral even on failure; private logs/results remain and the temporary +URI file is removed. + +The runner prints its artifacts directory. `summary.json` records build/server +versions, source commit/dirty status, image identity, suite exits and cleanup. +Each Python suite has a `result.json`; real-backend assertions are in `fault.log`. +Artifacts contain no credentials intended for publication. No personal machine +paths, generated database IDs or PIDs are required in this document. + +Optional environment: `LAKE_E2E_IMAGE`, `TIDB_E2E_VERSION`, `E2E_PORT_OFFSET` +(default 10000), `E2E_CDC_BINARY`, and an existing `OUT_DIR` parent. The managed +runner does not adopt inherited `LAKE_URI_FILE`, `LAKE_NATIVE_TEST_URI`, capture +address or source-port settings. A source/capture already running outside this +runner is not its test environment. + +### External authorized warehouse and baseline probes + +The [acceptance runner](../../tests/integration_tests/lake_native/run.py) can also +use an explicitly authorized test warehouse and already-owned local TiUP/capture. +Set `LAKE_URI_FILE` to a private mode-0600 file, `CDC_SERVER` (default local port +18300), `MYSQL_PORT` (default 14000) and a fresh `E2E_RUN_DIR` per invocation. +Never pass real credentials on argv or point these tests at production. + +```sh +python3 tests/integration_tests/lake_native/run.py +python3 tests/integration_tests/lake_native/extended.py +python3 tests/integration_tests/lake_native/options.py --manual-key +python3 tests/integration_tests/lake_native/options.py +``` + +External-mode failures retain the isolated database/feed for inspection. After +confirming old requests have terminated, use the same environment/run directory +with `run.py --cleanup`. This mode does not stop borrowed TiUP/capture services; +that remains the operator's responsibility. Its optional drained restart needs +`E2E_CDC_BINARY`, `E2E_CDC_DATA_DIR` and `E2E_PD` identifying owned processes. + +The optional [baseline overlay](../../tests/integration_tests/lake_native/baseline.py) +requires `BEND_CDC_DIR` to identify a clean, fixed `b648162` checkout and the same +private URI/run-directory setup. It modifies no baseline files. It exercises the +old core on an authorized HTTPS endpoint, not the entire S3 listing/decoding +pipeline. Its legacy driver/API identifiers must remain unchanged. + +### Focused local checks + +```sh +python3 -m unittest discover -s tests/integration_tests/lake_native -p 'test_*.py' +go test -p 2 -race ./downstreamadapter/sink/lake ./downstreamadapter/sink ./pkg/sink/spool ./pkg/errors ./pkg/config ./downstreamadapter/sink/metrics -count=1 -timeout=180s +go test -p 2 -race ./pkg/common/event -run 'TestTableSchemaStore|TestNeedTableNameStore' -count=1 +go test -p 2 ./pkg/metrics -run TestDownstreamTypeFromSinkURI -count=1 +tools/bin/golangci-lint run --timeout 5m ./downstreamadapter/sink/lake/... ./downstreamadapter/sink/metrics/... +``` + +The local managed suite supplies `LAKE_NATIVE_TEST_URI` to the Go live tests via +the environment. Without explicit opt-in these tests skip; an offline HTTP +fixture is not equivalent to real SQL execution. + +## 10. E2E results and evidence boundaries + +### Completed acceptance record + +On 2026-10-26 the Lake-named implementation with the review fixes passed all nine +suites on macOS arm64, Go 1.26.9, TiDB v8.5.1 and the pinned compatible backend +above. Validation used the review working tree on top of `a3fb6259a`, based on +upstream `b9179182f`. These are local results, not a claim about current CI. +The acceptance configuration deliberately limits the shared source pool to one +event while exercising multiple source tables; normal contention does not stop +the task. + +| Suite | Assertions | Result | +|---|---|---| +| Core acceptance | 9: CRUD; PK/UK changes; composite/quoted keys; type fidelity; column DDL; TRUNCATE; large transaction; drained restart; DROP cleanup | PASS | +| Extended DDL | 11: non-null unique handle; INT-to-wide-VARCHAR; defaults/nullability; widening; literal backfill; CHANGE; multi-operation projection | PASS | +| Explicit conflict key | 1: forced keyless CRUD/ADD COLUMN and single-native-key override | PASS | +| Skipped DDL | 2: RENAME and DROP DATABASE preserve target objects while later DML/checkpoints progress | PASS | +| Expected rejections | 4 separate probes: partition dispatcher, genuinely keyless/no override, narrowing, expression default | PASS as rejection, not feature support | +| Real-backend Go tests | 3 parent tests: replay/fences, partial raw/MERGE response loss, DDL interruption recovery | PASS | + +There are 27 Python scenarios plus three live Go parent tests. The acceptance +run observed **41 actual disk-spool readbacks**, with a 1200-row transaction and +64 KiB spool quota. Comparisons are key-based and check actual values, not merely +row counts. They include 32-column scalar/binary/decimal/JSON/temporal fidelity; +FLOAT uses tolerance while exact values, null distinctions and schema defaults +are explicitly checked. The drained restart creates a source DML/DDL backlog +and resumes from the original safe checkpoint, without overriding its timestamp. + +Real response-loss injection delegates SQL to the actual server, independently +observes a matching terminal success, then drops that response before the sink +sees it. Partial raw testing commits the first INSERT and blocks the second +before send. Unknown paths issue no early flush or blind retry. Recovery is +allowed because the test knows the old terminal outcome and has ended the old +sink lifecycle. This is not a general automatic UNKNOWN recovery service. + +| DDL | Lost-response boundaries | Count | +|---|---|---| +| CREATE | pending journal, target create, raw create, applied journal | 4 | +| ALTER | pending, ADD, RENAME, MODIFY, raw ensure, applied | 6 | +| TRUNCATE | pending, raw drop, target truncate, raw recreate, applied | 5 | +| DROP | pending, target drop, raw drop, applied | 4 | + +All **19 DDL boundaries** passed. Non-DROP recovery verifies new rows and rejects +old DML/applied-DDL replay without clearing new data. DROP verifies target/raw +absence. These are commit-boundary response-loss tests, not every OS instruction +point, server failure during execution or cross-capture failover. + +All suite-owned database/feed cleanup passed. Independent checks found no old +or restarted capture PIDs, no owned backend containers and no URI file. The +current binary was built with `make cdc GOBUILD='CGO_ENABLED=1 GO111MODULE=on go build -p 2'` to reuse the local Go cache, then supplied through `E2E_CDC_BINARY`. +Production build defaults were not changed. + +A separate 2026-10-25 authorized staging run also passed the core/DDL matrix and +response-loss probes. Fixed-version baseline core probes identified the original +scope boundaries: non-null unique keys are valid handles, table rename/database +drop are skipped rather than projected, and INT32-to-wide-VARCHAR must remain +supported. They do not prove complete old-S3-chain equivalence or seamless live +journal takeover. + +### Review regressions and remaining limits + +Review regression validation passed: + +- Event-count, byte-budget and control-filled lane-queue pressure wait without + stopping the sink or bypassing the shared quota; independent workers drain + them. Close/fatal failure terminates waiters with zero false flush callbacks. +- Admission/Close race scenarios passed 30 repeated race-enabled runs. +- Plaintext configuration accepts only loopback IPs/localhost; remote, wildcard + and misleading hostname cases are rejected before authentication. +- Real loopback/wildcard listeners are rejected before source startup/SQL; a + closed listener permits a new run. +- Nine service-free Python lifecycle tests passed, including streaming hashing + and early minimum-version rejection. All runners parse with Python 3.10 + grammar; an older installed interpreter exits before building services. +- Six focused Go race-test packages, schema-store/metrics tests, scoped vet and + fresh-cache lint passed (zero lint issues). Go/import/Python/shell formatting + and repository-relative document-link checks passed. + +No Linux/amd64 CI execution, production deployment, general HA/fencing proof, +long-running churn/capacity bound, cloud gateway/S3 qualification, cost benchmark +or performance/SLO acceptance is implied by these tests. Pinning a multi-platform +image is not evidence that a second platform was tested. Latest tombstones and +old-query exclusivity remain operational responsibilities, not closed risks. diff --git a/downstreamadapter/sink/lake/client.go b/downstreamadapter/sink/lake/client.go new file mode 100644 index 0000000000..3b0cdcf979 --- /dev/null +++ b/downstreamadapter/sink/lake/client.go @@ -0,0 +1,218 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "strings" + "sync/atomic" + "time" + + "github.com/google/uuid" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/writelease" +) + +const maxResponseBytes = 8 << 20 + +type queryPage struct { + ID string `json:"id"` + NodeID string `json:"node_id"` + State string `json:"state"` + Error *struct { + Code int `json:"code"` + } `json:"error"` + Data json.RawMessage `json:"data"` + NextURI string `json:"next_uri"` + FinalURI string `json:"final_uri"` +} + +// httpClient intentionally has no POST retry, redirect or driver-level retry. +// Every statement is autocommit; DDL is never placed inside a DML transaction. +type httpClient struct { + opts options + transport *http.Client + gate atomic.Pointer[writelease.Gate] +} + +func newHTTPClient(o options) *httpClient { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConnsPerHost = o.workers + 2 + return &httpClient{opts: o, transport: &http.Client{Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}} +} + +func (c *httpClient) mutate(ctx context.Context, sql string, args []any) (mutationResult, error) { + _, state, err := c.query(ctx, sql, args, false) + return state, err +} + +func (c *httpClient) selectRows(ctx context.Context, sql string, args []any) ([][]*string, error) { + rows, state, err := c.query(ctx, sql, args, true) + if err != nil { + return nil, err + } + if state.state != committed { + return nil, errors.ErrLakeExecution.GenWithStackByArgs() + } + return rows, nil +} + +func (c *httpClient) query(ctx context.Context, sql string, args []any, readData bool) ([][]*string, mutationResult, error) { + ctx, cancel := context.WithTimeout(ctx, c.opts.queryTimeout) + defer cancel() + if err := writelease.WaitForWrite(ctx, c.gate.Load()); err != nil { + return nil, mutationResult{state: notCommitted}, errors.WrapError(errors.ErrLakeExecution, err) + } + id := uuid.NewString() + payload := struct { + SQL string `json:"sql"` + Params []any `json:"params,omitempty"` + Session any `json:"session"` + Pagination any `json:"pagination"` + }{sql, args, map[string]any{"database": "default", "settings": map[string]string{"timezone": "UTC"}}, map[string]int{"wait_time_secs": 1, "max_rows_per_page": 1024, "max_rows_in_buffer": 1024}} + body, err := json.Marshal(payload) + if err != nil { + return nil, mutationResult{state: notCommitted}, errors.WrapError(errors.ErrLakeExecution, err) + } + method, path := http.MethodPost, "/v1/query" + node, route := "", "" + var rows [][]*string + var total int + state := mutationResult{state: commitUnknown} + for { + page, h, err := c.request(ctx, method, path, body, id, node, route) + if err != nil { + return nil, state, err + } + // A pre-start SQL rejection has no server query ID. All started + // queries, especially successful mutations, must echo our identity. + preStartFailure := method == http.MethodPost && page.State == "Failed" && page.ID == "" + if page.ID != id && !preStartFailure { + return nil, mutationResult{state: commitUnknown}, errors.ErrLakeProtocol.GenWithStackByArgs(id, "query identity mismatch") + } + if page.NodeID != "" { + node = page.NodeID + } + if v := h.Get("X-DATABEND-ROUTE-HINT"); v != "" { + route = v + } + if page.State == "Failed" { + code := 0 + if page.Error != nil { + code = page.Error.Code + } + return nil, mutationResult{state: notCommitted}, errors.ErrLakeProtocol.GenWithStackByArgs(id, "server terminal failure code "+jsonInt(code)) + } + if page.Error != nil || (page.State != "Succeeded" && page.State != "Starting" && page.State != "Running") { + return nil, mutationResult{state: commitUnknown}, errors.ErrLakeProtocol.GenWithStackByArgs(id, "missing or inconsistent execution state") + } + if page.State == "Succeeded" { + state.state = committed + if !readData { + c.finalize(ctx, page.FinalURI, id, node, route) + return nil, state, nil + } + } + if readData && len(page.Data) > 0 { + var data [][]*string + if err := json.Unmarshal(page.Data, &data); err != nil { + return nil, state, errors.ErrLakeProtocol.GenWithStackByArgs(id, "invalid result data") + } + // Administrative reads are intentionally bounded, independently of paging. + total += len(page.Data) + if total > maxResponseBytes { + return nil, state, errors.ErrLakeProtocol.GenWithStackByArgs(id, "result exceeds metadata budget") + } + rows = append(rows, data...) + } + if page.NextURI == "" || strings.HasSuffix(page.NextURI, "/final") { + if state.state != committed { + return nil, state, errors.ErrLakeProtocol.GenWithStackByArgs(id, "no terminal execution evidence") + } + c.finalize(ctx, page.FinalURI, id, node, route) + return rows, state, nil + } + method, path, body = http.MethodGet, page.NextURI, nil + select { + case <-ctx.Done(): + return nil, state, errors.ErrLakeProtocol.GenWithStackByArgs(id, "query polling interrupted; verify server termination before resuming") + case <-time.After(10 * time.Millisecond): + } + } +} + +func jsonInt(n int) string { b, _ := json.Marshal(n); return string(b) } + +func (c *httpClient) request(ctx context.Context, method, path string, body []byte, id, node, route string) (queryPage, http.Header, error) { + // Never forward credentials to an absolute or network-path URI supplied by a server. + u, err := url.Parse(path) + if err != nil || u.IsAbs() || u.Host != "" || !strings.HasPrefix(path, "/v1/query") { + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "invalid query continuation URI") + } + req, err := http.NewRequestWithContext(ctx, method, c.opts.endpoint+path, bytes.NewReader(body)) // #nosec G704 -- endpoint is operator configured; continuation above is restricted to the same origin. + if err != nil { + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "invalid request") + } + req.SetBasicAuth(c.opts.username, c.opts.password) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-DATABEND-QUERY-ID", id) + if c.opts.warehouse != "" { + req.Header.Set("X-DATABEND-WAREHOUSE", c.opts.warehouse) + } + if node != "" { + req.Header.Set("X-DATABEND-STICKY-NODE", node) + req.Header.Set("X-DATABEND-NODE-ID", node) + } + if route != "" { + req.Header.Set("X-DATABEND-ROUTE-HINT", route) + } + if method == http.MethodPost { + if err := writelease.WaitForWrite(ctx, c.gate.Load()); err != nil { + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "write admission closed before request") + } + } + resp, err := c.transport.Do(req) // #nosec G704 -- intentional configured sink endpoint, same origin only and redirects disabled. + if err != nil { + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "transport failed; verify server termination before resuming") + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "HTTP status "+jsonInt(resp.StatusCode)) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) + if err != nil || len(data) > maxResponseBytes { + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "truncated or oversized response") + } + var page queryPage + if err := json.Unmarshal(data, &page); err != nil { + return queryPage{}, nil, errors.ErrLakeProtocol.GenWithStackByArgs(id, "invalid execution response") + } + return page, resp.Header, nil +} + +func (c *httpClient) finalize(ctx context.Context, path, id, node, route string) { + if path == "" { + return + } + // Releasing result buffers is observation/cleanup only, never a new mutation. + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + _, _, _ = c.request(ctx, http.MethodGet, path, nil, id, node, route) +} diff --git a/downstreamadapter/sink/lake/client_test.go b/downstreamadapter/sink/lake/client_test.go new file mode 100644 index 0000000000..54abb4c216 --- /dev/null +++ b/downstreamadapter/sink/lake/client_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/writelease" + "github.com/stretchr/testify/require" +) + +func TestHTTPClientRequiresTerminalEvidence(t *testing.T) { + for _, tc := range []struct { + name string + second any + want commitState + }{ + {"result before failure", map[string]any{"state": "Failed", "error": map[string]any{"code": 1001}}, notCommitted}, + {"result before disconnect", nil, commitUnknown}, + {"committed statistics malformed", map[string]any{"state": "Succeeded", "data": "not result rows", "stats": "not statistics"}, committed}, + } { + t.Run(tc.name, func(t *testing.T) { + var posts, gets atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := r.Header.Get("X-DATABEND-QUERY-ID") + if r.Method == "POST" { + posts.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{"id": id, "node_id": "node", "state": "Running", "data": [][]string{{"1"}}, "next_uri": "/v1/query/next"}) + return + } + gets.Add(1) + if tc.second == nil { + w.WriteHeader(503) + return + } + response := tc.second.(map[string]any) + response["id"] = id + _ = json.NewEncoder(w).Encode(response) + })) + defer server.Close() + c := newHTTPClient(options{endpoint: server.URL, queryTimeout: time.Second, workers: 1, username: "root"}) + defer c.transport.CloseIdleConnections() + result, err := c.mutate(context.Background(), "MERGE INTO TARGET", nil) + require.Equal(t, tc.want, result.state) + if tc.want == committed { + require.NoError(t, err) + } else { + require.Error(t, err) + } + require.EqualValues(t, 1, posts.Load()) + require.EqualValues(t, 1, gets.Load()) + }) + } +} + +func TestHTTPClientProtectsCredentialsAndDoesNotRetryPost(t *testing.T) { + var leaked atomic.Int64 + foreign := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { leaked.Add(1) })) + defer foreign.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"id": r.Header.Get("X-DATABEND-QUERY-ID"), "state": "Running", "next_uri": foreign.URL + "/v1/query/steal"}) + })) + defer server.Close() + c := newHTTPClient(options{endpoint: server.URL, username: "user", password: "SECRET", queryTimeout: time.Second}) + result, err := c.mutate(context.Background(), "INSERT SECRET_PAYLOAD", nil) + require.Error(t, err) + require.Equal(t, commitUnknown, result.state) + require.Zero(t, leaked.Load()) + require.NotContains(t, err.Error(), "SECRET") + gate := writelease.NewGate() + c.gate.Store(gate) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result, err = c.mutate(ctx, "DELETE TARGET", nil) + require.Error(t, err) + require.Equal(t, notCommitted, result.state) +} + +func TestHTTPClientRejectsWrongOrMissingSuccessfulIdentity(t *testing.T) { + for _, id := range []string{"", "another-query"} { + t.Run(id, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"id": id, "state": "Succeeded", "data": [][]string{{"1"}}}) + })) + defer server.Close() + c := newHTTPClient(options{endpoint: server.URL, queryTimeout: time.Second}) + result, err := c.mutate(context.Background(), "INSERT TARGET", nil) + require.Error(t, err) + require.Equal(t, commitUnknown, result.state) + }) + } +} diff --git a/downstreamadapter/sink/lake/completion.go b/downstreamadapter/sink/lake/completion.go new file mode 100644 index 0000000000..d899ba6c57 --- /dev/null +++ b/downstreamadapter/sink/lake/completion.go @@ -0,0 +1,94 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "sync" + "sync/atomic" +) + +// eventCompletion owns the source flush callback independently of spool entries. +// Spool release/discard/close must never call a fragment's applied method. +type eventCompletion struct { + mu sync.Mutex + pending int + sealed bool + failed bool + completed bool + callback func() +} + +type fragmentCompletion struct { + done atomic.Bool + parent *eventCompletion +} + +func newEventCompletion(callback func()) *eventCompletion { + return &eventCompletion{callback: callback} +} + +// register must happen before publishing a fragment to a worker. +func (c *eventCompletion) register() (*fragmentCompletion, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.sealed || c.failed || c.completed { + return nil, false + } + c.pending++ + return &fragmentCompletion{parent: c}, true +} + +func (c *eventCompletion) seal() { + c.mu.Lock() + c.sealed = true + callback := c.finishLocked() + c.mu.Unlock() + if callback != nil { + callback() + } +} + +func (c *eventCompletion) fail() { + c.mu.Lock() + c.failed = true + c.callback = nil + c.mu.Unlock() +} + +func (c *eventCompletion) finishLocked() func() { + if !c.sealed || c.failed || c.completed || c.pending != 0 { + return nil + } + c.completed = true + callback := c.callback + c.callback = nil + return callback +} + +// applied is called only with confirmed target application. Its caller may +// release the fragment payload immediately, even while the event is OPEN. +func (f *fragmentCompletion) applied() { + if !f.done.CompareAndSwap(false, true) { + return + } + c := f.parent + c.mu.Lock() + c.pending-- + callback := c.finishLocked() + c.mu.Unlock() + if callback != nil { + callback() + } +} diff --git a/downstreamadapter/sink/lake/completion_test.go b/downstreamadapter/sink/lake/completion_test.go new file mode 100644 index 0000000000..92d9eef5c7 --- /dev/null +++ b/downstreamadapter/sink/lake/completion_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCompletionRequiresSealAndAllFragments(t *testing.T) { + var calls atomic.Int64 + c := newEventCompletion(func() { calls.Add(1) }) + first, ok := c.register() + require.True(t, ok) + first.applied() + first.applied() + require.Zero(t, calls.Load(), "registered==applied while OPEN is not complete") + second, ok := c.register() + require.True(t, ok) + c.seal() + require.Zero(t, calls.Load()) + second.applied() + c.seal() + second.applied() + require.EqualValues(t, 1, calls.Load()) + _, ok = c.register() + require.False(t, ok) +} + +func TestCompletionFailureAndEmptyEvent(t *testing.T) { + var calls atomic.Int64 + c := newEventCompletion(func() { calls.Add(1) }) + f, ok := c.register() + require.True(t, ok) + c.fail() // Resource cleanup must not manufacture an application result. + c.seal() + f.applied() // Even a late worker result cannot revive a failed event. + require.Zero(t, calls.Load()) + _, ok = c.register() + require.False(t, ok) + empty := newEventCompletion(func() { calls.Add(1) }) + empty.seal() + empty.seal() + require.EqualValues(t, 1, calls.Load()) +} + +func TestCompletionConcurrentSealAndReentrantCallback(t *testing.T) { + for range 50 { + var calls atomic.Int64 + var last *fragmentCompletion + c := newEventCompletion(func() { + calls.Add(1) + last.applied() // Source callbacks may re-enter after completion. + }) + fragments := make([]*fragmentCompletion, 16) + for i := range fragments { + f, ok := c.register() + require.True(t, ok) + fragments[i] = f + } + last = fragments[len(fragments)-1] + var wg sync.WaitGroup + for _, f := range fragments { + wg.Go(func() { + f.applied() + f.applied() + }) + } + wg.Go(c.seal) + wg.Wait() + require.EqualValues(t, 1, calls.Load()) + } +} diff --git a/downstreamadapter/sink/lake/config.go b/downstreamadapter/sink/lake/config.go new file mode 100644 index 0000000000..eecdb057a8 --- /dev/null +++ b/downstreamadapter/sink/lake/config.go @@ -0,0 +1,162 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "net" + "net/url" + "strconv" + "strings" + "time" + + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/util" +) + +// All quotas are local to this sink instance. Exclusive is an operator assertion, +// not a distributed lock or permission to automatically fail over an old query. +type options struct { + endpoint, database, username, password, warehouse, spoolDir string + workers, batchRows, maxEntries, sourceEvents int + batchBytes, spoolBytes, sourceBytes int64 + memoryRatio float64 + flushInterval, queryTimeout time.Duration + remoteLimit int64 + timezone *time.Location + conflictKey string +} + +func parseOptions(uri *url.URL, cfg *config.ChangefeedConfig) (options, error) { + o := options{workers: 4, batchRows: 2048, batchBytes: 1 << 20, spoolBytes: 10 << 30, sourceBytes: 256 << 20, sourceEvents: 128, maxEntries: 4096, memoryRatio: 0.02, flushInterval: time.Second, queryTimeout: 2 * time.Minute} + bad := func() (options, error) { + return options{}, errors.ErrSinkURIInvalid.GenWithStackByArgs("invalid Lake options; require exclusive=true, bounded positive quotas and supported incremental settings") + } + if uri == nil || (config.GetScheme(uri) != config.LakeScheme && config.GetScheme(uri) != config.LakeHTTPScheme) || uri.Host == "" || uri.Fragment != "" { + return bad() + } + q, queryErr := url.ParseQuery(uri.RawQuery) + if queryErr != nil { + return bad() + } + allowed := map[string]bool{"exclusive": true, "workers": true, "batch-rows": true, "batch-bytes": true, "spool-bytes": true, "source-bytes": true, "source-events": true, "max-entries": true, "memory-ratio": true, "flush-interval": true, "query-timeout": true, "spool-dir": true, "warehouse": true, "remote-bytes-limit": true, "conflict-key": true} + for key, v := range q { + if !allowed[key] || len(v) != 1 { + return bad() + } + } + if q.Get("exclusive") != "true" { + return bad() + } + if q.Has("conflict-key") { + o.conflictKey = q.Get("conflict-key") + if strings.TrimSpace(o.conflictKey) == "" || strings.ContainsRune(o.conflictKey, 0) || len(o.conflictKey) > 256 { + return bad() + } + } + o.timezone = time.UTC + if cfg != nil { + if cfg.TimeZone != "" { + var err error + o.timezone, err = util.GetTimezone(cfg.TimeZone) + if err != nil { + return bad() + } + } + if cfg.EnableTableAcrossNodes || cfg.EnableSyncPoint || cfg.EnableActiveActive || cfg.IsLowLatencyMode() { + return bad() + } + if cfg.Consistent != nil && cfg.Consistent.Level != nil && *cfg.Consistent.Level != "none" && *cfg.Consistent.Level != "" { + return bad() + } + if cfg.SinkConfig != nil { + if cfg.SinkConfig.TxnAtomicity != nil && *cfg.SinkConfig.TxnAtomicity != "none" && *cfg.SinkConfig.TxnAtomicity != "" { + return bad() + } + for _, r := range cfg.SinkConfig.DispatchRules { + if r != nil && (r.TargetSchema != "" || r.TargetTable != "") { + return bad() + } + } + if len(cfg.SinkConfig.ColumnSelectors) > 0 { + return bad() + } + } + } + for key, p := range map[string]*int{"workers": &o.workers, "batch-rows": &o.batchRows, "source-events": &o.sourceEvents, "max-entries": &o.maxEntries} { + if q.Has(key) { + n, err := strconv.Atoi(q.Get(key)) + if err != nil || n <= 0 { + return bad() + } + *p = n + } + } + for key, p := range map[string]*int64{"batch-bytes": &o.batchBytes, "spool-bytes": &o.spoolBytes, "source-bytes": &o.sourceBytes, "remote-bytes-limit": &o.remoteLimit} { + if q.Has(key) { + n, err := strconv.ParseInt(q.Get(key), 10, 64) + if err != nil || n <= 0 { + return bad() + } + *p = n + } + } + for key, p := range map[string]*time.Duration{"flush-interval": &o.flushInterval, "query-timeout": &o.queryTimeout} { + if q.Has(key) { + d, err := time.ParseDuration(q.Get(key)) + if err != nil || d <= 0 { + return bad() + } + *p = d + } + } + if q.Has("memory-ratio") { + n, err := strconv.ParseFloat(q.Get("memory-ratio"), 64) + if err != nil || !(n > 0 && n < 1) { + return bad() + } + o.memoryRatio = n + } + if o.workers > 64 || o.batchRows > 100000 || o.sourceEvents > 65536 || o.maxEntries > 65536 || o.batchBytes < 1024 || o.batchBytes > 64<<20 || o.spoolBytes < o.batchBytes*2 || o.sourceBytes < o.batchBytes || o.spoolBytes > 1<<50 { + return bad() + } + o.database = strings.Trim(uri.Path, "/") + if strings.Contains(o.database, "/") || strings.ContainsRune(o.database, 0) { + return bad() + } + o.username = "root" + if uri.User != nil { + o.username = uri.User.Username() + o.password, _ = uri.User.Password() + } + endpoint := *uri + endpoint.Path = "" + endpoint.User = nil + endpoint.RawQuery = "" + endpoint.RawPath = "" + endpoint.Scheme = "https" + if config.GetScheme(uri) == config.LakeHTTPScheme { + host := uri.Hostname() + ip := net.ParseIP(host) + if !strings.EqualFold(host, "localhost") && (ip == nil || !ip.IsLoopback()) { + return bad() + } + endpoint.Scheme = "http" + } + o.endpoint = endpoint.String() + o.warehouse = q.Get("warehouse") + o.spoolDir = q.Get("spool-dir") + return o, nil +} diff --git a/downstreamadapter/sink/lake/config_test.go b/downstreamadapter/sink/lake/config_test.go new file mode 100644 index 0000000000..9844b1d0c0 --- /dev/null +++ b/downstreamadapter/sink/lake/config_test.go @@ -0,0 +1,83 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "net/url" + "testing" + + "github.com/pingcap/ticdc/pkg/config" + "github.com/stretchr/testify/require" +) + +func TestLakeOptions(t *testing.T) { + uri, err := url.Parse("lake://user:pass@warehouse/lake?exclusive=true&workers=2&batch-bytes=4096&spool-bytes=16384") + require.NoError(t, err) + o, err := parseOptions(uri, nil) + require.NoError(t, err) + require.Equal(t, "https://warehouse", o.endpoint) + require.Equal(t, "lake", o.database) + require.Equal(t, 2, o.workers) + require.EqualValues(t, 16384, o.spoolBytes) + lakeURI, err := url.Parse("lake://user:pass@warehouse:443/default?exclusive=true&warehouse=default") + require.NoError(t, err) + lake, err := parseOptions(lakeURI, nil) + require.NoError(t, err) + require.Equal(t, "https://warehouse:443", lake.endpoint) + for _, raw := range []string{ + "lake://host", "lake://host?exclusive=false", "lake://host?exclusive=true&workers=0", "lake://host?exclusive=true&memory-ratio=NaN", "lake://host?exclusive=true&batch-bytes=4096&spool-bytes=4096", "lake://host?exclusive=true&raw-change=true", "lake://host?exclusive=true&workers=1&workers=2", "lake://host?exclusive=true&query-timeout=0s", + } { + u, err := url.Parse(raw) + require.NoError(t, err) + _, err = parseOptions(u, nil) + require.Error(t, err, raw) + } + _, err = parseOptions(uri, &config.ChangefeedConfig{EnableTableAcrossNodes: true}) + require.Error(t, err) + _, err = parseOptions(uri, &config.ChangefeedConfig{EnableSyncPoint: true}) + require.Error(t, err) +} + +func TestConflictKeyOption(t *testing.T) { + for _, key := range []string{"id", "BusinessID", "a\"b"} { + u, err := url.Parse("lake://host/default?exclusive=true&conflict-key=" + url.QueryEscape(key)) + require.NoError(t, err) + o, err := parseOptions(u, &config.ChangefeedConfig{ForceReplicate: true}) + require.NoError(t, err) + require.Equal(t, key, o.conflictKey) + } + for _, raw := range []string{"", " ", "bad\x00name"} { + u, err := url.Parse("lake://host/default?exclusive=true&conflict-key=" + url.QueryEscape(raw)) + require.NoError(t, err) + _, err = parseOptions(u, nil) + require.Error(t, err) + } +} + +func TestLakePlaintextRequiresLoopback(t *testing.T) { + for _, host := range []string{"localhost", "LOCALHOST:8000", "127.0.0.1:8000", "127.1.2.3", "[::1]:8000", "[::ffff:127.0.0.1]:8000"} { + u, err := url.Parse("lake+http://user:pass@" + host + "/default?exclusive=true") + require.NoError(t, err) + o, err := parseOptions(u, nil) + require.NoError(t, err, host) + require.Equal(t, "http://"+host, o.endpoint) + } + for _, host := range []string{"warehouse.example:8000", "localhost.example", "127.0.0.1.example", "10.0.0.1", "0.0.0.0", "[::]", "[2001:db8::1]"} { + u, err := url.Parse("lake+http://user:pass@" + host + "?exclusive=true") + require.NoError(t, err) + _, err = parseOptions(u, nil) + require.Error(t, err, host) + } +} diff --git a/downstreamadapter/sink/lake/integration_test.go b/downstreamadapter/sink/lake/integration_test.go new file mode 100644 index 0000000000..690c852694 --- /dev/null +++ b/downstreamadapter/sink/lake/integration_test.go @@ -0,0 +1,94 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "context" + "net/url" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/stretchr/testify/require" +) + +// Explicit opt-in only. This test creates and drops its own unique database; +// never use a production warehouse or the running bend-cdc target for it. +func TestLakeNativeLiveIntegration(t *testing.T) { + raw := os.Getenv("LAKE_NATIVE_TEST_URI") + if raw == "" { + t.Skip("LAKE_NATIVE_TEST_URI not configured; no live Lake evidence") + } + uri, err := url.Parse(raw) + require.NoError(t, err) + database := "ticdc_native_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + uri.Path = "/" + database + uri.RawPath = "" + q := uri.Query() + q.Set("exclusive", "true") + q.Set("workers", "1") + q.Set("flush-interval", "10ms") + q.Set("spool-dir", t.TempDir()) + uri.RawQuery = q.Encode() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + require.NoError(t, Verify(ctx, &config.ChangefeedConfig{}, uri)) + s, err := New(ctx, common.NewChangeFeedIDWithName(database, "default"), &config.ChangefeedConfig{}, uri, 0) + require.NoError(t, err) + t.Cleanup(func() { + s.Close() + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, _ = s.client.mutate(cleanupCtx, "DROP DATABASE IF EXISTS "+quoteIdentifier(database), nil) + }) + done := runRuntime(t, s) + var callbacks atomic.Int64 + apply := func(e *commonEvent.DMLEvent) { + e.PostTxnFlushed = []func(){func() { callbacks.Add(1) }} + before := callbacks.Load() + s.AddDMLEvent(e) + require.Eventually(t, func() bool { return callbacks.Load() == before+1 }, 30*time.Second, 10*time.Millisecond) + } + info := nativeTestInfo() + apply(nativeTestEvent(info, 100, 2)) + rows, err := s.client.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM "+quoteIdentifier(database)+`."t"`, nil) + require.NoError(t, err) + require.Equal(t, "2", *rows[0][0]) + deleted := nativeTestEvent(info, 130, 1) + deleted.RowTypes[0] = common.RowTypeDelete + apply(deleted) + // A historical UPSERT must not resurrect a retained tombstone. + apply(nativeTestEvent(info, 100, 1)) + rows, err = s.client.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM "+quoteIdentifier(database)+`."t"`, nil) + require.NoError(t, err) + require.Equal(t, "1", *rows[0][0]) + ddl := &commonEvent.DDLEvent{Type: byte(model.ActionTruncateTable), TableInfo: info, FinishedTs: 150} + require.NoError(t, s.WriteBlockEvent(ddl)) + apply(nativeTestEvent(info, 120, 2)) // Applied epoch rejects old DML without projection. + apply(nativeTestEvent(info, 160, 1)) + require.NoError(t, s.WriteBlockEvent(ddl)) // Applied replay must not truncate the new row. + rows, err = s.client.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM "+quoteIdentifier(database)+`."t"`, nil) + require.NoError(t, err) + require.Equal(t, "1", *rows[0][0]) + s.Close() + require.ErrorIs(t, <-done, context.Canceled) +} diff --git a/downstreamadapter/sink/lake/lane.go b/downstreamadapter/sink/lake/lane.go new file mode 100644 index 0000000000..25d43adc71 --- /dev/null +++ b/downstreamadapter/sink/lake/lane.go @@ -0,0 +1,340 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "context" + "encoding/json" + "time" + + "github.com/pingcap/ticdc/pkg/errors" + codec "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/sink/spool" +) + +type spoolRow struct { + ID string `json:"id"` + Payload string `json:"payload"` + CommitTs uint64 `json:"commit_ts"` + Action rowAction `json:"action"` +} +type pendingFragment struct { + entry *spool.Entry + completion *fragmentCompletion + count int + bytes int64 +} +type pendingTable struct { + state *tableState + fragments []pendingFragment + rows int + bytes int64 +} +type tableLane struct { + sink *Sink + input chan workItem + schemas *schemaManager + batches map[string]*pendingTable + entries int +} + +func (l *tableLane) run(ctx context.Context) error { + timer := time.NewTicker(l.sink.opts.flushInterval) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return context.Cause(ctx) + case <-timer.C: + if err := l.flushAll(ctx); err != nil { + return err + } + case item := <-l.input: + l.sink.mu.Lock() + l.sink.notifyAdmissionLocked() + l.sink.mu.Unlock() + var err error + switch { + case item.event != nil: + err = l.encode(ctx, &item) + case item.ddl != nil: + err = l.sink.statistics.RecordDDLExecution(func() (string, error) { return "lake", l.schemas.applyDDL(ctx, item.ddl) }) + default: + err = l.flushAll(ctx) + } + if err != nil { + l.sink.stop(err) + err = context.Cause(l.sink.ctx) + } + if item.done != nil { + item.done <- err + } + if err != nil { + return err + } + } + } +} + +func (l *tableLane) encode(ctx context.Context, item *workItem) (err error) { + event := item.event + item.event = nil + released := false + release := func() { + if !released { + released = true + l.sink.releaseSource(item.charge) + } + } + defer release() + s, err := schemaFromNative(event.TableInfo, l.sink.opts.database, l.sink.opts.conflictKey) + if err != nil { + return err + } + state, skip, err := l.schemas.prepareDML(ctx, s, event.CommitTs) + if err != nil { + return err + } + enqueue, flush := event.DetachPostCallbacks() + tracker := newEventCompletion(flush) + defer func() { + if err != nil { + tracker.fail() + } + }() + if skip { + event = nil + release() + enqueue() + tracker.seal() + return nil + } + var blob []byte + rows := 0 + publish := func(last bool) error { + if rows == 0 { + if last { + enqueue() + } + return nil + } + blob = append(blob, ']') + fragment, ok := tracker.register() + if !ok { + return errors.ErrLakeInvalidData.GenWithStackByArgs("fragment registered after completion") + } + var callback func() + if last { + callback = enqueue + } + err := l.enqueue(ctx, state, blob, rows, fragment, callback) + blob = nil + rows = 0 + return err + } + count := int32(0) + for { + if ctx.Err() != nil { + return context.Cause(ctx) + } + change, ok := event.GetNextRow() + if !ok { + break + } + row, err := normalizeChange(event.TableInfo, s, change, event.CommitTs, l.sink.opts.timezone) + if err != nil { + return err + } + encoded, err := json.Marshal(spoolRow{row.id, row.payload, row.commitTs, row.action}) + if err != nil { + return errors.WrapError(errors.ErrLakeInvalidData, err, "cannot encode spool row") + } + if int64(len(encoded)+2) > l.sink.opts.batchBytes { + return errors.ErrLakeInvalidData.GenWithStackByArgs("single encoded row exceeds batch byte limit") + } + if rows > 0 && (int64(len(blob)+len(encoded)+2) > l.sink.opts.batchBytes || rows >= l.sink.opts.batchRows) { + if err := publish(false); err != nil { + return err + } + } + if rows == 0 { + blob = append(blob, '[') + } else { + blob = append(blob, ',') + } + blob = append(blob, encoded...) + rows++ + count++ + } + if count != event.Len() { + return errors.ErrLakeInvalidData.GenWithStackByArgs("incomplete native row iteration") + } + event = nil + release() // Only callback closures and immutable bounded fragments remain. + if err := publish(true); err != nil { + return err + } + tracker.seal() + return nil +} + +func (l *tableLane) enqueue(ctx context.Context, state *tableState, blob []byte, count int, completion *fragmentCompletion, enqueue func()) error { + msg := &codec.Message{Value: blob} // Never attach source success to spool.Callback. + msgs := []*codec.Message{msg} + if spool.AccountingBytes(msgs) > l.sink.opts.spoolBytes { + return errors.ErrLakeInvalidData.GenWithStackByArgs("encoded fragment exceeds spool quota") + } + for { + action, entry, err := l.sink.spool.TryEnqueue(msgs, enqueue) + if err != nil { + return err + } + if action == spool.EnqueueActionAcceptedOversized { + return errors.ErrLakeInvalidData.GenWithStackByArgs("spool size invariant violated") + } + if action == spool.EnqueueActionWaitDiskQuota { + // Drain every local table, not just the large event. Other lanes keep + // draining on their own timers; released segment files remain charged. + if err := l.flushAll(ctx); err != nil { + return err + } + if err := l.sink.spool.WaitForDiskQuota(ctx, msgs); err != nil { + return err + } + continue + } + key := schemaKey(state.schema) + batch := l.batches[key] + if batch == nil { + batch = &pendingTable{state: state} + l.batches[key] = batch + } + batch.fragments = append(batch.fragments, pendingFragment{entry, completion, count, int64(len(blob))}) + batch.rows += count + batch.bytes += int64(len(blob)) + l.entries++ + if batch.bytes >= l.sink.opts.batchBytes || batch.rows >= l.sink.opts.batchRows { + return l.flush(ctx, key) + } + if l.entries >= l.sink.opts.maxEntries { + return l.flushAll(ctx) + } + return nil + } +} + +func (l *tableLane) flushAll(ctx context.Context) error { + for key := range l.batches { + if err := l.flush(ctx, key); err != nil { + return err + } + } + return nil +} + +func (l *tableLane) flush(ctx context.Context, key string) error { + batch := l.batches[key] + if batch == nil { + return nil + } + // Bound remote retained-raw admission by monitoring only. The measurement is + // neither a cross-capture reservation nor an absolute physical-cost ceiling. + if l.sink.opts.remoteLimit > 0 { + rows, err := l.sink.client.selectRows(ctx, "SELECT COALESCE(SUM(data_compressed_size), 0)::VARCHAR FROM system.tables WHERE database = ? AND ends_with(name, '_cdc_raw')", []any{batch.state.schema.database}) + if err != nil { + return err + } + n, err := parseRemoteBytes(rows) + if err != nil { + return err + } + if n >= l.sink.opts.remoteLimit { + return errors.ErrLakeInvalidData.GenWithStackByArgs("remote raw monitoring threshold reached; expand budget or reinitialize manually") + } + } + var rows []normalizedRow + for _, f := range batch.fragments { + reader, err := l.sink.spool.NewMessageReader(f.entry) + if err != nil { + return err + } + _, data, _, ok, err := reader.Next() + if err != nil { + return err + } + if !ok { + return errors.ErrLakeInvalidData.GenWithStackByArgs("empty spool fragment") + } + var decoded []spoolRow + if err := json.Unmarshal(data, &decoded); err != nil { + return errors.WrapError(errors.ErrLakeInvalidData, err, "cannot decode spool fragment") + } + if len(decoded) != f.count { + return errors.ErrLakeInvalidData.GenWithStackByArgs("incomplete spool fragment") + } + for _, r := range decoded { + rows = append(rows, normalizedRow{r.ID, r.Payload, r.CommitTs, r.Action}) + } + } + var result applyResult + err := l.sink.statistics.RecordBatchExecution(func() (int, int64, error) { + var err error + result, err = batch.state.writer.applyBatch(ctx, rows, time.Now()) + return len(rows), batch.bytes, err + }) + if result.applied { + delete(l.batches, key) + l.entries -= len(batch.fragments) + // Reclaim space separately from source completion; OPEN events keep only + // counters, never already-applied payloads. Release never invokes success. + for _, f := range batch.fragments { + l.sink.spool.Release(f.entry) + f.completion.applied() + } + } + return err +} + +func (l *tableLane) cleanup() { + for _, b := range l.batches { + for _, f := range b.fragments { + f.completion.parent.fail() + l.sink.spool.Release(f.entry) + } + } + clear(l.batches) + for { + select { + case item := <-l.input: + if item.event != nil { + l.sink.releaseSource(item.charge) + } + default: + return + } + } +} + +// parseRemoteBytes rejects malformed measurements instead of treating them as 0. +func parseRemoteBytes(rows [][]*string) (int64, error) { + if len(rows) != 1 || len(rows[0]) != 1 || rows[0][0] == nil { + return 0, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid remote raw measurement") + } + var n int64 + if err := json.Unmarshal([]byte(*rows[0][0]), &n); err != nil || n < 0 { + return 0, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid remote raw byte count") + } + return n, nil +} diff --git a/downstreamadapter/sink/lake/live_fault_test.go b/downstreamadapter/sink/lake/live_fault_test.go new file mode 100644 index 0000000000..1568489aab --- /dev/null +++ b/downstreamadapter/sink/lake/live_fault_test.go @@ -0,0 +1,405 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" + "github.com/stretchr/testify/require" +) + +// liveResponseCut delegates to the REAL warehouse, then discards a chosen +// Succeeded response at the client transport boundary. It never simulates SQL +// execution or invents terminal success. A before-send cut is used only to prove +// partial raw INSERTs; recovery is permitted only after this test knows all old +// requests have either not been sent or reached a matching real terminal state. +// This is response-loss/process-state replacement injection, not a server crash. +type liveResponseCut struct { + base http.RoundTripper + matches func(string, []any) bool + at int + before bool + mu sync.Mutex + posts int + cutID string + terminal string + fired bool +} + +func (f *liveResponseCut) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Method == http.MethodPost { + r, err := req.GetBody() + if err != nil { + return nil, err + } + var payload struct { + SQL string `json:"sql"` + Params []any `json:"params"` + } + err = json.NewDecoder(r).Decode(&payload) + _ = r.Close() + if err != nil { + return nil, err + } + f.mu.Lock() + if f.matches(payload.SQL, payload.Params) { + f.posts++ + if f.posts == f.at { + f.cutID = req.Header.Get("X-DATABEND-QUERY-ID") + } + } + cutBefore := f.before && f.cutID == req.Header.Get("X-DATABEND-QUERY-ID") + if cutBefore { + f.fired = true + } + f.mu.Unlock() + if cutBefore { + return nil, io.ErrUnexpectedEOF + } + } + response, err := f.base.RoundTrip(req) + if err != nil { + return nil, err + } + data, err := io.ReadAll(response.Body) + _ = response.Body.Close() + if err != nil { + return nil, err + } + response.Body = io.NopCloser(bytes.NewReader(data)) + var page queryPage + if json.Unmarshal(data, &page) == nil { + f.mu.Lock() + cut := !f.before && !f.fired && page.ID != "" && page.ID == f.cutID && page.State == "Succeeded" + if cut { + f.terminal, f.fired = page.ID, true + } + f.mu.Unlock() + if cut { + _ = response.Body.Close() + return nil, io.ErrUnexpectedEOF + } + } + return response, nil +} + +func liveFaultEnvironment(t *testing.T) (context.Context, options, *httpClient) { + t.Helper() + raw := os.Getenv("LAKE_NATIVE_TEST_URI") + if raw == "" { + t.Skip("explicit LAKE_NATIVE_TEST_URI is required; no real warehouse evidence") + } + u, err := url.Parse(raw) + require.NoError(t, err) + u.Path = "/ticdc_fault_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + u.RawPath = "" + q := u.Query() + q.Set("exclusive", "true") + q.Set("workers", "1") + q.Set("batch-rows", "2") + q.Set("flush-interval", "10ms") + q.Set("spool-dir", t.TempDir()) + u.RawQuery = q.Encode() + o, err := parseOptions(u, &config.ChangefeedConfig{}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + t.Cleanup(cancel) + client := newHTTPClient(o) + r, err := client.mutate(ctx, "CREATE DATABASE "+quoteIdentifier(o.database), nil) + require.NoError(t, err) + require.Equal(t, committed, r.state) + t.Cleanup(func() { + cleanup, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + r, err := client.mutate(cleanup, "DROP DATABASE IF EXISTS "+quoteIdentifier(o.database), nil) + require.NoError(t, err) + require.Equal(t, committed, r.state) + }) + t.Log("owned real Lake database", o.database) + return ctx, o, client +} + +func liveRuntime(ctx context.Context, t *testing.T, o options, name string) *Sink { + t.Helper() + u, err := url.Parse(os.Getenv("LAKE_NATIVE_TEST_URI")) + require.NoError(t, err) + u.Path, u.RawPath = "/"+o.database, "" + q := u.Query() + q.Set("exclusive", "true") + q.Set("workers", "1") + q.Set("flush-interval", "10ms") + q.Set("spool-dir", t.TempDir()) + u.RawQuery = q.Encode() + s, err := New(ctx, common.NewChangeFeedIDWithName(name, "default"), &config.ChangefeedConfig{}, u, 0) + require.NoError(t, err) + t.Cleanup(s.Close) + return s +} + +func liveNamedInfo(name string) *common.TableInfo { + base := nativeTestInfo() + return common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr(name), PKIsHandle: true, Columns: base.GetColumns()}) +} + +func liveCount(ctx context.Context, t *testing.T, c *httpClient, name string, expected string) { + t.Helper() + rows, err := c.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM "+name, nil) + require.NoError(t, err) + require.Equal(t, expected, *rows[0][0]) +} + +func TestLakeNativeLiveRawPartialAndMergeResponseLoss(t *testing.T) { + ctx, o, admin := liveFaultEnvironment(t) + t.Run("partial_raw_before_second_insert", func(t *testing.T) { + info := liveNamedInfo("partial_raw") + schema, err := schemaFromNative(info, o.database, "") + require.NoError(t, err) + client := newHTTPClient(o) + m := newSchemaManager(client, o) + state, _, err := m.prepareDML(ctx, schema, 100) + require.NoError(t, err) + cut := &liveResponseCut{base: client.transport.Transport, at: 2, before: true, matches: func(sql string, _ []any) bool { return strings.HasPrefix(sql, "INSERT INTO "+schema.rawName()) }} + client.transport.Transport = cut + var rows []normalizedRow + for i := range 4 { + r, err := normalizeNativeRow(schema, insertAction, 100, map[string]any{"id": i, "value": "row"}) + require.NoError(t, err) + rows = append(rows, r) + } + applied, err := state.writer.applyBatch(ctx, rows, time.Now()) + require.Error(t, err) + require.False(t, applied.applied) + require.True(t, cut.fired) + require.Equal(t, 2, cut.posts, "unknown raw must not be retried") + liveCount(ctx, t, admin, schema.rawName(), "2") + liveCount(ctx, t, admin, schema.targetName(), "0") + // Replacement process state; the second request was never sent and the + // first request had a matching terminal success. Reapply ALL source rows. + m = newSchemaManager(admin, o) + state, _, err = m.prepareDML(ctx, schema, 100) + require.NoError(t, err) + applied, err = state.writer.applyBatch(ctx, rows, time.Now()) + require.NoError(t, err) + require.True(t, applied.applied) + liveCount(ctx, t, admin, schema.targetName(), "4") + liveCount(ctx, t, admin, schema.rawName(), "4") + }) + t.Run("real_merge_committed_terminal_response_discarded", func(t *testing.T) { + info := liveNamedInfo("merge_lost") + s := liveRuntime(ctx, t, o, "merge_lost") + schema, err := schemaFromNative(info, o.database, "") + require.NoError(t, err) + cut := &liveResponseCut{base: s.client.transport.Transport, at: 1, matches: func(sql string, _ []any) bool { + return strings.HasPrefix(sql, "MERGE INTO "+schema.targetName()+" a USING") + }} + s.client.transport.Transport = cut + done := runRuntime(t, s) + var flushed atomic.Int64 + e := nativeTestEvent(info, 100, 2) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + select { + case err := <-done: + require.Error(t, err) + case <-time.After(90 * time.Second): + t.Fatal("unknown real MERGE did not terminate old sink lifecycle") + } + s.Close() + require.Zero(t, flushed.Load(), "commit without client evidence must not flush") + require.False(t, s.IsNormal()) + require.True(t, cut.fired) + require.NotEmpty(t, cut.terminal, "test must know matching server terminal BEFORE recovery") + require.Equal(t, 1, cut.posts) + t.Log("discarded real MERGE Succeeded response", cut.terminal) + liveCount(ctx, t, admin, schema.targetName(), "2") + m := newSchemaManager(admin, o) + state, _, err := m.prepareDML(ctx, schema, 100) + require.NoError(t, err) + var rows []normalizedRow + for i := range 2 { + r, err := normalizeNativeRow(schema, insertAction, 100, map[string]any{"id": i, "value": "payload"}) + require.NoError(t, err) + rows = append(rows, r) + } + applied, err := state.writer.applyBatch(ctx, rows, time.Now()) + require.NoError(t, err) + require.True(t, applied.applied) + liveCount(ctx, t, admin, schema.targetName(), "2") + liveCount(ctx, t, admin, schema.rawName(), "2") + }) +} + +func TestLakeNativeLiveDDLInterruptionRecovery(t *testing.T) { + ctx, o, admin := liveFaultEnvironment(t) + for _, kind := range []string{"create", "alter", "truncate", "drop"} { + cuts := []string{"pending", "target", "raw", "applied"} + if kind == "alter" { + cuts = []string{"pending", "add", "rename", "modify", "raw", "applied"} + } + if kind == "truncate" { + cuts = []string{"pending", "drop_raw", "truncate", "raw", "applied"} + } + for _, point := range cuts { + t.Run(kind+"/"+point, func(t *testing.T) { + info := liveNamedInfo(kind + "_" + point) + schema, err := schemaFromNative(info, o.database, "") + require.NoError(t, err) + s := liveRuntime(ctx, t, o, schema.table) + client := s.client + m := s.lanes[0].schemas + if kind != "create" { + state, _, err := m.prepareDML(ctx, schema, 100) + require.NoError(t, err) + r, err := normalizeNativeRow(schema, insertAction, 100, map[string]any{"id": 1, "value": "old"}) + require.NoError(t, err) + _, err = state.writer.applyBatch(ctx, []normalizedRow{r}, time.Now()) + require.NoError(t, err) + } + e := &commonEvent.DDLEvent{TableInfo: info, FinishedTs: 150} + switch kind { + case "create": + e.Type = byte(model.ActionCreateTable) + case "truncate": + e.Type = byte(model.ActionTruncateTable) + case "drop": + e.Type = byte(model.ActionDropTable) + case "alter": + e.Type = byte(model.ActionMultiSchemaChange) + e.Query = "ALTER TABLE t ADD COLUMN extra BIGINT NULL, RENAME COLUMN value TO label, MODIFY COLUMN label TEXT NULL" + cols := append([]*model.ColumnInfo(nil), info.GetColumns()...) + copyValue := *cols[1] + copyValue.Name = ast.NewCIStr("label") + cols[1] = ©Value + cols = append(cols, &model.ColumnInfo{ID: 3, Name: ast.NewCIStr("extra"), FieldType: *types.NewFieldType(mysql.TypeLonglong), State: model.StatePublic}) + e.TableInfo = common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr(schema.table), PKIsHandle: true, Columns: cols}) + } + cut := &liveResponseCut{base: client.transport.Transport, at: 1, matches: func(sql string, args []any) bool { + switch point { + case "pending", "applied": + return strings.HasPrefix(sql, "MERGE INTO "+schema.journalName()) && len(args) == 3 && args[1] == (point == "applied") + case "target": + if kind == "drop" { + return sql == "DROP TABLE IF EXISTS "+schema.targetName() + } + return strings.HasPrefix(sql, "CREATE TABLE IF NOT EXISTS "+schema.targetName()+" (") + case "raw": + if kind == "drop" { + return sql == "DROP TABLE IF EXISTS "+schema.rawName() + } + return strings.HasPrefix(sql, "CREATE TABLE IF NOT EXISTS "+schema.rawName()+" (") + case "drop_raw": + return sql == "DROP TABLE IF EXISTS "+schema.rawName() + case "truncate": + return sql == "TRUNCATE TABLE "+schema.targetName() + default: + return strings.HasPrefix(sql, "ALTER TABLE "+schema.targetName()+" "+strings.ToUpper(point)+" COLUMN ") + } + }} + client.transport.Transport = cut + var callbacks atomic.Int64 + e.PostTxnFlushed = []func(){func() { callbacks.Add(1) }} + done := runRuntime(t, s) + require.Error(t, s.WriteBlockEvent(e)) + select { + case err := <-done: + require.Error(t, err) + case <-time.After(90 * time.Second): + t.Fatal("unknown DDL did not terminate old sink lifecycle") + } + s.Close() + require.Zero(t, callbacks.Load(), "unknown DDL must not flush") + require.True(t, cut.fired) + require.Equal(t, 1, cut.posts) + require.NotEmpty(t, cut.terminal) + t.Log("cut after real committed DDL", point, cut.terminal) + // The test has independent matching terminal proof for the old + // request. Replace ALL local schema/DDL state; reuse only Lake data. + m = newSchemaManager(admin, o) + require.NoError(t, m.applyDDL(ctx, e)) + post, err := schemaFromNative(e.TableInfo, o.database, "") + require.NoError(t, err) + r, err := m.loadEpoch(ctx, post) + require.NoError(t, err) + require.EqualValues(t, 150, r.epoch) + require.True(t, r.applied) + if kind == "drop" { + exists, err := m.targetExists(ctx, post) + require.NoError(t, err) + require.False(t, exists) + rows, err := admin.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM system.tables WHERE database=? AND name IN (?, ?)", []any{post.database, post.table, post.table + "_cdc_raw"}) + require.NoError(t, err) + require.Equal(t, "0", *rows[0][0]) + return + } + state, skip, err := m.prepareDML(ctx, post, 160) + require.NoError(t, err) + require.False(t, skip) + payload := map[string]any{"id": 2, "value": "new"} + if kind == "alter" { + payload = map[string]any{"id": 2, "label": "new", "extra": 7} + } + row, err := normalizeNativeRow(post, insertAction, 160, payload) + require.NoError(t, err) + _, err = state.writer.applyBatch(ctx, []normalizedRow{row}, time.Now()) + require.NoError(t, err) + // Replaying an applied DDL must not clear new-generation data. + require.NoError(t, m.applyDDL(ctx, e)) + expected := "1" + if kind == "alter" { + expected = "2" + } + liveCount(ctx, t, admin, post.targetName(), expected) + projection := "id::VARCHAR, value::VARCHAR" + want := [][]string{{"2", "new"}} + if kind == "alter" { + projection = "id::VARCHAR, label::VARCHAR, COALESCE(extra::VARCHAR, 'SQL_NULL')" + want = [][]string{{"1", "old", "SQL_NULL"}, {"2", "new", "7"}} + } + actual, err := admin.selectRows(ctx, "SELECT "+projection+" FROM "+post.targetName()+" ORDER BY id", nil) + require.NoError(t, err) + values := make([][]string, len(actual)) + for i, row := range actual { + for _, cell := range row { + require.NotNil(t, cell) + values[i] = append(values[i], *cell) + } + } + require.Equal(t, want, values) + _, skip, err = m.prepareDML(ctx, post, 120) + require.NoError(t, err) + require.True(t, skip, "old DML must not enter post-DDL schema") + }) + } + } +} diff --git a/downstreamadapter/sink/lake/metrics.go b/downstreamadapter/sink/lake/metrics.go new file mode 100644 index 0000000000..a95347cdb8 --- /dev/null +++ b/downstreamadapter/sink/lake/metrics.go @@ -0,0 +1,46 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + sinkmetrics "github.com/pingcap/ticdc/downstreamadapter/sink/metrics" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/sink/spool" +) + +func newSpoolMetrics(id common.ChangeFeedID) *spool.Metrics { + k, f := id.Keyspace(), id.Name() + return &spool.Metrics{ + MemoryBytes: sinkmetrics.LakeBufferBytes.WithLabelValues(k, f, "memory"), + DiskBytes: sinkmetrics.LakeBufferBytes.WithLabelValues(k, f, "disk"), + PendingPostEnqueue: sinkmetrics.LakeBufferEntries.WithLabelValues(k, f, "enqueue"), + DiskQuotaWaiters: sinkmetrics.LakeBufferEntries.WithLabelValues(k, f, "waiters"), + SegmentCount: sinkmetrics.LakeBufferEntries.WithLabelValues(k, f, "segments"), + DiskQuotaWait: sinkmetrics.LakeSpoolDuration.WithLabelValues(k, f), + LoadedBytes: sinkmetrics.LakeSpoolLoad.WithLabelValues(k, f), + RotatedCount: sinkmetrics.LakeSpoolRotations.WithLabelValues(k, f), + Close: func() { + for _, kind := range []string{"memory", "disk"} { + sinkmetrics.LakeBufferBytes.DeleteLabelValues(k, f, kind) + } + for _, kind := range []string{"enqueue", "waiters", "segments"} { + sinkmetrics.LakeBufferEntries.DeleteLabelValues(k, f, kind) + } + sinkmetrics.LakeSpoolDuration.DeleteLabelValues(k, f) + sinkmetrics.LakeSpoolLoad.DeleteLabelValues(k, f) + sinkmetrics.LakeSpoolRotations.DeleteLabelValues(k, f) + }, + } +} diff --git a/downstreamadapter/sink/lake/native.go b/downstreamadapter/sink/lake/native.go new file mode 100644 index 0000000000..dac524b6d9 --- /dev/null +++ b/downstreamadapter/sink/lake/native.go @@ -0,0 +1,263 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/chunk" +) + +func schemaFromNative(info *common.TableInfo, database, conflictKey string) (tableSchema, error) { + if info == nil { + return tableSchema{}, errors.ErrLakeInvalidData.GenWithStackByArgs("missing authoritative table metadata") + } + if info.IsPartitionTable() { + return tableSchema{}, errors.ErrLakeInvalidData.GenWithStackByArgs("partition dispatchers are outside the single-table writer proof; partition DDL is unsupported") + } + s := tableSchema{database: info.GetSchemaName(), table: info.GetTableName(), primaryKey: info.GetPrimaryKeyColumnNames(), source: info.GetSchemaName() + "." + info.GetTableName()} + if database != "" { + s.database = database + } + if strings.HasSuffix(s.table, "_cdc_raw") || strings.HasSuffix(s.table, "_cdc_schema_epoch") { + return tableSchema{}, errors.ErrLakeInvalidData.GenWithStackByArgs("source table uses a reserved CDC suffix") + } + for _, c := range info.GetColumns() { + if !common.IsColCDCVisible(c) { + continue + } + typ, enc, err := nativeColumnType(c) + if err != nil { + return tableSchema{}, err + } + def, expr, err := nativeDefault(c, enc) + if err != nil { + return tableSchema{}, err + } + s.columns = append(s.columns, column{name: c.Name.O, encoding: enc, sourceID: c.ID, typ: typ, nullable: !mysql.HasNotNullFlag(c.GetFlag()) || c.IsGenerated(), defaultSQL: def, defaultExpr: expr}) + } + if conflictKey != "" { + // Match bend-cdc's persisted single-column override. Never silently + // replace a composite key with only one of its members. + if len(s.primaryKey) > 1 { + return tableSchema{}, errors.ErrLakeInvalidData.GenWithStackByArgs("conflict-key cannot override a composite source key") + } + actual := "" + for _, col := range s.columns { + if strings.EqualFold(col.name, conflictKey) { + actual = col.name + break + } + } + if actual == "" { + return tableSchema{}, errors.ErrLakeInvalidData.GenWithStackByArgs("conflict-key must name a visible source column") + } + s.primaryKey = []string{actual} + } + return s, s.validate() +} + +func nativeColumnType(c *model.ColumnInfo) (string, columnEncoding, error) { + signed := func(a, b string) string { + if mysql.HasUnsignedFlag(c.GetFlag()) { + return b + } + return a + } + switch c.GetType() { + case mysql.TypeTiny: + return signed("INT8", "UINT8"), scalarColumn, nil + case mysql.TypeShort: + return signed("INT16", "UINT16"), scalarColumn, nil + case mysql.TypeInt24, mysql.TypeLong: + return signed("INT32", "UINT32"), scalarColumn, nil + case mysql.TypeLonglong: + return signed("INT64", "UINT64"), scalarColumn, nil + case mysql.TypeYear: + return "INT16", scalarColumn, nil + case mysql.TypeFloat: + return "FLOAT32", scalarColumn, nil + case mysql.TypeDouble: + return "FLOAT64", scalarColumn, nil + case mysql.TypeNewDecimal: + p, scale := c.GetFlen(), c.GetDecimal() + if p <= 0 || p > 76 || scale < 0 || scale > p { + return "", 0, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid decimal metadata") + } + return fmt.Sprintf("DECIMAL(%d,%d)", p, scale), scalarColumn, nil + case mysql.TypeDate, mysql.TypeNewDate: + return "DATE", scalarColumn, nil + case mysql.TypeDatetime, mysql.TypeTimestamp: + return "TIMESTAMP", scalarColumn, nil + case mysql.TypeDuration, mysql.TypeEnum, mysql.TypeSet: + return "VARCHAR", scalarColumn, nil + case mysql.TypeBit: + return "UINT64", scalarColumn, nil + case mysql.TypeJSON: + return "VARIANT", jsonColumn, nil + case mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: + if c.GetCharset() == "binary" { + return "BINARY", binaryColumn, nil + } + return "VARCHAR", scalarColumn, nil + default: + return "", 0, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported native column type") + } +} + +func nativeValues(info *common.TableInfo, row chunk.Row, location *time.Location) (map[string]any, error) { + if location == nil { + location = time.UTC + } + values := make(map[string]any, len(info.GetColumns())) + for _, col := range info.GetColumns() { + if !common.IsColCDCVisible(col) { + continue + } + offset, ok := info.GetRowColumnsOffset()[col.ID] + if !ok || offset >= row.Len() { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("row does not match authoritative schema") + } + d := row.GetDatum(offset, &col.FieldType) + var v any + if !d.IsNull() { + switch col.GetType() { + case mysql.TypeEnum: + v = d.GetMysqlEnum().Name + case mysql.TypeSet: + v = d.GetMysqlSet().Name + case mysql.TypeBit: + n, err := d.GetMysqlBit().ToInt(types.DefaultStmtNoWarningContext) + if err != nil { + return nil, errors.WrapError(errors.ErrLakeInvalidData, err, "invalid bit datum") + } + v = strconv.FormatUint(n, 10) + case mysql.TypeJSON: + v = d.GetMysqlJSON().String() + case mysql.TypeNewDecimal: + v = d.GetMysqlDecimal().String() + case mysql.TypeDate, mysql.TypeNewDate, mysql.TypeDatetime: + v = d.GetMysqlTime().String() + case mysql.TypeTimestamp: + stamp := d.GetMysqlTime() + if !stamp.IsZero() { + if err := stamp.ConvertTimeZone(location, time.UTC); err != nil { + return nil, errors.WrapError(errors.ErrLakeInvalidData, err, "cannot normalize timestamp timezone") + } + } + v = stamp.String() + case mysql.TypeDuration: + v = d.GetMysqlDuration().String() + case mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: + v = d.GetBytes() + default: + text, err := d.ToString() + if err != nil { + return nil, errors.WrapError(errors.ErrLakeInvalidData, err, "cannot format native datum") + } + v = text + } + } + values[col.Name.O] = v + } + return values, nil +} + +func normalizeChange(info *common.TableInfo, schema tableSchema, change commonEvent.RowChange, ts uint64, location *time.Location) (normalizedRow, error) { + action := rowAction(change.RowType.String()) + image := change.Row + if change.RowType == common.RowTypeDelete { + image = change.PreRow + } + values, err := nativeValues(info, image, location) + if err != nil { + return normalizedRow{}, err + } + row, err := normalizeNativeRow(schema, action, ts, values) + if err != nil { + return normalizedRow{}, err + } + if change.RowType == common.RowTypeUpdate { + before, err := nativeValues(info, change.PreRow, location) + if err != nil { + return normalizedRow{}, err + } + old, err := normalizeNativeRow(schema, deleteAction, ts, before) + if err != nil { + return normalizedRow{}, err + } + if old.id != row.id { + return normalizedRow{}, errors.ErrLakeInvalidData.GenWithStackByArgs("conflict key changed in an unsplit UPDATE; use a stable conflict-key or split change mode") + } + } + return row, nil +} + +func sqlString(s string) string { + return "'" + strings.ReplaceAll(strings.ReplaceAll(s, `\`, `\\`), "'", "''") + "'" +} + +func nativeDefault(c *model.ColumnInfo, encoding columnEncoding) (string, bool, error) { + value := c.GetDefaultValue() + if c.IsGenerated() || value == nil { + return "", c.DefaultIsExpr, nil + } + var text string + switch v := value.(type) { + case string: + text = v + case []byte: + text = string(v) + case bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + text = fmt.Sprint(v) + default: + return "", false, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported column default representation") + } + expr := c.DefaultIsExpr + if (c.GetType() == mysql.TypeTimestamp || c.GetType() == mysql.TypeDatetime) && strings.HasPrefix(strings.ToUpper(text), "CURRENT_TIMESTAMP") { + expr = true + } + if expr { + return "", true, nil + } // Initial DML carries every value explicitly. + if c.GetType() == mysql.TypeBit { + n, err := types.BinaryLiteral([]byte(text)).ToInt(types.DefaultStmtNoWarningContext) + if err != nil { + return "", false, errors.WrapError(errors.ErrLakeInvalidData, err, "invalid bit default") + } + return strconv.FormatUint(n, 10), false, nil + } + if encoding == binaryColumn { + return "unhex(" + sqlString(hex.EncodeToString([]byte(text))) + ")", false, nil + } + if encoding == jsonColumn { + if !json.Valid([]byte(text)) { + return "", false, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid JSON default") + } + return "parse_json(" + sqlString(text) + ")", false, nil + } + return sqlString(text), false, nil +} diff --git a/downstreamadapter/sink/lake/native_test.go b/downstreamadapter/sink/lake/native_test.go new file mode 100644 index 0000000000..ed262eec63 --- /dev/null +++ b/downstreamadapter/sink/lake/native_test.go @@ -0,0 +1,282 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "encoding/hex" + "encoding/json" + "strconv" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/chunk" + "github.com/stretchr/testify/require" +) + +func nativeTestInfo() *common.TableInfo { + id := types.NewFieldType(mysql.TypeLonglong) + id.SetFlag(mysql.PriKeyFlag | mysql.NotNullFlag) + value := types.NewFieldType(mysql.TypeVarchar) + value.SetCharset("utf8mb4") + return common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr("t"), PKIsHandle: true, Columns: []*model.ColumnInfo{{ID: 1, Name: ast.NewCIStr("id"), FieldType: *id, State: model.StatePublic}, {ID: 2, Name: ast.NewCIStr("value"), FieldType: *value, State: model.StatePublic}}}) +} + +func nativeTestEvent(info *common.TableInfo, ts uint64, n int) *commonEvent.DMLEvent { + e := commonEvent.NewDMLEvent(common.NewDispatcherID(), 1, ts-1, ts, info) + e.Rows = chunk.NewChunkWithCapacity(info.GetFieldSlice(), n) + for i := range n { + e.Rows.AppendInt64(0, int64(i)) + e.Rows.AppendString(1, "payload") + e.RowTypes = append(e.RowTypes, common.RowTypeInsert) + } + e.Length = int32(n) + return e +} + +func TestNativeDatumAdapterAndRawChangeContract(t *testing.T) { + info := nativeTestInfo() + enumType := types.NewFieldType(mysql.TypeEnum) + enumType.SetElems([]string{"2", "x"}) + info = common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr("t"), PKIsHandle: true, Columns: []*model.ColumnInfo{info.GetColumns()[0], {ID: 2, Name: ast.NewCIStr("value"), FieldType: *enumType, State: model.StatePublic}}}) + s, err := schemaFromNative(info, "", "") + require.NoError(t, err) + chk := chunk.NewChunkWithCapacity(info.GetFieldSlice(), 2) + chk.AppendInt64(0, 1) + chk.AppendEnum(1, types.Enum{Name: "2", Value: 1}) + chk.AppendInt64(0, 2) + chk.AppendEnum(1, types.Enum{Name: "x", Value: 2}) + row, err := normalizeChange(info, s, commonEvent.RowChange{Row: chk.GetRow(0), RowType: common.RowTypeInsert}, 100, time.UTC) + require.NoError(t, err) + var payload map[string]any + require.NoError(t, json.Unmarshal([]byte(row.payload), &payload)) + require.Equal(t, "2", payload["value"]) + _, err = normalizeChange(info, s, commonEvent.RowChange{Row: chk.GetRow(1), PreRow: chk.GetRow(0), RowType: common.RowTypeUpdate}, 100, time.UTC) + require.ErrorContains(t, err, "split change mode") + // The normal mode keeps split DELETE and INSERT as distinct, ordered actions. + del, err := normalizeChange(info, s, commonEvent.RowChange{PreRow: chk.GetRow(0), RowType: common.RowTypeDelete}, 100, time.UTC) + require.NoError(t, err) + ins, err := normalizeChange(info, s, commonEvent.RowChange{Row: chk.GetRow(1), RowType: common.RowTypeInsert}, 100, time.UTC) + require.NoError(t, err) + require.Equal(t, "1", del.id) + require.Equal(t, "2", ins.id) +} + +func TestNativeTypeMapping(t *testing.T) { + for _, tc := range []struct { + typ byte + unsigned bool + want string + }{ + {mysql.TypeTiny, false, "INT8"}, {mysql.TypeTiny, true, "UINT8"}, {mysql.TypeShort, false, "INT16"}, {mysql.TypeLong, true, "UINT32"}, {mysql.TypeLonglong, true, "UINT64"}, {mysql.TypeYear, false, "INT16"}, {mysql.TypeDuration, false, "VARCHAR"}, {mysql.TypeDate, false, "DATE"}, {mysql.TypeTimestamp, false, "TIMESTAMP"}, {mysql.TypeJSON, false, "VARIANT"}, + } { + ft := types.NewFieldType(tc.typ) + if tc.unsigned { + ft.SetFlag(mysql.UnsignedFlag) + } + typ, _, err := nativeColumnType(&model.ColumnInfo{FieldType: *ft}) + require.NoError(t, err) + require.Equal(t, tc.want, typ) + } + ft := types.NewFieldType(mysql.TypeNewDecimal) + ft.SetFlen(65) + ft.SetDecimal(30) + typ, _, err := nativeColumnType(&model.ColumnInfo{FieldType: *ft}) + require.NoError(t, err) + require.Equal(t, "DECIMAL(65,30)", typ) + ft = types.NewFieldType(mysql.TypeBlob) + ft.SetCharset("binary") + typ, enc, err := nativeColumnType(&model.ColumnInfo{FieldType: *ft}) + require.NoError(t, err) + require.Equal(t, "BINARY", typ) + require.Equal(t, binaryColumn, enc) +} + +func TestNativeDatumValuesPreserveExactRepresentations(t *testing.T) { + base := nativeTestInfo().GetColumns()[0] + makeColumn := func(id int64, name string, tp byte) *model.ColumnInfo { + ft := types.NewFieldType(tp) + return &model.ColumnInfo{ID: id, Name: ast.NewCIStr(name), FieldType: *ft, State: model.StatePublic} + } + binary := makeColumn(2, "binary", mysql.TypeBlob) + binary.SetCharset("binary") + decimal := makeColumn(3, "decimal", mysql.TypeNewDecimal) + decimal.SetFlen(65) + decimal.SetDecimal(30) + document := makeColumn(4, "document", mysql.TypeJSON) + set := makeColumn(5, "set_label", mysql.TypeSet) + elems := make([]string, 64) + for i := range elems { + elems[i] = strconv.Itoa(i) + } + set.SetElems(elems) + stamp := makeColumn(6, "stamp", mysql.TypeTimestamp) + stamp.SetDecimal(6) + null := makeColumn(7, "sql_null", mysql.TypeJSON) + jsonNull := makeColumn(8, "json_null", mysql.TypeJSON) + info := common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr("typed"), PKIsHandle: true, Columns: []*model.ColumnInfo{base, binary, decimal, document, set, stamp, null, jsonNull}}) + s, err := schemaFromNative(info, "", "") + require.NoError(t, err) + chk := chunk.NewChunkWithCapacity(info.GetFieldSlice(), 1) + chk.AppendInt64(0, 1) + data := make([]byte, 256) + for i := range data { + data[i] = byte(i) + } + chk.AppendBytes(1, data) + dec := new(types.MyDecimal) + require.NoError(t, dec.FromString([]byte("12345678901234567890.123456789"))) + chk.AppendMyDecimal(2, dec) + doc, err := types.ParseBinaryJSONFromString(`{"uint":18446744073709551615}`) + require.NoError(t, err) + chk.AppendJSON(3, doc) + chk.AppendSet(4, types.Set{Name: "63", Value: uint64(1) << 63}) + chk.AppendTime(5, types.NewTime(types.FromDate(2026, 10, 25, 1, 2, 3, 123456), mysql.TypeTimestamp, 6)) + chk.AppendNull(6) + nilDoc, err := types.ParseBinaryJSONFromString("null") + require.NoError(t, err) + chk.AppendJSON(7, nilDoc) + row, err := normalizeChange(info, s, commonEvent.RowChange{Row: chk.GetRow(0), RowType: common.RowTypeInsert}, 100, time.UTC) + require.NoError(t, err) + var payload map[string]any + require.NoError(t, json.Unmarshal([]byte(row.payload), &payload)) + require.Equal(t, hex.EncodeToString(data), payload["binary"]) + require.Equal(t, "12345678901234567890.123456789", payload["decimal"]) + require.Equal(t, doc.String(), payload["document"]) + require.Equal(t, "63", payload["set_label"]) + require.Equal(t, "2026-10-25 01:02:03.123456", payload["stamp"]) + require.Nil(t, payload["sql_null"]) + require.Equal(t, "null", payload["json_null"]) +} + +func TestNativeDefaultEncoding(t *testing.T) { + col := nativeTestInfo().GetColumns()[1].Clone() + col.DefaultValue = []byte("a'\\b") + def, expr, err := nativeDefault(col, scalarColumn) + require.NoError(t, err) + require.False(t, expr) + require.Equal(t, `'a''\\b'`, def) + def, expr, err = nativeDefault(col, binaryColumn) + require.NoError(t, err) + require.False(t, expr) + require.Equal(t, `unhex('61275c62')`, def) + ft := types.NewFieldType(mysql.TypeBit) + col.FieldType = *ft + require.NoError(t, col.SetDefaultValue(string([]byte{0xff, 0xff}))) + def, expr, err = nativeDefault(col, scalarColumn) + require.NoError(t, err) + require.False(t, expr) + require.Equal(t, "65535", def) + ft = types.NewFieldType(mysql.TypeTimestamp) + col.FieldType = *ft + col.DefaultValue = "CURRENT_TIMESTAMP" + def, expr, err = nativeDefault(col, scalarColumn) + require.NoError(t, err) + require.True(t, expr) + require.Empty(t, def) +} + +func TestNativeTimestampUsesSourceTimezoneButDatetimeKeepsWallClock(t *testing.T) { + id := nativeTestInfo().GetColumns()[0] + timestampType := types.NewFieldType(mysql.TypeTimestamp) + timestampType.SetDecimal(6) + datetimeType := types.NewFieldType(mysql.TypeDatetime) + datetimeType.SetDecimal(6) + info := common.WrapTableInfo("lake", &model.TableInfo{ID: 1, Name: ast.NewCIStr("time_values"), PKIsHandle: true, Columns: []*model.ColumnInfo{id, {ID: 2, Name: ast.NewCIStr("stamp"), FieldType: *timestampType, State: model.StatePublic}, {ID: 3, Name: ast.NewCIStr("wall"), FieldType: *datetimeType, State: model.StatePublic}}}) + schema, err := schemaFromNative(info, "", "") + require.NoError(t, err) + chk := chunk.NewChunkWithCapacity(info.GetFieldSlice(), 1) + chk.AppendInt64(0, 1) + chk.AppendTime(1, types.NewTime(types.FromDate(2026, 10, 25, 8, 2, 3, 456789), mysql.TypeTimestamp, 6)) + chk.AppendTime(2, types.NewTime(types.FromDate(2026, 10, 25, 8, 2, 3, 456789), mysql.TypeDatetime, 6)) + row, err := normalizeChange(info, schema, commonEvent.RowChange{Row: chk.GetRow(0), RowType: common.RowTypeInsert}, 100, time.FixedZone("UTC+8", 8*60*60)) + require.NoError(t, err) + var values map[string]any + require.NoError(t, json.Unmarshal([]byte(row.payload), &values)) + require.Equal(t, "2026-10-25 00:02:03.456789", values["stamp"]) + require.Equal(t, "2026-10-25 08:02:03.456789", values["wall"]) +} + +// TiCDC's GetPrimaryKeyColumnNames intentionally falls back to a NOT NULL +// UNIQUE index. Do not describe or accidentally narrow this to declared PKs. +func TestNativeImplicitUniqueConflictKey(t *testing.T) { + base := nativeTestInfo() + value := *base.GetColumns()[1] + value.Offset = 1 + id := *base.GetColumns()[0] + id.FieldType = *types.NewFieldType(mysql.TypeLonglong) + id.SetFlag(mysql.NotNullFlag) + info := common.WrapTableInfo("lake", &model.TableInfo{ + ID: 2, Name: ast.NewCIStr("implicit_key"), + Columns: []*model.ColumnInfo{&id, &value}, + Indices: []*model.IndexInfo{{ + ID: 1, Name: ast.NewCIStr("uk_id"), Unique: true, State: model.StatePublic, + Columns: []*model.IndexColumn{{Name: id.Name, Offset: 0, Length: types.UnspecifiedLength}}, + }}, + }) + schema, err := schemaFromNative(info, "", "") + require.NoError(t, err) + require.Equal(t, []string{"id"}, schema.primaryKey) + nullableID := id + nullableID.SetFlag(0) + info = common.WrapTableInfo("lake", &model.TableInfo{ + ID: 3, Name: ast.NewCIStr("nullable_unique"), Columns: []*model.ColumnInfo{&nullableID, &value}, + Indices: []*model.IndexInfo{{ + ID: 1, Name: ast.NewCIStr("uk_id"), Unique: true, State: model.StatePublic, + Columns: []*model.IndexColumn{{Name: id.Name, Offset: 0, Length: types.UnspecifiedLength}}, + }}, + }) + _, err = schemaFromNative(info, "", "") + require.Error(t, err, "nullable UNIQUE is not an implicit conflict key") +} + +func TestNativeConfiguredConflictKey(t *testing.T) { + base := nativeTestInfo() + id, value := *base.GetColumns()[0], *base.GetColumns()[1] + id.SetFlag(0) + value.Offset = 1 + info := common.WrapTableInfo("lake", &model.TableInfo{ID: 7, Name: ast.NewCIStr("keyless"), Columns: []*model.ColumnInfo{&id, &value}}) + _, err := schemaFromNative(info, "", "") + require.Error(t, err) + schema, err := schemaFromNative(info, "", "ID") + require.NoError(t, err) + require.Equal(t, []string{"id"}, schema.primaryKey) + row, err := normalizeNativeRow(schema, insertAction, 100, map[string]any{"id": 1, "value": "v"}) + require.NoError(t, err) + require.NotEmpty(t, row.id) + _, err = normalizeNativeRow(schema, insertAction, 100, map[string]any{"id": nil, "value": "v"}) + require.Error(t, err, "the configured column may be nullable in source metadata, but identity values must not be NULL") + _, err = schemaFromNative(info, "", "missing") + require.Error(t, err) + // The explicit single-column override also matches bend-cdc on keyed tables. + schema, err = schemaFromNative(base, "", "VALUE") + require.NoError(t, err) + require.Equal(t, []string{"value"}, schema.primaryKey) + composite := common.WrapTableInfo("lake", &model.TableInfo{ID: 8, Name: ast.NewCIStr("composite"), Columns: []*model.ColumnInfo{&id, &value}, Indices: []*model.IndexInfo{{ID: 1, Primary: true, Unique: true, State: model.StatePublic, Columns: []*model.IndexColumn{{Name: id.Name, Offset: 0}, {Name: value.Name, Offset: 1}}}}}) + _, err = schemaFromNative(composite, "", "id") + require.ErrorContains(t, err, "composite") + // Arbitrary logical keys are not necessarily split by TiCDC's PK/UK scanner. + // A mutated key in an unsplit UPDATE must stop instead of orphaning old rows. + schema, err = schemaFromNative(info, "", "id") + require.NoError(t, err) + event := nativeTestEvent(info, 100, 2) + _, err = normalizeChange(info, schema, commonEvent.RowChange{PreRow: event.Rows.GetRow(0), Row: event.Rows.GetRow(1), RowType: common.RowTypeUpdate}, 100, time.UTC) + require.ErrorContains(t, err, "stable conflict-key") +} diff --git a/downstreamadapter/sink/lake/row.go b/downstreamadapter/sink/lake/row.go new file mode 100644 index 0000000000..ab5f3fe92d --- /dev/null +++ b/downstreamadapter/sink/lake/row.go @@ -0,0 +1,245 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package lake owns the native Lake sink implementation. Its row and SQL +// layer is ported from bend-cdc's TiDB path, without the Canal JSON decoder. +package lake + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "math" + "sort" + "strings" + "unicode/utf8" + + "github.com/pingcap/ticdc/pkg/errors" +) + +type columnEncoding uint8 + +const ( + scalarColumn columnEncoding = iota + binaryColumn + jsonColumn +) + +type column struct { + name string + encoding columnEncoding + sourceID int64 + typ string + nullable bool + defaultSQL string + defaultExpr bool +} + +// tableSchema describes the target projection, not an inferred source schema. +type tableSchema struct { + database string + source string + table string + columns []column + primaryKey []string +} + +func (s tableSchema) validate() error { + validName := func(name string) bool { return name != "" && !strings.ContainsRune(name, 0) } + if !validName(s.database) || !validName(s.table) || len(s.columns) == 0 || len(s.primaryKey) == 0 { + return errors.ErrLakeInvalidData.GenWithStackByArgs("table metadata and a primary key are required") + } + columns := make(map[string]struct{}, len(s.columns)) + folded := make(map[string]struct{}, len(s.columns)) + for _, col := range s.columns { + if !validName(col.name) || col.encoding > jsonColumn { + return errors.ErrLakeInvalidData.GenWithStackByArgs("invalid column metadata") + } + lower := strings.ToLower(col.name) + if _, ok := folded[lower]; ok { + return errors.ErrLakeInvalidData.GenWithStackByArgs("duplicate column name") + } + folded[lower] = struct{}{} + columns[col.name] = struct{}{} + } + keys := make(map[string]struct{}, len(s.primaryKey)) + for _, key := range s.primaryKey { + if _, ok := columns[key]; !ok { + return errors.ErrLakeInvalidData.GenWithStackByArgs("primary key is missing from columns") + } + if _, ok := keys[key]; ok { + return errors.ErrLakeInvalidData.GenWithStackByArgs("duplicate primary key column") + } + keys[key] = struct{}{} + } + return nil +} + +func (s tableSchema) canonicalKeys() []string { + keys := append([]string(nil), s.primaryKey...) + sort.Slice(keys, func(i, j int) bool { return strings.ToLower(keys[i]) < strings.ToLower(keys[j]) }) + return keys +} + +type rowAction string + +const ( + insertAction rowAction = "insert" + updateAction rowAction = "update" + deleteAction rowAction = "delete" +) + +// normalizedRow is an owned, immutable raw payload. The SQL writer cannot +// accidentally run Canal ordinal/Latin-1 decoding on this representation again. +type normalizedRow struct { + id string + payload string + commitTs uint64 + action rowAction +} + +// normalizeNativeRow accepts labels for ENUM/SET, original bytes for BINARY and +// exact JSON documents. The DMLEvent adapter must extract these from typed datum +// values; this function deliberately does not infer ordinal encodings. +func normalizeNativeRow(schema tableSchema, action rowAction, commitTs uint64, values map[string]any) (normalizedRow, error) { + if err := schema.validate(); err != nil { + return normalizedRow{}, err + } + if commitTs == 0 || commitTs > math.MaxInt64 { + return normalizedRow{}, errors.ErrLakeInvalidData.GenWithStackByArgs("commit TSO cannot be represented in raw BIGINT") + } + if action != insertAction && action != updateAction && action != deleteAction { + return normalizedRow{}, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported row action") + } + data := make(map[string]any, len(values)) + for _, col := range schema.columns { + value, exists := values[col.name] + if !exists { + if action == deleteAction { + continue // A tombstone needs the old primary key, not an after image. + } + return normalizedRow{}, errors.ErrLakeInvalidData.GenWithStackByArgs("incomplete after image") + } + normalized, err := normalizeNativeValue(col.encoding, value) + if err != nil { + return normalizedRow{}, err + } + data[col.name] = normalized + } + for name := range values { + if _, exists := data[name]; !exists { + return normalizedRow{}, errors.ErrLakeInvalidData.GenWithStackByArgs("value has no column metadata") + } + } + id, err := conflictIdentity(data, schema.canonicalKeys()) + if err != nil { + return normalizedRow{}, err + } + payload, err := json.Marshal(data) + if err != nil { + return normalizedRow{}, errors.WrapError(errors.ErrLakeInvalidData, err, "cannot encode normalized row") + } + return normalizedRow{id: id, payload: string(payload), commitTs: commitTs, action: action}, nil +} + +func normalizeNativeValue(encoding columnEncoding, value any) (any, error) { + if value == nil { + return nil, nil + } + if encoding == binaryColumn { + switch v := value.(type) { + case []byte: + return hex.EncodeToString(v), nil + case string: + return hex.EncodeToString([]byte(v)), nil // Original bytes, not Canal Latin-1. + default: + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("binary datum must contain bytes") + } + } + if encoding == jsonColumn { + var document []byte + switch v := value.(type) { + case string: + document = []byte(v) + case []byte: + document = v + case json.RawMessage: + document = v + default: + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("JSON datum must contain an exact document") + } + if !json.Valid(document) { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid JSON document") + } + return string(document), nil // Includes JSON null, distinct from SQL NULL. + } + switch v := value.(type) { + case string: + if !utf8.ValidString(v) { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid text encoding") + } + return strings.Clone(v), nil + case []byte: + if !utf8.Valid(v) { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid text encoding") + } + return string(v), nil + case bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + return value, nil + case json.Number: + text := string(v) + if len(text) == 0 || !json.Valid([]byte(text)) || (text[0] != '-' && (text[0] < '0' || text[0] > '9')) { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid exact JSON number") + } + return json.Number(strings.Clone(string(v))), nil + case float32: + if !math.IsNaN(float64(v)) && !math.IsInf(float64(v), 0) { + return v, nil + } + case float64: + if !math.IsNaN(v) && !math.IsInf(v, 0) { + return v, nil + } + } + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported native scalar datum") +} + +// conflictIdentity preserves bend-cdc's single-key text and canonical composite +// JSON shape. It intentionally contains no arrival time, batch ID or local seq. +func conflictIdentity(values map[string]any, keys []string) (string, error) { + type keyPart struct { + Name string `json:"name"` + Value string `json:"value"` + } + parts := make([]keyPart, 0, len(keys)) + for _, key := range keys { + value, ok := values[key] + if !ok || value == nil { + return "", errors.ErrLakeInvalidData.GenWithStackByArgs("primary key is missing or null") + } + text := fmt.Sprint(value) + if len(keys) == 1 { + if strings.TrimSpace(text) == "" { + return "", errors.ErrLakeInvalidData.GenWithStackByArgs("empty primary key") + } + return text, nil + } + parts = append(parts, keyPart{Name: key, Value: text}) + } + encoded, err := json.Marshal(parts) + if err != nil { + return "", errors.WrapError(errors.ErrLakeInvalidData, err, "cannot encode primary key") + } + return string(encoded), nil +} diff --git a/downstreamadapter/sink/lake/row_test.go b/downstreamadapter/sink/lake/row_test.go new file mode 100644 index 0000000000..41d9e7ac69 --- /dev/null +++ b/downstreamadapter/sink/lake/row_test.go @@ -0,0 +1,140 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "encoding/hex" + "encoding/json" + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func testSchema() tableSchema { + return tableSchema{database: "lake", table: "orders", columns: []column{{name: "id"}, {name: "value"}}, primaryKey: []string{"id"}} +} + +func testRow(t *testing.T, action rowAction, ts uint64, id int) normalizedRow { + t.Helper() + row, err := normalizeNativeRow(testSchema(), action, ts, map[string]any{"id": id, "value": "payload"}) + require.NoError(t, err) + return row +} + +func TestNativeRowNormalizesExactlyOnce(t *testing.T) { + schema := tableSchema{ + database: "lake", table: "native_values", + columns: []column{ + {name: "id"}, + {name: "enum_label"}, + {name: "set_label"}, + {name: "binary", encoding: binaryColumn}, + {name: "document", encoding: jsonColumn}, + {name: "sql_null", encoding: jsonColumn}, + {name: "json_null", encoding: jsonColumn}, + {name: "decimal"}, + }, + primaryKey: []string{"id"}, + } + bytes := make([]byte, 256) + for i := range bytes { + bytes[i] = byte(i) + } + wantHex := hex.EncodeToString(bytes) + document := `{"uint":18446744073709551615,"decimal":12345678901234567890.123456789}` + row, err := normalizeNativeRow(schema, insertAction, 123, map[string]any{ + "id": uint64(math.MaxUint64), "enum_label": "2", "set_label": "last", + "binary": bytes, "document": json.RawMessage(document), "sql_null": nil, "json_null": "null", + "decimal": "12345678901234567890.123456789", + }) + require.NoError(t, err) + clear(bytes) // The payload must no longer reference source chunk bytes. + var values map[string]json.RawMessage + require.NoError(t, json.Unmarshal([]byte(row.payload), &values)) + require.JSONEq(t, `"2"`, string(values["enum_label"]), "a numeric ENUM label is not an ordinal") + require.JSONEq(t, `"last"`, string(values["set_label"])) + require.JSONEq(t, `"`+wantHex+`"`, string(values["binary"])) + require.Equal(t, "18446744073709551615", string(values["id"])) + require.Equal(t, "18446744073709551615", row.id) + var actualDocument string + require.NoError(t, json.Unmarshal(values["document"], &actualDocument)) + require.Equal(t, document, actualDocument) + require.Equal(t, "null", string(values["sql_null"])) + require.Equal(t, `"null"`, string(values["json_null"])) + require.JSONEq(t, `"12345678901234567890.123456789"`, string(values["decimal"])) +} + +func TestNativeRowRetainsCompositeIdentity(t *testing.T) { + schema := tableSchema{ + database: "lake", table: "events", + columns: []column{{name: "tenant"}, {name: "id"}}, primaryKey: []string{"tenant", "id"}, + } + row, err := normalizeNativeRow(schema, deleteAction, 100, map[string]any{"tenant": "a|b", "id": int64(42)}) + require.NoError(t, err) + require.Equal(t, `[{"name":"id","value":"42"},{"name":"tenant","value":"a|b"}]`, row.id) + schema.primaryKey = []string{"id", "tenant"} + replayed, err := normalizeNativeRow(schema, deleteAction, 100, map[string]any{"tenant": "a|b", "id": "42"}) + require.NoError(t, err) + require.Equal(t, row.id, replayed.id, "typed and Canal-style string keys retain the baseline identity") +} + +func TestNativeRowRejectsInvalidInputs(t *testing.T) { + for _, tc := range []struct { + name string + action rowAction + ts uint64 + values map[string]any + }{ + {"zero TSO", insertAction, 0, map[string]any{"id": 1, "value": "v"}}, + {"TSO overflow", insertAction, math.MaxUint64, map[string]any{"id": 1, "value": "v"}}, + {"unknown action", "replace", 1, map[string]any{"id": 1, "value": "v"}}, + {"missing after image", updateAction, 1, map[string]any{"id": 1}}, + {"null key", insertAction, 1, map[string]any{"id": nil, "value": "v"}}, + {"empty key", insertAction, 1, map[string]any{"id": "", "value": "v"}}, + {"extra column", insertAction, 1, map[string]any{"id": 1, "value": "v", "extra": 1}}, + {"non finite", insertAction, 1, map[string]any{"id": 1, "value": math.Inf(1)}}, + {"invalid number", insertAction, 1, map[string]any{"id": 1, "value": json.Number("not-a-number")}}, + {"invalid UTF8", insertAction, 1, map[string]any{"id": 1, "value": string([]byte{0xff})}}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := normalizeNativeRow(testSchema(), tc.action, tc.ts, tc.values) + require.Error(t, err) + }) + } + _, err := normalizeNativeRow(testSchema(), deleteAction, 1, map[string]any{"id": 1}) + require.NoError(t, err, "a tombstone need not invent a full after image") + for _, encoding := range []columnEncoding{binaryColumn, jsonColumn} { + _, err := normalizeNativeValue(encoding, 123) + require.Error(t, err) + } + _, err = normalizeNativeValue(jsonColumn, "not-json") + require.Error(t, err) +} + +func TestTableMetadataValidation(t *testing.T) { + for _, change := range []func(*tableSchema){ + func(s *tableSchema) { s.database = "" }, + func(s *tableSchema) { s.table = "bad\x00name" }, + func(s *tableSchema) { s.primaryKey = nil }, + func(s *tableSchema) { s.primaryKey = []string{"missing"} }, + func(s *tableSchema) { s.primaryKey = []string{"id", "id"} }, + func(s *tableSchema) { s.columns = append(s.columns, column{name: "ID"}) }, + } { + s := testSchema() + change(&s) + require.Error(t, s.validate()) + } +} diff --git a/downstreamadapter/sink/lake/runtime_test.go b/downstreamadapter/sink/lake/runtime_test.go new file mode 100644 index 0000000000..d7290b058c --- /dev/null +++ b/downstreamadapter/sink/lake/runtime_test.go @@ -0,0 +1,586 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/sink/spool" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/stretchr/testify/require" +) + +// apiFixture models only the HTTP boundary and journal persistence. It does not +// execute Lake SQL and must not be cited as proof of server atomicity. +type apiFixture struct { + mu sync.Mutex + columns []columnDef + epoch uint64 + applied bool + plan string + sql []string + rawRows int + merges int + truncates int + unknownMerge bool + unknownRawAt int + targetMissing bool + remoteBytes string + maxAddTime string + failAfterTruncate bool +} + +func newAPIFixture(t *testing.T) (*apiFixture, *httptest.Server) { + s, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + f := &apiFixture{columns: toSchemaDef(s).Columns} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + SQL string `json:"sql"` + Params []json.RawMessage `json:"params"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + w.WriteHeader(400) + return + } + f.mu.Lock() + defer f.mu.Unlock() + f.sql = append(f.sql, req.SQL) + data := [][]string{} + switch { + case strings.HasPrefix(req.SQL, "SELECT epoch"): + if f.epoch > 0 { + data = [][]string{{strconv.FormatUint(f.epoch, 10), strconv.FormatBool(f.applied), f.plan}} + } + case strings.HasPrefix(req.SQL, "SELECT COALESCE(SUM(data_compressed_size)"): + value := f.remoteBytes + if value == "" { + value = "0" + } + data = [][]string{{value}} + case strings.HasPrefix(req.SQL, "SELECT COUNT(*)::VARCHAR FROM system.tables"): + value := "1" + if f.targetMissing { + value = "0" + } + data = [][]string{{value}} + case strings.HasPrefix(req.SQL, "CREATE TABLE") && !strings.Contains(req.SQL, "_cdc_"): + if f.targetMissing { + f.targetMissing = false + var p schemaPlan + _ = json.Unmarshal([]byte(f.plan), &p) + if len(p.Schema.Columns) > 0 { + f.columns = p.Schema.Columns + } + } + case strings.HasPrefix(req.SQL, "DESCRIBE"): + for _, c := range f.columns { + nullable := "NO" + if c.Nullable { + nullable = "YES" + } + data = append(data, []string{c.Name, c.Type, nullable}) + } + case strings.HasPrefix(req.SQL, "MERGE") && strings.Contains(req.SQL, "_cdc_schema_epoch"): + var e uint64 + var a bool + var p string + _ = json.Unmarshal(req.Params[0], &e) + _ = json.Unmarshal(req.Params[1], &a) + _ = json.Unmarshal(req.Params[2], &p) + if f.failAfterTruncate && a { + f.failAfterTruncate = false + w.WriteHeader(503) + return + } + f.epoch, f.applied, f.plan = e, a, p + case strings.HasPrefix(req.SQL, "INSERT"): + f.rawRows += len(req.Params) / 6 + if f.unknownRawAt > 0 && f.rawRows >= f.unknownRawAt { + w.WriteHeader(503) + return + } + case strings.HasPrefix(req.SQL, "MERGE"): + if f.unknownMerge { + w.WriteHeader(503) + return + } + f.merges++ + case strings.HasPrefix(req.SQL, "TRUNCATE"): + f.truncates++ + f.rawRows = 0 + case strings.HasPrefix(req.SQL, "DROP TABLE") && strings.Contains(req.SQL, "_cdc_raw"): + f.rawRows = 0 + case strings.HasPrefix(req.SQL, "ALTER TABLE"): + var plan schemaPlan + _ = json.Unmarshal([]byte(f.plan), &plan) + f.columns = plan.Schema.Columns + case strings.HasPrefix(req.SQL, "SELECT COALESCE(MAX(add_time)"): + data = [][]string{{f.maxAddTime}} + case strings.HasPrefix(req.SQL, "SELECT ?"): + data = [][]string{{"1"}} + } + _ = json.NewEncoder(w).Encode(map[string]any{"id": r.Header.Get("X-DATABEND-QUERY-ID"), "state": "Succeeded", "data": data}) + })) + t.Cleanup(server.Close) + return f, server +} + +func runtimeSink(t *testing.T, server *httptest.Server, extra string) *Sink { + t.Helper() + uri, err := url.Parse(strings.Replace(server.URL, "http://", "lake+http://", 1) + "?exclusive=true&workers=1&flush-interval=1h&query-timeout=1s&spool-dir=" + url.QueryEscape(t.TempDir()) + extra) + require.NoError(t, err) + s, err := New(context.Background(), common.NewChangeFeedIDWithName("lake-test", "default"), &config.ChangefeedConfig{}, uri, 0) + require.NoError(t, err) + t.Cleanup(s.Close) + return s +} + +func runRuntime(t *testing.T, s *Sink) <-chan error { + t.Helper() + done := make(chan error, 1) + go func() { done <- s.Run(context.Background()) }() + return done +} + +func waitEnqueued(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatal("event not enqueued") + } +} + +func TestRuntimeBatchApplyAndDDLBarrier(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + done := runRuntime(t, s) + var flushed atomic.Int64 + for _, ts := range []uint64{100, 120} { + e := nativeTestEvent(nativeTestInfo(), ts, 2) + enqueued := make(chan struct{}) + e.PostTxnEnqueued = []func(){func() { close(enqueued) }} + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + waitEnqueued(t, enqueued) + } + require.Zero(t, flushed.Load(), "raw acceptance is not target application") + e := &commonEvent.DDLEvent{Type: byte(model.ActionAddIndex), TableInfo: nativeTestInfo(), FinishedTs: 150} + require.NoError(t, s.FlushDMLBeforeBlock(e)) + require.EqualValues(t, 2, flushed.Load()) + f.mu.Lock() + require.Equal(t, 4, f.rawRows) + require.Equal(t, 1, f.merges) + f.mu.Unlock() + require.NoError(t, s.WriteBlockEvent(e)) + f.mu.Lock() + require.EqualValues(t, 150, f.epoch) + require.True(t, f.applied) + for _, sql := range f.sql { + require.NotContains(t, sql, "STREAM") + } + f.mu.Unlock() + s.Close() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestRuntimeUnknownMergeFailsLifecycleWithoutFlush(t *testing.T) { + f, server := newAPIFixture(t) + f.unknownMerge = true + s := runtimeSink(t, server, "&batch-rows=1") + done := runRuntime(t, s) + var flushed atomic.Int64 + e := nativeTestEvent(nativeTestInfo(), 100, 1) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + select { + case err := <-done: + require.Error(t, err) + require.Equal(t, errors.ErrChangefeedUnretryable.RFCCode(), errors.ErrorCode(err)) + require.Contains(t, err.Error(), "request") + case <-time.After(5 * time.Second): + t.Fatal("unknown mutation did not terminate Run") + } + require.Zero(t, flushed.Load()) + require.False(t, s.IsNormal()) + f.mu.Lock() + n := len(f.sql) + f.mu.Unlock() + s.AddDMLEvent(nativeTestEvent(nativeTestInfo(), 120, 1)) + f.mu.Lock() + require.Len(t, f.sql, n) + f.mu.Unlock() +} + +func TestRuntimeLargeEventStreamsAndReclaimsBeforeSeal(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "&batch-bytes=1024&spool-bytes=4096&memory-ratio=0.01") + done := runRuntime(t, s) + var flushed atomic.Int64 + e := nativeTestEvent(nativeTestInfo(), 100, 150) + enqueued := make(chan struct{}) + e.PostTxnEnqueued = []func(){func() { close(enqueued) }} + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + waitEnqueued(t, enqueued) + require.NoError(t, s.FlushDMLBeforeBlock(&commonEvent.DDLEvent{})) + require.EqualValues(t, 1, flushed.Load()) + f.mu.Lock() + require.Equal(t, 150, f.rawRows) + require.Greater(t, f.merges, 1) + f.mu.Unlock() + s.Close() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestRuntimeFatalAdmissionAndCloseNeverFlush(t *testing.T) { + _, server := newAPIFixture(t) + t.Run("oversized event", func(t *testing.T) { + s := runtimeSink(t, server, "&source-bytes=1048576") + e := nativeTestEvent(nativeTestInfo(), 100, 100000) + var flushed atomic.Int64 + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + require.False(t, s.IsNormal()) + err := s.Run(context.Background()) + require.Error(t, err) + require.Equal(t, errors.ErrChangefeedUnretryable.RFCCode(), errors.ErrorCode(err)) + require.Zero(t, flushed.Load()) + }) + t.Run("close pending", func(t *testing.T) { + s := runtimeSink(t, server, "") + done := runRuntime(t, s) + e := nativeTestEvent(nativeTestInfo(), 100, 1) + var flushed atomic.Int64 + enqueued := make(chan struct{}) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + e.PostTxnEnqueued = []func(){func() { close(enqueued) }} + s.AddDMLEvent(e) + waitEnqueued(t, enqueued) + s.Close() + require.ErrorIs(t, <-done, context.Canceled) + require.Zero(t, flushed.Load()) + }) +} + +func TestSpoolPressureDrainsOtherTablesWithoutSuccessOnDiscard(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "&batch-bytes=1024&spool-bytes=4096&memory-ratio=0.01") + lane := s.lanes[0] + var flush atomic.Int64 + tracker := newEventCompletion(func() { flush.Add(1) }) + for i := range 40 { + schema := testSchema() + schema.table = "table" + strconv.Itoa(i) + writer, err := newTableWriter(schema, func(context.Context, string, []any) (mutationResult, error) { + return mutationResult{state: committed}, nil + }, 10, 1, 0) + require.NoError(t, err) + row := testRow(t, insertAction, 100, i) + data, err := json.Marshal([]spoolRow{{row.id, row.payload, row.commitTs, row.action}}) + require.NoError(t, err) + fragment, ok := tracker.register() + require.True(t, ok) + require.NoError(t, lane.enqueue(context.Background(), &tableState{schema: schema, writer: writer}, data, 1, fragment, nil)) + } + require.Zero(t, flush.Load(), "already applied OPEN fragments do not acknowledge the whole event") + tracker.seal() + lane.cleanup() + require.Zero(t, flush.Load(), "fatal cleanup cannot acknowledge retained fragments") + require.EqualValues(t, 0, spool.AccountingBytes(nil)) +} + +func TestConcurrentAdmissionAndCloseReleaseAllSources(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + done := runRuntime(t, s) + info := nativeTestInfo() + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { s.AddDMLEvent(nativeTestEvent(info, 100, 1)) }) + } + s.Close() + wg.Wait() + require.ErrorIs(t, <-done, context.Canceled) + s.mu.Lock() + defer s.mu.Unlock() + require.Zero(t, s.sourceBytes) + require.Zero(t, s.sourceEvents) +} + +func TestInternalTimeoutRemainsFatalNotLifecycleCancel(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + err := errors.WrapError(errors.ErrLakeExecution, context.DeadlineExceeded) + s.stop(err) + require.True(t, errors.Is(context.Cause(s.ctx), context.DeadlineExceeded)) + err = s.Run(context.Background()) + require.Error(t, err) + require.Equal(t, errors.ErrChangefeedUnretryable.RFCCode(), errors.ErrorCode(err)) +} + +func TestRuntimePartialRawCommitDoesNotConfirmOrRetry(t *testing.T) { + f, server := newAPIFixture(t) + f.unknownRawAt = 2 + s := runtimeSink(t, server, "&batch-rows=1") + done := runRuntime(t, s) + var flush atomic.Int64 + e := nativeTestEvent(nativeTestInfo(), 100, 3) + e.PostTxnFlushed = []func(){func() { flush.Add(1) }} + s.AddDMLEvent(e) + select { + case err := <-done: + require.Error(t, err) + case <-time.After(5 * time.Second): + t.Fatal("partial raw failure did not stop") + } + require.Zero(t, flush.Load()) + f.mu.Lock() + defer f.mu.Unlock() + require.Equal(t, 2, f.rawRows, "unknown raw SQL must not be resent or continue to the third fragment") + require.Equal(t, 1, f.merges, "only the first fragment had a confirmed MERGE") +} + +func TestRuntimeUsesRetainedTimestampAfterRestart(t *testing.T) { + f, server := newAPIFixture(t) + f.maxAddTime = "2099-01-02 03:04:05.999999" + s := runtimeSink(t, server, "") + info := nativeTestInfo() + schema, err := schemaFromNative(info, "", "") + require.NoError(t, err) + state, skip, err := s.lanes[0].schemas.prepareDML(context.Background(), schema, 100) + require.NoError(t, err) + require.False(t, skip) + require.Equal(t, "2099-01-02 03:04:05.999999", state.writer.clock.last.Format("2006-01-02 15:04:05.000000")) + _, args := schema.insertRawSQL([]normalizedRow{testRow(t, deleteAction, 100, 1)}, &state.writer.clock, time.Now()) + require.Equal(t, "2099-01-02 03:04:06.000000", args[3], "recovery must not allocate a lower same-TSO tie breaker") +} + +func TestRuntimeLowTrafficAndTransportEpochDoesNotDiscardAdmittedData(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + s.opts.flushInterval = 10 * time.Millisecond + done := runRuntime(t, s) + var flushed atomic.Int64 + for _, epoch := range []uint64{1, 2} { + e := nativeTestEvent(nativeTestInfo(), 100+epoch, 1) + e.Epoch = epoch + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + } + require.Eventually(t, func() bool { return flushed.Load() == 2 }, 5*time.Second, time.Millisecond) + s.Close() + require.ErrorIs(t, <-done, context.Canceled) + require.EqualValues(t, 2, flushed.Load()) +} + +func TestRemoteRawMonitoringThresholdStopsBeforeNextInsert(t *testing.T) { + f, server := newAPIFixture(t) + f.remoteBytes = "100" + s := runtimeSink(t, server, "&remote-bytes-limit=100&batch-rows=1") + done := runRuntime(t, s) + var flushed atomic.Int64 + e := nativeTestEvent(nativeTestInfo(), 100, 1) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + select { + case err := <-done: + require.ErrorContains(t, err, "monitoring threshold") + case <-time.After(5 * time.Second): + t.Fatal("remote threshold did not stop admission") + } + require.Zero(t, flushed.Load()) + f.mu.Lock() + defer f.mu.Unlock() + require.Zero(t, f.rawRows) +} + +func TestRuntimeConfiguredConflictKeyReachesDMLAndJournal(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "&conflict-key=VaLuE") + done := runRuntime(t, s) + info := nativeTestInfo() + ddl := &commonEvent.DDLEvent{Type: byte(model.ActionCreateTable), TableInfo: info, FinishedTs: 90} + require.NoError(t, s.WriteBlockEvent(ddl)) + var flushed atomic.Int64 + e := nativeTestEvent(info, 100, 1) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + s.AddDMLEvent(e) + require.NoError(t, s.FlushDMLBeforeBlock(nil)) + require.EqualValues(t, 1, flushed.Load()) + f.mu.Lock() + var plan schemaPlan + require.NoError(t, json.Unmarshal([]byte(f.plan), &plan)) + require.Equal(t, []string{"value"}, plan.Schema.Keys) + require.Equal(t, 1, f.rawRows) + f.mu.Unlock() + s.Close() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestRuntimeSkippedDDLDrainsAndContinues(t *testing.T) { + for _, action := range []model.ActionType{model.ActionRenameTable, model.ActionRenameTables, model.ActionDropSchema} { + t.Run(action.String(), func(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + done := runRuntime(t, s) + var dml, ddl atomic.Int64 + e := nativeTestEvent(nativeTestInfo(), 100, 1) + e.PostTxnFlushed = []func(){func() { dml.Add(1) }} + s.AddDMLEvent(e) + // No TableInfo is intentional: DB/multi-table events need no new + // metadata/progress protocol just to be explicitly skipped. + ignored := &commonEvent.DDLEvent{Type: byte(action), SchemaName: "lake", TableName: "t", FinishedTs: 150, PostTxnFlushed: []func(){func() { ddl.Add(1) }}} + require.NoError(t, s.WriteBlockEvent(ignored)) + require.EqualValues(t, 1, dml.Load(), "prior DML must drain before skipped DDL callback") + require.EqualValues(t, 1, ddl.Load()) + f.mu.Lock() + require.Zero(t, f.epoch, "skipping must not invent a target schema epoch") + require.Equal(t, 1, f.merges) + before := len(f.sql) + f.mu.Unlock() + require.NoError(t, s.WriteBlockEvent(&commonEvent.DDLEvent{Type: byte(action), FinishedTs: 150})) + f.mu.Lock() + require.Len(t, f.sql, before, "no SQL must be executed for the skipped event itself") + f.mu.Unlock() + e = nativeTestEvent(nativeTestInfo(), 160, 1) + e.PostTxnFlushed = []func(){func() { dml.Add(1) }} + s.AddDMLEvent(e) + require.NoError(t, s.FlushDMLBeforeBlock(nil)) + require.EqualValues(t, 2, dml.Load()) + require.True(t, s.IsNormal()) + s.Close() + require.ErrorIs(t, <-done, context.Canceled) + }) + } +} + +func TestSkippedDDLDoesNotBypassUnknownDML(t *testing.T) { + f, server := newAPIFixture(t) + f.unknownMerge = true + s := runtimeSink(t, server, "") + done := runRuntime(t, s) + s.AddDMLEvent(nativeTestEvent(nativeTestInfo(), 100, 1)) + var flushed atomic.Int64 + ddl := &commonEvent.DDLEvent{Type: byte(model.ActionDropSchema), FinishedTs: 150, PostTxnFlushed: []func(){func() { flushed.Add(1) }}} + require.Error(t, s.WriteBlockEvent(ddl)) + require.Zero(t, flushed.Load()) + require.Error(t, <-done) + s.Close() +} + +func TestRuntimeSharedAdmissionBackpressure(t *testing.T) { + for _, budget := range []string{"events", "bytes", "lane queue"} { + t.Run(budget, func(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "&source-events=2&batch-bytes=1024&spool-bytes=4096") + var flushed atomic.Int64 + first := nativeTestEvent(nativeTestInfo(), 100, 1) + second := nativeTestEvent(nativeTestInfo(), 120, 1) + first.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + second.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + wantFlushed := int64(2) + charge := sourceEventBytes(first) + require.Equal(t, charge, sourceEventBytes(second)) + switch budget { + case "events": + s.opts.sourceEvents = 1 + s.AddDMLEvent(first) + case "bytes": + s.opts.sourceBytes = charge + charge/2 + s.AddDMLEvent(first) + case "lane queue": + // Existing barrier items are not part of the DML source pool. + for range cap(s.lanes[0].input) { + s.lanes[0].input <- workItem{done: make(chan error, 1)} + } + wantFlushed = 1 + } + admitted := make(chan struct{}) + go func() { + s.AddDMLEvent(second) + close(admitted) + }() + select { + case <-admitted: + t.Fatal("full shared admission must wait, not fail or bypass quota") + case <-time.After(50 * time.Millisecond): + } + require.True(t, s.IsNormal(), "ordinary pressure must not stop the feed") + require.Zero(t, flushed.Load()) + s.mu.Lock() + bytes, events := s.sourceBytes, s.sourceEvents + s.mu.Unlock() + require.LessOrEqual(t, bytes, s.opts.sourceBytes) + require.LessOrEqual(t, events, s.opts.sourceEvents) + done := runRuntime(t, s) + waitEnqueued(t, admitted) + require.NoError(t, s.FlushDMLBeforeBlock(nil)) + require.Equal(t, wantFlushed, flushed.Load()) + s.Close() + require.ErrorIs(t, <-done, context.Canceled) + }) + } +} + +func TestRuntimeAdmissionWaitStopsWithoutFlush(t *testing.T) { + for _, fatal := range []bool{false, true} { + t.Run(strconv.FormatBool(fatal), func(t *testing.T) { + _, server := newAPIFixture(t) + s := runtimeSink(t, server, "&source-events=1") + var flushed atomic.Int64 + for i := range 2 { + e := nativeTestEvent(nativeTestInfo(), uint64(100+i), 1) + e.PostTxnFlushed = []func(){func() { flushed.Add(1) }} + if i == 0 { + s.AddDMLEvent(e) + continue + } + returned := make(chan struct{}) + go func() { s.AddDMLEvent(e); close(returned) }() + select { + case <-returned: + t.Fatal("source pool must apply backpressure") + case <-time.After(50 * time.Millisecond): + } + if fatal { + s.stop(errors.ErrLakeExecution.GenWithStackByArgs()) + } + s.Close() + waitEnqueued(t, returned) + } + require.Zero(t, flushed.Load()) + s.mu.Lock() + require.Zero(t, s.sourceBytes) + require.Zero(t, s.sourceEvents) + s.mu.Unlock() + }) + } +} diff --git a/downstreamadapter/sink/lake/schema.go b/downstreamadapter/sink/lake/schema.go new file mode 100644 index 0000000000..927255f768 --- /dev/null +++ b/downstreamadapter/sink/lake/schema.go @@ -0,0 +1,707 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "context" + "encoding/json" + "fmt" + "math" + "reflect" + "strconv" + "strings" + "time" + + "github.com/pingcap/log" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + // Install TiDB AST literal constructors used by parser.New below. + _ "github.com/pingcap/tidb/pkg/types/parser_driver" + "go.uber.org/zap" +) + +// The only durable journal is the existing per-table schema epoch. Plan stores +// translated, bounded column operations so a pending DDL can resume before DML. +// It is not an applied DML ledger or an owner-fencing mechanism. +type columnDef struct { + Name string `json:"name"` + Type string `json:"type"` + Nullable bool `json:"nullable"` + Default string `json:"default,omitempty"` + Encoding columnEncoding `json:"encoding"` + SourceID int64 `json:"source_id"` + DefaultExpr bool `json:"default_expr,omitempty"` +} +type schemaDef struct { + Database string `json:"database"` + Table string `json:"table"` + Source string `json:"source"` + Columns []columnDef `json:"columns"` + Keys []string `json:"keys"` +} +type schemaStep struct { + Kind string `json:"kind"` + OldName string `json:"old_name,omitempty"` + Column columnDef `json:"column"` +} +type schemaPlan struct { + Kind string `json:"kind"` + Schema schemaDef `json:"schema"` + Steps []schemaStep `json:"steps,omitempty"` +} +type epochRecord struct { + epoch uint64 + applied bool + plan schemaPlan +} +type tableState struct { + schema tableSchema + epoch uint64 + writer *tableWriter + dropped bool +} +type schemaManager struct { + client *httpClient + opts options + tables map[string]*tableState +} + +func newSchemaManager(c *httpClient, o options) *schemaManager { + return &schemaManager{client: c, opts: o, tables: make(map[string]*tableState)} +} +func schemaKey(s tableSchema) string { return s.database + "\x00" + s.table } +func toSchemaDef(s tableSchema) schemaDef { + d := schemaDef{Database: s.database, Table: s.table, Source: s.source, Keys: s.primaryKey} + for _, c := range s.columns { + d.Columns = append(d.Columns, columnDef{c.name, c.typ, c.nullable, c.defaultSQL, c.encoding, c.sourceID, c.defaultExpr}) + } + return d +} + +func (d schemaDef) schema() tableSchema { + s := tableSchema{database: d.Database, table: d.Table, source: d.Source, primaryKey: d.Keys} + for _, c := range d.Columns { + s.columns = append(s.columns, column{name: c.Name, typ: c.Type, nullable: c.Nullable, defaultSQL: c.Default, encoding: c.Encoding, sourceID: c.SourceID, defaultExpr: c.DefaultExpr}) + } + return s +} + +func (c columnDef) definition() string { + result := quoteIdentifier(c.Name) + " " + c.Type + if c.Nullable { + result += " NULL" + } else { + result += " NOT NULL" + } + if c.Default != "" { + result += " DEFAULT " + c.Default + } + return result +} + +func (s tableSchema) createTargetSQL() string { + cols := make([]string, 0, len(s.columns)) + for _, c := range toSchemaDef(s).Columns { + cols = append(cols, c.definition()) + } + return "CREATE TABLE IF NOT EXISTS " + s.targetName() + " (" + strings.Join(cols, ", ") + ")" +} + +func (s tableSchema) journalName() string { + return quoteIdentifier(s.database) + "." + quoteIdentifier(s.table+"_cdc_schema_epoch") +} + +func (m *schemaManager) exec(ctx context.Context, sql string, args ...any) error { + result, err := m.client.mutate(ctx, sql, args) + if result.state == commitUnknown { + if err == nil { + err = errors.ErrLakeCommitUnknown.GenWithStackByArgs() + } else { + err = errors.WrapError(errors.ErrLakeCommitUnknown, err) + } + return errors.WrapChangefeedUnretryableErr(err) + } + if err != nil { + return err + } + if result.state != committed { + return errors.ErrLakeExecution.GenWithStackByArgs() + } + return nil +} + +func (m *schemaManager) loadEpoch(ctx context.Context, s tableSchema) (epochRecord, error) { + if err := m.exec(ctx, "CREATE DATABASE IF NOT EXISTS "+quoteIdentifier(s.database)); err != nil { + return epochRecord{}, err + } + if err := m.exec(ctx, "CREATE TABLE IF NOT EXISTS "+s.journalName()+" (epoch BIGINT, applied BOOLEAN, plan VARCHAR)"); err != nil { + return epochRecord{}, err + } + rows, err := m.client.selectRows(ctx, "SELECT epoch, applied, plan FROM "+s.journalName(), nil) + if err != nil { + return epochRecord{}, err + } + if len(rows) == 0 { + return epochRecord{}, nil + } + if len(rows) != 1 || len(rows[0]) != 3 || rows[0][0] == nil || rows[0][1] == nil || rows[0][2] == nil { + return epochRecord{}, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid schema epoch journal") + } + e, err := strconv.ParseUint(*rows[0][0], 10, 64) + if err != nil || e == 0 || e > math.MaxInt64 { + return epochRecord{}, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid applied schema epoch") + } + applied, err := strconv.ParseBool(*rows[0][1]) + if err != nil { + return epochRecord{}, errors.WrapError(errors.ErrLakeInvalidData, err, "invalid epoch status") + } + r := epochRecord{epoch: e, applied: applied} + if err := json.Unmarshal([]byte(*rows[0][2]), &r.plan); err != nil { + return epochRecord{}, errors.WrapError(errors.ErrLakeInvalidData, err, "invalid epoch plan") + } + saved := r.plan.Schema.schema() + if err := saved.validate(); err != nil { + return epochRecord{}, err + } + if saved.database != s.database || saved.table != s.table || saved.source != s.source { + return epochRecord{}, errors.ErrLakeInvalidData.GenWithStackByArgs("target belongs to another source or schema journal") + } + return r, nil +} + +func (m *schemaManager) saveEpoch(ctx context.Context, s tableSchema, r epochRecord) error { + plan, err := json.Marshal(r.plan) + if err != nil { + return errors.WrapError(errors.ErrLakeInvalidData, err, "invalid schema plan") + } + return m.exec(ctx, "MERGE INTO "+s.journalName()+" t USING (SELECT ?::BIGINT AS epoch, ?::BOOLEAN AS applied, ?::VARCHAR AS plan) s ON true WHEN MATCHED THEN UPDATE * WHEN NOT MATCHED THEN INSERT *", int64(r.epoch), r.applied, string(plan)) +} + +func canonicalType(t string) string { + t = strings.ToUpper(strings.ReplaceAll(t, " ", "")) + if inner, ok := strings.CutPrefix(t, "NULLABLE("); ok { + t = strings.TrimSuffix(inner, ")") + } + switch t { + case "TINYINTUNSIGNED": + return "UINT8" + case "SMALLINTUNSIGNED": + return "UINT16" + case "INTUNSIGNED", "INTEGERUNSIGNED": + return "UINT32" + case "BIGINTUNSIGNED": + return "UINT64" + case "TINYINT": + return "INT8" + case "SMALLINT": + return "INT16" + case "INT", "INTEGER": + return "INT32" + case "BIGINT": + return "INT64" + case "STRING", "TEXT": + return "VARCHAR" + case "JSON": + return "VARIANT" + case "FLOAT": + return "FLOAT32" + case "DOUBLE": + return "FLOAT64" + } + return t +} + +func (m *schemaManager) targetExists(ctx context.Context, s tableSchema) (bool, error) { + rows, err := m.client.selectRows(ctx, "SELECT COUNT(*)::VARCHAR FROM system.tables WHERE database = ? AND name = ?", []any{s.database, s.table}) + if err != nil { + return false, err + } + count, err := parseRemoteBytes(rows) + if err != nil { + return false, err + } + if count > 1 { + return false, errors.ErrLakeInvalidData.GenWithStackByArgs("ambiguous target table metadata") + } + return count == 1, nil +} + +func (m *schemaManager) describe(ctx context.Context, s tableSchema) (map[string]columnDef, error) { + rows, err := m.client.selectRows(ctx, "DESCRIBE "+s.targetName(), nil) + if err != nil { + return nil, err + } + cols := make(map[string]columnDef, len(rows)) + for _, r := range rows { + if len(r) < 3 || r[0] == nil || r[1] == nil || r[2] == nil { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("incomplete target column metadata") + } + name := *r[0] + cols[name] = columnDef{Name: name, Type: canonicalType(*r[1]), Nullable: strings.EqualFold(*r[2], "YES") || strings.HasPrefix(strings.ToUpper(*r[1]), "NULLABLE(")} + } + return cols, nil +} + +func (m *schemaManager) verifyTarget(ctx context.Context, s tableSchema) error { + cols, err := m.describe(ctx, s) + if err != nil { + return err + } + if len(cols) != len(s.columns) { + return errors.ErrLakeInvalidData.GenWithStackByArgs("target column set differs from source metadata") + } + for _, c := range s.columns { + actual, ok := cols[c.name] + if !ok || canonicalType(actual.Type) != canonicalType(c.typ) || actual.Nullable != c.nullable { + return errors.ErrLakeInvalidData.GenWithStackByArgs("target type or nullability differs from source projection") + } + } + return nil +} + +func (m *schemaManager) initialize(ctx context.Context, s tableSchema) (*tableState, error) { + if state := m.tables[schemaKey(s)]; state != nil { + return state, nil + } + if len(m.tables) >= m.opts.maxEntries { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("table metadata capacity exhausted") + } + r, err := m.loadEpoch(ctx, s) + if err != nil { + return nil, err + } + if r.epoch > 0 && !r.applied { + if err := m.finishPlan(ctx, r); err != nil { + return nil, err + } + r.applied = true + } + effective := s + dropped := false + if r.epoch > 0 { + effective = r.plan.Schema.schema() + dropped = r.plan.Kind == "drop" + } + if !dropped { + if err := m.exec(ctx, effective.createTargetSQL()); err != nil { + return nil, err + } + if err := m.verifyTarget(ctx, effective); err != nil { + return nil, err + } + if err := m.exec(ctx, effective.createRawSQL()); err != nil { + return nil, err + } + } + writer, err := newTableWriter(effective, m.client.mutate, m.opts.batchRows, 3, r.epoch) + if err != nil { + return nil, err + } + if !dropped { + rows, err := m.client.selectRows(ctx, "SELECT COALESCE(MAX(add_time)::VARCHAR, '') FROM "+effective.rawName(), nil) + if err != nil { + return nil, err + } + if len(rows) != 1 || len(rows[0]) != 1 || rows[0][0] == nil { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid retained raw time watermark") + } + if text := *rows[0][0]; text != "" { + stamp, err := time.ParseInLocation("2006-01-02 15:04:05.999999", text, time.UTC) + if err != nil { + return nil, errors.WrapError(errors.ErrLakeInvalidData, err, "invalid retained raw timestamp") + } + writer.clock.last = stamp + } + } + state := &tableState{schema: effective, epoch: r.epoch, writer: writer, dropped: dropped} + m.tables[schemaKey(s)] = state + return state, nil +} + +func (m *schemaManager) prepareDML(ctx context.Context, s tableSchema, ts uint64) (*tableState, bool, error) { + state, err := m.initialize(ctx, s) + if err != nil { + return nil, false, err + } + if ts <= state.epoch { + return state, true, nil + } + if state.dropped || !reflect.DeepEqual(toSchemaDef(state.schema), toSchemaDef(s)) { + return nil, false, errors.ErrLakeInvalidData.GenWithStackByArgs("DML schema is not the applied target schema; replay its DDL first") + } + return state, false, nil +} + +func allowedModify(from, to columnDef) bool { + a, b := canonicalType(from.Type), canonicalType(to.Type) + // Match the existing conservative widening family. No implicit narrowing or + // binary/JSON conversions are inferred from a successful server cast. + return a == b || a == "INT8" && (b == "INT32" || b == "INT64") || a == "INT32" && (b == "INT64" || b == "VARCHAR") +} + +func makeColumnSteps(query string, s tableSchema, current map[string]columnDef) ([]schemaStep, error) { + stmt, err := parser.New().ParseOneStmt(query, "", "") + if err != nil { + return nil, errors.WrapError(errors.ErrLakeInvalidData, err, "cannot parse incremental DDL") + } + alter, ok := stmt.(*ast.AlterTableStmt) + if !ok { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported column DDL") + } + desired := make(map[string]columnDef) + for _, c := range toSchemaDef(s).Columns { + desired[strings.ToLower(c.Name)] = c + } + foldedCurrent := make(map[string]columnDef, len(current)) + for name, c := range current { + foldedCurrent[strings.ToLower(name)] = c + } + current = foldedCurrent + isKey := func(name string) bool { + for _, k := range s.primaryKey { + if strings.EqualFold(k, name) { + return true + } + } + return false + } + parameterizedCharacterSafe := func(oldName string, def *ast.ColumnDef) bool { + if desired[strings.ToLower(def.Name.Name.O)].Type != "VARCHAR" || def.Tp.GetFlen() <= 0 { + return true + } + // Preserve bend-cdc's INT -> VARCHAR widening without opening arbitrary + // character narrowing. Every signed INT32 fits in 11 decimal characters. + // Same-target VARCHAR changes still cannot prove old source width. + return canonicalType(current[strings.ToLower(oldName)].Type) == "INT32" && + def.Tp.GetType() == mysql.TypeVarchar && def.Tp.GetFlen() >= 11 + } + var steps []schemaStep + for _, spec := range alter.Specs { + add := func(kind, old, name string) error { + oldKey, nameKey := strings.ToLower(old), strings.ToLower(name) + c := desired[nameKey] + if (kind == "add" || kind == "modify") && c.DefaultExpr { + return errors.ErrLakeInvalidData.GenWithStackByArgs("expression default DDL is unsupported") + } + if kind == "drop" { + c = columnDef{Name: name} + } else if c.Name == "" { + return errors.ErrLakeInvalidData.GenWithStackByArgs("DDL references an invisible or missing column") + } + if (kind == "drop" || kind == "rename") && (isKey(old) || isKey(name)) { + return errors.ErrLakeInvalidData.GenWithStackByArgs("changing a conflict key is unsupported") + } + prev, present := current[oldKey] + if present { + old = prev.Name + } + if kind == "drop" { + c.Name = old + } + switch kind { + case "add": + if _, ok := current[nameKey]; ok { + return errors.ErrLakeInvalidData.GenWithStackByArgs("column already exists before new DDL") + } + if !c.Nullable && c.Default == "" { + return errors.ErrLakeInvalidData.GenWithStackByArgs("ADD NOT NULL requires a literal source default") + } + case "drop", "rename", "modify": + if !present { + return errors.ErrLakeInvalidData.GenWithStackByArgs("old column is missing before new DDL") + } + } + if kind == "modify" && (!allowedModify(prev, c) || (isKey(old) && len(s.primaryKey) > 1)) { + return errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported type transition or composite key modification") + } + if kind == "rename" { + if _, ok := current[nameKey]; ok { + return errors.ErrLakeInvalidData.GenWithStackByArgs("rename target already exists") + } + delete(current, oldKey) + } + switch kind { + case "drop": + delete(current, oldKey) + case "rename": + prev.Name = c.Name + current[nameKey] = prev // RENAME does not also change the old type. + default: + current[nameKey] = c + } + steps = append(steps, schemaStep{Kind: kind, OldName: old, Column: c}) + return nil + } + switch spec.Tp { + case ast.AlterTableAddColumns: + for _, c := range spec.NewColumns { + explicitDefault := false + for _, opt := range c.Options { + if opt.Tp == ast.ColumnOptionDefaultValue { + explicitDefault = true + } + } + col := desired[strings.ToLower(c.Name.Name.O)] + if !explicitDefault && !col.Nullable && col.Type == "VARCHAR" && col.Default == "" { + col.Default = "''" + desired[strings.ToLower(c.Name.Name.O)] = col + } + if err := add("add", "", c.Name.Name.O); err != nil { + return nil, err + } + } + case ast.AlterTableDropColumn: + if err := add("drop", spec.OldColumnName.Name.O, spec.OldColumnName.Name.O); err != nil { + return nil, err + } + case ast.AlterTableRenameColumn: + if err := add("rename", spec.OldColumnName.Name.O, spec.NewColumnName.Name.O); err != nil { + return nil, err + } + case ast.AlterTableModifyColumn: + if len(spec.NewColumns) != 1 { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid MODIFY") + } + def := spec.NewColumns[0] + if !parameterizedCharacterSafe(def.Name.Name.O, def) { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("parameterized character MODIFY may narrow source data") + } + name := spec.NewColumns[0].Name.Name.O + if err := add("modify", name, name); err != nil { + return nil, err + } + case ast.AlterTableChangeColumn: + if len(spec.NewColumns) != 1 { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid CHANGE") + } + old, name := spec.OldColumnName.Name.O, spec.NewColumns[0].Name.Name.O + if !parameterizedCharacterSafe(old, spec.NewColumns[0]) { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("parameterized character CHANGE may narrow source data") + } + if old != name { + if err := add("rename", old, name); err != nil { + return nil, err + } + } + if err := add("modify", name, name); err != nil { + return nil, err + } + case ast.AlterTableAddConstraint: + if spec.Constraint == nil || spec.Constraint.Tp == ast.ConstraintPrimaryKey { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("changing primary keys is unsupported") + } + case ast.AlterTableDropIndex, ast.AlterTableRenameIndex, ast.AlterTableIndexInvisible: // Secondary indexes have no target projection. + default: + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported DDL; target has not been changed") + } + } + return steps, nil +} + +func (m *schemaManager) finishPlan(ctx context.Context, r epochRecord) error { + s := r.plan.Schema.schema() + switch r.plan.Kind { + case "drop": + if err := m.exec(ctx, "DROP TABLE IF EXISTS "+s.rawName()); err != nil { + return err + } + if err := m.exec(ctx, "DROP TABLE IF EXISTS "+s.targetName()); err != nil { + return err + } + case "truncate", "create", "alter", "index": + if err := m.exec(ctx, s.createTargetSQL()); err != nil { + return err + } + if r.plan.Kind == "truncate" { + if err := m.exec(ctx, "DROP TABLE IF EXISTS "+s.rawName()); err != nil { + return err + } + if err := m.exec(ctx, "TRUNCATE TABLE "+s.targetName()); err != nil { + return err + } + } + for _, step := range r.plan.Steps { + cols, err := m.describe(ctx, s) + if err != nil { + return err + } + old, present := cols[step.OldName] + newCol, renamed := cols[step.Column.Name] + suffix := "" + switch step.Kind { + case "add": + if renamed { + if canonicalType(newCol.Type) != canonicalType(step.Column.Type) || newCol.Nullable != step.Column.Nullable { + return errors.ErrLakeInvalidData.GenWithStackByArgs("pending ADD has conflicting target representation") + } + continue + } + suffix = "ADD COLUMN " + step.Column.definition() + case "drop": + if !present { + continue + } + suffix = "DROP COLUMN " + quoteIdentifier(step.OldName) + case "rename": + if !present && renamed { + continue + } + if !present || renamed { + return errors.ErrLakeInvalidData.GenWithStackByArgs("pending RENAME is ambiguous") + } + suffix = "RENAME COLUMN " + quoteIdentifier(step.OldName) + " TO " + quoteIdentifier(step.Column.Name) + case "modify": + if !present || !allowedModify(old, step.Column) { + return errors.ErrLakeInvalidData.GenWithStackByArgs("pending MODIFY has conflicting target representation") + } + suffix = "MODIFY COLUMN " + step.Column.definition() + default: + return errors.ErrLakeInvalidData.GenWithStackByArgs("invalid durable DDL operation") + } + if err := m.exec(ctx, "ALTER TABLE "+s.targetName()+" "+suffix); err != nil { + return err + } + } + if err := m.verifyTarget(ctx, s); err != nil { + return err + } + if err := m.exec(ctx, s.createRawSQL()); err != nil { + return err + } + default: + return errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported durable DDL plan") + } + r.applied = true + return m.saveEpoch(ctx, s, r) +} + +func (m *schemaManager) applyDDL(ctx context.Context, e *commonEvent.DDLEvent) error { + if e.IsBootstrap { + return nil + } // Metadata bootstrap is not proof of an applied DDL. + if e.NotSync || e.TiDBOnly { + return nil + } + if e.FinishedTs == 0 || e.FinishedTs > math.MaxInt64 { + return errors.ErrLakeInvalidData.GenWithStackByArgs("invalid DDL FinishedTs") + } + switch model.ActionType(e.Type) { + case model.ActionRenameTable, model.ActionRenameTables, model.ActionDropSchema: + // Compatibility with bend-cdc's non-projecting path. The outer sink + // has already drained prior DML; only these explicitly opted-out DDLs + // succeed without a target mutation or an invented applied epoch. + log.Warn("Lake sink skipped source DDL and kept target objects unchanged", + zap.String("ddlType", model.ActionType(e.Type).String()), + zap.String("schema", e.SchemaName), zap.String("table", e.TableName), + zap.Uint64("finishedTs", e.FinishedTs)) + return nil + } + if model.ActionType(e.Type) == model.ActionCreateSchema { + db := e.SchemaName + if m.opts.database != "" { + db = m.opts.database + } + if db == "" || strings.ContainsRune(db, 0) { + return errors.ErrLakeInvalidData.GenWithStackByArgs("invalid schema name") + } + return m.exec(ctx, "CREATE DATABASE IF NOT EXISTS "+quoteIdentifier(db)) + } + if e.TableInfo == nil { + return errors.ErrLakeInvalidData.GenWithStackByArgs("DDL without table metadata is outside the incremental projection") + } + s, err := schemaFromNative(e.TableInfo, m.opts.database, m.opts.conflictKey) + if err != nil { + return err + } + r, err := m.loadEpoch(ctx, s) + if err != nil { + return err + } + if r.epoch > e.FinishedTs || r.epoch == e.FinishedTs && r.applied { + return nil + } + if r.epoch > 0 && !r.applied { + if err := m.finishPlan(ctx, r); err != nil { + return err + } + delete(m.tables, schemaKey(s)) + if r.epoch == e.FinishedTs { + return nil + } + } + plan := schemaPlan{Schema: toSchemaDef(s)} + switch model.ActionType(e.Type) { + case model.ActionCreateTable, model.ActionRecoverTable: + plan.Kind = "create" + case model.ActionTruncateTable: + plan.Kind = "truncate" + case model.ActionDropTable: + plan.Kind = "drop" + case model.ActionAddIndex, model.ActionDropIndex, model.ActionRenameIndex, model.ActionAlterIndexVisibility: + plan.Kind = "index" + case model.ActionAddColumn, model.ActionDropColumn, model.ActionModifyColumn, model.ActionMultiSchemaChange: + plan.Kind = "alter" + exists, err := m.targetExists(ctx, s) + if err != nil { + return err + } + if !exists { + // An actual DDL is sufficient authoritative metadata to create an + // empty projection; this is not historical-table discovery. + stmt, parseErr := parser.New().ParseOneStmt(e.Query, "", "") + if parseErr != nil { + return errors.WrapError(errors.ErrLakeInvalidData, parseErr, "cannot parse incremental DDL") + } + alter, ok := stmt.(*ast.AlterTableStmt) + if !ok { + return errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported incremental column DDL") + } + for _, spec := range alter.Specs { + switch spec.Tp { + case ast.AlterTableAddColumns, ast.AlterTableDropColumn, ast.AlterTableModifyColumn, ast.AlterTableChangeColumn, ast.AlterTableRenameColumn: + default: + return errors.ErrLakeInvalidData.GenWithStackByArgs("unsupported DDL on missing projection") + } + } + plan.Kind = "create" + break + } + current, err := m.describe(ctx, s) + if err != nil { + return err + } + plan.Steps, err = makeColumnSteps(e.Query, s, current) + if err != nil { + return err + } + default: + return errors.ErrLakeInvalidData.GenWithStackByArgs(fmt.Sprintf("unsupported DDL action %d", e.Type)) + } + // No new mutation is issued for rejected DDLs. Callers have already drained + // every affected lane; all old SQL must be terminated before a manual resume. + r = epochRecord{epoch: e.FinishedTs, plan: plan} + if err := m.saveEpoch(ctx, s, r); err != nil { + return err + } + if err := m.finishPlan(ctx, r); err != nil { + return err + } + delete(m.tables, schemaKey(s)) + return nil +} diff --git a/downstreamadapter/sink/lake/schema_test.go b/downstreamadapter/sink/lake/schema_test.go new file mode 100644 index 0000000000..5202c8ac9c --- /dev/null +++ b/downstreamadapter/sink/lake/schema_test.go @@ -0,0 +1,207 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "context" + "encoding/json" + "strings" + "testing" + + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/stretchr/testify/require" +) + +func TestColumnDDLTranslationIsConservative(t *testing.T) { + s, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + s.columns = append(s.columns, column{name: "extra", typ: "INT64", nullable: true, sourceID: 3}) + current := func() map[string]columnDef { + return map[string]columnDef{"id": {Name: "id", Type: "INT64"}, "value": {Name: "value", Type: "VARCHAR", Nullable: true}} + } + steps, err := makeColumnSteps("ALTER TABLE t ADD COLUMN extra BIGINT", s, current()) + require.NoError(t, err) + require.Len(t, steps, 1) + require.Equal(t, "add", steps[0].Kind) + for _, sql := range []string{"ALTER TABLE t RENAME COLUMN id TO value", "ALTER TABLE t DROP PRIMARY KEY", "ALTER TABLE t ADD COLUMN missing INT", "ALTER TABLE t TRUNCATE PARTITION p0"} { + _, err := makeColumnSteps(sql, s, current()) + require.Error(t, err, sql) + } + s.columns[2].nullable = false + _, err = makeColumnSteps("ALTER TABLE t ADD COLUMN extra BIGINT NOT NULL", s, current()) + require.Error(t, err) + s.columns[2].defaultSQL = "'42'" + _, err = makeColumnSteps("ALTER TABLE t ADD COLUMN extra BIGINT NOT NULL DEFAULT 42", s, current()) + require.NoError(t, err) + s.columns[2].defaultExpr = true + _, err = makeColumnSteps("ALTER TABLE t ADD COLUMN extra BIGINT", s, current()) + require.Error(t, err) +} + +func TestPendingTruncateRecoveryAndAppliedReplay(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + m := s.lanes[0].schemas + info := nativeTestInfo() + schema, err := schemaFromNative(info, "", "") + require.NoError(t, err) + _, skip, err := m.prepareDML(context.Background(), schema, 100) + require.NoError(t, err) + require.False(t, skip) + f.mu.Lock() + require.Zero(t, f.epoch, "bootstrap must not invent an applied epoch") + f.failAfterTruncate = true + f.mu.Unlock() + ddl := &commonEvent.DDLEvent{Type: byte(model.ActionTruncateTable), TableInfo: info, FinishedTs: 150} + require.Error(t, m.applyDDL(context.Background(), ddl)) + f.mu.Lock() + require.EqualValues(t, 150, f.epoch) + require.False(t, f.applied) + require.Equal(t, 1, f.truncates) + f.mu.Unlock() + // Simulate a fresh sink only AFTER the old request has a verified terminal + // state (the fixture failed before saving applied). Transport cancellation is + // not that proof in production. + m = newSchemaManager(s.client, s.opts) + _, skip, err = m.prepareDML(context.Background(), schema, 120) + require.NoError(t, err) + require.True(t, skip) + f.mu.Lock() + require.True(t, f.applied) + require.Equal(t, 2, f.truncates) + f.mu.Unlock() + _, skip, err = m.prepareDML(context.Background(), schema, 160) + require.NoError(t, err) + require.False(t, skip) + require.NoError(t, m.applyDDL(context.Background(), ddl)) + f.mu.Lock() + require.Equal(t, 2, f.truncates, "applied replay must not clear newer data") + f.mu.Unlock() +} + +func TestPendingColumnResumeBeforeNewDML(t *testing.T) { + f, server := newAPIFixture(t) + s := runtimeSink(t, server, "") + schema, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + schema.columns = append(schema.columns, column{name: "extra", typ: "INT64", nullable: true, sourceID: 3}) + plan := schemaPlan{Kind: "alter", Schema: toSchemaDef(schema), Steps: []schemaStep{{Kind: "add", Column: columnDef{Name: "extra", Type: "INT64", Nullable: true}}}} + raw, err := json.Marshal(plan) + require.NoError(t, err) + f.mu.Lock() + f.epoch = 150 + f.applied = false + f.plan = string(raw) + f.mu.Unlock() + m := s.lanes[0].schemas + _, skip, err := m.prepareDML(context.Background(), schema, 160) + require.NoError(t, err) + require.False(t, skip) + f.mu.Lock() + require.True(t, f.applied) + require.Len(t, f.columns, 3) + f.mu.Unlock() + _, skip, err = m.prepareDML(context.Background(), schema, 140) + require.NoError(t, err) + require.True(t, skip) +} + +func TestAddNotNullTextKeepsBaselineImplicitDefault(t *testing.T) { + s, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + s.columns = append(s.columns, column{name: "text", typ: "VARCHAR", sourceID: 3}) + current := func() map[string]columnDef { + return map[string]columnDef{"id": {Name: "id", Type: "INT64"}, "value": {Name: "value", Type: "VARCHAR", Nullable: true}} + } + steps, err := makeColumnSteps("ALTER TABLE t ADD COLUMN text TEXT NOT NULL", s, current()) + require.NoError(t, err) + require.Equal(t, "''", steps[0].Column.Default) + _, err = makeColumnSteps("ALTER TABLE t ADD COLUMN text TEXT NOT NULL DEFAULT NULL", s, current()) + require.Error(t, err) +} + +func TestChangeColumnValidatesOldTypeBeforePending(t *testing.T) { + s, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + s.columns[1].name = "NewName" + s.columns[1].typ = "INT8" + current := map[string]columnDef{"id": {Name: "id", Type: "INT64"}, "value": {Name: "value", Type: "INT64", Nullable: true}} + _, err = makeColumnSteps("ALTER TABLE t CHANGE COLUMN VALUE newname TINYINT", s, current) + require.Error(t, err, "rename must not masquerade as a completed type change") + s.columns[1].typ = "INT64" + current["value"] = columnDef{Name: "value", Type: "INT32", Nullable: true} + steps, err := makeColumnSteps("ALTER TABLE t CHANGE COLUMN VALUE newname BIGINT", s, current) + require.NoError(t, err) + require.Len(t, steps, 2) + require.Equal(t, "value", steps[0].OldName) + require.Equal(t, "NewName", steps[0].Column.Name) + require.Equal(t, "NewName", steps[1].OldName) +} + +func TestActualDDLCanCreateMissingProjectionWithoutFullImport(t *testing.T) { + f, server := newAPIFixture(t) + f.targetMissing = true + s := runtimeSink(t, server, "") + e := &commonEvent.DDLEvent{Type: byte(model.ActionModifyColumn), TableInfo: nativeTestInfo(), FinishedTs: 150, Query: "ALTER TABLE t MODIFY COLUMN value TEXT"} + require.NoError(t, s.lanes[0].schemas.applyDDL(context.Background(), e)) + f.mu.Lock() + defer f.mu.Unlock() + require.False(t, f.targetMissing) + require.True(t, f.applied) + require.EqualValues(t, 150, f.epoch) + var plan schemaPlan + require.NoError(t, json.Unmarshal([]byte(f.plan), &plan)) + require.Equal(t, "create", plan.Kind) + require.Empty(t, plan.Steps) +} + +// Lake DESCRIBE uses SQL aliases rather than internal UINT* spelling. +func TestCanonicalDescribeUnsignedTypes(t *testing.T) { + for source, want := range map[string]string{"TINYINT UNSIGNED": "UINT8", "SMALLINT UNSIGNED": "UINT16", "INT UNSIGNED": "UINT32", "BIGINT UNSIGNED": "UINT64", "Nullable(BIGINT UNSIGNED)": "UINT64", "Decimal(65, 30)": "DECIMAL(65,30)"} { + require.Equal(t, want, canonicalType(source)) + } +} + +func TestParameterizedCharacterWideningKeepsBaselineIntTransition(t *testing.T) { + for _, tc := range []struct { + query, from string + allowed bool + }{ + {"ALTER TABLE t MODIFY COLUMN value VARCHAR(200) NOT NULL DEFAULT '7'", "INT32", true}, + {"ALTER TABLE t MODIFY COLUMN value VARCHAR(11)", "INT32", true}, + {"ALTER TABLE t MODIFY COLUMN value VARCHAR(10)", "INT32", false}, + {"ALTER TABLE t MODIFY COLUMN value CHAR(200)", "INT32", false}, + {"ALTER TABLE t MODIFY COLUMN value VARCHAR(200)", "VARCHAR", false}, + {"ALTER TABLE t MODIFY COLUMN value VARCHAR(200)", "INT64", false}, + {"ALTER TABLE t CHANGE COLUMN value label VARCHAR(200)", "INT32", true}, + } { + t.Run(tc.query+"/"+tc.from, func(t *testing.T) { + schema, err := schemaFromNative(nativeTestInfo(), "", "") + require.NoError(t, err) + if strings.Contains(tc.query, "CHANGE") { + schema.columns[1].name = "label" + } + _, err = makeColumnSteps(tc.query, schema, map[string]columnDef{ + "id": {Name: "id", Type: "INT64"}, "value": {Name: "value", Type: tc.from, Nullable: true}, + }) + if tc.allowed { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} diff --git a/downstreamadapter/sink/lake/sink.go b/downstreamadapter/sink/lake/sink.go new file mode 100644 index 0000000000..747568b9a3 --- /dev/null +++ b/downstreamadapter/sink/lake/sink.go @@ -0,0 +1,307 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "context" + "hash/fnv" + "net/url" + "sync" + "sync/atomic" + + "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/pkg/sink/spool" + "github.com/pingcap/ticdc/pkg/writelease" + "golang.org/x/sync/errgroup" +) + +type workItem struct { + event *commonEvent.DMLEvent + charge int64 + ddl *commonEvent.DDLEvent + done chan error +} + +// Sink owns a bounded source admission pool, shared spool and a fixed number +// of serial table lanes. No transport epoch owns data after admission. +type Sink struct { + opts options + client *httpClient + spool *spool.Spool + lanes []*tableLane + statistics *metrics.Statistics + ctx context.Context + cancel context.CancelCauseFunc + normal atomic.Bool + stopOnce, cleanupOnce sync.Once + mu sync.Mutex + sourceBytes int64 + sourceEvents int + admissionChanged chan struct{} + started bool + closed bool + done chan struct{} + ddlMu sync.Mutex +} + +func Verify(ctx context.Context, cfg *config.ChangefeedConfig, uri *url.URL) error { + o, err := parseOptions(uri, cfg) + if err != nil { + return err + } + c := newHTTPClient(o) + defer c.transport.CloseIdleConnections() + rows, err := c.selectRows(ctx, "SELECT ?::BIGINT", []any{int64(1)}) + if err != nil { + return err + } + if len(rows) != 1 || len(rows[0]) != 1 || rows[0][0] == nil || *rows[0][0] != "1" { + return errors.ErrLakeInvalidData.GenWithStackByArgs("server must support HTTP bound parameters and terminal query states") + } + return nil +} + +func New(ctx context.Context, id common.ChangeFeedID, cfg *config.ChangefeedConfig, uri *url.URL, keyspaceID uint32) (*Sink, error) { + o, err := parseOptions(uri, cfg) + if err != nil { + return nil, err + } + buffer, err := spool.New(id, spool.WithRootDir(o.spoolDir), spool.WithDirectoryNamespace("lake-sink-spool", config.GetGlobalServerConfig().AdvertiseAddr), spool.WithMetrics(newSpoolMetrics(id)), spool.WithDiskQuotaBytes(o.spoolBytes), spool.WithSegmentBytes(min(64<<20, o.spoolBytes/4)), spool.WithMemoryRatio(o.memoryRatio)) + if err != nil { + return nil, err + } + ctx, cancel := context.WithCancelCause(ctx) + s := &Sink{opts: o, client: newHTTPClient(o), spool: buffer, ctx: ctx, cancel: cancel, done: make(chan struct{}), admissionChanged: make(chan struct{}), statistics: metrics.NewStatistics(id, keyspaceID, "lake")} + s.normal.Store(true) + for i := 0; i < o.workers; i++ { + s.lanes = append(s.lanes, &tableLane{sink: s, input: make(chan workItem, o.sourceEvents), schemas: newSchemaManager(s.client, o), batches: make(map[string]*pendingTable)}) + } + return s, nil +} +func (s *Sink) SinkType() common.SinkType { return common.LakeSinkType } +func (s *Sink) IsNormal() bool { return s.normal.Load() && s.ctx.Err() == nil } +func (s *Sink) BatchCount() int { return min(s.opts.sourceEvents, s.opts.batchRows) } +func (s *Sink) BatchBytes() int { return int(min(s.opts.sourceBytes/2, s.opts.batchBytes)) } +func (s *Sink) SetWriteGate(gate *writelease.Gate) { s.client.gate.Store(gate) } +func (s *Sink) SetTableSchemaStore(*commonEvent.TableSchemaStore) {} +func (s *Sink) AddCheckpointTs(uint64) {} + +// stop marks an internal failure as unretryable, even when its cause is a +// timeout. Only stopLifecycle is used for externally requested cancellation. +func (s *Sink) stop(err error) { + s.stopLifecycle(errors.WrapChangefeedUnretryableErr(err)) +} + +func (s *Sink) stopLifecycle(err error) { + s.stopOnce.Do(func() { s.normal.Store(false); s.cancel(err) }) +} + +func (s *Sink) lane(database, table string) *tableLane { + h := fnv.New64a() + _, _ = h.Write([]byte(database + "\x00" + table)) + return s.lanes[h.Sum64()%uint64(len(s.lanes))] +} + +func (s *Sink) releaseSource(charge int64) { + s.mu.Lock() + s.sourceBytes -= charge + s.sourceEvents-- + s.notifyAdmissionLocked() + s.mu.Unlock() +} + +// notifyAdmissionLocked broadcasts a quota/queue change without allocating +// goroutines or retaining a second waiting-event queue. The caller holds mu. +func (s *Sink) notifyAdmissionLocked() { + close(s.admissionChanged) + s.admissionChanged = make(chan struct{}) +} + +func sourceEventBytes(e *commonEvent.DMLEvent) int64 { + charge := e.Rows.MemoryUsage() + max(e.GetSize(), int64(0)) + int64(len(e.RowTypes))*8 + int64(len(e.RowKeys))*24 + int64(len(e.Checksum))*64 + 1024 + for _, key := range e.RowKeys { + charge += int64(len(key)) + } + return charge +} + +func (s *Sink) AddDMLEvent(e *commonEvent.DMLEvent) { + if !s.IsNormal() { + return + } + if e == nil || e.TableInfo == nil || e.Rows == nil { + s.stop(errors.ErrLakeInvalidData.GenWithStackByArgs("invalid native DML event")) + return + } + // Shared chunks are conservatively charged in full for each admitted event. + // Waiting callers still own their event under the upstream memory budget. + charge := sourceEventBytes(e) + if charge > s.opts.sourceBytes { + s.stop(errors.ErrLakeInvalidData.GenWithStackByArgs("single source event exceeds admission byte limit")) + return + } + db := e.TableInfo.GetSchemaName() + if s.opts.database != "" { + db = s.opts.database + } + lane := s.lane(db, e.TableInfo.GetTableName()) + for { + s.mu.Lock() + if !s.IsNormal() || s.closed { + s.mu.Unlock() + return + } + if charge <= s.opts.sourceBytes-s.sourceBytes && s.sourceEvents < s.opts.sourceEvents { + s.sourceBytes += charge + s.sourceEvents++ + select { + case lane.input <- workItem{event: e, charge: charge}: + s.mu.Unlock() + return + default: + // A control/barrier item may occupy this lane. Roll back the + // reservation and wait for the independent lane worker. + s.sourceBytes -= charge + s.sourceEvents-- + } + } + changed := s.admissionChanged + s.mu.Unlock() + select { + case <-s.ctx.Done(): + return + case <-changed: + } + } +} + +func (s *Sink) waitWork(lane *tableLane, item workItem) error { + item.done = make(chan error, 1) + select { + case <-s.ctx.Done(): + return context.Cause(s.ctx) + case lane.input <- item: + } + select { + case <-s.ctx.Done(): + return context.Cause(s.ctx) + case err := <-item.done: + return err + } +} + +func (s *Sink) FlushDMLBeforeBlock(commonEvent.BlockEvent) error { + if !s.IsNormal() { + return context.Cause(s.ctx) + } + for _, lane := range s.lanes { + if err := s.waitWork(lane, workItem{}); err != nil { + return err + } + } + return nil +} + +func (s *Sink) WriteBlockEvent(event commonEvent.BlockEvent) error { + s.ddlMu.Lock() + defer s.ddlMu.Unlock() + if err := s.FlushDMLBeforeBlock(event); err != nil { + return err + } + e, ok := event.(*commonEvent.DDLEvent) + if !ok { + err := errors.ErrLakeInvalidData.GenWithStackByArgs("syncpoints are not supported") + s.stop(err) + return err + } + db, table := e.SchemaName, e.TableName + if e.TableInfo != nil { + db, table = e.TableInfo.GetSchemaName(), e.TableInfo.GetTableName() + } + if s.opts.database != "" { + db = s.opts.database + } + if err := s.waitWork(s.lane(db, table), workItem{ddl: e}); err != nil { + return err + } + event.PostFlush() + return nil +} + +func (s *Sink) cleanup() { + s.cleanupOnce.Do(func() { + // Pair with AddDMLEvent's admission critical section. Once normal is + // false, no producer may publish behind cleanup's final queue drain. + s.mu.Lock() + s.closed = true + s.mu.Unlock() + for _, lane := range s.lanes { + lane.cleanup() + } + s.spool.Close() + s.client.transport.CloseIdleConnections() + s.statistics.Close() + }) +} + +func (s *Sink) Run(ctx context.Context) error { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return context.Cause(s.ctx) + } + if s.started { + s.mu.Unlock() + return errors.ErrLakeInvalidData.GenWithStackByArgs("sink lifecycle cannot be reused") + } + s.started = true + s.mu.Unlock() + defer close(s.done) + defer s.cleanup() + stop := context.AfterFunc(ctx, func() { s.stopLifecycle(context.Cause(ctx)) }) + defer stop() + g, _ := errgroup.WithContext(s.ctx) + for _, lane := range s.lanes { + g.Go(func() error { + err := lane.run(s.ctx) + if err != nil { + s.stop(err) + } + return err + }) + } + _ = g.Wait() + s.normal.Store(false) + // The first cause already distinguishes a normal external stop from an + // unretryable internal failure. No nested timeout can downgrade the latter. + return context.Cause(s.ctx) +} + +func (s *Sink) Close() { + s.stopLifecycle(context.Canceled) + s.mu.Lock() + s.closed = true + started := s.started + s.mu.Unlock() + if started { + <-s.done + } else { + s.cleanup() + } +} diff --git a/downstreamadapter/sink/lake/sql.go b/downstreamadapter/sink/lake/sql.go new file mode 100644 index 0000000000..66ccaabf2a --- /dev/null +++ b/downstreamadapter/sink/lake/sql.go @@ -0,0 +1,126 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "fmt" + "strings" + "time" +) + +func quoteIdentifier(name string) string { + return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` +} + +func (s tableSchema) targetName() string { + return quoteIdentifier(s.database) + "." + quoteIdentifier(s.table) +} + +func (s tableSchema) rawName() string { + return quoteIdentifier(s.database) + "." + quoteIdentifier(s.table+"_cdc_raw") +} + +func (s tableSchema) createRawSQL() string { + return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( +id VARCHAR(255), table_name VARCHAR(255), raw_data JSON, +add_time TIMESTAMP, event_pos BIGINT, action VARCHAR(20))`, s.rawName()) +} + +// addTimeClock assigns timestamps in the precision actually sent to Lake. +// Comparing nanoseconds before formatting microseconds would lose tie ordering. +type addTimeClock struct { + last time.Time +} + +func (c *addTimeClock) next(candidate time.Time) time.Time { + candidate = candidate.UTC().Truncate(time.Microsecond) + if !c.last.IsZero() && !candidate.After(c.last) { + candidate = c.last.Add(time.Microsecond) + } + c.last = candidate + return candidate +} + +func (s tableSchema) insertRawSQL(rows []normalizedRow, clock *addTimeClock, now time.Time) (string, []any) { + placeholders := make([]string, len(rows)) + args := make([]any, 0, len(rows)*6) + for i, row := range rows { + placeholders[i] = "(?, ?, ?, ?, ?, ?)" + args = append(args, row.id, s.table, row.payload, + clock.next(now).Format("2006-01-02 15:04:05.000000"), int64(row.commitTs), string(row.action)) + } + return "INSERT INTO " + s.rawName() + + " (id, table_name, raw_data, add_time, event_pos, action) VALUES " + strings.Join(placeholders, ", "), args +} + +func (s tableSchema) mergeSQL(epoch uint64) string { + projections := make([]string, 0, len(s.columns)) + for _, col := range s.columns { + // A source name is a literal JSON object key, not a JSONPath. Quoting + // it as an SQL identifier still leaves embedded quotes unescaped in + // Lake's get_by_keypath conversion. GET uses the exact string key. + path := "get(s.raw_data, " + sqlString(col.name) + ")" + alias := quoteIdentifier(col.name) + switch col.encoding { + case binaryColumn: + projections = append(projections, fmt.Sprintf("unhex(%s) AS %s", path, alias)) + case jsonColumn: + projections = append(projections, fmt.Sprintf( + "CASE WHEN %[1]s IS NULL OR is_null_value(%[1]s) THEN NULL ELSE parse_json(%[1]s::VARCHAR) END AS %[2]s", path, alias)) + default: + projections = append(projections, path+" AS "+alias) + } + } + // Keep the operation discriminator separate from arbitrary source names. + operation := "__ticdc_operation" + for { + collision := false + for _, c := range s.columns { + if strings.EqualFold(c.name, operation) { + collision = true + break + } + } + if !collision { + break + } + operation += "_" + } + op := quoteIdentifier(operation) + joins := make([]string, 0, len(s.primaryKey)) + for _, key := range s.canonicalKeys() { + quoted := quoteIdentifier(key) + joins = append(joins, "a."+quoted+" = b."+quoted) + } + // The source is deliberately raw, not a stream. This is bend-cdc's strict + // ordering fallback, including latest tombstones and the applied DDL fence. + return fmt.Sprintf(`MERGE INTO %s a USING ( +SELECT %s, s.action AS %s FROM %s s +WHERE s.event_pos > %d AND NOT EXISTS ( +SELECT 1 FROM %s h WHERE h.id = s.id AND ( +h.event_pos > s.event_pos OR (h.event_pos = s.event_pos AND h.add_time > s.add_time))) +QUALIFY ROW_NUMBER() OVER (PARTITION BY s.id ORDER BY s.event_pos DESC, s.add_time DESC) = 1 +) b ON %s +WHEN MATCHED AND b.%s != 'delete' THEN UPDATE * +WHEN MATCHED AND b.%s = 'delete' THEN DELETE +WHEN NOT MATCHED AND b.%s != 'delete' THEN INSERT *`, + s.targetName(), strings.Join(projections, ", "), op, s.rawName(), epoch, s.rawName(), strings.Join(joins, " AND "), op, op, op) +} + +func (s tableSchema) compactRawSQL() string { + return fmt.Sprintf(`DELETE FROM %s AS stale WHERE EXISTS ( +SELECT 1 FROM %s AS newer WHERE newer.id = stale.id AND ( +newer.event_pos > stale.event_pos OR (newer.event_pos = stale.event_pos AND newer.add_time > stale.add_time)))`, s.rawName(), s.rawName()) +} diff --git a/downstreamadapter/sink/lake/sql_test.go b/downstreamadapter/sink/lake/sql_test.go new file mode 100644 index 0000000000..2d48f8ac97 --- /dev/null +++ b/downstreamadapter/sink/lake/sql_test.go @@ -0,0 +1,92 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestRawInsertUsesParametersAndStoredPrecision(t *testing.T) { + schema := testSchema() + rows := []normalizedRow{ + testRow(t, deleteAction, 100, 1), testRow(t, insertAction, 100, 1), testRow(t, updateAction, 100, 1), + } + var clock addTimeClock + now := time.Date(2026, 10, 24, 1, 2, 3, 456789001, time.UTC) + statement, args := schema.insertRawSQL(rows, &clock, now) + require.Equal(t, 18, len(args)) + require.Equal(t, 3, strings.Count(statement, "(?, ?, ?, ?, ?, ?)")) + require.NotContains(t, statement, "payload") + require.Equal(t, []any{"2026-10-24 01:02:03.456789", "2026-10-24 01:02:03.456790", "2026-10-24 01:02:03.456791"}, []any{args[3], args[9], args[15]}) + require.Equal(t, "delete", args[5]) + require.Equal(t, "insert", args[11]) + _, later := schema.insertRawSQL(rows[:1], &clock, now.Add(-time.Hour)) + require.Equal(t, "2026-10-24 01:02:03.456792", later[3], "clock rollback must not reverse source tie order") +} + +func TestFullRawMergeSQLKeepsBaselineSemantics(t *testing.T) { + schema := tableSchema{ + database: `lake"db`, table: `order"s`, + columns: []column{{name: "tenant"}, {name: "id"}, {name: `raw"bytes`, encoding: binaryColumn}, {name: "json", encoding: jsonColumn}}, + primaryKey: []string{"tenant", "id"}, + } + require.NoError(t, schema.validate()) + statement := schema.mergeSQL(500) + for _, want := range []string{ + `MERGE INTO "lake""db"."order""s" a`, + `FROM "lake""db"."order""s_cdc_raw" s`, + `s.event_pos > 500`, `h.event_pos > s.event_pos`, `h.add_time > s.add_time`, + `PARTITION BY s.id ORDER BY s.event_pos DESC, s.add_time DESC`, + `a."id" = b."id" AND a."tenant" = b."tenant"`, + `unhex(get(s.raw_data, 'raw"bytes'))`, `is_null_value(get(s.raw_data, 'json'))`, `parse_json(get(s.raw_data, 'json')::VARCHAR)`, + `WHEN MATCHED AND b."__ticdc_operation" = 'delete' THEN DELETE`, + `WHEN NOT MATCHED AND b."__ticdc_operation" != 'delete' THEN INSERT *`, + } { + require.Contains(t, statement, want) + } + require.NotContains(t, statement, "_cdc_stream") + require.NotContains(t, statement, "LIMIT") + require.NotContains(t, statement, "TRY_") + compact := schema.compactRawSQL() + require.Contains(t, compact, `newer.id = stale.id`) + require.Contains(t, compact, `newer.event_pos > stale.event_pos`) + require.NotContains(t, compact, "NOW()", "latest tombstones are not TTL deleted") + require.NotContains(t, compact, ">=", "equal physical replays must not delete every latest row") + require.NotContains(t, schema.createRawSQL(), "STREAM") +} + +func TestMergeOperationAliasDoesNotCollideWithSourceColumn(t *testing.T) { + schema := testSchema() + schema.columns = append(schema.columns, column{name: "action"}, column{name: "__ticdc_operation"}) + statement := schema.mergeSQL(0) + require.Contains(t, statement, `get(s.raw_data, 'action') AS "action"`) + require.Contains(t, statement, `s.action AS "__ticdc_operation_"`) + require.Contains(t, statement, `b."__ticdc_operation_" = 'delete'`) +} + +func TestMergeProjectionUsesLiteralJSONKeyNotKeyPath(t *testing.T) { + schema := testSchema() + schema.columns = append(schema.columns, column{name: `a"b`}, column{name: "a.b"}, column{name: "O'Reilly"}, column{name: `a\b`}) + sql := schema.mergeSQL(0) + require.Contains(t, sql, `get(s.raw_data, 'a"b') AS "a""b"`) + require.Contains(t, sql, `get(s.raw_data, 'a.b') AS "a.b"`) + require.Contains(t, sql, `get(s.raw_data, 'O''Reilly') AS "O'Reilly"`) + require.Contains(t, sql, `get(s.raw_data, 'a\\b') AS "a\b"`) + require.NotContains(t, sql, "s.raw_data:") +} diff --git a/downstreamadapter/sink/lake/writer.go b/downstreamadapter/sink/lake/writer.go new file mode 100644 index 0000000000..f2cdd635ce --- /dev/null +++ b/downstreamadapter/sink/lake/writer.go @@ -0,0 +1,160 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "context" + "encoding/json" + "math" + "sync" + "time" + + "github.com/pingcap/ticdc/pkg/errors" +) + +type commitState uint8 + +const ( + commitUnknown commitState = iota // Missing evidence must never mean success. + committed + notCommitted +) + +// mutationResult is produced at the client boundary, before any inner retry. +// observationErr is valid only after commit has independently been confirmed. +type mutationResult struct { + state commitState + retryable bool + observationErr error +} + +// executeMutation must read the server terminal state, not just the first result +// page. The HTTP adapter supplies it; transport errors alone must not +// be labelled notCommitted. Arguments and SQL must not be included in errors. +type executeMutation func(context.Context, string, []any) (mutationResult, error) + +type applyResult struct { + // applied remains true if only subsequent raw compaction fails. A worker + // must not undo/redo already applied source completion to repair maintenance. + applied bool +} + +// tableWriter is a serial raw/merge lane. It owns no source callbacks; its worker +// completes fragments only when applyResult.applied is true. +type tableWriter struct { + mu sync.Mutex + schema tableSchema + execute executeMutation + batchRows int + maxAttempts int + epoch uint64 + clock addTimeClock + written uint64 + merged uint64 + failed error +} + +func newTableWriter(schema tableSchema, execute executeMutation, batchRows, maxAttempts int, epoch uint64) (*tableWriter, error) { + if err := schema.validate(); err != nil { + return nil, err + } + if execute == nil || batchRows <= 0 || maxAttempts <= 0 || epoch > math.MaxInt64 { + return nil, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid table writer configuration") + } + // Keep metadata owned even if the caller reuses its slices for a later DDL. + schema.columns = append([]column(nil), schema.columns...) + schema.primaryKey = append([]string(nil), schema.primaryKey...) + return &tableWriter{schema: schema, execute: execute, batchRows: batchRows, maxAttempts: maxAttempts, epoch: epoch}, nil +} + +// applyBatch requires the target/raw tables and the durable schema epoch to +// have been initialized by the table worker. It preserves source row order and +// never writes a row at/before the applied epoch into the new projection. +func (w *tableWriter) applyBatch(ctx context.Context, rows []normalizedRow, now time.Time) (applyResult, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.failed != nil { + return applyResult{}, w.failed + } + // Validate the complete batch before issuing any mutation. + for _, row := range rows { + if row.id == "" || !json.Valid([]byte(row.payload)) || row.commitTs == 0 || row.commitTs > math.MaxInt64 || + (row.action != insertAction && row.action != updateAction && row.action != deleteAction) { + return applyResult{}, errors.ErrLakeInvalidData.GenWithStackByArgs("invalid normalized row") + } + } + pending := make([]normalizedRow, 0, len(rows)) + for _, row := range rows { + if row.commitTs > w.epoch { + pending = append(pending, row) + } + } + if len(pending) == 0 { + // Only a proven applied DDL fence permits this no-op, not schema bootstrap. + return applyResult{applied: true}, nil + } + for start := 0; start < len(pending); start += w.batchRows { + end := min(start+w.batchRows, len(pending)) + statement, args := w.schema.insertRawSQL(pending[start:end], &w.clock, now) + if err := w.runMutation(ctx, statement, args); err != nil { + w.failed = err + return applyResult{}, err + } + w.written++ + } + if err := w.runMutation(ctx, w.schema.mergeSQL(w.epoch), nil); err != nil { + w.failed = err + return applyResult{}, err + } + w.merged = w.written + result := applyResult{applied: true} + if err := w.runMutation(ctx, w.schema.compactRawSQL(), nil); err != nil { + w.failed = err + return result, err + } + return result, nil +} + +func (w *tableWriter) runMutation(ctx context.Context, statement string, args []any) error { + for attempt := 0; attempt < w.maxAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return errors.WrapError(errors.ErrLakeExecution, err) + } + result, err := w.execute(ctx, statement, args) + switch result.state { + case committed: + if err != nil { + // Contradictory execution evidence is not an observation error. + return errors.WrapError(errors.ErrLakeCommitUnknown, err) + } + // A confirmed commit is not undone by result statistics parsing. + return nil + case notCommitted: + if !result.retryable || attempt+1 == w.maxAttempts { + if err != nil { + return errors.WrapError(errors.ErrLakeExecution, err) + } + return errors.ErrLakeExecution.GenWithStackByArgs() + } + default: + // Check before transient-error handling: no unknown request is resent. + if err != nil { + return errors.WrapError(errors.ErrLakeCommitUnknown, err) + } + return errors.ErrLakeCommitUnknown.GenWithStackByArgs() + } + } + return errors.ErrLakeExecution.GenWithStackByArgs() +} diff --git a/downstreamadapter/sink/lake/writer_test.go b/downstreamadapter/sink/lake/writer_test.go new file mode 100644 index 0000000000..38af6384cf --- /dev/null +++ b/downstreamadapter/sink/lake/writer_test.go @@ -0,0 +1,176 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lake + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestTableWriterAppliesOnlyAfterAllRawAndMerge(t *testing.T) { + var statements []string + execute := func(_ context.Context, statement string, args []any) (mutationResult, error) { + statements = append(statements, statement) + if strings.HasPrefix(statement, "INSERT") { + require.Len(t, args, 6) + } + return mutationResult{state: committed}, nil + } + w, err := newTableWriter(testSchema(), execute, 1, 3, 0) + require.NoError(t, err) + result, err := w.applyBatch(context.Background(), []normalizedRow{ + testRow(t, deleteAction, 100, 1), testRow(t, insertAction, 100, 2), + }, time.Now()) + require.NoError(t, err) + require.True(t, result.applied) + require.Len(t, statements, 4) + require.True(t, strings.HasPrefix(statements[0], "INSERT")) + require.True(t, strings.HasPrefix(statements[1], "INSERT")) + require.True(t, strings.HasPrefix(statements[2], "MERGE")) + require.True(t, strings.HasPrefix(statements[3], "DELETE")) + require.EqualValues(t, 2, w.written) + require.Equal(t, w.written, w.merged) +} + +func TestTableWriterStopsUnknownBeforeInnerRetry(t *testing.T) { + for _, failAt := range []int{1, 2, 3} { + t.Run([]string{"", "first raw", "second raw", "merge"}[failAt], func(t *testing.T) { + calls := 0 + execute := func(context.Context, string, []any) (mutationResult, error) { + calls++ + if calls == failAt { + return mutationResult{state: commitUnknown, retryable: true}, context.DeadlineExceeded + } + return mutationResult{state: committed}, nil + } + w, err := newTableWriter(testSchema(), execute, 1, 10, 0) + require.NoError(t, err) + rows := []normalizedRow{testRow(t, insertAction, 100, 1), testRow(t, updateAction, 120, 2)} + result, err := w.applyBatch(context.Background(), rows, time.Now()) + require.Error(t, err) + code, ok := errors.RFCCode(err) + require.True(t, ok) + require.Equal(t, errors.ErrLakeCommitUnknown.RFCCode(), code) + require.False(t, result.applied) + require.Equal(t, failAt, calls, "unknown must not reach an inner retry, MERGE, or compaction") + require.Zero(t, w.merged) + _, err = w.applyBatch(context.Background(), rows, time.Now()) + require.Error(t, err) + require.Equal(t, failAt, calls, "a stopped lane cannot take a new batch") + }) + } +} + +func TestTableWriterRetriesOnlyConfirmedNotCommitted(t *testing.T) { + calls := 0 + execute := func(context.Context, string, []any) (mutationResult, error) { + calls++ + if calls == 1 { + return mutationResult{state: notCommitted, retryable: true}, errors.ErrLakeInvalidData.GenWithStackByArgs("confirmed rollback") + } + return mutationResult{state: committed, observationErr: errors.ErrLakeInvalidData.GenWithStackByArgs("statistics unavailable")}, nil + } + w, err := newTableWriter(testSchema(), execute, 1, 2, 0) + require.NoError(t, err) + result, err := w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 100, 1)}, time.Now()) + require.NoError(t, err) + require.True(t, result.applied) + require.Equal(t, 4, calls, "one proven rollback retry, then raw/MERGE/compaction; no statistics retry") + + calls = 0 + w, err = newTableWriter(testSchema(), func(context.Context, string, []any) (mutationResult, error) { + calls++ + return mutationResult{state: notCommitted, retryable: true}, errors.ErrLakeInvalidData.GenWithStackByArgs("confirmed rollback") + }, 1, 2, 0) + require.NoError(t, err) + result, err = w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 100, 1)}, time.Now()) + require.Error(t, err) + require.False(t, result.applied) + require.Equal(t, 2, calls) +} + +func TestTableWriterDoesNotRevokeAppliedOnCompactionFailure(t *testing.T) { + calls := 0 + execute := func(context.Context, string, []any) (mutationResult, error) { + calls++ + if calls == 3 { + return mutationResult{state: notCommitted}, errors.ErrLakeInvalidData.GenWithStackByArgs("compaction denied") + } + return mutationResult{state: committed}, nil + } + w, err := newTableWriter(testSchema(), execute, 10, 1, 0) + require.NoError(t, err) + result, err := w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 100, 1)}, time.Now()) + require.Error(t, err) + require.True(t, result.applied, "MERGE committed; maintenance does not roll it back") + require.Equal(t, w.written, w.merged) + _, err = w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 120, 2)}, time.Now()) + require.Error(t, err) + require.Equal(t, 3, calls) +} + +func TestTableWriterRejectsMissingOrContradictoryCommitEvidence(t *testing.T) { + for _, tc := range []struct { + name string + result mutationResult + err error + }{ + {name: "missing terminal evidence"}, + {name: "contradictory evidence", result: mutationResult{state: committed}, err: errors.ErrLakeInvalidData.GenWithStackByArgs("execution failed")}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + w, err := newTableWriter(testSchema(), func(context.Context, string, []any) (mutationResult, error) { + calls++ + return tc.result, tc.err + }, 10, 3, 0) + require.NoError(t, err) + result, err := w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 100, 1)}, time.Now()) + require.Error(t, err) + code, ok := errors.RFCCode(err) + require.True(t, ok) + require.Equal(t, errors.ErrLakeCommitUnknown.RFCCode(), code) + require.False(t, result.applied) + require.Equal(t, 1, calls) + }) + } +} + +func TestTableWriterAppliedEpochAndValidation(t *testing.T) { + calls := 0 + w, err := newTableWriter(testSchema(), func(context.Context, string, []any) (mutationResult, error) { + calls++ + return mutationResult{state: committed}, nil + }, 10, 1, 100) + require.NoError(t, err) + result, err := w.applyBatch(context.Background(), []normalizedRow{testRow(t, insertAction, 100, 1)}, time.Now()) + require.NoError(t, err) + require.True(t, result.applied) + require.Zero(t, calls) + _, err = w.applyBatch(context.Background(), []normalizedRow{{id: "1", payload: "not-json", commitTs: 120, action: insertAction}}, time.Now()) + require.Error(t, err) + require.Zero(t, calls) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result, err = w.applyBatch(ctx, []normalizedRow{testRow(t, insertAction, 120, 1)}, time.Now()) + require.Error(t, err) + require.False(t, result.applied) + require.Zero(t, calls) +} diff --git a/downstreamadapter/sink/metrics/lake.go b/downstreamadapter/sink/metrics/lake.go new file mode 100644 index 0000000000..cbc594c851 --- /dev/null +++ b/downstreamadapter/sink/metrics/lake.go @@ -0,0 +1,29 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +var ( + LakeBufferBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: "ticdc", Subsystem: "lake", Name: "buffer_bytes", Help: "Local Lake memory and physical spool bytes."}, []string{"namespace", "changefeed", "kind"}) + LakeBufferEntries = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: "ticdc", Subsystem: "lake", Name: "buffer_entries", Help: "Local Lake pending enqueue, segment and quota waiter counts."}, []string{"namespace", "changefeed", "kind"}) + LakeSpoolDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{Namespace: "ticdc", Subsystem: "lake", Name: "spool_wait_seconds", Help: "Lake spool quota wait time.", Buckets: prometheus.ExponentialBuckets(0.001, 2, 20)}, []string{"namespace", "changefeed"}) + LakeSpoolLoad = prometheus.NewHistogramVec(prometheus.HistogramOpts{Namespace: "ticdc", Subsystem: "lake", Name: "spool_load_bytes", Help: "Lake encoded bytes loaded from local spool.", Buckets: prometheus.ExponentialBuckets(1024, 4, 9)}, []string{"namespace", "changefeed"}) + LakeSpoolRotations = prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: "ticdc", Subsystem: "lake", Name: "spool_rotations_total", Help: "Lake local spool segment rotations."}, []string{"namespace", "changefeed"}) +) + +func InitLakeMetrics(registry *prometheus.Registry) { + registry.MustRegister(LakeBufferBytes, LakeBufferEntries, LakeSpoolDuration, LakeSpoolLoad, LakeSpoolRotations) +} diff --git a/downstreamadapter/sink/sink.go b/downstreamadapter/sink/sink.go index a62aea599e..c51c36c827 100644 --- a/downstreamadapter/sink/sink.go +++ b/downstreamadapter/sink/sink.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/ticdc/downstreamadapter/sink/blackhole" "github.com/pingcap/ticdc/downstreamadapter/sink/cloudstorage" "github.com/pingcap/ticdc/downstreamadapter/sink/kafka" + "github.com/pingcap/ticdc/downstreamadapter/sink/lake" "github.com/pingcap/ticdc/downstreamadapter/sink/mysql" "github.com/pingcap/ticdc/downstreamadapter/sink/pulsar" "github.com/pingcap/ticdc/pkg/common" @@ -72,6 +73,8 @@ func New(ctx context.Context, cfg *config.ChangefeedConfig, changefeedID common. return pulsar.New(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive, keyspaceID) case config.S3Scheme, config.FileScheme, config.GCSScheme, config.GSScheme, config.AzblobScheme, config.AzureScheme, config.CloudStorageNoopScheme: return cloudstorage.New(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive, cfg.EnableTableAcrossNodes, nil, keyspaceID) + case config.LakeScheme, config.LakeHTTPScheme: + return lake.New(ctx, changefeedID, cfg, sinkURI, keyspaceID) case config.BlackHoleScheme: return blackhole.New(changefeedID, keyspaceID) } @@ -97,6 +100,8 @@ func Verify(ctx context.Context, cfg *config.ChangefeedConfig, changefeedID comm return pulsar.Verify(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive) case config.S3Scheme, config.FileScheme, config.GCSScheme, config.GSScheme, config.AzblobScheme, config.AzureScheme, config.CloudStorageNoopScheme: return cloudstorage.Verify(ctx, changefeedID, sinkURI, cfg.SinkConfig, cfg.CaseSensitive, cfg.EnableTableAcrossNodes) + case config.LakeScheme, config.LakeHTTPScheme: + return lake.Verify(ctx, cfg, sinkURI) case config.BlackHoleScheme: return nil } diff --git a/downstreamadapter/sink/sink_test.go b/downstreamadapter/sink/sink_test.go index 48c347da9a..88bfc4bcb8 100644 --- a/downstreamadapter/sink/sink_test.go +++ b/downstreamadapter/sink/sink_test.go @@ -15,9 +15,11 @@ package sink import ( "context" + "net/url" "testing" "github.com/pingcap/ticdc/pkg/common" + commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/config" "github.com/stretchr/testify/require" ) @@ -75,3 +77,19 @@ func requireInvalidSinkURIError(t *testing.T, err error) { require.Contains(t, errMsg, `parse ""`) require.Contains(t, errMsg, "invalid URL escape") } + +func TestLakeFactoryRegistration(t *testing.T) { + cfg := &config.ChangefeedConfig{SinkURI: "lake+http://127.0.0.1:1?exclusive=true&spool-dir=" + url.QueryEscape(t.TempDir())} + s, err := New(context.Background(), cfg, common.NewChangefeedID(common.DefaultKeyspaceName), common.DefaultKeyspaceID) + require.NoError(t, err) + defer s.Close() + require.Equal(t, common.LakeSinkType, s.SinkType()) + store := commonEvent.NewTableSchemaStore(nil, s.SinkType(), false) + s.SetTableSchemaStore(store) + require.False(t, config.IsStorageScheme("lake")) + require.False(t, config.IsMySQLCompatibleScheme("lake")) + cfg.SinkURI = "lake://user:verysecure@localhost?exclusive=false" + _, err = New(context.Background(), cfg, common.NewChangefeedID(common.DefaultKeyspaceName), common.DefaultKeyspaceID) + require.Error(t, err) + require.NotContains(t, err.Error(), "verysecure") +} diff --git a/pkg/common/event/table_schema_store.go b/pkg/common/event/table_schema_store.go index 588f33b7fa..20277e1c6f 100644 --- a/pkg/common/event/table_schema_store.go +++ b/pkg/common/event/table_schema_store.go @@ -91,7 +91,7 @@ func newTableSchemaStoreRequirements( updateTableIDs: false, needTableNames: true, } - case commonType.CloudStorageSinkType, commonType.BlackHoleSinkType: + case commonType.CloudStorageSinkType, commonType.BlackHoleSinkType, commonType.LakeSinkType: return tableSchemaStoreRequirements{ needTableIDs: false, updateTableIDs: false, diff --git a/pkg/common/event/table_schema_store_test.go b/pkg/common/event/table_schema_store_test.go index fd56f35adc..716da0cc8a 100644 --- a/pkg/common/event/table_schema_store_test.go +++ b/pkg/common/event/table_schema_store_test.go @@ -253,3 +253,12 @@ func TestTableSchemaStoreNonMysqlTableIDsBootstrapOnly(t *testing.T) { require.ElementsMatch(t, []int64{10, 20}, store.GetAllNormalTableIds()) } + +func TestTableSchemaStoreWhenLakeSink(t *testing.T) { + store := NewTableSchemaStore(nil, common.LakeSinkType, false) + require.NotNil(t, store) + require.Nil(t, store.TableIDStore) + require.Nil(t, store.tableNameStore) + store.AddEvent(&DDLEvent{FinishedTs: 100}) + require.Empty(t, store.GetAllTableIds()) +} diff --git a/pkg/common/types.go b/pkg/common/types.go index 638d75305d..acae4d5b36 100644 --- a/pkg/common/types.go +++ b/pkg/common/types.go @@ -344,6 +344,7 @@ const ( CloudStorageSinkType BlackHoleSinkType RedoSinkType + LakeSinkType ) type RowType byte diff --git a/pkg/config/sink_protocol.go b/pkg/config/sink_protocol.go index 113d66eab9..bd24688c73 100644 --- a/pkg/config/sink_protocol.go +++ b/pkg/config/sink_protocol.go @@ -114,6 +114,10 @@ func (p Protocol) String() string { const ( // KafkaScheme indicates the scheme is kafka. KafkaScheme = "kafka" + // LakeScheme is the native Lake HTTPS warehouse sink. + LakeScheme = "lake" + // LakeHTTPScheme is the explicit plaintext local/test warehouse sink. + LakeHTTPScheme = "lake+http" // KafkaSSLScheme indicates the scheme is kafka+ssl. KafkaSSLScheme = "kafka+ssl" // BlackHoleScheme indicates the scheme is blackhole. diff --git a/pkg/errors/helper.go b/pkg/errors/helper.go index 19a6bc05f6..b5e6188bfe 100644 --- a/pkg/errors/helper.go +++ b/pkg/errors/helper.go @@ -102,6 +102,8 @@ func RFCCode(err error) (errors.RFCErrorCode, bool) { } var changefeedUnRetryableErrors = []*errors.Error{ + ErrLakeInvalidData, + ErrLakeCommitUnknown, ErrExpressionColumnNotFound, ErrExpressionParseFailed, ErrSchemaSnapshotNotFound, diff --git a/pkg/errors/lake.go b/pkg/errors/lake.go new file mode 100644 index 0000000000..d9962b72a4 --- /dev/null +++ b/pkg/errors/lake.go @@ -0,0 +1,27 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package errors + +import "github.com/pingcap/errors" + +var ( + ErrLakeProtocol = errors.Normalize("Lake request %s: %s", errors.RFCCodeText("CDC:ErrLakeProtocol")) + ErrLakeInvalidData = errors.Normalize( + "invalid Lake sink data: %s", errors.RFCCodeText("CDC:ErrLakeInvalidData")) + ErrLakeExecution = errors.Normalize( + "Lake mutation failed", errors.RFCCodeText("CDC:ErrLakeExecution")) + ErrLakeCommitUnknown = errors.Normalize( + "Lake mutation commit state is unknown", errors.RFCCodeText("CDC:ErrLakeCommitUnknown")) +) diff --git a/pkg/metrics/downstream_type.go b/pkg/metrics/downstream_type.go index 399337cbe6..9afc5dcfed 100644 --- a/pkg/metrics/downstream_type.go +++ b/pkg/metrics/downstream_type.go @@ -59,6 +59,8 @@ func downstreamTypeFromScheme(scheme string) string { return config.AzblobScheme case config.CloudStorageNoopScheme: return config.CloudStorageNoopScheme + case config.LakeScheme, config.LakeHTTPScheme: + return config.LakeScheme case config.BlackHoleScheme: return config.BlackHoleScheme case "": diff --git a/pkg/metrics/downstream_type_test.go b/pkg/metrics/downstream_type_test.go index 50cf8d718d..f6fe517e72 100644 --- a/pkg/metrics/downstream_type_test.go +++ b/pkg/metrics/downstream_type_test.go @@ -45,6 +45,9 @@ func TestDownstreamTypeFromSinkURI(t *testing.T) { {sinkURI: "azblob://bucket/prefix", expectedLabel: "azblob"}, {sinkURI: "noop://bucket/prefix", expectedLabel: "noop"}, {sinkURI: "blackhole:///", expectedLabel: "blackhole"}, + {sinkURI: "lake://warehouse", expectedLabel: "lake"}, + {sinkURI: "lake://warehouse:443/default", expectedLabel: "lake"}, + {sinkURI: "lake+http://127.0.0.1:8000", expectedLabel: "lake"}, {sinkURI: "127.0.0.1:3306", expectedLabel: "unknown"}, {sinkURI: "mysql://[::1", expectedLabel: "unknown"}, {sinkURI: "", expectedLabel: "unknown"}, diff --git a/pkg/metrics/init.go b/pkg/metrics/init.go index 79e5cedc68..ef5db0b60e 100644 --- a/pkg/metrics/init.go +++ b/pkg/metrics/init.go @@ -44,6 +44,7 @@ func InitMetrics(registry *prometheus.Registry) { kafka.InitMetrics(registry) gc.InitMetrics(registry) metrics.InitCloudStorageMetrics(registry) + metrics.InitLakeMetrics(registry) initRedoMetrics(registry) initDDLMetrics(registry) } diff --git a/pkg/sink/spool/spool.go b/pkg/sink/spool/spool.go index 4ec7f90410..bd90faf98b 100644 --- a/pkg/sink/spool/spool.go +++ b/pkg/sink/spool/spool.go @@ -1010,3 +1010,9 @@ func calculateEntrySize(msgs []*common.Message) entrySize { } return size } + +// AccountingBytes returns the encoded size charged by TryEnqueue. Callers may +// reject oversized entries before the optional in-memory bypass is considered. +func AccountingBytes(msgs []*common.Message) int64 { + return calculateEntrySize(msgs).accountingBytes +} diff --git a/pkg/sink/spool/spool_test.go b/pkg/sink/spool/spool_test.go index 5f92e615a7..696f54c504 100644 --- a/pkg/sink/spool/spool_test.go +++ b/pkg/sink/spool/spool_test.go @@ -1031,3 +1031,9 @@ func TestTryEnqueueReturnsOversizedActionWithInMemoryEntry(t *testing.T) { require.True(t, entry.InMemory()) require.False(t, entry.IsSpilled()) } + +func TestAccountingBytesMatchesAdmissionEncoding(t *testing.T) { + messages := []*common.Message{{Key: []byte("key"), Value: []byte("value")}, {Value: []byte("second")}} + require.EqualValues(t, len(serializeMessages(messages)), AccountingBytes(messages)) + require.Zero(t, AccountingBytes(nil)) +} diff --git a/tests/integration_tests/lake_native/.gitignore b/tests/integration_tests/lake_native/.gitignore new file mode 100644 index 0000000000..c18dd8d83c --- /dev/null +++ b/tests/integration_tests/lake_native/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/tests/integration_tests/lake_native/baseline.py b/tests/integration_tests/lake_native/baseline.py new file mode 100644 index 0000000000..20e61d586a --- /dev/null +++ b/tests/integration_tests/lake_native/baseline.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Read-only Go overlay for real Lake parity probes of fixed bend-cdc b648162. + +Uses the same private LAKE_URI_FILE/E2E_RUN_DIR as run.py; no baseline files are +changed. Requires the independent bend-cdc module at BEND_CDC_DIR. +""" + +import json +import os +import pathlib +import subprocess +import urllib.parse + + +def main(): + credential = pathlib.Path(os.environ["LAKE_URI_FILE"]) + if credential.stat().st_mode & 0o077: + raise RuntimeError("LAKE_URI_FILE must have private 0600 permissions") + raw = credential.read_text().strip() + uri = urllib.parse.urlparse(raw) + root = pathlib.Path(os.environ["E2E_RUN_DIR"]) + root.mkdir(parents=True, exist_ok=True) + os.chmod(root, 0o700) + log = root / "bend-live-parity.log" + if log.exists(): + raise RuntimeError("refusing to overwrite previous baseline evidence") + repo = pathlib.Path(os.environ["BEND_CDC_DIR"]).resolve() + head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=repo, text=True + ).strip() + if head != "b6481621021e82aee659ad88e8e6c00f6dd11906": + raise RuntimeError( + "baseline checkout must be fixed b648162, not HEAD of another version" + ) + dirty = subprocess.check_output( + ["git", "status", "--porcelain"], cwd=repo, text=True + ) + if dirty: + raise RuntimeError("baseline must be clean; overlay does not modify checkout") + fixture = pathlib.Path(__file__).with_name("baseline_parity_test.go.txt").resolve() + overlay = root / "bend-overlay.json" + overlay.write_text( + json.dumps( + { + "Replace": { + str(repo / "pkg/source/tidb/zz_native_parity_overlay_test.go"): str( + fixture + ) + } + } + ) + ) + q = urllib.parse.parse_qs(uri.query) + # This authorized endpoint exposes the Basic-auth Query API directly. + q.update(login=["disable"], sslmode=["enable"]) + env = os.environ.copy() + env["TIDB_DDL_INTEGRATION_DSN"] = uri._replace( + scheme="databend", query=urllib.parse.urlencode(q, doseq=True) + ).geturl() + env.setdefault("GOMAXPROCS", "4") + with log.open("w") as output: + os.chmod(log, 0o600) + result = subprocess.run( + [ + "go", + "test", + "-p", + "2", + "-tags", + "integration", + "-overlay", + str(overlay), + "./pkg/source/tidb", + "-run", + "^(TestBendParityBaselineLive|TestDatabendSchemaIndexWarningAndTruncate)$", + "-v", + "-count=1", + "-timeout=12m", + ], + cwd=repo, + env=env, + stdout=output, + stderr=subprocess.STDOUT, + ) + # Driver errors are private, but scrub literal credentials from evidence too. + text = log.read_text().replace(raw, "[REDACTED_URI]") + for password in (uri.password, urllib.parse.unquote(uri.password or "")): + if password: + text = text.replace(password, "[REDACTED]") + log.write_text(text) + print("BASELINE", "PASS" if result.returncode == 0 else "FAIL", log) + raise SystemExit(result.returncode) + + +if __name__ == "__main__": + main() diff --git a/tests/integration_tests/lake_native/baseline_parity_test.go.txt b/tests/integration_tests/lake_native/baseline_parity_test.go.txt new file mode 100644 index 0000000000..bdadb4a3f2 --- /dev/null +++ b/tests/integration_tests/lake_native/baseline_parity_test.go.txt @@ -0,0 +1,120 @@ +package tidb + +import ( + "context" + "database/sql" + "fmt" + _ "github.com/datafuselabs/databend-go" + "github.com/stretchr/testify/require" + "os" + "testing" + "time" +) + +// Temporary test overlay against the untouched fixed bend-cdc checkout. +func TestBendParityBaselineLive(t *testing.T) { + dsn := os.Getenv("TIDB_DDL_INTEGRATION_DSN") + if dsn == "" { + t.Skip("requires authorized Lake") + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute) + defer cancel() + db, err := sql.Open("databend", dsn) + require.NoError(t, err) + defer db.Close() + name := fmt.Sprintf("bend_parity_%d", time.Now().UnixNano()) + _, err = db.ExecContext(ctx, "CREATE DATABASE "+name) + require.NoError(t, err) + defer func() { + _, err := db.ExecContext(context.Background(), "DROP DATABASE IF EXISTS "+name) + require.NoError(t, err) + }() + t.Log("owned baseline database", name) + sink, err := NewDatabendMergeSink(&DatabendMergeSinkConfig{DSN: dsn, TargetDatabase: name, AllowDelete: true, MergeInterval: time.Hour}) + require.NoError(t, err) + defer sink.Close() + id := TableID{Database: "source", Table: "partitioned"} + columns := []ColumnSchema{{Name: "id", Type: "INT", IsPk: "true", Nullable: "false"}, {Name: "v", Type: "INT", Nullable: "true"}} + apply := func(t *testing.T, epoch int64, query string) { + require.NoError(t, sink.ApplySchema(ctx, id, &SchemaMetaFile{TableVersion: epoch, Query: query, Columns: columns})) + } + t.Run("logical_partition_metadata_and_DML", func(t *testing.T) { + apply(t, 100, "CREATE TABLE partitioned(id INT PRIMARY KEY,v INT) PARTITION BY HASH(id) PARTITIONS 2") + _, err := sink.WriteBatch(ctx, id, []*CanalJSONMessage{{Type: "INSERT", PKNames: []string{"id"}, Data: []map[string]interface{}{{"id": "1", "v": "12"}}, TiDBExtension: &CanalJSONTiDBExtension{CommitTS: 110}}}) + require.NoError(t, err) + require.NoError(t, sink.Flush(ctx, id)) + var count int + require.NoError(t, db.QueryRowContext(ctx, "SELECT count(*) FROM "+name+".partitioned").Scan(&count)) + require.Equal(t, 1, count) + }) + t.Run("rename_is_skipped_not_projected", func(t *testing.T) { + apply(t, 120, "RENAME TABLE partitioned TO renamed") + var count int + require.NoError(t, db.QueryRowContext(ctx, "SELECT count(*) FROM system.tables WHERE database=? AND name='renamed'", name).Scan(&count)) + require.Zero(t, count) + require.NoError(t, db.QueryRowContext(ctx, "SELECT count(*) FROM "+name+".partitioned").Scan(&count)) + require.Equal(t, 1, count) + }) + t.Run("drop_database_is_skipped_not_projected", func(t *testing.T) { + apply(t, 130, "DROP DATABASE source") + var count int + require.NoError(t, db.QueryRowContext(ctx, "SELECT count(*) FROM "+name+".partitioned").Scan(&count)) + require.Equal(t, 1, count) + }) + t.Run("default_and_nullability", func(t *testing.T) { + columns[1].Nullable = "false" + v := "7" + columns[1].Default = &v + apply(t, 140, "ALTER TABLE partitioned MODIFY COLUMN v INT NOT NULL DEFAULT 7") + columns[1].Nullable = "true" + columns[1].Default = nil + apply(t, 150, "ALTER TABLE partitioned MODIFY COLUMN v INT NULL DEFAULT NULL") + rows, err := db.QueryContext(ctx, "DESCRIBE "+name+".partitioned") + require.NoError(t, err) + defer rows.Close() + var observed bool + for rows.Next() { + var col, typ, nullable, def, extra string + require.NoError(t, rows.Scan(&col, &typ, &nullable, &def, &extra)) + if col == "v" { + t.Log("baseline default/nullability", typ, nullable, def) + require.Equal(t, "YES", nullable) + require.Equal(t, "NULL", def) + observed = true + } + } + require.NoError(t, rows.Err()) + require.True(t, observed) + }) + t.Run("int_to_parameterized_varchar", func(t *testing.T) { + columns[1].Type = "VARCHAR(200)" + v := "7" + columns[1].Default = &v + apply(t, 160, "ALTER TABLE partitioned MODIFY COLUMN v VARCHAR(200) NULL DEFAULT '7'") + var value string + require.NoError(t, db.QueryRowContext(ctx, "SELECT v FROM "+name+".partitioned").Scan(&value)) + require.Equal(t, "12", value) + }) + t.Run("no_PK_requires_explicit_conflictKey", func(t *testing.T) { + plain, err := NewDatabendMergeSink(&DatabendMergeSinkConfig{DSN: dsn, TargetDatabase: name}) + require.NoError(t, err) + defer plain.Close() + table := TableID{Database: "source", Table: "no_pk"} + meta := &SchemaMetaFile{TableVersion: 100, Query: "CREATE TABLE no_pk(id INT,v INT)", Columns: []ColumnSchema{{Name: "id", Type: "INT"}, {Name: "v", Type: "INT"}}} + require.Error(t, plain.ApplySchema(ctx, table, meta)) + explicit, err := NewDatabendMergeSink(&DatabendMergeSinkConfig{DSN: dsn, TargetDatabase: name, ConflictKey: "id", AllowDelete: true, MergeInterval: time.Hour}) + require.NoError(t, err) + defer explicit.Close() + require.NoError(t, explicit.ApplySchema(ctx, table, meta)) + _, err = explicit.WriteBatch(ctx, table, []*CanalJSONMessage{{Type: "INSERT", Data: []map[string]interface{}{{"id": "1", "v": "22"}}, TiDBExtension: &CanalJSONTiDBExtension{CommitTS: 110}}}) + require.NoError(t, err) + require.NoError(t, explicit.Flush(ctx, table)) + var count int + require.NoError(t, db.QueryRowContext(ctx, "SELECT count(*) FROM "+name+".no_pk").Scan(&count)) + require.Equal(t, 1, count) + }) + t.Run("multi_column_is_rejected_before_execution", func(t *testing.T) { + _, err := SchemaChanges(&SchemaMetaFile{Query: "ALTER TABLE partitioned ADD COLUMN a INT,ADD COLUMN b INT", Columns: []ColumnSchema{{Name: "a", Type: "INT"}, {Name: "b", Type: "INT"}}}) + require.ErrorContains(t, err, "multi-operation") + }) +} diff --git a/tests/integration_tests/lake_native/extended.py b/tests/integration_tests/lake_native/extended.py new file mode 100644 index 0000000000..e23a99e20a --- /dev/null +++ b/tests/integration_tests/lake_native/extended.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Opt-in real TiUP/Lake DDL matrix and explicit unsupported-scope probes. + +Run after run.py's documented local setup. Each invocation owns a fresh random +source/target database/feed. A rejected probe PASS means rejection was verified, +not that the corresponding feature is supported. +""" + +import argparse +import json +import time + +from run import Run, quote + + +class ExtendedRun(Run): + def __init__(self, force_replicate=False): + self.force_replicate = force_replicate + super().__init__() + + def api(self, path, data=None, method=None): + if path == "/api/v2/changefeeds" and data and self.force_replicate: + data["replica_config"]["force_replicate"] = True + return super().api(path, data, method) + + def schema(self, table, column, nullable, default=None): + self.check_feed() + rows = self.lake("DESCRIBE " + quote(self.database) + "." + quote(table)) + row = next(r for r in rows if r[0] == column) + assert row[2] == ("YES" if nullable else "NO"), row + if default is not None: + actual = row[3].strip("()' ") + assert actual.upper() == default.upper(), (column, actual, default) + self.results.setdefault("schema_observations", []).append(row) + self.save() + + def unique_without_pk(self): + db = "`" + self.database + "`" + self.local( + f"CREATE TABLE {db}.implicit_key(id INT NOT NULL UNIQUE,v INT);" + f"INSERT INTO {db}.implicit_key VALUES(1,5),(3,8)" + ) + self.simple("implicit_key", ["id", "v"]) + self.local(f"UPDATE {db}.implicit_key SET id=2,v=6 WHERE id=1") + self.simple("implicit_key", ["id", "v"]) + self.local(f"DELETE FROM {db}.implicit_key WHERE id=2") + self.simple("implicit_key", ["id", "v"]) + self.results["implicit_key_config"] = { + "force_replicate": self.feed_status() + .get("config", {}) + .get("force_replicate", False), + "declared_primary_key": False, + "not_null_unique_key": "id", + } + self.save() + + def int_to_varchar(self): + db = "`" + self.database + "`" + self.local( + f"CREATE TABLE {db}.varchar_widen(id INT PRIMARY KEY,v INT NOT NULL DEFAULT 7);INSERT INTO {db}.varchar_widen VALUES(1,-2147483648)" + ) + self.simple("varchar_widen", ["id", "v"]) + self.local( + f"ALTER TABLE {db}.varchar_widen MODIFY COLUMN v VARCHAR(200) NOT NULL DEFAULT '7';INSERT INTO {db}.varchar_widen VALUES(2,'0012')" + ) + self.simple("varchar_widen", ["id", "v"]) + self.schema("varchar_widen", "v", False, "7") + + def tests(self): + self.case( + "NOT NULL UNIQUE without declared PK, including key change", + self.unique_without_pk, + ) + self.case( + "INT to sufficiently wide VARCHAR(200) with literal default", + self.int_to_varchar, + ) + db = "`" + self.database + "`" + table = db + ".matrix" + self.local( + f"CREATE TABLE {table}(id INT PRIMARY KEY,n INT NOT NULL DEFAULT 1,s VARCHAR(32),tiny TINYINT);" + f"INSERT INTO {table}(id,s,tiny) VALUES(1,'seed',12)" + ) + self.simple("matrix", ["id", "n", "s", "tiny"]) + + def execute(sql, names): + self.local(sql) + self.simple("matrix", names) + + names = ["id", "n", "s", "tiny"] + + def default_change(): + execute( + f"ALTER TABLE {table} MODIFY COLUMN n INT NOT NULL DEFAULT 7;" + f"INSERT INTO {table}(id,s) VALUES(2,'default7')", + names, + ) + self.schema("matrix", "n", False, "7") + + self.case( + "same-type MODIFY literal default preserves old values", default_change + ) + + def nullable(): + execute( + f"ALTER TABLE {table} MODIFY COLUMN n INT NULL DEFAULT NULL;" + f"INSERT INTO {table}(id,s) VALUES(3,'null')", + names, + ) + self.schema("matrix", "n", True, "NULL") + + self.case("NOT NULL to NULL with explicit DEFAULT NULL", nullable) + + def not_null(): + execute(f"UPDATE {table} SET n=8 WHERE n IS NULL", names) + execute( + f"ALTER TABLE {table} MODIFY COLUMN n INT NOT NULL DEFAULT 9;" + f"INSERT INTO {table}(id) VALUES(4)", + names, + ) + self.schema("matrix", "n", False, "9") + + self.case("NULL to NOT NULL after removing source nulls", not_null) + + def implicit_nullable(): + execute( + f"ALTER TABLE {table} MODIFY COLUMN n BIGINT DEFAULT 11;" + f"INSERT INTO {table}(id,n) VALUES(5,NULL)", + names, + ) + self.schema("matrix", "n", True, "11") + + self.case( + "MODIFY complete definition implicitly removes NOT NULL", implicit_nullable + ) + + def default_removed(): + execute( + f"ALTER TABLE {table} MODIFY COLUMN n BIGINT NULL;" + f"INSERT INTO {table}(id) VALUES(6)", + names, + ) + self.schema("matrix", "n", True, "NULL") + + self.case( + "MODIFY omitted DEFAULT removes previous source default", default_removed + ) + + def widening(): + execute(f"ALTER TABLE {table} MODIFY COLUMN tiny INT", names) + execute( + f"ALTER TABLE {table} MODIFY COLUMN tiny BIGINT;" + f"UPDATE {table} SET tiny=9223372036854775806 WHERE id=1", + names, + ) + execute(f"ALTER TABLE {table} MODIFY COLUMN s TEXT", names) + + self.case("TINYINT to INT to BIGINT and VARCHAR to TEXT", widening) + + def add_defaults(): + execute( + f"ALTER TABLE {table} ADD COLUMN amount DECIMAL(20,6) NOT NULL DEFAULT 1.234567", + names + ["amount"], + ) + execute( + f"ALTER TABLE {table} ADD COLUMN optional INT NULL DEFAULT NULL", + names + ["amount", "optional"], + ) + self.schema("matrix", "optional", True, "NULL") + execute( + f"ALTER TABLE {table} ADD COLUMN note VARCHAR(64) NOT NULL DEFAULT 'O\\'Reilly\\\\path'", + names + ["amount", "optional", "note"], + ) + + self.case("ADD numeric/text/NULL literal defaults and backfill", add_defaults) + + def change(): + execute( + f"ALTER TABLE {table} CHANGE COLUMN tiny counter BIGINT NULL DEFAULT 17;" + f"INSERT INTO {table}(id) VALUES(7)", + ["id", "n", "s", "counter", "amount", "optional", "note"], + ) + self.schema("matrix", "counter", True, "17") + + self.case("CHANGE rename plus full default/nullability definition", change) + + def multi(): + execute( + f"ALTER TABLE {table} ADD COLUMN extra INT NOT NULL DEFAULT 42," + " DROP COLUMN optional, RENAME COLUMN note TO message;" + f"UPDATE {table} SET extra=43 WHERE id=1", + ["id", "n", "s", "counter", "amount", "extra", "message"], + ) + + self.case("multi-operation ADD/DROP/RENAME final projection", multi) + + def reject(self, name): + db = "`" + self.database + "`" + target = quote(self.database) + '."probe"' + before = None + if name not in ("partition", "no-pk"): + self.local( + f"CREATE TABLE {db}.probe(id INT PRIMARY KEY,v INT DEFAULT 1);" + f"INSERT INTO {db}.probe VALUES(1,7)" + ) + self.simple("probe", ["id", "v"]) + before = self.lake("SELECT * FROM " + target) + sql = { + "partition": f"CREATE TABLE {db}.probe(id INT PRIMARY KEY,v INT) PARTITION BY HASH(id) PARTITIONS 2", + "no-pk": f"CREATE TABLE {db}.probe(id INT,v INT)", + "narrow": f"ALTER TABLE {db}.probe MODIFY COLUMN v TINYINT", + "expression-default": f"ALTER TABLE {db}.probe ADD COLUMN ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP", + }[name] + self.local(sql) + if name in ("partition", "no-pk"): + self.local(f"INSERT INTO {db}.probe VALUES(1,5)") + deadline = time.monotonic() + 65 + while time.monotonic() < deadline: + status = self.feed_status() + if status.get("state") == "failed": + break + time.sleep(0.5) + else: + raise AssertionError( + "probe did not fail closed: " + + self.sanitize( + json.dumps( + {k: status.get(k) for k in ("state", "error", "checkpoint_ts")} + ) + ) + ) + # Verify a stable failed lifecycle, not only a transient reported error. + checkpoint = status["checkpoint_ts"] + time.sleep(2) + after = self.feed_status() + assert after.get("state") == "failed", self.sanitize(after) + assert after["checkpoint_ts"] == checkpoint + self.results["rejection"] = { + "case": name, + "state": after["state"], + "checkpoint_ts": checkpoint, + "error": self.sanitize(after.get("error")), + } + if before is not None: + assert self.lake("SELECT * FROM " + target) == before + else: + assert self.lake( + "SELECT COUNT(*)::VARCHAR FROM system.tables WHERE database=? AND name='probe'", + [self.database], + ) == [["0"]] + self.save() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--reject", + choices=[ + "partition", + "no-pk", + "narrow", + "expression-default", + ], + ) + parser.add_argument("--unique-only", action="store_true") + parser.add_argument("--varchar-only", action="store_true") + args = parser.parse_args() + run = ExtendedRun(force_replicate=args.reject == "no-pk") + try: + run.setup() + if args.reject: + run.case( + "expected rejection: " + args.reject, lambda: run.reject(args.reject) + ) + elif args.varchar_only: + run.case( + "INT to sufficiently wide VARCHAR(200) with literal default", + run.int_to_varchar, + ) + elif args.unique_only: + run.case( + "NOT NULL UNIQUE without declared PK, including key change", + run.unique_without_pk, + ) + else: + run.tests() + run.cleanup() + run.results["status"] = "PASS" + run.save() + except Exception as error: + run.results["status"] = "FAIL" + run.results["error"] = run.sanitize(error) + run.save() + print("ERROR", run.sanitize(error), flush=True) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/tests/integration_tests/lake_native/local.py b/tests/integration_tests/lake_native/local.py new file mode 100755 index 0000000000..c0e2ffdb67 --- /dev/null +++ b/tests/integration_tests/lake_native/local.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python3 +"""Owned local TiUP + TiCDC + Docker Lake lifecycle for native sink E2E. + +Unlike run.py's external-warehouse mode, this runner always creates its own +loopback-only Lake, credentials, source and capture. No cloud URI is read. +""" + +import argparse +import hashlib +import json +import os +import pathlib +import shlex +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import time +import urllib.request +import uuid + +HERE = pathlib.Path(__file__).resolve().parent +REPO = HERE.parents[2] +# Fixed multi-architecture release, not a moving latest/nightly tag. +IMAGE = ( + "datafuselabs/databend:v1.2.925-patch-13@sha256:" + "7f93f005aa0d626ebc7554d142b0a6b6ebb90e085b0d5bd0d97470084c42d41b" +) +SUITES = { + "acceptance": ["run.py"], + "ddl": ["extended.py"], + "manual-key": ["options.py", "--manual-key"], + "skipped-ddl": ["options.py"], + **{ + f"reject-{name}": ["extended.py", "--reject", name] + for name in ("partition", "no-pk", "narrow", "expression-default") + }, + "fault": None, +} + + +def wait_ready(check, process, timeout, description): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process is not None and process.poll() is not None: + raise RuntimeError( + description + " exited before becoming ready; inspect logs" + ) + try: + if check(): + return + except (OSError, ValueError, RuntimeError, subprocess.SubprocessError): + pass + time.sleep(0.5) + raise RuntimeError(description + " readiness timeout; inspect logs") + + +def free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def require_python(): + if sys.version_info < (3, 10): + raise SystemExit("Lake E2E requires Python 3.10 or newer") + + +def check_source_port(port): + # On BSD/macOS a specific-address SO_REUSEADDR bind can coexist with a + # wildcard listener. Detect a live loopback/wildcard service before binding. + with socket.socket() as probe: + probe.settimeout(1) + if probe.connect_ex(("127.0.0.1", port)) == 0: + raise RuntimeError("source port %d is already in use" % port) + with socket.socket() as sock: + # TIME_WAIT is not a live listener; allow immediate reruns. + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", port)) + + +def file_sha256(path): + # file_digest is only available in Python 3.11+. + digest = hashlib.sha256() + with path.open("rb") as binary: + for chunk in iter(lambda: binary.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +class Local: + def __init__(self, root, image, version): + self.root = root + self.image = image + self.version = version + self.processes = [] + self.container = "ticdc-lake-e2e-" + uuid.uuid4().hex[:12] + self.container_created = False + self.tag = "native-e2e-" + uuid.uuid4().hex[:12] + self.env = os.environ.copy() + # Never inherit a cloud target or another runner's capture/restart options. + for key in ( + "LAKE_URI_FILE", + "LAKE_NATIVE_TEST_URI", + "E2E_CDC_BINARY", + "E2E_CDC_DATA_DIR", + "E2E_PD", + "CDC_SERVER", + "MYSQL_PORT", + ): + self.env.pop(key, None) + self.summary = {"status": "RUNNING", "suites": []} + self.binary = None + self.offset = int(os.environ.get("E2E_PORT_OFFSET", "10000")) + if not 0 <= self.offset <= 30000: + raise RuntimeError("E2E_PORT_OFFSET must be in [0,30000]") + + def command(self, args, **kwargs): + kwargs.setdefault("timeout", 900) + return subprocess.run(args, check=True, env=self.env, **kwargs) + + def start(self, args, log): + with (self.root / log).open("wb") as output: + process = subprocess.Popen( + args, + env=self.env, + stdout=output, + stderr=subprocess.STDOUT, + start_new_session=True, + cwd=REPO, + ) + self.processes.append(process) + return process + + def save(self): + (self.root / "summary.json").write_text( + json.dumps(self.summary, indent=2) + "\n" + ) + + def prepare(self): + for program in ("docker", "tiup", "mysql", "go"): + if not shutil.which(program): + raise RuntimeError("missing local prerequisite: " + program) + self.command( + ["docker", "info"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + # Fixed source ports must not silently attach to an existing playground. + for base in (4000, 10080, 2379, 2380, 20160, 20180): + check_source_port(base + self.offset) + self.summary["source_commit"] = self.command( + ["git", "-C", str(REPO), "rev-parse", "HEAD"], + capture_output=True, + text=True, + ).stdout.strip() + self.summary["source_dirty"] = bool( + self.command( + ["git", "-C", str(REPO), "status", "--porcelain"], + capture_output=True, + text=True, + ).stdout.strip() + ) + supplied = os.environ.get("E2E_CDC_BINARY") + if supplied: + self.binary = pathlib.Path(supplied).resolve(strict=True) + else: + self.binary = self.root / "cdc" + build = self.start(["make", "cdc"], "build.log") + if build.wait(timeout=900): + raise RuntimeError( + "current-branch TiCDC build failed; inspect build.log" + ) + shutil.copy2(REPO / "bin/cdc", self.binary) + self.summary["cdc_binary_supplied"] = bool(supplied) + self.summary["cdc_version"] = self.command( + [str(self.binary), "version"], capture_output=True, text=True + ).stdout.strip() + self.summary["cdc_sha256"] = file_sha256(self.binary) + print("Starting owned local Lake background container", flush=True) + with (self.root / "pull.log").open("w") as log: + self.command( + ["docker", "pull", self.image], stdout=log, stderr=subprocess.STDOUT + ) + self.summary["lake_image"] = json.loads( + self.command( + ["docker", "image", "inspect", self.image], + capture_output=True, + text=True, + ).stdout + )[0]["Id"] + self.summary["lake_image_tag"] = self.image + # No host network, no cloud storage, no MinIO download, no exposed password. + password = uuid.uuid4().hex + self.env["QUERY_DEFAULT_PASSWORD"] = password + self.command( + [ + "docker", + "create", + "--name", + self.container, + "--label", + "ticdc.e2e=" + self.tag, + "--publish", + "127.0.0.1::8000", + "--env", + "QUERY_STORAGE_TYPE=fs", + "--env", + "QUERY_DEFAULT_USER=ticdc_e2e", + "--env", + "QUERY_DEFAULT_PASSWORD", + self.image, + ], + stdout=subprocess.DEVNULL, + ) + self.container_created = True + self.command(["docker", "start", self.container], stdout=subprocess.DEVNULL) + address = self.command( + ["docker", "port", self.container, "8000/tcp"], + capture_output=True, + text=True, + ).stdout.strip() + uri = f"lake+http://ticdc_e2e:{password}@{address}/default?exclusive=true" + credentials = self.root / "local-uri" + credentials.write_text(uri) + credentials.chmod(0o600) + self.env["LAKE_URI_FILE"] = str(credentials) + self.env["LAKE_NATIVE_TEST_URI"] = uri + self.env.pop("QUERY_DEFAULT_PASSWORD") + self.env["MYSQL_PORT"] = str(4000 + self.offset) + self.env["E2E_RUN_DIR"] = str(self.root / "readiness") + # Reuse the exact HTTP/terminal-state adapter used by the assertions. + from run import Run + + old_env = os.environ.copy() + try: + os.environ.update(self.env) + probe = Run() + finally: + os.environ.clear() + os.environ.update(old_env) + wait_ready(lambda: probe.lake("SELECT 1") == [["1"]], None, 90, "Lake") + self.summary["lake_version"] = probe.lake("SELECT version()")[0][0] + if probe.lake("SELECT ?::BIGINT", [1]) != [["1"]]: + raise RuntimeError("Lake must support Query API bound parameters") + print("Starting owned local TiDB/PD/TiKV", flush=True) + source = self.start( + [ + "tiup", + "playground", + self.version, + "--tag", + self.tag, + "--db", + "1", + "--pd", + "1", + "--kv", + "1", + "--tiflash", + "0", + "--ticdc", + "0", + "--without-monitor", + "--host", + "127.0.0.1", + "--port-offset", + str(self.offset), + ], + "tiup.log", + ) + wait_ready(lambda: probe.local("SELECT 1") == "1", source, 180, "TiDB") + # Owned source only: legacy index construction works on low-space laptops. + probe.local( + "SET GLOBAL tidb_ddl_enable_fast_reorg=OFF;SET GLOBAL tidb_enable_dist_task=OFF" + ) + self.summary["source_version"] = probe.local("SELECT version()") + self.summary["source_adjustments"] = [ + "tidb_ddl_enable_fast_reorg=OFF", + "tidb_enable_dist_task=OFF", + ] + port = free_port() + self.env.update( + CDC_SERVER=f"http://127.0.0.1:{port}", + E2E_CDC_BINARY=str(self.binary), + E2E_CDC_DATA_DIR=str(self.root / "cdc-data"), + E2E_PD=f"http://127.0.0.1:{2379 + self.offset}", + ) + capture = self.start( + [ + str(self.binary), + "server", + "--newarch", + "--tz", + "UTC", + "--pd", + self.env["E2E_PD"], + "--addr", + f"127.0.0.1:{port}", + "--data-dir", + self.env["E2E_CDC_DATA_DIR"], + "--log-file", + str(self.root / "cdc.log"), + ], + "cdc-console.log", + ) + + def capture_ready(): + with urllib.request.urlopen( + self.env["CDC_SERVER"] + "/api/v2/status", timeout=5 + ) as response: + if not json.load(response).get("is_owner", False): + return False + # Owner election precedes coordinator bootstrap. Do not submit the + # first create request while the read API still returns HTTP 500. + with urllib.request.urlopen( + self.env["CDC_SERVER"] + "/api/v2/changefeeds", timeout=5 + ) as response: + return response.status == 200 + + wait_ready(capture_ready, capture, 90, "TiCDC") + self.save() + + def suites(self, names): + for name in names: + self.env["E2E_RUN_DIR"] = str(self.root / name) + if name == "fault": + args = [ + "go", + "test", + "-p", + "2", + "-race", + "./downstreamadapter/sink/lake", + "-run", + "TestLakeNativeLive", + "-v", + "-count=1", + "-timeout=20m", + ] + else: + args = [sys.executable, str(HERE / SUITES[name][0]), *SUITES[name][1:]] + print("Running " + name, flush=True) + started = time.monotonic() + child = self.start(args, name + ".log") + # Reap the old capture while acceptance's owned restart waits for exit. + while child.poll() is None: + for process in self.processes: + process.poll() + time.sleep(0.2) + record = { + "name": name, + "exit_code": child.returncode, + "seconds": round(time.monotonic() - started, 2), + } + self.summary["suites"].append(record) + self.save() + if child.returncode: + raise RuntimeError( + name + " failed; inspect " + str(self.root / (name + ".log")) + ) + + def cleanup(self): + # A restarted capture is launched by run.py, not this process. Check both + # its generated PID path and exact binary before signalling it. + restarted = self.root / "acceptance/restart-pid" + errors = [] + if restarted.exists(): + try: + pid = int(restarted.read_text()) + command = subprocess.run( + ["ps", "-p", str(pid), "-o", "command="], + capture_output=True, + text=True, + ) + argv = shlex.split(command.stdout.strip()) + if command.returncode == 0: + data_dir = ( + argv[argv.index("--data-dir") + 1] + if "--data-dir" in argv + and argv.index("--data-dir") + 1 < len(argv) + else "" + ) + if ( + len(argv) < 2 + or pathlib.Path(argv[0]).resolve() != self.binary.resolve() + or argv[1] != "server" + or pathlib.Path(data_dir).resolve() + != (self.root / "cdc-data").resolve() + ): + raise RuntimeError("refusing to stop unowned restarted capture") + os.kill(pid, signal.SIGTERM) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.2) + else: + raise RuntimeError("restarted capture did not terminate") + except Exception as error: + errors.append(str(error)) + for process in reversed(self.processes): + try: + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=35) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + except ProcessLookupError: + pass + except Exception as error: + errors.append("process cleanup: " + type(error).__name__) + # TiUP may have already exited after a child startup failure. Its unique + # tag confines cleanup to this run; never use broad pkill/tiup clean --all. + if (self.root / "tiup.log").exists(): + try: + result = subprocess.run( + ["tiup", "clean", self.tag], + env=self.env, + capture_output=True, + text=True, + timeout=60, + ) + (self.root / "tiup-clean.log").write_text(result.stdout + result.stderr) + if result.returncode: + errors.append("TiUP tag cleanup failed") + except Exception as error: + errors.append("TiUP cleanup: " + type(error).__name__) + if self.container_created: + try: + with (self.root / "lake.log").open("wb") as log: + subprocess.run( + ["docker", "logs", self.container], + stdout=log, + stderr=subprocess.STDOUT, + timeout=30, + ) + except Exception as error: + errors.append("Lake logs: " + type(error).__name__) + try: + result = subprocess.run( + ["docker", "rm", "--force", "--volumes", self.container], + capture_output=True, + text=True, + timeout=45, + ) + if result.returncode: + errors.append("Lake container cleanup failed") + except Exception as error: + errors.append("Lake cleanup: " + type(error).__name__) + (self.root / "local-uri").unlink(missing_ok=True) + self.summary["cleanup"] = "PASS" if not errors else errors + if errors: + self.summary["status"] = "FAIL" + self.save() + + +def main(): + require_python() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--suite", + choices=SUITES, + action="append", + help="repeat to select suites; default all", + ) + args = parser.parse_args() + os.umask(0o077) + root = pathlib.Path( + tempfile.mkdtemp(prefix="ticdc-lake-local-", dir=os.environ.get("OUT_DIR")) + ) + print("Local E2E artifacts: " + str(root), flush=True) + runner = Local( + root, + os.environ.get("LAKE_E2E_IMAGE", IMAGE), + os.environ.get("TIDB_E2E_VERSION", "v8.5.1"), + ) + + def interrupted(signum, _frame): + raise InterruptedError("local E2E interrupted by signal " + str(signum)) + + signal.signal(signal.SIGTERM, interrupted) + signal.signal(signal.SIGINT, interrupted) + try: + runner.prepare() + runner.suites(args.suite or list(SUITES)) + runner.summary["status"] = "PASS" + except Exception as error: + # Do not include inherited env, query SQL or credentials in the summary. + runner.summary["status"] = "FAIL" + runner.summary["error"] = type(error).__name__ + ": " + str(error) + print(runner.summary["error"], file=sys.stderr) + finally: + runner.cleanup() + print("RESULT " + str(root / "summary.json"), flush=True) + return 0 if runner.summary["status"] == "PASS" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/integration_tests/lake_native/options.py b/tests/integration_tests/lake_native/options.py new file mode 100644 index 0000000000..b83f0c501e --- /dev/null +++ b/tests/integration_tests/lake_native/options.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Real-source acceptance for conflict-key and explicitly skipped source DDL. + +Uses the same private environment and isolated ownership rules as run.py. +""" + +import argparse +import json +import time +import urllib.parse + +from run import Run, quote + + +class OptionsRun(Run): + def __init__(self, manual=False): + self.manual = manual + super().__init__() + if manual: + q = urllib.parse.parse_qs(self.parsed.query) + q["conflict-key"] = ["ID"] # Match authoritative source casing. + self.parsed = self.parsed._replace( + query=urllib.parse.urlencode(q, doseq=True) + ) + + def api(self, path, data=None, method=None): + if self.manual and path == "/api/v2/changefeeds" and data: + data["replica_config"]["force_replicate"] = True + return super().api(path, data, method) + + def drained_marker(self): + marker = int( + self.local( + "BEGIN;SELECT COUNT(*) FROM mysql.tidb;SELECT @@tidb_current_ts;COMMIT" + ).splitlines()[-1] + ) + assert marker > 0 + deadline = time.monotonic() + 75 + while self.check_feed()["checkpoint_ts"] < marker: + if time.monotonic() > deadline: + raise AssertionError("checkpoint did not pass skipped DDL") + time.sleep(0.3) + self.results.setdefault("drained_markers", []).append(marker) + self.save() + + def test_manual(self): + db = "`" + self.database + "`" + self.local( + f"CREATE TABLE {db}.keyless(id BIGINT,v VARCHAR(64));INSERT INTO {db}.keyless VALUES(1,'one'),(2,'two')" + ) + self.simple("keyless", ["id", "v"]) + self.local( + f"UPDATE {db}.keyless SET v='updated' WHERE id=1;DELETE FROM {db}.keyless WHERE id=2;INSERT INTO {db}.keyless VALUES(3,'three')" + ) + self.simple("keyless", ["id", "v"]) + self.local( + f"ALTER TABLE {db}.keyless ADD COLUMN extra BIGINT NULL;INSERT INTO {db}.keyless VALUES(4,'four',42)" + ) + self.simple("keyless", ["id", "v", "extra"]) + journal = self.lake( + "SELECT plan FROM " + quote(self.database) + '."keyless_cdc_schema_epoch"' + ) + + plan = json.loads(journal[0][0]) + assert plan["schema"]["keys"] == ["id"], plan + self.results["manual_key"] = { + "source_primary_key": False, + "source_unique_key": False, + "effective_key": plan["schema"]["keys"], + "force_replicate": self.feed_status()["config"].get("force_replicate"), + } + # bend-cdc also permits replacing a single native PK, not a composite. + self.local( + f"CREATE TABLE {db}.override_key(other_id INT PRIMARY KEY,id BIGINT,v VARCHAR(64));INSERT INTO {db}.override_key VALUES(10,1,'one'),(20,2,'two')" + ) + self.simple("override_key", ["other_id", "id", "v"]) + self.local( + f"UPDATE {db}.override_key SET v='changed' WHERE other_id=10;DELETE FROM {db}.override_key WHERE other_id=20" + ) + self.simple("override_key", ["other_id", "id", "v"]) + self.drained_marker() + self.save() + + def test_rename(self): + db = "`" + self.database + "`" + target = quote(self.database) + self.local( + f"CREATE TABLE {db}.original(id INT PRIMARY KEY,v VARCHAR(64));INSERT INTO {db}.original VALUES(1,'old')" + ) + self.simple("original", ["id", "v"]) + self.local(f"RENAME TABLE {db}.original TO {db}.renamed") + self.drained_marker() + assert self.lake("SELECT id::VARCHAR,v FROM " + target + '."original"') == [ + ["1", "old"] + ] + assert self.lake( + "SELECT COUNT(*)::VARCHAR FROM system.tables WHERE database=? AND name='renamed'", + [self.database], + ) == [["0"]] + # No historical rows are moved. Subsequent DML gets a new projection. + self.local( + f"UPDATE {db}.renamed SET v='new' WHERE id=1;INSERT INTO {db}.renamed VALUES(2,'inserted')" + ) + self.simple("renamed", ["id", "v"]) + assert self.lake("SELECT id::VARCHAR,v FROM " + target + '."original"') == [ + ["1", "old"] + ] + self.results["rename"] = { + "source_renamed": True, + "old_target_preserved": True, + "new_target_populated_by_subsequent_dml": True, + } + self.save() + + def test_drop_database(self): + db = "`" + self.database + "`" + target = quote(self.database) + before = self.lake( + "SELECT id::VARCHAR,v FROM " + target + '."renamed" ORDER BY id' + ) + self.local(f"DROP DATABASE {db}") + self.drained_marker() + assert ( + self.lake("SELECT id::VARCHAR,v FROM " + target + '."renamed" ORDER BY id') + == before + ) + assert self.lake("SELECT id::VARCHAR,v FROM " + target + '."original"') == [ + ["1", "old"] + ] + self.local( + f"CREATE DATABASE {db};CREATE TABLE {db}.after_drop(id INT PRIMARY KEY,v INT);INSERT INTO {db}.after_drop VALUES(1,42)" + ) + self.simple("after_drop", ["id", "v"]) + self.drained_marker() + self.results["drop_database"] = { + "target_database_preserved": True, + "existing_target_data_preserved": True, + "subsequent_source_database_recreation_and_new_table_dml": True, + } + self.save() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--manual-key", action="store_true") + args = parser.parse_args() + run = OptionsRun(manual=args.manual_key) + try: + run.setup() + if args.manual_key: + run.case( + "force replicated keyless DML/DDL and single native key override", + run.test_manual, + ) + else: + run.case( + "RENAME skipped, old target preserved and subsequent DML continues", + run.test_rename, + ) + run.case( + "DROP DATABASE skipped, checkpoint and subsequent DML continue", + run.test_drop_database, + ) + run.cleanup() + run.results["status"] = "PASS" + run.save() + except Exception as error: + run.results["status"] = "FAIL" + run.results["error"] = run.sanitize(error) + run.save() + print("ERROR", run.sanitize(error), flush=True) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/tests/integration_tests/lake_native/run.py b/tests/integration_tests/lake_native/run.py new file mode 100755 index 0000000000..bdd8a66bb0 --- /dev/null +++ b/tests/integration_tests/lake_native/run.py @@ -0,0 +1,769 @@ +#!/usr/bin/env python3 +"""Opt-in TiUP -> native TiCDC -> isolated Lake database acceptance test. + +Requires an already-running local TiUP and current-branch CDC server. Credentials +are read from LAKE_URI_FILE (mode 0600), never argv or repository config. Only the +random native_e2e_* database and this run's changefeed are created/dropped. +""" + +import base64 +import datetime +import decimal +import json +import os +import pathlib +import re +import signal +import shlex +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + + +def quote(name): + return '"' + name.replace('"', '""') + '"' + + +class Run: + def __init__(self, existing=False): + credential_file = pathlib.Path(os.environ["LAKE_URI_FILE"]) + if credential_file.stat().st_mode & 0o077: + raise RuntimeError("LAKE_URI_FILE must have private 0600 permissions") + self.uri = credential_file.read_text().strip() + self.parsed = urllib.parse.urlparse(self.uri) + self.root = pathlib.Path(os.environ["E2E_RUN_DIR"]) + self.root.mkdir(parents=True, exist_ok=True) + os.chmod(self.root, 0o700) + self.cdc = os.environ.get("CDC_SERVER", "http://127.0.0.1:18300") + self.port = os.environ.get("MYSQL_PORT", "14000") + self.database = ( + "native_e2e_" + + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S_") + + uuid.uuid4().hex[:8] + ) + self.feed = self.database.replace("_", "-") + # Plain HTTP is only enabled by the sink's explicit test scheme. + scheme = "http" if self.parsed.scheme == "lake+http" else "https" + self.base = scheme + "://" + self.parsed.netloc.rsplit("@", 1)[-1] + self.results = { + "database": self.database, + "changefeed": self.feed, + "cases": [], + "started_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + } + self.feed_created = False + self.remote_created = False + if existing: + saved = json.loads((self.root / "result.json").read_text()) + database = saved["database"] + if not re.fullmatch( + r"native_e2e_[0-9]{8}_[0-9]{6}_[0-9a-f]{8}", database + ) or saved["changefeed"] != database.replace("_", "-"): + raise RuntimeError( + "refusing cleanup outside this runner's isolated namespace" + ) + self.results = saved + self.database = database + self.feed = saved["changefeed"] + self.remote_created = True + self.feed_created = True + else: + if (self.root / "result.json").exists(): + raise RuntimeError( + "E2E_RUN_DIR already contains a result; use a new directory or --cleanup" + ) + self.save() + + def sanitize(self, text): + return ( + str(text) + .replace(self.uri, "[REDACTED_URI]") + .replace(self.parsed.password or "\0", "[REDACTED]") + ) + + def save(self): + path = self.root / "result.json" + path.write_text(json.dumps(self.results, indent=2, ensure_ascii=False)) + os.chmod(path, 0o600) + + def local(self, sql): + proc = subprocess.run( + [ + "mysql", + "-h", + "127.0.0.1", + "-P", + self.port, + "-u", + "root", + "--default-character-set=utf8mb4", + "-N", + "-B", + "--raw", + "-e", + "SET time_zone='+00:00';" + sql, + ], + text=True, + capture_output=True, + timeout=40, + ) + if proc.returncode: + raise RuntimeError("local SQL failed: " + self.sanitize(proc.stderr)) + return proc.stdout.strip() + + def api(self, path, data=None, method=None): + req = urllib.request.Request( + self.cdc + path, + data=None if data is None else json.dumps(data).encode(), + method=method, + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=60) as response: + raw = response.read() + return json.loads(raw) if raw else {} + except urllib.error.HTTPError as error: + raise RuntimeError( + "CDC HTTP %d: %s" % (error.code, self.sanitize(error.read().decode())) + ) from None + + def lake(self, sql, params=None): + """No mutation retries: an ambiguous response stops the test for inspection.""" + query_id = str(uuid.uuid4()) + auth = ( + urllib.parse.unquote(self.parsed.username) + + ":" + + urllib.parse.unquote(self.parsed.password) + ) + headers = { + "Authorization": "Basic " + base64.b64encode(auth.encode()).decode(), + "Content-Type": "application/json", + "X-DATABEND-WAREHOUSE": urllib.parse.parse_qs(self.parsed.query).get( + "warehouse", ["default"] + )[0], + "X-DATABEND-QUERY-ID": query_id, + } + body = { + "sql": sql, + "session": {"database": "default", "settings": {"timezone": "UTC"}}, + "pagination": {"wait_time_secs": 1, "max_rows_per_page": 1024}, + } + if params is not None: + body["params"] = params + req = urllib.request.Request( + self.base + "/v1/query", data=json.dumps(body).encode(), headers=headers + ) + rows = [] + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(req, timeout=40) as response: + result = json.load(response) + route = response.headers.get("X-DATABEND-ROUTE-HINT") + except urllib.error.HTTPError as error: + raise RuntimeError( + "Lake HTTP %d request %s: %s" + % (error.code, query_id, self.sanitize(error.read(1024).decode())) + ) from None + if result.get("state") == "Failed" or result.get("error"): + raise RuntimeError( + "Lake SQL request %s failed: %s" + % (query_id, self.sanitize(result.get("error"))) + ) + if result.get("id") != query_id: + raise RuntimeError("Lake query identity mismatch: " + query_id) + rows.extend(result.get("data") or []) + path = result.get("next_uri") + if result.get("state") == "Succeeded" and ( + not path or path.endswith("/final") + ): + final = result.get("final_uri") + if final and final.startswith("/v1/query/"): + try: + with urllib.request.urlopen( + urllib.request.Request(self.base + final, headers=headers), + timeout=5, + ) as response: + response.read() + except (OSError, urllib.error.HTTPError): + pass # Result cleanup does not revoke confirmed query success. + return rows + if not path or not path.startswith("/v1/query/") or path.startswith("//"): + raise RuntimeError("Missing terminal evidence for " + query_id) + headers["X-DATABEND-STICKY-NODE"] = result.get("node_id", "") + headers["X-DATABEND-NODE-ID"] = result.get("node_id", "") + if route: + headers["X-DATABEND-ROUTE-HINT"] = route + req = urllib.request.Request(self.base + path, headers=headers) + time.sleep(0.1) + raise RuntimeError("Lake query terminal state timeout: " + query_id) + + def feed_status(self): + return self.api("/api/v2/changefeeds/" + self.feed) + + def check_feed(self): + status = self.feed_status() + if status.get("state") in ("failed", "error", "warning") or status.get("error"): + raise RuntimeError( + "changefeed unhealthy: " + + self.sanitize( + json.dumps( + { + "state": status.get("state"), + "error": status.get("error"), + "checkpoint_ts": status.get("checkpoint_ts"), + } + ) + ) + ) + return status + + def case(self, name, action): + started = time.monotonic() + record = {"name": name} + try: + action() + record.update(status="PASS", seconds=round(time.monotonic() - started, 3)) + print("PASS", name, record["seconds"], flush=True) + except Exception as error: + record.update( + status="FAIL", + seconds=round(time.monotonic() - started, 3), + error=self.sanitize(error), + ) + print("FAIL", name, record["error"], flush=True) + raise + finally: + self.results["cases"].append(record) + self.save() + + def compare(self, table, columns, numeric=(), json_columns=(), timeout=75): + # JSON_OBJECT keeps control characters/newlines and SQL NULL unambiguous. + source = ( + "SELECT JSON_OBJECT(" + + ",".join("'%s',%s" % (name, expr) for name, expr, _ in columns) + + ") FROM `%s`.`%s` ORDER BY id" % (self.database, table) + ) + expected = [ + json.loads(line) for line in self.local(source).splitlines() if line + ] + target = ( + "SELECT " + + ",".join(expr for _, _, expr in columns) + + " FROM " + + quote(self.database) + + "." + + quote(table) + + " ORDER BY id" + ) + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + self.check_feed() + try: + data = self.lake(target) + actual = [dict(zip([c[0] for c in columns], row)) for row in data] + + # Lake may bind ORDER BY id to the projected VARCHAR alias. + # Compare the key-indexed set, never numeric-vs-text sort order. + def key(row): + return str(row["id"]), str(row.get("tenant", "")) + + expected.sort(key=key) + actual.sort(key=key) + if len(expected) != len(actual): + raise AssertionError( + "row count expected %d actual %d" % (len(expected), len(actual)) + ) + for left, right in zip(expected, actual): + for name in left: + a, b = left[name], right[name] + if a is None or b is None: + assert a is b, (name, a, b) + elif name in json_columns: + assert json.loads( + a, parse_float=decimal.Decimal + ) == json.loads(b, parse_float=decimal.Decimal), ( + name, + a, + b, + ) + elif name in numeric: + da, db = decimal.Decimal(str(a)), decimal.Decimal(str(b)) + assert abs(da - db) <= max( + abs(da), decimal.Decimal(1) + ) * decimal.Decimal("0.000001"), (name, a, b) + else: + assert str(a) == str(b), (name, a, b) + return + except (RuntimeError, AssertionError) as error: + last = self.sanitize(error) + time.sleep(0.5) + raise AssertionError("target did not converge: " + last) + + def simple(self, table, names): + self.compare( + table, + [ + (name, "CAST(`%s` AS CHAR)" % name, quote(name) + "::VARCHAR") + for name in names + ], + ) + + def setup(self): + self.results["source_version"] = self.local("SELECT version()") + self.results["cdc_status"] = self.api("/api/v2/status") + self.results["lake_version"] = self.lake("SELECT version(), ?::BIGINT", [1])[0] + self.local("CREATE DATABASE `%s`" % self.database) + self.lake("CREATE DATABASE " + quote(self.database)) + self.remote_created = True + q = urllib.parse.parse_qs(self.parsed.query) + q.update( + { + "exclusive": ["true"], + "workers": ["1"], + "batch-rows": ["64"], + "batch-bytes": ["32768"], + "flush-interval": ["200ms"], + "spool-bytes": ["65536"], + "memory-ratio": ["0.001"], + "source-bytes": ["268435456"], + "source-events": ["1"], + "spool-dir": [str(self.root / "spool")], + } + ) + sink = self.parsed._replace( + path="/" + self.database, query=urllib.parse.urlencode(q, doseq=True) + ).geturl() + self.api( + "/api/v2/changefeeds", + { + "changefeed_id": self.feed, + "sink_uri": sink, + "replica_config": { + "filter": {"rules": [self.database + ".*"]}, + "sink": {"transaction_atomicity": "none"}, + "scheduler": {"enable_table_across_nodes": False}, + }, + }, + ) + self.feed_created = True + self.save() + time.sleep(2) + self.check_feed() + + def tests(self): + db = "`" + self.database + "`" + self.case( + "DML insert/update/delete and nonclustered PK/UK changes", + lambda: self.test_keys(db), + ) + self.case( + "Composite primary key and escaped identifiers", + lambda: self.test_composite(db), + ) + self.case( + "Quoted identifiers and operation discriminator collision", + lambda: self.test_identifiers(db), + ) + self.case( + "Column scalar/binary/decimal/JSON/ENUM/SET/temporal fidelity", + lambda: self.test_types(db), + ) + self.case("DDL add/default/rename/widen/drop", lambda: self.test_ddl(db)) + self.case( + "TRUNCATE new generation and subsequent DML", lambda: self.test_truncate(db) + ) + self.case( + "Large transaction and low traffic batching", lambda: self.test_batch(db) + ) + if os.environ.get("E2E_CDC_BINARY"): + self.case( + "Graceful CDC process restart with paused DML/DDL backlog", + lambda: self.test_restart(db), + ) + self.case("DROP TABLE cleanup", lambda: self.test_drop(db)) + self.results["last_status"] = self.feed_status() + # Never persist the full sink URI returned by a future API version. + self.results["last_status"] = json.loads( + self.sanitize(json.dumps(self.results["last_status"])) + ) + self.save() + + def test_keys(self, db): + self.local( + f"CREATE TABLE {db}.keys_test (id BIGINT PRIMARY KEY NONCLUSTERED,email VARCHAR(128) UNIQUE, value VARCHAR(255)); INSERT INTO {db}.keys_test VALUES (1,'a','first'),(2,'b','second')" + ) + self.simple("keys_test", ["id", "email", "value"]) + self.local(f"UPDATE {db}.keys_test SET email='c',value='changed' WHERE id=1") + self.simple("keys_test", ["id", "email", "value"]) + self.local(f"UPDATE {db}.keys_test SET id=3 WHERE id=1") + self.simple("keys_test", ["id", "email", "value"]) + self.local( + f"BEGIN; DELETE FROM {db}.keys_test WHERE id=2; INSERT INTO {db}.keys_test VALUES (2,'new','reused'); COMMIT;" + ) + self.simple("keys_test", ["id", "email", "value"]) + self.local(f"DELETE FROM {db}.keys_test WHERE id=3") + self.simple("keys_test", ["id", "email", "value"]) + + def test_composite(self, db): + self.local( + f"CREATE TABLE {db}.composite_test(id INT, tenant VARCHAR(64), `action` VARCHAR(64), PRIMARY KEY(tenant,id)); INSERT INTO {db}.composite_test VALUES (1,'a|b','insert'),(2,'a|b','update')" + ) + self.simple("composite_test", ["id", "tenant", "action"]) + self.local( + f"UPDATE {db}.composite_test SET tenant='b|c',action='delete' WHERE id=1; DELETE FROM {db}.composite_test WHERE id=2" + ) + self.simple("composite_test", ["id", "tenant", "action"]) + + def test_identifiers(self, db): + self.local( + f"""CREATE TABLE {db}.`quoted"table` (id INT PRIMARY KEY,`a"b` VARCHAR(64),`__ticdc_operation` VARCHAR(64),`action` VARCHAR(64)); INSERT INTO {db}.`quoted"table` VALUES(1,'quoted-column','source-value','insert')""" + ) + self.simple('quoted"table', ["id", 'a"b', "__ticdc_operation", "action"]) + self.local( + f"""UPDATE {db}.`quoted"table` SET `action`='delete',`a"b`='changed' WHERE id=1""" + ) + self.simple('quoted"table', ["id", 'a"b', "__ticdc_operation", "action"]) + + def test_types(self, db): + sets = ",".join("'s%d'" % i for i in range(64)) + definitions = ( + """id INT PRIMARY KEY, s_tiny TINYINT, u_tiny TINYINT UNSIGNED, + s_small SMALLINT, u_small SMALLINT UNSIGNED,s_medium MEDIUMINT,u_medium MEDIUMINT UNSIGNED, + s_int INT,u_int INT UNSIGNED,s_big BIGINT,u_big BIGINT UNSIGNED, + d DECIMAL(65,30),f FLOAT,g DOUBLE, label ENUM('2','x'),labels SET(%s), + bytes BLOB,txt TEXT,doc JSON,sql_null JSON,json_null JSON, + dt DATETIME(6),ts TIMESTAMP(6) NULL,day DATE,duration TIME(6),yr YEAR,bits BIT(64),fixed_bytes BINARY(8),var_bytes VARBINARY(32),char_value CHAR(8),bool_value BOOLEAN, + generated_value BIGINT GENERATED ALWAYS AS (s_big + 1) STORED, + virtual_value BIGINT GENERATED ALWAYS AS (id * 2) VIRTUAL""" + % sets + ) + self.local(f"CREATE TABLE {db}.types_test({definitions})") + hexbytes = bytes(range(256)).hex() + self.local(f"""INSERT INTO {db}.types_test(id,s_tiny,u_tiny,s_small,u_small,s_medium,u_medium,s_int,u_int,s_big,u_big,d,f,g,label,labels,bytes,txt,doc,sql_null,json_null,dt,ts,day,duration,yr,bits,fixed_bytes,var_bytes,char_value,bool_value) + VALUES (1,-128,255,-32768,65535,-8388608,16777215,-2147483648,4294967295,9223372036854775806,18446744073709551615, + 12345678901234567890123456789012345.123456789012345678901234567890,1.234567,1.234567890123,'2','s63',UNHEX('{hexbytes}'), + 'δΈ­ζ–‡πŸ˜€ quote\\\' slash\\\\ newline\\n',JSON_OBJECT('uint',18446744073709551615,'arr',JSON_ARRAY(NULL,TRUE),'label','2'),NULL,CAST('null' AS JSON), + '2026-10-25 01:02:03.123456','2026-10-25 01:02:03.654321','2026-10-25','-123:45:56.123456',2026,b'1111111111111111111111111111111111111111111111111111111111111111',UNHEX('000102ff'),UNHEX('00feff'),'char',TRUE); + INSERT INTO {db}.types_test(id) VALUES (2); + INSERT INTO {db}.types_test(id,s_tiny,u_tiny,s_small,u_small,s_medium,u_medium,s_int,u_int,s_big,u_big,d) + VALUES(3,127,0,32767,0,8388607,0,2147483647,0,-9223372036854775808,0,-99999999999999999999999999999999999.999999999999999999999999999999)""") + names = [ + "id", + "s_tiny", + "u_tiny", + "s_small", + "u_small", + "s_medium", + "u_medium", + "s_int", + "u_int", + "s_big", + "u_big", + "d", + "f", + "g", + "label", + "labels", + "txt", + "dt", + "ts", + "day", + "duration", + "yr", + "generated_value", + "char_value", + "bool_value", + ] + cols = [(n, "CAST(`%s` AS CHAR)" % n, quote(n) + "::VARCHAR") for n in names] + cols = [ + (name, "CAST(CAST(`%s` AS DOUBLE) AS CHAR)" % name, target) + if name in ("f", "g") + else (name, source, target) + for name, source, target in cols + ] + cols += [ + ("bytes", "HEX(bytes)", "upper(hex(" + quote("bytes") + "))"), + ("bits", "CAST(bits+0 AS CHAR)", quote("bits") + "::VARCHAR"), + ( + "fixed_bytes", + "HEX(fixed_bytes)", + "upper(hex(" + quote("fixed_bytes") + "))", + ), + ("var_bytes", "HEX(var_bytes)", "upper(hex(" + quote("var_bytes") + "))"), + ] + cols += [ + ( + n, + "CAST(`%s` AS CHAR)" % n, + "CASE WHEN is_null_value(" + + quote(n) + + ") THEN 'null' ELSE " + + quote(n) + + "::VARCHAR END", + ) + for n in ["doc", "sql_null", "json_null"] + ] + self.compare( + "types_test", + cols, + numeric=("f", "g"), + json_columns=("doc", "sql_null", "json_null"), + ) + described = self.lake("DESCRIBE " + quote(self.database) + '."types_test"') + self.results["types_describe"] = described + assert "virtual_value" not in [r[0] for r in described] + self.local( + f"UPDATE {db}.types_test SET label='x',labels='s0,s63',d=-1.000000000000000000000000000001,doc=JSON_OBJECT('changed',true),bytes=UNHEX('00ff'),txt=NULL WHERE id=1" + ) + self.compare( + "types_test", + cols, + numeric=("f", "g"), + json_columns=("doc", "sql_null", "json_null"), + ) + + def test_ddl(self, db): + self.local( + f"CREATE TABLE {db}.ddl_test(id INT PRIMARY KEY,value INT); INSERT INTO {db}.ddl_test VALUES(1,42)" + ) + self.simple("ddl_test", ["id", "value"]) + self.local( + f"ALTER TABLE {db}.ddl_test ADD COLUMN extra VARCHAR(64) NOT NULL DEFAULT 'hello'" + ) + self.simple("ddl_test", ["id", "value", "extra"]) + self.local(f"ALTER TABLE {db}.ddl_test RENAME COLUMN value TO amount") + self.simple("ddl_test", ["id", "amount", "extra"]) + self.local( + f"ALTER TABLE {db}.ddl_test MODIFY COLUMN amount BIGINT; UPDATE {db}.ddl_test SET amount=9223372036854775806 WHERE id=1" + ) + self.simple("ddl_test", ["id", "amount", "extra"]) + self.local( + f"ALTER TABLE {db}.ddl_test DROP COLUMN extra; INSERT INTO {db}.ddl_test VALUES(2,-12)" + ) + self.simple("ddl_test", ["id", "amount"]) + self.local( + f"CREATE INDEX amount_index ON {db}.ddl_test(amount);ALTER TABLE {db}.ddl_test DROP INDEX amount_index" + ) + self.local(f"ALTER TABLE {db}.ddl_test ADD COLUMN text_value TEXT NOT NULL") + self.simple("ddl_test", ["id", "amount", "text_value"]) + + def test_truncate(self, db): + self.local( + f"CREATE TABLE {db}.truncate_test(id INT PRIMARY KEY,value VARCHAR(64));INSERT INTO {db}.truncate_test VALUES(1,'old')" + ) + self.simple("truncate_test", ["id", "value"]) + self.local( + f"TRUNCATE TABLE {db}.truncate_test; INSERT INTO {db}.truncate_test VALUES(2,'new')" + ) + self.simple("truncate_test", ["id", "value"]) + + def test_batch(self, db): + self.local(f"CREATE TABLE {db}.batch_test(id INT PRIMARY KEY,value TEXT)") + values = ",".join("(%d,'%s')" % (i, "x" * 256) for i in range(1200)) + # Use stdin for large synthetic transactions, rather than shell/argv limits. + sql = ( + "SET time_zone='+00:00';BEGIN;INSERT INTO %s.batch_test VALUES%s;COMMIT;" + % (db, values) + ) + proc = subprocess.run( + ["mysql", "-h", "127.0.0.1", "-P", self.port, "-u", "root"], + input=sql, + text=True, + capture_output=True, + timeout=60, + ) + assert proc.returncode == 0, proc.stderr + self.simple("batch_test", ["id", "value"]) + self.local( + f"UPDATE {db}.batch_test SET value='tail' WHERE id=1199;DELETE FROM {db}.batch_test WHERE id<100" + ) + self.simple("batch_test", ["id", "value"]) + with urllib.request.urlopen(self.cdc + "/metrics", timeout=5) as response: + text = response.read().decode() + count = 0 + for line in text.splitlines(): + if ( + line.startswith("ticdc_lake_spool_load_bytes_count{") + and ('changefeed="' + self.feed + '"') in line + ): + count += float(line.rsplit(" ", 1)[-1]) + self.results["spool_disk_load_count"] = count + assert count > 0, "large transaction did not exercise disk spool" + + def wait_state(self, expected, timeout=60): + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + try: + status = self.feed_status() + if status.get("state") == expected: + return status + last = status.get("state", "missing state") + except (OSError, RuntimeError) as error: + # GET is safe to repeat while the restarted coordinator finishes + # bootstrap. Never retry an ambiguous mutation POST here. + last = self.sanitize(error) + time.sleep(0.3) + raise AssertionError("changefeed did not reach " + expected + ": " + last) + + def test_restart(self, db): + self.local( + f"CREATE TABLE {db}.restart_test(id INT PRIMARY KEY,value VARCHAR(64));INSERT INTO {db}.restart_test VALUES(1,'before'),(2,'keep')" + ) + self.simple("restart_test", ["id", "value"]) + marker = int( + self.local( + f"BEGIN;SELECT COUNT(*) FROM {db}.restart_test;SELECT @@tidb_current_ts;COMMIT" + ).splitlines()[-1] + ) + assert marker > 0, ( + "source checkpoint marker must be a real transaction read TSO" + ) + deadline = time.monotonic() + 60 + while self.check_feed().get("checkpoint_ts", 0) < marker: + if time.monotonic() > deadline: + raise AssertionError( + "checkpoint did not drain before controlled restart" + ) + time.sleep(0.3) + self.api("/api/v2/changefeeds/" + self.feed + "/pause", {}, "POST") + self.wait_state("stopped") + status = self.api("/api/v2/status") + pid = status["pid"] + binary = os.path.realpath(os.environ["E2E_CDC_BINARY"]) + command = subprocess.check_output( + ["ps", "-p", str(pid), "-o", "command="], text=True + ).strip() + argv = shlex.split(command) + if len(argv) < 2 or os.path.realpath(argv[0]) != binary or argv[1] != "server": + raise RuntimeError("refusing to stop an unowned CDC process") + os.kill(pid, signal.SIGTERM) + deadline = time.monotonic() + 45 + while True: + try: + os.kill(pid, 0) + except ProcessLookupError: + break + if time.monotonic() > deadline: + raise AssertionError("old local CDC process did not terminate") + time.sleep(0.2) + # This is a drained, exclusive restart, not ambiguous-query failover. + self.local( + f"UPDATE {db}.restart_test SET value='queued' WHERE id=1;INSERT INTO {db}.restart_test VALUES(3,'queued-insert');ALTER TABLE {db}.restart_test ADD COLUMN extra BIGINT NULL" + ) + console = (self.root / "restart-console.log").open("w") + # Keep the original owned capture's endpoint, including an ephemeral + # port assigned by local.py; never bind another runner's fixed port. + address = urllib.parse.urlparse(self.cdc).netloc + process = subprocess.Popen( + [ + binary, + "server", + "--newarch", + "--pd", + os.environ.get("E2E_PD", "http://127.0.0.1:12379"), + "--addr", + address, + "--advertise-addr", + address, + "--data-dir", + os.environ["E2E_CDC_DATA_DIR"], + "--log-file", + str(self.root / "restarted-cdc.log"), + "--tz", + "UTC", + ], + stdout=console, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + console.close() + (self.root / "restart-pid").write_text(str(process.pid)) + deadline = time.monotonic() + 60 + while True: + try: + if self.api("/api/v2/status").get("is_owner"): + break + except (OSError, RuntimeError): + pass + if time.monotonic() > deadline: + raise AssertionError("new local CDC did not become ready") + time.sleep(0.5) + self.wait_state("stopped") + self.api("/api/v2/changefeeds/" + self.feed + "/resume", {}, "POST") + self.wait_state("normal") + self.simple("restart_test", ["id", "value", "extra"]) + self.results["restart"] = { + "old_pid": pid, + "new_pid": process.pid, + "before_checkpoint": marker, + "after_checkpoint": self.feed_status().get("checkpoint_ts"), + } + self.save() + + def test_drop(self, db): + self.local( + f"CREATE TABLE {db}.drop_test(id INT PRIMARY KEY);INSERT INTO {db}.drop_test VALUES(1)" + ) + self.simple("drop_test", ["id"]) + self.local(f"DROP TABLE {db}.drop_test") + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + self.check_feed() + rows = self.lake( + "SELECT COUNT(*)::VARCHAR FROM system.tables WHERE database=? AND name IN (?,?)", + [self.database, "drop_test", "drop_test_cdc_raw"], + ) + if rows == [["0"]]: + return + time.sleep(0.5) + raise AssertionError("DROP left target/raw tables") + + def cleanup(self): + # Failures intentionally keep this isolated run for diagnosis. An operator + # can remove just this feed and native_e2e_* database after inspecting. + if self.feed_created: + status = self.feed_status() + if status.get("state") == "normal": + self.api("/api/v2/changefeeds/" + self.feed + "/pause", {}, "POST") + self.wait_state("stopped") + self.api("/api/v2/changefeeds/" + self.feed, method="DELETE") + if self.remote_created: + assert self.database.startswith("native_e2e_") + self.lake("DROP DATABASE IF EXISTS " + quote(self.database)) + self.local("DROP DATABASE IF EXISTS `%s`" % self.database) + self.results["cleanup"] = "PASS" + self.save() + + +if __name__ == "__main__": + run = Run(existing="--cleanup" in sys.argv[1:]) + if "--cleanup" in sys.argv[1:]: + run.cleanup() + print("CLEANED", run.database, flush=True) + sys.exit(0) + try: + run.setup() + run.tests() + run.cleanup() + run.results["status"] = "PASS" + run.save() + print("RESULT", str(run.root / "result.json"), flush=True) + except Exception as error: + run.results["status"] = "FAIL" + run.results["error"] = run.sanitize(error) + run.save() + print( + "ERROR", + run.sanitize(error), + "RESULT", + str(run.root / "result.json"), + flush=True, + ) + sys.exit(1) diff --git a/tests/integration_tests/lake_native/run.sh b/tests/integration_tests/lake_native/run.sh new file mode 100755 index 0000000000..9bbbb47444 --- /dev/null +++ b/tests/integration_tests/lake_native/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# The generic integration runner also discovers this case for other sink types. +set -euo pipefail +if [ "${1:-lake}" != lake ]; then + echo "lake_native: requires the lake sink suite; skipping" + exit 0 +fi +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$CUR/../../.." +exec python3 "$CUR/local.py" diff --git a/tests/integration_tests/lake_native/test_local.py b/tests/integration_tests/lake_native/test_local.py new file mode 100644 index 0000000000..274a1241c4 --- /dev/null +++ b/tests/integration_tests/lake_native/test_local.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Service-free checks for the local E2E lifecycle and target selection.""" + +import json +import hashlib +import os +import pathlib +import shlex +import socket +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from local import Local, check_source_port, file_sha256, require_python, wait_ready +from run import Run + + +class LocalRunnerTest(unittest.TestCase): + def test_source_ports_reject_live_loopback_and_wildcard_listeners(self): + for address in ("127.0.0.1", "0.0.0.0"): + with self.subTest(address=address), socket.socket() as listener: + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind((address, 0)) + listener.listen() + port = listener.getsockname()[1] + with self.assertRaisesRegex(RuntimeError, "already in use"): + check_source_port(port) + # A previous connection's TIME_WAIT does not forbid a new run. + check_source_port(port) + + def test_python_requirement_and_chunked_digest(self): + with patch("local.sys.version_info", (3, 9)): + with self.assertRaisesRegex(SystemExit, "Python 3.10"): + require_python() + with patch("local.sys.version_info", (3, 10)): + require_python() + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "binary" + data = b"lake-test" * (1 << 19) + path.write_bytes(data) + self.assertEqual(file_sha256(path), hashlib.sha256(data).hexdigest()) + + def test_owned_background_process_is_stopped(self): + with tempfile.TemporaryDirectory() as directory: + runner = Local(pathlib.Path(directory), "test-image", "v8.5.1") + child = runner.start( + [sys.executable, "-c", "import time; time.sleep(60)"], "owned.log" + ) + try: + runner.cleanup() + self.assertIsNotNone(child.poll()) + self.assertEqual(runner.summary["cleanup"], "PASS") + finally: + if child.poll() is None: + child.kill() + child.wait() + + def test_never_inherits_cloud_or_other_capture(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch.dict( + os.environ, + { + "LAKE_URI_FILE": "/not/read/cloud-secret", + "LAKE_NATIVE_TEST_URI": "must-not-use", + "CDC_SERVER": "https://not-local.invalid", + "E2E_CDC_BINARY": "/unowned/cdc", + "MYSQL_PORT": "1234", + }, + ), + ): + runner = Local(pathlib.Path(directory), "test-image", "v8.5.1") + for name in ( + "LAKE_URI_FILE", + "LAKE_NATIVE_TEST_URI", + "CDC_SERVER", + "E2E_CDC_BINARY", + "MYSQL_PORT", + ): + self.assertNotIn(name, runner.env) + runner.summary["status"] = "PASS" + runner.cleanup() + self.assertEqual( + json.loads((runner.root / "summary.json").read_text())["cleanup"], + "PASS", + ) + + def test_explicit_http_scheme_only(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + credential = root / "uri" + credential.write_text("lake+http://user:password@127.0.0.1:18000/default") + credential.chmod(0o600) + with patch.dict( + os.environ, + {"LAKE_URI_FILE": str(credential), "E2E_RUN_DIR": str(root / "http")}, + ): + self.assertEqual(Run().base, "http://127.0.0.1:18000") + credential.write_text("lake://user:password@lake.example/default") + with patch.dict( + os.environ, + {"LAKE_URI_FILE": str(credential), "E2E_RUN_DIR": str(root / "https")}, + ): + self.assertEqual(Run().base, "https://lake.example") + + def test_readiness_retries_service_errors_but_checks_exit(self): + attempts = iter([RuntimeError("not ready"), True]) + + def probe(): + value = next(attempts) + if isinstance(value, Exception): + raise value + return value + + with patch("local.time.sleep"): + wait_ready(probe, None, 1, "test-service") + process = subprocess.Popen(["true"]) + process.wait() + with self.assertRaisesRegex(RuntimeError, "exited before"): + wait_ready(lambda: True, process, 1, "test-service") + + def test_refuses_unowned_restart_pid_and_still_removes_credential(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / "acceptance").mkdir() + (root / "acceptance/restart-pid").write_text(str(os.getpid())) + (root / "local-uri").write_text("secret") + runner = Local(root, "test-image", "v8.5.1") + runner.binary = root / "unowned-cdc" + with patch("local.os.kill") as kill: + runner.cleanup() + kill.assert_not_called() + self.assertEqual(runner.summary["status"], "FAIL") + self.assertIn("unowned restarted capture", runner.summary["cleanup"][0]) + self.assertFalse((root / "local-uri").exists()) + + def test_restart_ownership_compares_canonical_paths(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / "acceptance").mkdir() + (root / "acceptance/restart-pid").write_text("12345") + (root / "sub").mkdir() + runner = Local(root, "test-image", "v8.5.1") + runner.binary = root / "sub/../cdc" + command = shlex.join( + [ + str(runner.binary.resolve()), + "server", + "--data-dir", + str(root / "cdc-data"), + ] + ) + with ( + patch( + "local.subprocess.run", + return_value=subprocess.CompletedProcess([], 0, stdout=command), + ), + patch("local.os.kill", side_effect=[None, ProcessLookupError]) as kill, + ): + runner.cleanup() + self.assertEqual(kill.call_args_list[0].args[0], 12345) + self.assertEqual(runner.summary["cleanup"], "PASS") + + def test_log_failure_does_not_skip_container_removal(self): + with tempfile.TemporaryDirectory() as directory: + runner = Local(pathlib.Path(directory), "test-image", "v8.5.1") + runner.container_created = True + + def execute(args, **kwargs): + if args[:2] == ["docker", "logs"]: + raise subprocess.TimeoutExpired(args, 30) + return subprocess.CompletedProcess(args, 0) + + with patch("local.subprocess.run", side_effect=execute) as run: + runner.cleanup() + self.assertEqual(run.call_args_list[-1].args[0][:2], ["docker", "rm"]) + self.assertEqual(runner.summary["status"], "FAIL") + self.assertIn("Lake logs", runner.summary["cleanup"][0]) + + +if __name__ == "__main__": + unittest.main()