Skip to content

Commit d167b03

Browse files
committed
feat: add Debezium twin-test for OLR compatibility verification
Run two Debezium Server instances (LogMiner vs OLR adapter) against Oracle XE 21c and compare CDC outputs. 14/16 scenarios pass; 2 fail due to known LogMiner vs OLR behavioral differences (BINARY_FLOAT support, LOB handling strategy).
1 parent 0d8b3b2 commit d167b03

11 files changed

Lines changed: 953 additions & 0 deletions

tests/debezium/.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
lib/
2+
output/
3+
data/

tests/debezium/Makefile

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
TESTS_DIR := $(abspath ..)
2+
SCRIPTS_DIR := $(TESTS_DIR)/scripts
3+
SCENARIOS := $(wildcard $(TESTS_DIR)/0-inputs/*.sql)
4+
SCENARIO ?=
5+
OJDBC_VERSION := 21.14.0.0
6+
OJDBC_URL := https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/$(OJDBC_VERSION)/ojdbc8-$(OJDBC_VERSION).jar
7+
8+
.PHONY: up down test-debezium download-ojdbc
9+
10+
download-ojdbc:
11+
@mkdir -p lib
12+
@test -f lib/ojdbc8.jar || \
13+
(echo "Downloading ojdbc8-$(OJDBC_VERSION).jar..." && \
14+
wget -q -O lib/ojdbc8.jar $(OJDBC_URL))
15+
16+
up: download-ojdbc
17+
docker compose up -d --wait
18+
19+
down:
20+
docker compose down -v
21+
22+
test-debezium:
23+
ifdef SCENARIO
24+
$(SCRIPTS_DIR)/run-debezium.sh $(SCENARIO)
25+
else
26+
@for sql in $(SCENARIOS); do \
27+
name=$$(basename "$$sql" .sql); \
28+
echo "=== Debezium twin-test: $$name ==="; \
29+
$(SCRIPTS_DIR)/run-debezium.sh "$$name" || exit 1; \
30+
done
31+
endif
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Debezium Server — LogMiner adapter → HTTP sink
2+
debezium.sink.type=http
3+
debezium.sink.http.url=http://receiver:8080/logminer
4+
5+
debezium.format.value=json
6+
debezium.format.value.schemas.enable=false
7+
debezium.format.key=json
8+
debezium.format.key.schemas.enable=false
9+
10+
debezium.source.connector.class=io.debezium.connector.oracle.OracleConnector
11+
debezium.source.database.connection.adapter=logminer
12+
debezium.source.database.hostname=oracle
13+
debezium.source.database.port=1521
14+
debezium.source.database.user=c##dbzuser
15+
debezium.source.database.password=dbz
16+
debezium.source.database.dbname=XE
17+
debezium.source.database.pdb.name=XEPDB1
18+
debezium.source.topic.prefix=logminer
19+
debezium.source.schema.include.list=OLR_TEST
20+
debezium.source.snapshot.mode=no_data
21+
debezium.source.log.mining.strategy=online_catalog
22+
23+
debezium.source.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore
24+
debezium.source.offset.storage.file.filename=/debezium/data/offsets.dat
25+
debezium.source.offset.flush.interval.ms=0
26+
debezium.source.schema.history.internal=io.debezium.storage.file.history.FileSchemaHistory
27+
debezium.source.schema.history.internal.file.filename=/debezium/data/schema-history.dat
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Debezium Server — OLR adapter → HTTP sink
2+
debezium.sink.type=http
3+
debezium.sink.http.url=http://receiver:8080/olr
4+
5+
debezium.format.value=json
6+
debezium.format.value.schemas.enable=false
7+
debezium.format.key=json
8+
debezium.format.key.schemas.enable=false
9+
10+
debezium.source.connector.class=io.debezium.connector.oracle.OracleConnector
11+
debezium.source.database.connection.adapter=olr
12+
debezium.source.openlogreplicator.source=XE
13+
debezium.source.openlogreplicator.host=olr
14+
debezium.source.openlogreplicator.port=5000
15+
debezium.source.database.hostname=oracle
16+
debezium.source.database.port=1521
17+
debezium.source.database.user=c##dbzuser
18+
debezium.source.database.password=dbz
19+
debezium.source.database.dbname=XE
20+
debezium.source.database.pdb.name=XEPDB1
21+
debezium.source.topic.prefix=olr
22+
debezium.source.schema.include.list=OLR_TEST
23+
debezium.source.snapshot.mode=no_data
24+
25+
debezium.source.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore
26+
debezium.source.offset.storage.file.filename=/debezium/data/offsets.dat
27+
debezium.source.offset.flush.interval.ms=0
28+
debezium.source.schema.history.internal=io.debezium.storage.file.history.FileSchemaHistory
29+
debezium.source.schema.history.internal.file.filename=/debezium/data/schema-history.dat
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
{
2+
"version": "1.9.0",
3+
"log-level": 4,
4+
"memory": {
5+
"min-mb": 64,
6+
"max-mb": 256
7+
},
8+
"source": [
9+
{
10+
"alias": "SOURCE",
11+
"name": "XE",
12+
"reader": {
13+
"type": "online",
14+
"user": "c##dbzuser",
15+
"password": "dbz",
16+
"server": "//oracle:1521/XEPDB1"
17+
},
18+
"format": {
19+
"type": "debezium",
20+
"scn-type": 1,
21+
"timestamp-type": 1,
22+
"user-type": 0,
23+
"redo-thread": 0
24+
},
25+
"filter": {
26+
"table": [
27+
{"owner": "OLR_TEST", "table": ".*"}
28+
]
29+
}
30+
}
31+
],
32+
"target": [
33+
{
34+
"alias": "DEBEZIUM",
35+
"source": "SOURCE",
36+
"writer": {
37+
"type": "network",
38+
"uri": "0.0.0.0:5000"
39+
}
40+
}
41+
]
42+
}

tests/debezium/docker-compose.yaml

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
services:
2+
oracle:
3+
image: gvenzl/oracle-xe:21.3.0-slim-faststart
4+
container_name: dbz-oracle
5+
ports:
6+
- "${ORACLE_PORT:-1522}:1521"
7+
environment:
8+
ORACLE_PASSWORD: oracle
9+
APP_USER: olr_test
10+
APP_USER_PASSWORD: olr_test
11+
volumes:
12+
- ./oracle-init:/container-entrypoint-initdb.d
13+
- oracle-data:/opt/oracle/oradata
14+
healthcheck:
15+
test: ["CMD", "healthcheck.sh"]
16+
interval: 10s
17+
timeout: 5s
18+
retries: 30
19+
20+
receiver:
21+
image: python:3.12-slim
22+
container_name: dbz-receiver
23+
command: ["python3", "/app/debezium-receiver.py"]
24+
ports:
25+
- "8080:8080"
26+
environment:
27+
OUTPUT_DIR: /app/output
28+
volumes:
29+
- ../scripts/debezium-receiver.py:/app/debezium-receiver.py:ro
30+
- ./output:/app/output
31+
32+
olr:
33+
image: olr-dev:${OLR_IMAGE_TAG:-latest}
34+
container_name: dbz-olr
35+
entrypoint: ["/bin/bash", "-c", "mkdir -p /olr-data/checkpoint && exec /opt/OpenLogReplicator/OpenLogReplicator \"$@\"", "--"]
36+
command: ["-r", "-f", "/config/olr-config.json"]
37+
working_dir: /olr-data
38+
group_add:
39+
- "54321" # oinstall group — read access to Oracle redo logs
40+
depends_on:
41+
oracle:
42+
condition: service_healthy
43+
tmpfs:
44+
- /olr-data:uid=1000,gid=54322
45+
volumes:
46+
- ./config/olr-config.json:/config/olr-config.json:ro
47+
- oracle-data:/opt/oracle/oradata:ro
48+
49+
dbz-logminer:
50+
image: quay.io/debezium/server:3.0
51+
container_name: dbz-logminer
52+
depends_on:
53+
oracle:
54+
condition: service_healthy
55+
receiver:
56+
condition: service_started
57+
volumes:
58+
- ./config/application-logminer.properties:/debezium/config/application.properties:ro
59+
- ./lib/ojdbc8.jar:/debezium/lib/ojdbc8.jar:ro
60+
- dbz-logminer-data:/debezium/data
61+
62+
dbz-olr:
63+
image: quay.io/debezium/server:3.0
64+
container_name: dbz-olr-adapter
65+
depends_on:
66+
olr:
67+
condition: service_started
68+
receiver:
69+
condition: service_started
70+
volumes:
71+
- ./config/application-olr.properties:/debezium/config/application.properties:ro
72+
- ./lib/ojdbc8.jar:/debezium/lib/ojdbc8.jar:ro
73+
- dbz-olr-data:/debezium/data
74+
75+
volumes:
76+
oracle-data:
77+
dbz-logminer-data:
78+
dbz-olr-data:
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
# Enable archivelog mode and supplemental logging for OLR testing.
3+
# Runs as part of gvenzl/oracle-xe container initialization.
4+
5+
sqlplus -S / as sysdba <<'SQL'
6+
SHUTDOWN IMMEDIATE;
7+
STARTUP MOUNT;
8+
ALTER DATABASE ARCHIVELOG;
9+
ALTER DATABASE OPEN;
10+
ALTER DATABASE ADD SUPPLEMENTAL LOG DATA;
11+
ALTER SYSTEM SET db_recovery_file_dest_size=10G;
12+
13+
-- Archive to shared volume so OLR container can read them
14+
ALTER SYSTEM SET log_archive_dest_1='LOCATION=/opt/oracle/oradata/XE/archive' SCOPE=BOTH;
15+
HOST mkdir -p /opt/oracle/oradata/XE/archive
16+
17+
-- Grant PDB user access to v$database for SCN queries in test scenarios
18+
ALTER SESSION SET CONTAINER=XEPDB1;
19+
GRANT SELECT ON SYS.V_$DATABASE TO olr_test;
20+
SQL
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/bin/bash
2+
# Create Debezium user with LogMiner privileges + sentinel table.
3+
# Runs after 01-setup.sh during gvenzl/oracle-xe container initialization.
4+
5+
sqlplus -S / as sysdba <<'SQL'
6+
-- Create common user for Debezium LogMiner access (CDB level)
7+
CREATE USER c##dbzuser IDENTIFIED BY dbz
8+
DEFAULT TABLESPACE USERS
9+
QUOTA UNLIMITED ON USERS
10+
CONTAINER=ALL;
11+
12+
-- Session and container
13+
GRANT CREATE SESSION TO c##dbzuser CONTAINER=ALL;
14+
GRANT SET CONTAINER TO c##dbzuser CONTAINER=ALL;
15+
16+
-- LogMiner specific
17+
GRANT LOGMINING TO c##dbzuser CONTAINER=ALL;
18+
GRANT EXECUTE ON DBMS_LOGMNR TO c##dbzuser CONTAINER=ALL;
19+
GRANT EXECUTE ON DBMS_LOGMNR_D TO c##dbzuser CONTAINER=ALL;
20+
21+
-- V$ views for log mining
22+
GRANT SELECT ON V_$DATABASE TO c##dbzuser CONTAINER=ALL;
23+
GRANT SELECT ON V_$LOG TO c##dbzuser CONTAINER=ALL;
24+
GRANT SELECT ON V_$LOG_HISTORY TO c##dbzuser CONTAINER=ALL;
25+
GRANT SELECT ON V_$LOGMNR_LOGS TO c##dbzuser CONTAINER=ALL;
26+
GRANT SELECT ON V_$LOGMNR_CONTENTS TO c##dbzuser CONTAINER=ALL;
27+
GRANT SELECT ON V_$LOGFILE TO c##dbzuser CONTAINER=ALL;
28+
GRANT SELECT ON V_$ARCHIVED_LOG TO c##dbzuser CONTAINER=ALL;
29+
GRANT SELECT ON V_$ARCHIVE_DEST_STATUS TO c##dbzuser CONTAINER=ALL;
30+
GRANT SELECT ON V_$TRANSACTION TO c##dbzuser CONTAINER=ALL;
31+
GRANT SELECT ON V_$THREAD TO c##dbzuser CONTAINER=ALL;
32+
GRANT SELECT ON V_$PARAMETER TO c##dbzuser CONTAINER=ALL;
33+
GRANT SELECT ON V_$NLS_PARAMETERS TO c##dbzuser CONTAINER=ALL;
34+
GRANT SELECT ON V_$TIMEZONE_NAMES TO c##dbzuser CONTAINER=ALL;
35+
36+
-- General access
37+
GRANT SELECT ANY TABLE TO c##dbzuser CONTAINER=ALL;
38+
GRANT SELECT ANY TRANSACTION TO c##dbzuser CONTAINER=ALL;
39+
GRANT SELECT ANY DICTIONARY TO c##dbzuser CONTAINER=ALL;
40+
GRANT FLASHBACK ANY TABLE TO c##dbzuser CONTAINER=ALL;
41+
GRANT SELECT_CATALOG_ROLE TO c##dbzuser CONTAINER=ALL;
42+
GRANT EXECUTE_CATALOG_ROLE TO c##dbzuser CONTAINER=ALL;
43+
GRANT LOCK ANY TABLE TO c##dbzuser CONTAINER=ALL;
44+
GRANT CREATE TABLE TO c##dbzuser CONTAINER=ALL;
45+
GRANT ALTER ANY TABLE TO c##dbzuser CONTAINER=ALL;
46+
GRANT CREATE SEQUENCE TO c##dbzuser CONTAINER=ALL;
47+
48+
-- OLR needs SELECT+FLASHBACK on SYS dictionary base tables (AS OF SCN queries)
49+
-- These must be granted at PDB level since CONTAINER=ALL doesn't propagate for SYS objects
50+
ALTER SESSION SET CONTAINER=XEPDB1;
51+
52+
GRANT SELECT, FLASHBACK ON SYS.CCOL$ TO c##dbzuser;
53+
GRANT SELECT, FLASHBACK ON SYS.CDEF$ TO c##dbzuser;
54+
GRANT SELECT, FLASHBACK ON SYS.COL$ TO c##dbzuser;
55+
GRANT SELECT, FLASHBACK ON SYS.DEFERRED_STG$ TO c##dbzuser;
56+
GRANT SELECT, FLASHBACK ON SYS.ECOL$ TO c##dbzuser;
57+
GRANT SELECT, FLASHBACK ON SYS.LOB$ TO c##dbzuser;
58+
GRANT SELECT, FLASHBACK ON SYS.LOBCOMPPART$ TO c##dbzuser;
59+
GRANT SELECT, FLASHBACK ON SYS.LOBFRAG$ TO c##dbzuser;
60+
GRANT SELECT, FLASHBACK ON SYS.OBJ$ TO c##dbzuser;
61+
GRANT SELECT, FLASHBACK ON SYS.TAB$ TO c##dbzuser;
62+
GRANT SELECT, FLASHBACK ON SYS.TABCOMPART$ TO c##dbzuser;
63+
GRANT SELECT, FLASHBACK ON SYS.TABPART$ TO c##dbzuser;
64+
GRANT SELECT, FLASHBACK ON SYS.TABSUBPART$ TO c##dbzuser;
65+
GRANT SELECT, FLASHBACK ON SYS.TS$ TO c##dbzuser;
66+
GRANT SELECT, FLASHBACK ON SYS.USER$ TO c##dbzuser;
67+
GRANT SELECT, FLASHBACK ON XDB.XDB$TTSET TO c##dbzuser;
68+
69+
-- OLR also needs FLASHBACK on XDB token tables (dynamic names like X$NMxxxx, X$PTxxxx, X$QNxxxx)
70+
BEGIN
71+
FOR t IN (SELECT table_name FROM dba_tables WHERE owner='XDB'
72+
AND (table_name LIKE 'X$NM%' OR table_name LIKE 'X$PT%' OR table_name LIKE 'X$QN%')) LOOP
73+
EXECUTE IMMEDIATE 'GRANT SELECT, FLASHBACK ON XDB.' || t.table_name || ' TO c##dbzuser';
74+
END LOOP;
75+
END;
76+
/
77+
78+
GRANT DBA TO c##dbzuser;
79+
80+
-- Sentinel table for completion detection
81+
82+
BEGIN
83+
EXECUTE IMMEDIATE 'CREATE TABLE olr_test.DEBEZIUM_SENTINEL (
84+
id NUMBER PRIMARY KEY,
85+
marker VARCHAR2(100)
86+
)';
87+
EXCEPTION WHEN OTHERS THEN
88+
IF SQLCODE = -955 THEN NULL; ELSE RAISE; END IF;
89+
END;
90+
/
91+
92+
ALTER TABLE olr_test.DEBEZIUM_SENTINEL ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS;
93+
SQL

0 commit comments

Comments
 (0)