Skip to content

Commit bb13b9b

Browse files
rophyrophy
committed
test: add regression test framework with multi-environment support
Docker-based test infrastructure for OLR regression testing: - Dockerfile.dev: dev image with all deps + gtest - Per-environment configs in tests/1-environments/ (free-23 first) - generate.sh: 7-stage pipeline (SQL → redo capture → schema → LogMiner → OLR → compare → golden file) - test_pipeline.cpp: parameterized gtest, auto-discovers fixtures from both 2-prebuilt/ (Git LFS) and 3-generated/ (CI artifact) - compare.py: content-based OLR vs LogMiner matching - CI: sql-tests-free23 (generates fixtures), redo-log-tests (replays) - Includes basic-crud scenario as initial SQL test input
1 parent dabeb4b commit bb13b9b

20 files changed

Lines changed: 2381 additions & 0 deletions

‎.dockerignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
.git/
2+
.github/
3+
oracle-rac/
4+
tests/0-inputs/
5+
tests/1-environments/
6+
tests/2-prebuilt/
7+
tests/3-generated/
8+
tests/.work/
9+
tests/scripts/

‎.gitattributes‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
tests/2-prebuilt/redo/** filter=lfs diff=lfs merge=lfs -text
2+
tests/2-prebuilt/schema/** filter=lfs diff=lfs merge=lfs -text
3+
tests/2-prebuilt/expected/** filter=lfs diff=lfs merge=lfs -text

‎.github/workflows/redo-log-tests.yaml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: Redo Log Tests
2+
3+
on:
4+
push:
5+
branches: [master]
6+
pull_request:
7+
branches: [master]
8+
9+
env:
10+
CACHE_IMAGE: ghcr.io/${{ github.repository_owner }}/openlogreplicator:ci
11+
12+
jobs:
13+
test:
14+
runs-on: ubuntu-latest
15+
timeout-minutes: 15
16+
permissions:
17+
packages: write
18+
steps:
19+
- uses: actions/checkout@v4
20+
with:
21+
lfs: true
22+
23+
- uses: docker/setup-buildx-action@v3
24+
25+
- name: Log in to ghcr.io
26+
uses: docker/login-action@v3
27+
with:
28+
registry: ghcr.io
29+
username: ${{ github.actor }}
30+
password: ${{ secrets.GITHUB_TOKEN }}
31+
32+
- name: Build OLR image
33+
uses: docker/build-push-action@v6
34+
with:
35+
context: .
36+
file: Dockerfile.dev
37+
load: true
38+
tags: olr-dev:latest
39+
cache-from: type=registry,ref=${{ env.CACHE_IMAGE }}
40+
cache-to: type=registry,ref=${{ env.CACHE_IMAGE }},mode=max
41+
build-args: |
42+
BUILD_TYPE=Debug
43+
UIDOLR=1001
44+
GIDOLR=1001
45+
WITHORACLE=1
46+
WITHKAFKA=1
47+
WITHPROTOBUF=1
48+
WITHPROMETHEUS=1
49+
WITHTESTS=1
50+
51+
- name: Download fixtures from sql-tests-free23
52+
uses: dawidd6/action-download-artifact@v6
53+
with:
54+
workflow: sql-tests-free23.yaml
55+
name: test-artifacts-free-23
56+
path: tests/3-generated/
57+
if_no_artifact_found: warn
58+
59+
- name: Run redo log tests
60+
run: make test-redo

‎.github/workflows/sql-tests-free23.yaml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: "SQL Tests: Oracle Free 23"
2+
3+
on:
4+
push:
5+
branches: [master]
6+
workflow_dispatch:
7+
8+
env:
9+
CACHE_IMAGE: ghcr.io/${{ github.repository_owner }}/openlogreplicator:ci
10+
ORACLE_TARGET: free-23
11+
12+
jobs:
13+
generate:
14+
runs-on: ubuntu-latest
15+
timeout-minutes: 45
16+
permissions:
17+
packages: write
18+
steps:
19+
- uses: actions/checkout@v4
20+
21+
- uses: docker/setup-buildx-action@v3
22+
23+
- name: Log in to ghcr.io
24+
uses: docker/login-action@v3
25+
with:
26+
registry: ghcr.io
27+
username: ${{ github.actor }}
28+
password: ${{ secrets.GITHUB_TOKEN }}
29+
30+
- name: Build OLR image
31+
uses: docker/build-push-action@v6
32+
with:
33+
context: .
34+
file: Dockerfile.dev
35+
load: true
36+
tags: olr-dev:latest
37+
cache-from: type=registry,ref=${{ env.CACHE_IMAGE }}
38+
cache-to: type=registry,ref=${{ env.CACHE_IMAGE }},mode=max
39+
build-args: |
40+
BUILD_TYPE=Debug
41+
UIDOLR=1001
42+
GIDOLR=1001
43+
WITHORACLE=1
44+
WITHKAFKA=1
45+
WITHPROTOBUF=1
46+
WITHPROMETHEUS=1
47+
WITHTESTS=1
48+
49+
- name: Start containers
50+
run: make -C tests/1-environments/${{ env.ORACLE_TARGET }} up
51+
52+
- name: Generate all fixtures
53+
run: make -C tests/1-environments/${{ env.ORACLE_TARGET }} test-sql
54+
55+
- name: Upload generated fixtures
56+
uses: actions/upload-artifact@v4
57+
with:
58+
name: test-artifacts-${{ env.ORACLE_TARGET }}
59+
path: tests/3-generated/
60+
retention-days: 90

‎.gitignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
build/
2+
cmake-build-*/
3+
.idea/
4+
.vscode/
5+
*.swp
6+
*~
7+
.DS_Store
8+
tests/3-generated/
9+
tests/.work/

‎CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,10 @@ if (WITH_PROTOBUF)
127127
endif ()
128128

129129
add_subdirectory(src)
130+
if (WITH_TESTS)
131+
enable_testing()
132+
add_subdirectory(tests)
133+
endif ()
130134

131135
target_link_libraries(OpenLogReplicator Threads::Threads)
132136

‎Dockerfile.dev‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
# syntax=docker/dockerfile:1
2+
#
3+
# Dockerfile.dev — Optimized for local dev builds
4+
# Splits dependencies into cached layers + uses ccache for incremental compilation.
5+
#
6+
# Usage:
7+
# GIDOLR=$(id -g) UIDOLR=$(id -u) docker compose build olr
8+
9+
ARG IMAGE=debian
10+
ARG VERSION=13.0
11+
FROM ${IMAGE}:${VERSION}
12+
13+
ARG ARCH=x86_64
14+
ARG GIDOLR=1001
15+
ARG UIDOLR=1001
16+
ARG GIDORA=54322
17+
ARG BUILD_TYPE=Debug
18+
ARG WITHKAFKA
19+
ARG WITHPROMETHEUS
20+
ARG WITHORACLE
21+
ARG WITHPROTOBUF
22+
ARG WITHTESTS
23+
24+
ENV LC_ALL=C
25+
ENV LANG=en_US.UTF-8
26+
ENV ORACLE_MAJOR=23
27+
ENV ORACLE_MINOR=26
28+
ENV PROTOBUF_VERSION_DIR=21.12
29+
ENV PROTOBUF_VERSION=3.21.12
30+
ENV RAPIDJSON_VERSION=1.1.0
31+
ENV LIBRDKAFKA_VERSION=2.13.0
32+
ENV PROMETHEUS_VERSION=1.3.0
33+
ENV OPENLOGREPLICATOR_VERSION=local
34+
ENV LD_LIBRARY_PATH=/opt/instantclient_${ORACLE_MAJOR}_${ORACLE_MINOR}:/opt/librdkafka/lib:/opt/prometheus/lib:/opt/protobuf/lib
35+
ENV BUILDARGS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DWITH_RAPIDJSON=/opt/rapidjson -S ../ -B ./"
36+
ENV BUILDARGS="${BUILDARGS}${WITHKAFKA:+ -DWITH_RDKAFKA=/opt/librdkafka}"
37+
ENV BUILDARGS="${BUILDARGS}${WITHPROMETHEUS:+ -DWITH_PROMETHEUS=/opt/prometheus}"
38+
ENV BUILDARGS="${BUILDARGS}${WITHORACLE:+ -DWITH_OCI=/opt/instantclient_${ORACLE_MAJOR}_${ORACLE_MINOR}}"
39+
ENV BUILDARGS="${BUILDARGS}${WITHPROTOBUF:+ -DWITH_PROTOBUF=/opt/protobuf}"
40+
ENV BUILDARGS="${BUILDARGS}${WITHTESTS:+ -DWITH_TESTS=ON}"
41+
ENV COMPILEKAFKA="${WITHKAFKA:+1}"
42+
ENV COMPILEPROMETHEUS="${WITHPROMETHEUS:+1}"
43+
ENV COMPILEORACLE="${WITHORACLE:+1}"
44+
ENV COMPILEPROTOBUF="${WITHPROTOBUF:+1}"
45+
ENV DEBIAN_FRONTEND=noninteractive
46+
47+
COPY scripts/run.sh /opt
48+
49+
# Layer 1: System packages + ccache
50+
RUN set -eu && \
51+
apt-get update && \
52+
apt-get -y install file gcc g++ libaio1t64 libasan8 libubsan1 libtool libz-dev make patch unzip wget cmake git curl ccache && \
53+
ln -s libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1
54+
55+
# Layer 2: RapidJSON
56+
RUN set -eu && \
57+
cd /opt && \
58+
wget -q https://github.com/Tencent/rapidjson/archive/refs/tags/v${RAPIDJSON_VERSION}.tar.gz && \
59+
tar xzf v${RAPIDJSON_VERSION}.tar.gz && \
60+
rm v${RAPIDJSON_VERSION}.tar.gz && \
61+
ln -s rapidjson-${RAPIDJSON_VERSION} rapidjson && \
62+
if [ "${RAPIDJSON_VERSION}" = "1.1.0" ]; then \
63+
cd rapidjson && \
64+
wget -q https://github.com/Tencent/rapidjson/commit/3b2441b87f99ab65f37b141a7b548ebadb607b96.diff && \
65+
patch -p1 < 3b2441b87f99ab65f37b141a7b548ebadb607b96.diff && \
66+
rm 3b2441b87f99ab65f37b141a7b548ebadb607b96.diff ; \
67+
fi
68+
69+
# Layer 3: Oracle Instant Client
70+
RUN set -eu && \
71+
if [ "${COMPILEORACLE}" != "" ]; then \
72+
cd /opt && \
73+
wget -q https://download.oracle.com/otn_software/linux/instantclient/${ORACLE_MAJOR}${ORACLE_MINOR}000/instantclient-basic-linux.x64-${ORACLE_MAJOR}.${ORACLE_MINOR}.0.0.0.zip && \
74+
unzip -o instantclient-basic-linux.x64-${ORACLE_MAJOR}.${ORACLE_MINOR}.0.0.0.zip && \
75+
rm instantclient-basic-linux.x64-${ORACLE_MAJOR}.${ORACLE_MINOR}.0.0.0.zip && \
76+
rm -rf META-INF && \
77+
wget -q https://download.oracle.com/otn_software/linux/instantclient/${ORACLE_MAJOR}${ORACLE_MINOR}000/instantclient-sdk-linux.x64-${ORACLE_MAJOR}.${ORACLE_MINOR}.0.0.0.zip && \
78+
unzip -o instantclient-sdk-linux.x64-${ORACLE_MAJOR}.${ORACLE_MINOR}.0.0.0.zip && \
79+
rm instantclient-sdk-linux.x64-${ORACLE_MAJOR}.${ORACLE_MINOR}.0.0.0.zip && \
80+
rm -rf META-INF ; \
81+
fi
82+
83+
# Layer 4: Protobuf (~5-7 min, cached unless version changes)
84+
RUN set -eu && \
85+
if [ "${COMPILEPROTOBUF}" != "" ]; then \
86+
cd /opt && \
87+
wget -q https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION_DIR}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
88+
tar xzf protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
89+
rm protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
90+
cd /opt/protobuf-${PROTOBUF_VERSION} && \
91+
./configure --prefix=/opt/protobuf && \
92+
make && \
93+
make install ; \
94+
fi
95+
96+
# Layer 5: librdkafka (~2-3 min, cached unless version changes)
97+
RUN set -eu && \
98+
if [ "${COMPILEKAFKA}" != "" ]; then \
99+
cd /opt && \
100+
wget -q https://github.com/confluentinc/librdkafka/archive/refs/tags/v${LIBRDKAFKA_VERSION}.tar.gz && \
101+
tar xzf v${LIBRDKAFKA_VERSION}.tar.gz && \
102+
rm v${LIBRDKAFKA_VERSION}.tar.gz && \
103+
cd /opt/librdkafka-${LIBRDKAFKA_VERSION} && \
104+
./configure --prefix=/opt/librdkafka && \
105+
make && \
106+
make install ; \
107+
fi
108+
109+
# Layer 6: Prometheus
110+
RUN set -eu && \
111+
if [ "${COMPILEPROMETHEUS}" != "" ]; then \
112+
cd /opt && \
113+
wget -q https://github.com/jupp0r/prometheus-cpp/releases/download/v${PROMETHEUS_VERSION}/prometheus-cpp-with-submodules.tar.gz && \
114+
tar xzf prometheus-cpp-with-submodules.tar.gz && \
115+
rm prometheus-cpp-with-submodules.tar.gz && \
116+
cd /opt/prometheus-cpp-with-submodules && \
117+
mkdir _build && \
118+
cd _build && \
119+
cmake .. -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/opt/prometheus -DENABLE_PUSH=OFF -DENABLE_COMPRESSION=OFF && \
120+
cmake --build . --parallel 4 && \
121+
ctest -V && \
122+
cmake --install . ; \
123+
fi
124+
125+
# --- Source changes only invalidate from here down ---
126+
COPY . /opt/OpenLogReplicator-local
127+
128+
# Layer 7: Build OLR (ccache persists across rebuilds via BuildKit cache mount)
129+
RUN --mount=type=cache,target=/root/.ccache,id=olr-ccache \
130+
set -eu && \
131+
export CCACHE_DIR=/root/.ccache && \
132+
cd /opt/OpenLogReplicator-local && \
133+
if [ "${COMPILEPROTOBUF}" != "" ]; then \
134+
cd proto && \
135+
/opt/protobuf/bin/protoc OraProtoBuf.proto --cpp_out=. && \
136+
mv OraProtoBuf.pb.cc ../src/common/OraProtoBuf.pb.cpp && \
137+
mv OraProtoBuf.pb.h ../src/common/OraProtoBuf.pb.h && \
138+
cd .. ; \
139+
fi && \
140+
mkdir cmake-build-${BUILD_TYPE}-${ARCH} && \
141+
cd cmake-build-${BUILD_TYPE}-${ARCH} && \
142+
cmake ${BUILDARGS} \
143+
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
144+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache && \
145+
cmake --build ./ --target OpenLogReplicator -j && \
146+
if [ "${WITHTESTS}" != "" ]; then \
147+
cmake --build ./ --target olr_tests -j ; \
148+
fi && \
149+
mkdir -p /opt/OpenLogReplicator/log /opt/OpenLogReplicator/tmp /opt/OpenLogReplicator/scripts && \
150+
cp ./OpenLogReplicator /opt/OpenLogReplicator && \
151+
cp -p /opt/OpenLogReplicator-local/scripts/gencfg.sql /opt/OpenLogReplicator/scripts/gencfg.sql
152+
153+
# Layer 8: User setup
154+
RUN set -eu && \
155+
mkdir -p /home/user1 && \
156+
groupadd -g ${GIDOLR} user1 && \
157+
if [ "${GIDOLR}" != "${GIDORA}" ]; then \
158+
groupadd -g ${GIDORA} oracle && \
159+
useradd -u ${UIDOLR} user1 -g user1 -G oracle -d /home/user1 ; \
160+
else \
161+
useradd -u ${UIDOLR} user1 -g user1 -d /home/user1 ; \
162+
fi && \
163+
chown -R user1:user1 /home/user1 && \
164+
chown -R user1:user1 /opt/OpenLogReplicator && \
165+
chown -R user1:user1 /opt/OpenLogReplicator-local/cmake-build-*
166+
167+
USER user1:oracle
168+
RUN set -eu && \
169+
export LD_LIBRARY_PATH=/opt/instantclient_${ORACLE_MAJOR}_${ORACLE_MINOR}:/opt/librdkafka/lib:/opt/prometheus/lib:/opt/protobuf/lib && \
170+
/opt/OpenLogReplicator/OpenLogReplicator --version
171+
172+
WORKDIR /opt/OpenLogReplicator
173+
ENTRYPOINT ["/opt/run.sh"]

‎Makefile‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
OLR_IMAGE ?= olr-dev:latest
2+
CACHE_IMAGE ?= ghcr.io/bersler/openlogreplicator:ci
3+
BUILD_TYPE ?= Debug
4+
OLR_BUILD_DIR ?= /opt/OpenLogReplicator-local/cmake-build-$(BUILD_TYPE)-x86_64
5+
6+
.PHONY: build test-redo clean
7+
8+
build:
9+
docker buildx build \
10+
--build-arg BUILD_TYPE=$(BUILD_TYPE) \
11+
--build-arg UIDOLR=$$(id -u) \
12+
--build-arg GIDOLR=$$(id -g) \
13+
--build-arg GIDORA=54322 \
14+
--build-arg WITHORACLE=1 \
15+
--build-arg WITHKAFKA=1 \
16+
--build-arg WITHPROTOBUF=1 \
17+
--build-arg WITHPROMETHEUS=1 \
18+
--build-arg WITHTESTS=1 \
19+
--cache-from type=registry,ref=$(CACHE_IMAGE) \
20+
-t $(OLR_IMAGE) \
21+
--load \
22+
-f Dockerfile.dev .
23+
24+
test-redo:
25+
docker run --rm \
26+
-v $(CURDIR)/tests:/opt/OpenLogReplicator-local/tests \
27+
--entrypoint bash $(OLR_IMAGE) \
28+
-c "ctest --test-dir $(OLR_BUILD_DIR) --output-on-failure"
29+
30+
clean:
31+
rm -rf tests/3-generated tests/.work

‎scripts/run.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/sh
2+
# Start script for Docker
3+
# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com)
4+
#
5+
# This file is part of OpenLogReplicator
6+
#
7+
# Open Log Replicator is free software; you can redistribute it and/or
8+
# modify it under the terms of the GNU General Public License as published
9+
# by the Free Software Foundation; either version 3, or (at your option)
10+
# any later version.
11+
#
12+
# Open Log Replicator is distributed in the hope that it will be useful,
13+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
15+
# Public License for more details.
16+
#
17+
# You should have received a copy of the GNU General Public License
18+
# along with Open Log Replicator; see the file LICENSE.txt If not see
19+
# <http://www.gnu.org/licenses/>.
20+
21+
cd /opt/OpenLogReplicator
22+
FLAG_FILE=/opt/OpenLogReplicator/log/.olr_started.flag
23+
if [ -x /opt/bin/OpenLogReplicator ]; then
24+
OLR_EXEC=/opt/bin/OpenLogReplicator
25+
else
26+
OLR_EXEC=./OpenLogReplicator
27+
fi
28+
29+
if [ ! -f "$FLAG_FILE" ]; then
30+
# first execution, provided arguments for startup
31+
touch "$FLAG_FILE"
32+
${OLR_EXEC} "$@" 2>&1 | tee -a /opt/OpenLogReplicator/log/OpenLogReplicator.err
33+
else
34+
# subsequent execution, start normally
35+
${OLR_EXEC} 2>&1 | tee -a /opt/OpenLogReplicator/log/OpenLogReplicator.err
36+
fi

0 commit comments

Comments
 (0)