diff --git a/jobs/colin-extract-refresh/.env.sample b/jobs/colin-extract-refresh/.env.sample new file mode 100644 index 0000000000..1e25ada2fc --- /dev/null +++ b/jobs/colin-extract-refresh/.env.sample @@ -0,0 +1,12 @@ + +DATABASE_USERNAME_COLIN_ORACLE= +DATABASE_PASSWORD_COLIN_ORACLE= +DATABASE_NAME_COLIN_ORACLE= +DATABASE_HOST_COLIN_ORACLE= +DATABASE_PORT_COLIN_ORACLE= + +DATABASE_USERNAME_COLIN_MIGR= +DATABASE_PASSWORD_COLIN_MIGR= +DATABASE_NAME_COLIN_MIGR= +DATABASE_HOST_COLIN_MIGR= +DATABASE_PORT_COLIN_MIGR= \ No newline at end of file diff --git a/jobs/colin-extract-refresh/Dockerfile b/jobs/colin-extract-refresh/Dockerfile new file mode 100644 index 0000000000..7d4ed21c84 --- /dev/null +++ b/jobs/colin-extract-refresh/Dockerfile @@ -0,0 +1,56 @@ +FROM python:3.11-slim + +ARG VCS_REF="missing" +ARG BUILD_DATE="missing" + +ENV VCS_REF=${VCS_REF} +ENV BUILD_DATE=${BUILD_DATE} +ENV PYTHONUNBUFFERED=1 + + + + + +LABEL org.label-schema.vcs-ref=${VCS_REF} \ + org.label-schema.build-date=${BUILD_DATE} + + +USER root + +ENV ORACLE_CLIENT_LIB_DIR=/opt/oracle/instantclient_21_4 +ENV LD_LIBRARY_PATH=${ORACLE_CLIENT_LIB_DIR} + + +WORKDIR /opt/oracle/ +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends unzip wget ca-certificates;\ + apt-get install -y --no-install-recommends libaio1 || apt-get install -y --no-install-recommends libaio1t64; \ + ln -sf /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1 || true; \ + ln -sf /lib/x86_64-linux-gnu/libaio.so.1t64 /lib/x86_64-linux-gnu/libaio.so.1 || true; \ + wget -q https://download.oracle.com/otn_software/linux/instantclient/214000/instantclient-basic-linux.x64-21.4.0.0.0dbru.zip; \ + unzip instantclient-basic-linux.x64-21.4.0.0.0dbru.zip; \ + rm instantclient-basic-linux.x64-21.4.0.0.0dbru.zip; \ + cd "${ORACLE_CLIENT_LIB_DIR}"; \ + rm -f *jdbc* *occi* *mysql* *README *jar uidrvci genezi adrci; \ + test -e libclntsh.so || ln -sf libclntsh.so.* libclntsh.so; \ + test -f libclntsh.so; \ + echo "${ORACLE_CLIENT_LIB_DIR}" > /etc/ld.so.conf.d/oracle-instantclient.conf; \ + ldconfig; \ + chmod 755 "${ORACLE_CLIENT_LIB_DIR}"; \ + rm -rf /var/lib/apt/lists/* + +# Create directories with proper permissions +RUN mkdir -p /opt/app-root && \ + chmod 755 /opt/app-root + +WORKDIR /opt/app-root + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +USER 1001 + +EXPOSE 8080 + +CMD [ "python", "/opt/app-root/test_connectivity.py" ] diff --git a/jobs/colin-extract-refresh/Makefile b/jobs/colin-extract-refresh/Makefile new file mode 100644 index 0000000000..a5a17449a0 --- /dev/null +++ b/jobs/colin-extract-refresh/Makefile @@ -0,0 +1,41 @@ +.PHONY: help setup install build build-nc run + +MKFILE_PATH:=$(abspath $(lastword $(MAKEFILE_LIST))) +CURRENT_ABS_DIR:=$(patsubst %/,%,$(dir $(MKFILE_PATH))) + +DOCKER_NAME:=colin-extract-refresh +TAG_NAME ?= dev +DOCKER_PLATFORM ?= --platform linux/amd64 + +################################################################################# +# COMMANDS -- Setup # +################################################################################# +setup: install ## Setup the project + +install: ## Install python virtual environment + test -f .venv/bin/activate || python3.11 -m venv $(CURRENT_ABS_DIR)/.venv ;\ + . .venv/bin/activate ;\ + pip install --upgrade pip ;\ + pip install -Ur requirements.txt + +run: ## Run the project in local + . .venv/bin/activate && python test_connectivity.py + +build: + docker build ${DOCKER_PLATFORM} -t $(DOCKER_NAME):$(TAG_NAME) $(CURRENT_ABS_DIR) \ + --build-arg VCS_REF=$$(git -C $(CURRENT_ABS_DIR) rev-parse --short HEAD 2>/dev/null || echo missing) \ + --build-arg BUILD_DATE=$$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +build-nc: + docker build ${DOCKER_PLATFORM} --no-cache -t $(DOCKER_NAME):$(TAG_NAME) $(CURRENT_ABS_DIR) + +run-docker: build + docker run --rm --env-file $(CURRENT_ABS_DIR)/.env $(DOCKER_NAME):$(TAG_NAME) + +################################################################################# +# Self Documenting Commands # +################################################################################# +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.DEFAULT_GOAL := help diff --git a/jobs/colin-extract-refresh/README.md b/jobs/colin-extract-refresh/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/jobs/colin-extract-refresh/checks/__init__.py b/jobs/colin-extract-refresh/checks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/jobs/colin-extract-refresh/checks/check_business.py b/jobs/colin-extract-refresh/checks/check_business.py new file mode 100644 index 0000000000..04345e0800 --- /dev/null +++ b/jobs/colin-extract-refresh/checks/check_business.py @@ -0,0 +1,28 @@ +import sys + +from sqlalchemy import create_engine, text +from config import get_named_config + +def run_check() -> int: + cfg = get_named_config() + if not all([cfg.DB_USER_COLIN_MIGR, cfg.DB_PASSWORD_COLIN_MIGR, cfg.DB_NAME_COLIN_MIGR, cfg.DB_HOST_COLIN_MIGR, cfg.DB_PORT_COLIN_MIGR]): + raise RuntimeError( + "Missing business env vars" + ) + print(f"[business-api] connecting to {cfg.SQLALCHEMY_DATABASE_URI_COLIN_MIGR}") + engine = create_engine(cfg.SQLALCHEMY_DATABASE_URI_COLIN_MIGR) + with engine.connect() as conn: + row = conn.execute(text("SELECT * FROM corporation LIMIT 1")).mappings().first() + if row is None: + print("no rows in business mig db") + else: + print(f"sample row: {dict(row)}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(run_check()) + except Exception as exc: + print(f"business db check failed: {exc}", file=sys.stderr) + raise diff --git a/jobs/colin-extract-refresh/checks/check_colin.py b/jobs/colin-extract-refresh/checks/check_colin.py new file mode 100644 index 0000000000..a5ad4697b6 --- /dev/null +++ b/jobs/colin-extract-refresh/checks/check_colin.py @@ -0,0 +1,37 @@ +import os +import sys +import oracledb +from sqlalchemy import create_engine, text +from config import get_named_config + +def _colin_oracle_init() -> None: + lib_dir = os.environ.get("ORACLE_CLIENT_LIB_DIR", "") + oracledb.init_oracle_client(lib_dir=lib_dir) + print('👷 Enable thick mode:', not oracledb.is_thin_mode()) + print('👷 Instant Client version:', oracledb.clientversion()) + + +def run_check() -> int: + cfg = get_named_config() + if not all([cfg.DB_USER_COLIN_ORACLE, cfg.DB_PASSWORD_COLIN_ORACLE, cfg.DB_NAME_COLIN_ORACLE, cfg.DB_HOST_COLIN_ORACLE, cfg.DB_PORT_COLIN_ORACLE]): + raise RuntimeError( + "Missing colin env vars" + ) + print(f"[business-api] connecting to {cfg.SQLALCHEMY_DATABASE_URI_COLIN_ORACLE}") + _colin_oracle_init() + engine = create_engine(cfg.SQLALCHEMY_DATABASE_URI_COLIN_ORACLE) + with engine.connect() as conn: + row = conn.execute(text("SELECT * FROM corporation FETCH FIRST 1 ROWS ONLY")).mappings().first() + if row is None: + print("no rows in COLIN db") + else: + print(f"COLIN sample row: {dict(row)}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(run_check()) + except Exception as exc: + print(f"business db check failed: {exc}", file=sys.stderr) + raise diff --git a/jobs/colin-extract-refresh/config.py b/jobs/colin-extract-refresh/config.py new file mode 100644 index 0000000000..7b68092720 --- /dev/null +++ b/jobs/colin-extract-refresh/config.py @@ -0,0 +1,146 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""All of the configuration for the service is captured here. + +All items are loaded, or have Constants defined here that +are loaded into the Flask configuration. +All modules and lookups get their configuration from the +Flask config, rather than reading environment variables directly +or by accessing this configuration directly. +""" +import os +import sys + +from dotenv import find_dotenv, load_dotenv + + +# this will load all the envars from a .env file located in the project root (api) +load_dotenv(find_dotenv()) + + +def _get_int(name: str, default: int = 0) -> int: + """Safe int env parsing that avoids None.isnumeric() crashes.""" + val = os.getenv(name) + return int(val) if (val and val.isnumeric()) else default + + +def _get_strict_int(name: str, default: int = 0) -> int: + """Parse an integer env var, raising when a non-blank value is invalid.""" + val = os.getenv(name) + if val is None or val.strip() == '': + return default + try: + return int(val) + except ValueError as exc: + raise ValueError(f'{name} must be a valid integer') from exc + + +def _get_bool(name: str, default: bool = False) -> bool: + """Safe bool env parsing (case-insensitive).""" + val = os.getenv(name) + if val is None: + return default + return val.strip().lower() == 'true' + + +def get_named_config(config_name: str = 'production'): + """Return the configuration object based on the name. + + :raise: KeyError: if an unknown configuration is requested + """ + if config_name in ['production', 'staging', 'default']: + config = ProdConfig() + elif config_name == 'testing': + config = TestConfig() + elif config_name == 'development': + config = DevConfig() + else: + raise KeyError(f'Unknown configuration: {config_name}') + return config + + +class _Config(): # pylint: disable=too-few-public-methods + """Base class configuration that should set reasonable defaults. + + Used as the base for all the other configurations. + """ + + DATA_LOAD_ENV = os.getenv('DATA_LOAD_ENV', '') + + # POSTGRESQL COLIN MIGRATION DB + DB_USER_COLIN_MIGR = os.getenv('DATABASE_USERNAME_COLIN_MIGR', '') + DB_PASSWORD_COLIN_MIGR = os.getenv('DATABASE_PASSWORD_COLIN_MIGR', '') + DB_NAME_COLIN_MIGR = os.getenv('DATABASE_NAME_COLIN_MIGR', '') + DB_HOST_COLIN_MIGR = os.getenv('DATABASE_HOST_COLIN_MIGR', '') + DB_PORT_COLIN_MIGR = os.getenv('DATABASE_PORT_COLIN_MIGR', '5432') + SQLALCHEMY_DATABASE_URI_COLIN_MIGR = 'postgresql://{user}:{password}@{host}:{port}/{name}'.format( + user=DB_USER_COLIN_MIGR, + password=DB_PASSWORD_COLIN_MIGR, + host=DB_HOST_COLIN_MIGR, + port=int(DB_PORT_COLIN_MIGR), + name=DB_NAME_COLIN_MIGR, + ) + SQLALCHEMY_TRACK_MODIFICATIONS = os.getenv('SQLALCHEMY_TRACK_MODIFICATIONS', False) + + + DATABASE_POOL_PRE_PING = os.getenv('DATABASE_POOL_PRE_PING', 'True') == 'True' + DATABASE_POOL_SIZE = os.getenv('DATABASE_POOL_SIZE', '5') + DATABASE_MAX_OVERFLOW = os.getenv('DATABASE_MAX_OVERFLOW', '10') + + SQLALCHEMY_ENGINE_OPTIONS = { + "pool_pre_ping": DATABASE_POOL_PRE_PING, + "pool_size": int(DATABASE_POOL_SIZE), + "max_overflow": int(DATABASE_MAX_OVERFLOW) + } + + # ORACLE COLIN DB + DB_USER_COLIN_ORACLE = os.getenv('DATABASE_USERNAME_COLIN_ORACLE', '') + DB_PASSWORD_COLIN_ORACLE = os.getenv('DATABASE_PASSWORD_COLIN_ORACLE', '') + DB_NAME_COLIN_ORACLE = os.getenv('DATABASE_NAME_COLIN_ORACLE', '') + DB_HOST_COLIN_ORACLE = os.getenv('DATABASE_HOST_COLIN_ORACLE', '') + DB_PORT_COLIN_ORACLE = os.getenv('DATABASE_PORT_COLIN_ORACLE', '1521') + SQLALCHEMY_DATABASE_URI_COLIN_ORACLE = 'oracle+oracledb://{user}:{password}@{host}:{port}/{name}'.format( + user=DB_USER_COLIN_ORACLE, + password=DB_PASSWORD_COLIN_ORACLE, + host=DB_HOST_COLIN_ORACLE, + port=int(DB_PORT_COLIN_ORACLE), + name=DB_NAME_COLIN_ORACLE, + ) + + TESTING = False + DEBUG = False + + +class DevConfig(_Config): # pylint: disable=too-few-public-methods + """Creates the Development Config object.""" + + TESTING = False + DEBUG = True + + +class TestConfig(_Config): # pylint: disable=too-few-public-methods + """In support of testing only. + + Used by the py.test suite + """ + + DEBUG = True + TESTING = True + + +class ProdConfig(_Config): # pylint: disable=too-few-public-methods + """Production environment configuration.""" + + TESTING = False + DEBUG = False diff --git a/jobs/colin-extract-refresh/openshift/Readme.md b/jobs/colin-extract-refresh/openshift/Readme.md new file mode 100644 index 0000000000..c8cd8e76bd --- /dev/null +++ b/jobs/colin-extract-refresh/openshift/Readme.md @@ -0,0 +1 @@ +# test extract job \ No newline at end of file diff --git a/jobs/colin-extract-refresh/openshift/templates/bc.yaml b/jobs/colin-extract-refresh/openshift/templates/bc.yaml new file mode 100644 index 0000000000..e0426c7dcb --- /dev/null +++ b/jobs/colin-extract-refresh/openshift/templates/bc.yaml @@ -0,0 +1,21 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + labels: + app: ${NAME} + name: ${NAME}-build +objects: +- apiVersion: v1 + kind: ImageStream + metadata: + name: ${NAME} + labels: + app: ${NAME} +parameters: + - description: | + The name assigned to all of the objects defined in this template. + You should keep this as default unless your know what your doing. + displayName: Name + name: NAME + required: true + value: colin-extract-refresh diff --git a/jobs/colin-extract-refresh/openshift/templates/cronjob.yaml b/jobs/colin-extract-refresh/openshift/templates/cronjob.yaml new file mode 100644 index 0000000000..c716ae715e --- /dev/null +++ b/jobs/colin-extract-refresh/openshift/templates/cronjob.yaml @@ -0,0 +1,125 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + labels: + name: ${NAME} + name: ${NAME}-cronjob +objects: +- kind: "CronJob" + apiVersion: "batch/v1" + metadata: + name: "${NAME}-${TAG}" + labels: + name: "${NAME}" + environment: "${TAG}" + role: "${ROLE}" + spec: + schedule: "${SCHEDULE}" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + jobTemplate: + metadata: + labels: + name: "${NAME}" + environment: "${TAG}" + role: "${ROLE}" + spec: + backoffLimit: 0 + activeDeadlineSeconds: 300 + template: + metadata: + labels: + name: "${NAME}" + environment: "${TAG}" + role: "${ROLE}" + spec: + containers: + - name: "${NAME}-${TAG}" + image: "${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/${NAME}:${IMAGE_TAG}" + imagePullPolicy: Always + command: + - python + - /opt/app-root/test_connectivity.py + envFrom: + - secretRef: + name: ${APP_SECRET_NAME} + resources: + requests: + cpu: "${CPU_REQUEST}" + memory: "${MEMORY_REQUEST}" + limits: + cpu: "${CPU_LIMIT}" + memory: "${MEMORY_LIMIT}" + restartPolicy: Never + terminationGracePeriodSeconds: 30 + +parameters: + + - name: NAME + displayName: Name + description: The name assigned to all of the OpenShift resources associated to the server instance. + required: true + value: colin-extract-refresh + + - name: TAG + displayName: Environment TAG name + description: The TAG name for this environment, e.g., dev, test, prod + value: dev + required: true + + - name: ROLE + displayName: Role + description: Role + required: true + value: job + + - name: IMAGE_NAMESPACE + displayName: Image Namespace + required: true + description: The namespace of the OpenShift project containing the imagestream for the application. + value: cc892f-tools + + - name: IMAGE_TAG + displayName: Environment TAG name + description: The TAG name for this environment, e.g., dev, test, prod + value: dev + required: true + + - name: IMAGE_REGISTRY + displayName: Image Registry + required: true + description: The image registry of the OpenShift project. + value: image-registry.openshift-image-registry.svc:5000 + + - name: APP_SECRET_NAME + displayName: App Secret Name + description: Secret with business-api and colin-api + required: true + value: colin-extract-refresh-secret + + - name: SCHEDULE + displayName: "Cron Schedule" + description: "Cron Schedule to Execute the Job (using local cluster system TZ) to run at 0:55 * * TUE-SAT at pacific time so it will be 7:55 at UTC" + value: "55 7 * * TUE-SAT" + required: true + + - name: CPU_REQUEST + displayName: CPU request + required: true + value: 50m + + - name: CPU_LIMIT + displayName: CPU limit + required: true + value: 200m + + - name: MEMORY_REQUEST + displayName: memory request + required: true + value: 128Mi + + - name: MEMORY_LIMIT + displayName: Memory Limit + required: true + value: 512Mi \ No newline at end of file diff --git a/jobs/colin-extract-refresh/requirements.txt b/jobs/colin-extract-refresh/requirements.txt new file mode 100644 index 0000000000..399c01c0d2 --- /dev/null +++ b/jobs/colin-extract-refresh/requirements.txt @@ -0,0 +1,4 @@ +SQLAlchemy==2.0.36 +psycopg2-binary==2.9.9 +oracledb==3.1.1 +python-dotenv==1.1.1 \ No newline at end of file diff --git a/jobs/colin-extract-refresh/setup.cfg b/jobs/colin-extract-refresh/setup.cfg new file mode 100644 index 0000000000..95a2b3967e --- /dev/null +++ b/jobs/colin-extract-refresh/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +exclude = .git,.venv +max-line-length = 120 diff --git a/jobs/colin-extract-refresh/setup.py b/jobs/colin-extract-refresh/setup.py new file mode 100644 index 0000000000..3f51a3d92b --- /dev/null +++ b/jobs/colin-extract-refresh/setup.py @@ -0,0 +1,22 @@ +# Copyright © 2019 Province of British Columbia. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Installer and setup for this module.""" + +from setuptools import find_packages, setup + + +setup( + name='colin-extract-refresh', + packages=find_packages() +) diff --git a/jobs/colin-extract-refresh/test_connectivity.py b/jobs/colin-extract-refresh/test_connectivity.py new file mode 100644 index 0000000000..84079f9aa2 --- /dev/null +++ b/jobs/colin-extract-refresh/test_connectivity.py @@ -0,0 +1,18 @@ +import sys + +from checks.check_business import run_check as run_business_check +from checks.check_colin import run_check as run_colin_check + +def main() -> int: + print("== running test_connectivity.py ==") + # run_business_check() + run_colin_check() + print("== done test_connectivity.py ==") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print(f"[test-connectivity] failed: {exc}", file=sys.stderr) \ No newline at end of file