"initial commit"

Author: Jakee4488

.github/workflows/cd.yml

@@ -0,0 +1,56 @@
+name: CD
+
+on:
+  workflow_dispatch:
+
+  push:
+    branches:
+        - 'main'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        environment: [acc, prd]
+    environment: ${{ matrix.environment }}
+    permissions:
+      contents: write # to push tag
+    env:
+      DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }}
+      DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
+      DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }}
+    steps:
+      - name: Checkout Source Code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
+
+      - name: Install Databricks CLI
+        uses: databricks/setup-cli@49580195afe1ccb06d195764a1d0ae9fabfe2edd #v0.246.0
+        with:
+          version: 0.246.0
+
+      - name: Configure Databricks CLI
+        run: |
+          mkdir -p ~/.databricks
+          cat > ~/.databrickscfg << EOF
+          [marvelous]
+          host = ${{ vars.DATABRICKS_HOST }}
+          client_id = ${{ secrets.DATABRICKS_CLIENT_ID }}
+          client_secret = ${{ secrets.DATABRICKS_CLIENT_SECRET }}
+          EOF
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 #v5.4.1
+
+      - name: Deploy to Databricks
+        env:
+          DATABRICKS_BUNDLE_ENV: ${{ matrix.environment }}
+        run: |
+            databricks bundle deploy \
+              --var="git_sha=${{ github.sha }}" \
+              --var="branch=${{ github.ref_name }}"
+            if [ "${{ matrix.environment }}" = "prd" ]; then
+            echo "VERSION=$(cat version.txt)"
+            git tag $VERSION
+            git push origin $VERSION
+            fi

.github/workflows/ci.yml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  pytest_and_checks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
+        with:
+          # Fetch all history for all branches and tags
+          fetch-depth: 0
+          ref: ${{ github.head_ref }}
+
+      - name: Git tag from version.txt
+        run: |
+          echo "VERSION=$(cat version.txt)"
+          git tag $VERSION
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 #v5.4.1
+
+      - name: Install the dependencies
+        run: |
+          uv sync --extra test
+
+      - name: Run pre-commit checks
+        run: |
+          uv run pre-commit run --all-files
+
+      - name: run pytest
+        run: |
+          uv run pytest -m "not ci_exclude"

.gitignore

@@ -0,0 +1,100 @@
+.venv/
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+.databricks/*
+.ruff_cache/*
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# mypy
+.mypy_cache/
+.dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Data, models, outputs
+*.h5
+*.hdf5
+*.tsv
+*.parquet
+*.pkl
+*.joblib
+*.npy
+*.npz
+*.log
+*.mlmodel
+*.onnx
+*.ckpt
+*.pt
+*.pth
+*.pb
+*.tflite
+*.metrics
+*.tensorboard
+runs/
+logs/
+output/
+outputs/
+checkpoints/
+
+# MLflow
+mlruns/
+
+# VSCode
+.vscode/
+
+# MacOS
+.DS_Store
+
+# Environment variables
+.env
+logs*

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+exclude: '^(notebooks|scripts)/.*'
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-json
+      - id: check-toml
+      - id: check-yaml
+        args: ["--unsafe"]
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: detect-private-key
+      - id: forbid-new-submodules
+      - id: check-docstring-first
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.5
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix, --show-fixes]
+      - id: ruff-format

README.md

@@ -0,0 +1,29 @@
+<h1 align="center">
+Marvelous MLOps Free End-to-end MLOps with Databricks Course
+
+## Set up your environment
+In this course, we use Databricks serverless [version 3](https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/three)
+
+In our examples, we use UV. Check out the documentation on how to install it: https://docs.astral.sh/uv/getting-started/installation/
+
+To create a new environment and create a lockfile, run:
+
+```
+uv sync --extra dev
+```
+
+
+
+# Data
+Using the [**Marvel Characters Dataset**](https://www.kaggle.com/datasets/mohitbansal31s/marvel-characters?resource=download) from Kaggle.
+
+This dataset contains detailed information about Marvel characters (e.g., name, powers, physical attributes, alignment, etc.).
+It is used to build classification and feature engineering models for various MLOps tasks, such as predicting character attributes or status.
+
+# Scripts
+
+- `01.process_data.py`: Loads and preprocesses the Marvel dataset, splits into train/test, and saves to the catalog.
+- `02.train_register_fe_model.py`: Performs feature engineering and trains the Marvel character model.
+- `03.deploy_model.py`: Deploys the trained Marvel model to a Databricks model serving endpoint.
+- `04.post_commit_status.py`: Posts status updates for Marvel integration tests to GitHub.
+- `05.refresh_monitor.py`: Refreshes monitoring tables and dashboards for Marvel model serving.