Merge pull request #39 from DataKitchen/release/2.11.3

Release/2.11.3

Author: datakitchen-devops

README.md

@@ -1,5 +1,5 @@
 # DataOps Observability
-![apache 2.0 license Badge](https://img.shields.io/badge/License%20-%20Apache%202.0%20-%20blue) ![PRs Badge](https://img.shields.io/badge/PRs%20-%20Welcome%20-%20green) [![Docker Pulls](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fhub.docker.com%2Fv2%2Frepositories%2Fdatakitchen%2Fdataops-testgen%2F&query=pull_count&style=flat&label=docker%20pulls&color=06A04A)](https://hub.docker.com/r/datakitchen/dataops-observability) [![Documentation](https://img.shields.io/badge/docs-On%20datakitchen.io-06A04A?style=flat)](https://docs.datakitchen.io/articles/#!dataops-observability-help/dataops-observability-help)
+![apache 2.0 license Badge](https://img.shields.io/badge/License%20-%20Apache%202.0%20-%20blue) ![PRs Badge](https://img.shields.io/badge/PRs%20-%20Welcome%20-%20green) [![Docker Pulls](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fhub.docker.com%2Fv2%2Frepositories%2Fdatakitchen%2Fdataops-testgen%2F&query=pull_count&style=flat&label=docker%20pulls&color=06A04A)](https://hub.docker.com/r/datakitchen/dataops-observability) [![Documentation](https://img.shields.io/badge/docs-On%20datakitchen.io-06A04A?style=flat)](https://docs.datakitchen.io/observability/what-is-observability/)
 [![Latest Version](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fhub.docker.com%2Fv2%2Frepositories%2Fdatakitchen%2Fdataops-observability-be%2Ftags%2F&query=results%5B0%5D.name&label=latest%20version&color=06A04A)](https://hub.docker.com/r/datakitchen/dataops-observability-be)
 [![Static Badge](https://img.shields.io/badge/Slack-Join%20Discussion-blue?style=flat&logo=slack)](https://data-observability-slack.datakitchen.io/join)
 
@@ -147,7 +147,7 @@ To resolve this error, do two things:
 ## Community
 
 ### Getting Started Guide
-We recommend you start by going through the [Data Observability Overview Demo](https://docs.datakitchen.io/articles/open-source-data-observability/data-observability-overview).
+We recommend you start by going through the [Data Observability Overview Demo](https://docs.datakitchen.io/tutorials/quickstart-demo/).
 
 ### Support
 For support requests, [join the Data Observability Slack](https://data-observability-slack.datakitchen.io/join) and ask post on #support channel.

common/entities/journey.py

@@ -17,6 +17,8 @@ class Journey(BaseEntity, AuditEntityMixin, AuditUpdateTimeEntityMixin):
 
     name = CharField(unique=False, null=True)
     description = CharField(null=True)
+    component_include_patterns = CharField(null=True)
+    component_exclude_patterns = CharField(null=True)
     project = ForeignKeyField(Project, backref="journeys", on_delete="CASCADE", null=False, index=True)
 
     @staticmethod

common/entity_services/journey_service.py

@@ -1,10 +1,11 @@
 __all__ = ["JourneyService"]
 
 import logging
+from fnmatch import fnmatch
 from typing import Optional
 from uuid import UUID
 
-from common.entities import Action, Company, Component, Journey, JourneyDagEdge, Organization, Project, Rule
+from common.entities import DB, Action, Company, Component, Journey, JourneyDagEdge, Organization, Project, Rule
 from common.entity_services.helpers import ComponentFilters, ListRules, Page
 from common.exceptions.service import MultipleActionsFound
 
@@ -64,6 +65,72 @@ def get_components_with_rules(journey_id: str, rules: ListRules, filters: Compon
 
         return Page[Component].get_paginated_results(query.distinct(), Component.key, rules)
 
+    @staticmethod
+    def parse_patterns(patterns_str: str | None) -> list[str]:
+        if not patterns_str:
+            return []
+        return [p.strip() for p in patterns_str.replace("\n", ",").split(",") if p.strip()]
+
+    @staticmethod
+    def get_components_matching_patterns(
+        project_id: str, include_patterns: list[str], exclude_patterns: list[str]
+    ) -> list[Component]:
+        if not include_patterns:
+            return []
+        components: list[Component] = list(Component.select().where(Component.project == project_id))
+        return [
+            c
+            for c in components
+            if any(fnmatch(c.key, p) for p in include_patterns) and not any(fnmatch(c.key, p) for p in exclude_patterns)
+        ]
+
+    @staticmethod
+    def apply_component_patterns(journey: Journey) -> None:
+        include_patterns = JourneyService.parse_patterns(journey.component_include_patterns)
+        exclude_patterns = JourneyService.parse_patterns(journey.component_exclude_patterns)
+
+        with DB.atomic():
+            existing_edges = list(
+                JourneyDagEdge.select().where(JourneyDagEdge.journey == journey, JourneyDagEdge.left.is_null(False))
+            )
+            JourneyDagEdge.delete().where(JourneyDagEdge.journey == journey).execute()
+
+            if not include_patterns:
+                return
+
+            matching = JourneyService.get_components_matching_patterns(
+                journey.project_id, include_patterns, exclude_patterns
+            )
+            matching_ids = {c.id for c in matching}
+
+            restored_edges = [e for e in existing_edges if e.left_id in matching_ids and e.right_id in matching_ids]
+            has_predecessor = {e.right_id for e in restored_edges}
+
+            for component in matching:
+                if component.id not in has_predecessor:
+                    JourneyDagEdge(journey=journey, left=None, right=component).save(force_insert=True)
+
+            for edge in restored_edges:
+                JourneyDagEdge(journey=journey, left=edge.left, right=edge.right).save(force_insert=True)
+
+    @staticmethod
+    def add_component_to_matching_journeys(component: Component) -> None:
+        journeys = list(
+            Journey.select().where(
+                Journey.project == component.project_id,
+                Journey.component_include_patterns.is_null(False),
+            )
+        )
+        for journey in journeys:
+            include_patterns = JourneyService.parse_patterns(journey.component_include_patterns)
+            exclude_patterns = JourneyService.parse_patterns(journey.component_exclude_patterns)
+            if not include_patterns:
+                continue
+            if any(fnmatch(component.key, p) for p in include_patterns) and not any(
+                fnmatch(component.key, p) for p in exclude_patterns
+            ):
+                JourneyDagEdge(journey=journey, left=None, right=component).save(force_insert=True)
+
     @staticmethod
     def get_upstream_nodes(journey: Journey, component_id: UUID) -> set:
         journey_dag = journey.journey_dag

common/tests/integration/entity_services/test_journey_service.py

@@ -8,6 +8,218 @@
 from common.exceptions.service import MultipleActionsFound
 
 
+# --- parse_patterns ---
+
+
+@pytest.mark.unit
+def test_parse_patterns_none():
+    assert JourneyService.parse_patterns(None) == []
+
+
+@pytest.mark.unit
+def test_parse_patterns_empty_string():
+    assert JourneyService.parse_patterns("") == []
+
+
+@pytest.mark.unit
+def test_parse_patterns_comma_separated():
+    assert JourneyService.parse_patterns("a,b,c") == ["a", "b", "c"]
+
+
+@pytest.mark.unit
+def test_parse_patterns_newline_separated():
+    assert JourneyService.parse_patterns("a\nb\nc") == ["a", "b", "c"]
+
+
+@pytest.mark.unit
+def test_parse_patterns_trims_whitespace():
+    assert JourneyService.parse_patterns("  a  ,  b  ") == ["a", "b"]
+
+
+@pytest.mark.unit
+def test_parse_patterns_skips_empty_entries():
+    assert JourneyService.parse_patterns("a,,b") == ["a", "b"]
+
+
+# --- get_components_matching_patterns ---
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_no_include_returns_empty(test_db, project, pipeline, pipeline_2):
+    result = JourneyService.get_components_matching_patterns(str(project.id), [], [])
+    assert result == []
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_wildcard_matches_all(test_db, project, pipeline, pipeline_2):
+    result = JourneyService.get_components_matching_patterns(str(project.id), ["*"], [])
+    assert {c.id for c in result} == {pipeline.id, pipeline_2.id}
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_exact_key(test_db, project, pipeline, pipeline_2):
+    result = JourneyService.get_components_matching_patterns(str(project.id), ["P1"], [])
+    assert len(result) == 1
+    assert result[0].id == pipeline.id
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_exclude_applied(test_db, project, pipeline, pipeline_2):
+    result = JourneyService.get_components_matching_patterns(str(project.id), ["*"], ["P2"])
+    assert len(result) == 1
+    assert result[0].id == pipeline.id
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_character_class_matches(test_db, project, pipeline, pipeline_2):
+    # [pP]* matches both P1 and P2 regardless of case
+    result = JourneyService.get_components_matching_patterns(str(project.id), ["[pP]*"], [])
+    assert {c.id for c in result} == {pipeline.id, pipeline_2.id}
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_character_class_negation(test_db, project, pipeline, pipeline_2):
+    # [!pP]* excludes keys starting with p or P — neither P1 nor P2 should match
+    result = JourneyService.get_components_matching_patterns(str(project.id), ["[!pP]*"], [])
+    assert result == []
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_character_class_exclude(test_db, project, pipeline, pipeline_2):
+    # include all, but exclude keys starting with p or P
+    result = JourneyService.get_components_matching_patterns(str(project.id), ["*"], ["[pP]*"])
+    assert result == []
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_character_class_exclude_negation(test_db, project, pipeline, pipeline_2):
+    # exclude *[!1] matches keys not ending in '1' — P2 is excluded, P1 remains
+    result = JourneyService.get_components_matching_patterns(str(project.id), ["*"], ["*[!1]"])
+    assert len(result) == 1
+    assert result[0].id == pipeline.id
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_question_mark_matches_any_single_char(test_db, project, pipeline, pipeline_2):
+    # P? matches any key of exactly two chars starting with P — both P1 and P2 match
+    result = JourneyService.get_components_matching_patterns(str(project.id), ["P?"], [])
+    assert {c.id for c in result} == {pipeline.id, pipeline_2.id}
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_question_mark_matches_specific(test_db, project, pipeline, pipeline_2):
+    # ?1 matches any two-char key ending in '1' — only P1 matches
+    result = JourneyService.get_components_matching_patterns(str(project.id), ["?1"], [])
+    assert len(result) == 1
+    assert result[0].id == pipeline.id
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_question_mark_exclude(test_db, project, pipeline, pipeline_2):
+    # exclude ?1 removes P1, leaving P2
+    result = JourneyService.get_components_matching_patterns(str(project.id), ["*"], ["?1"])
+    assert len(result) == 1
+    assert result[0].id == pipeline_2.id
+
+
+@pytest.mark.integration
+def test_get_components_matching_patterns_only_own_project(test_db, project, pipeline, organization, user):
+    from common.entities import Pipeline, Project
+
+    other_project = Project.create(name="Other", organization=organization, created_by=user)
+    Pipeline.create(name="Other P1", key="P1", project=other_project, created_by=user)
+    result = JourneyService.get_components_matching_patterns(str(project.id), ["P1"], [])
+    assert len(result) == 1
+    assert result[0].id == pipeline.id
+
+
+# --- apply_component_patterns ---
+
+
+@pytest.mark.integration
+def test_apply_component_patterns_no_include_clears_edges(test_db, journey, pipeline):
+    JourneyDagEdge.create(journey=journey, left=None, right=pipeline)
+    journey.component_include_patterns = None
+    journey.component_exclude_patterns = None
+    JourneyService.apply_component_patterns(journey)
+    assert JourneyDagEdge.select().where(JourneyDagEdge.journey == journey).count() == 0
+
+
+@pytest.mark.integration
+def test_apply_component_patterns_adds_matching_as_root_nodes(test_db, journey, pipeline, pipeline_2):
+    journey.component_include_patterns = "*"
+    journey.component_exclude_patterns = None
+    JourneyService.apply_component_patterns(journey)
+    edges = list(JourneyDagEdge.select().where(JourneyDagEdge.journey == journey))
+    assert len(edges) == 2
+    assert all(e.left_id is None for e in edges)
+
+
+@pytest.mark.integration
+def test_apply_component_patterns_preserves_existing_edges(test_db, journey, pipeline, pipeline_2):
+    JourneyDagEdge.create(journey=journey, left=pipeline, right=pipeline_2)
+    journey.component_include_patterns = "*"
+    journey.component_exclude_patterns = None
+    JourneyService.apply_component_patterns(journey)
+    edges = list(JourneyDagEdge.select().where(JourneyDagEdge.journey == journey))
+    # pipeline is root (no predecessor); pipeline->pipeline_2 edge is restored; pipeline_2 gets no root entry
+    assert len(edges) == 2
+    root_edges = [e for e in edges if e.left_id is None]
+    assert len(root_edges) == 1
+    assert root_edges[0].right_id == pipeline.id
+
+
+@pytest.mark.integration
+def test_apply_component_patterns_drops_edge_when_component_excluded(test_db, journey, pipeline, pipeline_2):
+    JourneyDagEdge.create(journey=journey, left=pipeline, right=pipeline_2)
+    journey.component_include_patterns = "*"
+    journey.component_exclude_patterns = "P2"
+    JourneyService.apply_component_patterns(journey)
+    edges = list(JourneyDagEdge.select().where(JourneyDagEdge.journey == journey))
+    assert len(edges) == 1
+    assert edges[0].right_id == pipeline.id
+
+
+# --- add_component_to_matching_journeys ---
+
+
+@pytest.mark.integration
+def test_add_component_to_matching_journeys_matches(test_db, journey, project, pipeline):
+    journey.component_include_patterns = "*"
+    journey.save()
+    JourneyService.add_component_to_matching_journeys(pipeline)
+    edges = list(JourneyDagEdge.select().where(JourneyDagEdge.journey == journey))
+    assert len(edges) == 1
+    assert edges[0].left_id is None
+    assert edges[0].right_id == pipeline.id
+
+
+@pytest.mark.integration
+def test_add_component_to_matching_journeys_no_include_skips(test_db, journey, pipeline):
+    JourneyService.add_component_to_matching_journeys(pipeline)
+    assert JourneyDagEdge.select().where(JourneyDagEdge.journey == journey).count() == 0
+
+
+@pytest.mark.integration
+def test_add_component_to_matching_journeys_excluded(test_db, journey, pipeline):
+    journey.component_include_patterns = "*"
+    journey.component_exclude_patterns = "P1"
+    journey.save()
+    JourneyService.add_component_to_matching_journeys(pipeline)
+    assert JourneyDagEdge.select().where(JourneyDagEdge.journey == journey).count() == 0
+
+
+@pytest.mark.integration
+def test_add_component_to_matching_journeys_only_matching_journey(test_db, journey, journey_2, pipeline):
+    journey.component_include_patterns = "P1"
+    journey.save()
+    journey_2.component_include_patterns = "other-*"
+    journey_2.save()
+    JourneyService.add_component_to_matching_journeys(pipeline)
+    assert JourneyDagEdge.select().where(JourneyDagEdge.journey == journey).count() == 1
+    assert JourneyDagEdge.select().where(JourneyDagEdge.journey == journey_2).count() == 0
+
+
 @pytest.mark.integration
 def test_get_rules_with_rules_journey_exists(test_db, journey, rule):
     rules_page = JourneyService.get_rules_with_rules(journey.id, ListRules())

deploy/docker/observability-be.dockerfile

@@ -32,7 +32,8 @@ RUN python3 -O -m pip install --no-deps /tmp/dk --prefix=/dk
 
 FROM ${BASE_IMAGE_URL}python:3.12.11-alpine3.22 AS runtime-image
 
-RUN apk update && apk upgrade && apk add --no-cache librdkafka=2.10.0-r0
+RUN apk update && apk upgrade && apk add --no-cache librdkafka=2.10.0-r0 \
+    && pip install --no-cache-dir --upgrade pip
 
 # Grab the pre-built app from the build-image. This way we don't have
 # excess laying around in the final image.

deploy/docker/observability-ui.dockerfile

@@ -8,7 +8,11 @@ COPY observability_ui/ /observability_ui
 RUN yarn
 RUN yarn build:ci
 
-FROM ${BASE_IMAGE_URL}nginxinc/nginx-unprivileged:alpine3.22
+FROM ${BASE_IMAGE_URL}nginxinc/nginx-unprivileged:alpine3.23
+
+USER root
+RUN apk upgrade --no-cache
+USER nginx
 
 WORKDIR /observability_ui
 

deploy/migrations/20260318_01_pQ7mR-add-journey-component-pattern-columns.py

@@ -0,0 +1,19 @@
+"""
+Add component_include_patterns and component_exclude_patterns columns to journey table
+"""
+
+from yoyo import step
+
+__depends__ = {"20240723_01_K3c3Q-renaming-the-agent-check-interval-column"}
+__transactional__ = False
+
+steps = [
+    step(
+        "ALTER TABLE `journey` ADD COLUMN `component_include_patterns` varchar(255) NULL",
+        "ALTER TABLE `journey` DROP COLUMN `component_include_patterns`",
+    ),
+    step(
+        "ALTER TABLE `journey` ADD COLUMN `component_exclude_patterns` varchar(255) NULL",
+        "ALTER TABLE `journey` DROP COLUMN `component_exclude_patterns`",
+    ),
+]

observability_api/endpoints/component_view.py

@@ -10,6 +10,7 @@
 
 from common.api.base_view import Permission
 from common.entities import BaseEntity, Project
+from common.entity_services import JourneyService
 from observability_api.endpoints.entity_view import BaseEntityView
 from observability_api.schemas.component_schemas import ComponentPatchSchema
 
@@ -55,6 +56,7 @@ def post(self, project_id: UUID) -> Response:
         component.created_by = self.user
         component.project = self.get_entity_or_fail(Project, Project.id == project_id)
         self.save_entity_or_fail(component, force_insert=True)
+        JourneyService.add_component_to_matching_journeys(component)
         return make_response(self.schema().dump(component), HTTPStatus.CREATED)
 
     @classmethod