fix: Semi aggregated query for hardware details summary endpoint

Author: alanpeixinho

backend/kernelCI_app/constants/localization.py

@@ -34,6 +34,7 @@ class ClientStrings:
     ISSUE_NOT_FOUND = "Issue not found"
     NO_ISSUE_FOUND = "No issues found"
     INVALID_JSON_BODY = "Invalid body, request body must be a valid json string"
+    INVALID_FILTERS = "Invalid filter key or value"
     ISSUE_EMPTY_LIST = "Invalid body, the issue list must not be empty"
     ISSUE_NO_EXTRA_DETAILS = (
         "No extra details found. Issue id has no incident or doesn't exist."

backend/kernelCI_app/helpers/filters.py

@@ -5,6 +5,7 @@
 from kernelCI_app.constants.general import UNCATEGORIZED_STRING
 from kernelCI_app.helpers.commonDetails import PossibleTabs
 from kernelCI_app.helpers.logger import log_message
+from kernelCI_app.models import StatusChoices
 from kernelCI_app.typeModels.databases import (
     StatusValues,
     failure_status_list,
@@ -22,6 +23,10 @@
 NULL_STRINGS = set(["null", UNKNOWN_STRING, "NULL"])
 
 
+def is_valid_status(status: str) -> bool:
+    return status in StatusChoices or status == "NULL"
+
+
 def is_status_failure(
     test_status: StatusValues, fail_list: list[StatusValues] = failure_status_list
 ) -> bool:
@@ -129,6 +134,12 @@ def is_issue_filtered_out(
     return not in_filter
 
 
+def is_filtered_out(value: str, filter_values: set[set]):
+    if filter_values and value not in filter_values:
+        return True
+    return False
+
+
 def should_filter_test_issue(
     *,
     issue_filters: set,
@@ -361,7 +372,7 @@ def __init__(self, data: Dict, process_body=False) -> None:
             "test.status": self._handle_test_status,
             "test.duration": self._handle_test_duration,
             "build.status": self._handle_build_status,
-            "build.duration": self._handle_build_duration,
+            "duration": self._handle_build_duration,
             "origin": self._handle_origins,
             "config_name": self._handle_config_name,
             "compiler": self._handle_compiler,
@@ -391,6 +402,14 @@ def __init__(self, data: Dict, process_body=False) -> None:
 
         self._process_filters()
 
+    def __repr__(self) -> str:
+        parts = ""
+        for parsed_filter in self.filters:
+            parts += "\n\t{},".format(
+                ", ".join([f"{key}={val}" for key, val in parsed_filter.items()])
+            )
+        return f"FilterParams({parts})"
+
     def _handle_boot_status(self, current_filter: ParsedFilter) -> None:
         self.filterBootStatus.add(current_filter["value"])
 

backend/kernelCI_app/helpers/hardwareDetails.py

@@ -509,8 +509,8 @@ def process_issue(
         incident_test_id=record["incidents__test_id"],
         build_status=record["build__status"],
         test_status=record["status"],
-        issue_comment=record["incidents__issue__comment"],
-        issue_report_url=record["incidents__issue__report_url"],
+        issue_comment=record.get("incidents__issue__comment"),
+        issue_report_url=record.get("incidents__issue__report_url"),
         is_failed_task=is_failed_task,
         issue_from=issue_from,
         task=task_issues_dict,

backend/kernelCI_app/queries/hardware.py

@@ -1,6 +1,7 @@
-from typing import TypedDict
+from typing import Optional, TypedDict
 from datetime import datetime
 from django.db import connection
+from itertools import repeat
 
 from kernelCI_app.helpers.database import dict_fetchall
 from kernelCI_app.cache import get_query_cache, set_query_cache
@@ -339,15 +340,250 @@ def get_hardware_details_data(
     return records
 
 
+def _get_build_duration_clause(builds_duration) -> str:
+    clause = ""
+
+    # builds
+    duration_min, duration_max = builds_duration
+    if duration_min:
+        clause += f"AND builds.duration >= {duration_min}\n"
+    if duration_max:
+        clause += f"AND builds.duration <= {duration_max}\n"
+
+    return clause
+
+
+def _get_boot_test_duration_clause(boots_duration, tests_duration) -> str:
+    clause = ""
+
+    # tests
+    duration_min, duration_max = tests_duration
+    if duration_min:
+        clause += f"AND tests.duration >= {duration_min}\n"
+    if duration_max:
+        clause += f"AND tests.duration <= {duration_max}\n"
+
+    # boots
+    duration_min, duration_max = boots_duration
+    if duration_min:
+        clause += (
+            "AND (NOT (tests.path like 'boot.%%' or tests.path = 'boot') "
+            f"OR tests.duration >= {duration_min})\n"
+        )
+    if duration_max:
+        clause += (
+            "AND (NOT (tests.path like 'boot.%%' or tests.path = 'boot') "
+            f"OR tests.duration <= {duration_max})\n"
+        )
+
+    return clause
+
+
+def get_hardware_details_filters(
+    *,
+    hardware_id: str,
+    origin: str,
+    start_datetime: datetime,
+    end_datetime: datetime,
+) -> dict[str, list]:
+
+    cache_key = "hardwareDetailsFilters"
+
+    tests_cache_params = {
+        "hardware_id": hardware_id,
+        "origin": origin,
+        "start_date": start_datetime,
+        "end_date": end_datetime,
+    }
+
+    query_rows = get_query_cache(cache_key, tests_cache_params)
+
+    if query_rows:
+        return query_rows[0]
+
+    query = """
+    SELECT array_agg(distinct builds.config_name) AS config_name,
+        array_agg(distinct builds.architecture) AS architecture,
+        array_agg(distinct builds.compiler) AS compiler,
+        array_agg(distinct tests.misc->>'runtime') AS lab
+    FROM
+        tests
+    INNER JOIN builds ON
+        tests.build_id = builds.id
+    WHERE
+    (
+        tests.environment_compatible @> ARRAY[%s]::TEXT[]
+        OR tests.environment_misc ->> 'platform' = %s
+    )
+        AND tests.origin = %s
+        AND tests.start_time >= %s
+        AND tests.start_time <= %s;
+    """
+    params = [
+        hardware_id,
+        hardware_id,
+        origin,
+        start_datetime,
+        end_datetime,
+    ]
+
+    with connection.cursor() as cursor:
+        cursor.execute(query, params)
+        query_rows = dict_fetchall(cursor)
+        set_query_cache(key=cache_key, params=tests_cache_params, rows=query_rows)
+        return query_rows[0]
+
+
+def get_hardware_details_summary(
+    *,
+    hardware_id: str,
+    origin: str,
+    trees_with_selected_commits: list[Tree],
+    builds_duration: (Optional[int], Optional[int]),
+    boots_duration: (Optional[int], Optional[int]),
+    tests_duration: (Optional[int], Optional[int]),
+    start_datetime: datetime,
+    end_datetime: datetime,
+):
+
+    cache_key = "hardwareDetailsSummary"
+
+    tests_cache_params = {
+        "hardware_id": hardware_id,
+        "origin": origin,
+        "trees": trees_with_selected_commits,
+        "start_date": start_datetime,
+        "end_date": end_datetime,
+        "builds_duration": builds_duration,
+        "boots_duration": boots_duration,
+        "tests_duration": tests_duration,
+    }
+
+    query_rows = get_query_cache(cache_key, tests_cache_params)
+
+    if query_rows:
+        return query_rows
+
+    commit_hashes = [tree.head_git_commit_hash for tree in trees_with_selected_commits]
+
+    builds_duration_clause = _get_build_duration_clause(builds_duration)
+    boots_tests_duration_clause = _get_boot_test_duration_clause(
+        boots_duration, tests_duration
+    )
+
+    query = """
+           (SELECT
+                 COUNT(distinct builds.id) AS count,
+                 checkouts.origin,
+                 builds.status AS status,
+                 count(incidents.id) AS known_issues,
+                 array[builds.compiler, builds.architecture] AS compiler_arch,
+                 builds.config_name,
+                 builds.misc->>'runtime' AS lab,
+                 tests.environment_misc->>'platform' AS platform,
+                 tests.environment_compatible,
+                 checkouts.origin,
+                 checkouts.tree_name,
+                 checkouts.git_repository_url,
+                 checkouts.git_commit_tags,
+                 checkouts.git_commit_name,
+                 checkouts.git_repository_branch,
+                 checkouts.git_commit_hash,
+                 true AS is_build,
+                 false AS is_test,
+                 false AS is_boot
+             FROM
+                builds
+            INNER JOIN tests ON
+                tests.build_id = builds.id
+            INNER JOIN checkouts ON
+                builds.checkout_id = checkouts.id
+            LEFT OUTER JOIN incidents ON
+                builds.id = incidents.build_id
+            WHERE
+                (
+                    builds.config_name IS NOT NULL
+                    AND builds.id not like 'maestro:dummy_%%'
+                    AND (tests.environment_compatible @> ARRAY[%s]::TEXT[]
+                    OR tests.environment_misc ->> 'platform' = %s)
+                )
+                AND builds.origin = %s
+                AND builds.start_time >= %s
+                AND builds.start_time <= %s
+                AND checkouts.git_commit_hash IN ({0}) {1}
+            GROUP BY checkouts.id, builds.status, tests.environment_compatible, compiler_arch,
+                builds.config_name, lab, platform, is_boot)
+            UNION ALL
+            (SELECT
+                 COUNT(*) AS tests_count,
+                 checkouts.origin,
+                 tests.status AS status,
+                 count(incidents.id) AS known_issues,
+                 array[builds.compiler, builds.architecture] AS compiler_arch,
+                 builds.config_name,
+                 tests.misc->>'runtime' AS lab,
+                 tests.environment_misc->>'platform' AS platform,
+                 tests.environment_compatible,
+                 checkouts.origin,
+                 checkouts.tree_name,
+                 checkouts.git_repository_url,
+                 checkouts.git_commit_tags,
+                 checkouts.git_commit_name,
+                 checkouts.git_repository_branch,
+                 checkouts.git_commit_hash,
+                 false AS is_build,
+                 true AS is_test,
+                 (tests.path like 'boot.%%' or tests.path = 'boot') AS is_boot
+             FROM
+                builds
+            inner JOIN tests ON
+                tests.build_id = builds.id
+            INNER JOIN checkouts ON
+                builds.checkout_id = checkouts.id
+            LEFT OUTER JOIN incidents ON
+                tests.id = incidents.test_id
+            WHERE
+                (
+                    (tests.environment_compatible @> ARRAY[%s]::TEXT[]
+                    OR tests.environment_misc ->> 'platform' = %s)
+                )
+                AND tests.origin = %s
+                AND tests.start_time >= %s
+                AND tests.start_time <= %s
+                AND checkouts.git_commit_hash IN ({0}) {2}
+            GROUP BY checkouts.id, tests.status, tests.environment_compatible, compiler_arch,
+                builds.config_name, lab, platform, is_boot);
+    """.format(
+        ",".join(repeat("%s", len(commit_hashes))),
+        builds_duration_clause,
+        boots_tests_duration_clause,
+    )
+
+    params = [
+        hardware_id,
+        hardware_id,
+        origin,
+        start_datetime,
+        end_datetime,
+        *commit_hashes,
+    ]
+
+    # TODO: check if we can reuse parameters to avoid double passing
+    params = [*params, *params]
+
+    with connection.cursor() as cursor:
+        cursor.execute(query, params)
+        query_rows = dict_fetchall(cursor)
+        set_query_cache(key=cache_key, params=tests_cache_params, rows=query_rows)
+        return query_rows
+
+
 def query_records(
     *, hardware_id: str, origin: str, trees: list[Tree], start_date: int, end_date: int
 ) -> list[dict] | None:
     commit_hashes = [tree.head_git_commit_hash for tree in trees]
 
-    # TODO Treat commit_hash collision (it can happen between repos)
-    with connection.cursor() as cursor:
-        cursor.execute(
-            """
+    query = """
             SELECT
                 tests.id,
                 tests.origin AS test_origin,
@@ -411,19 +647,22 @@ def query_records(
             ORDER BY
                 issues."_timestamp" DESC
             """.format(
-                ",".join(["%s"] * len(commit_hashes))
-            ),
-            [
-                hardware_id,
-                hardware_id,
-                origin,
-                start_date,
-                end_date,
-            ]
-            + commit_hashes,
-        )
+        ",".join(["%s"] * len(commit_hashes))
+    )
 
-        return dict_fetchall(cursor)
+    params = [
+        hardware_id,
+        hardware_id,
+        origin,
+        start_date,
+        end_date,
+    ] + commit_hashes
+
+    # TODO Treat commit_hash collision (it can happen between repos)
+    with connection.cursor() as cursor:
+        cursor.execute(query, params)
+        query_rows = dict_fetchall(cursor)
+        return query_rows
 
 
 def get_hardware_summary_data(