Skip to content

Commit 93f201f

Browse files
alanpeixinhoalanpeixinho
committed
Semi aggregated query for hardware details summary endpoint
1 parent 3bcaf9a commit 93f201f

6 files changed

Lines changed: 567 additions & 281 deletions

File tree

‎backend/kernelCI_app/helpers/filters.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,12 @@ def is_issue_filtered_out(
129129
return not in_filter
130130

131131

132+
def is_filtered_out(value: str, filter_values: set[set]):
133+
if filter_values and value not in filter_values:
134+
return True
135+
return False
136+
137+
132138
def should_filter_test_issue(
133139
*,
134140
issue_filters: set,
@@ -391,6 +397,14 @@ def __init__(self, data: Dict, process_body=False) -> None:
391397

392398
self._process_filters()
393399

400+
def __repr__(self) -> str:
401+
parts = ""
402+
for parsed_filter in self.filters:
403+
parts += "\n\t{},".format(
404+
", ".join([f"{key}={val}" for key, val in parsed_filter.items()])
405+
)
406+
return f"FilterParams({parts})"
407+
394408
def _handle_boot_status(self, current_filter: ParsedFilter) -> None:
395409
self.filterBootStatus.add(current_filter["value"])
396410

‎backend/kernelCI_app/helpers/hardwareDetails.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -509,8 +509,8 @@ def process_issue(
509509
incident_test_id=record["incidents__test_id"],
510510
build_status=record["build__status"],
511511
test_status=record["status"],
512-
issue_comment=record["incidents__issue__comment"],
513-
issue_report_url=record["incidents__issue__report_url"],
512+
issue_comment=record.get("incidents__issue__comment"),
513+
issue_report_url=record.get("incidents__issue__report_url"),
514514
is_failed_task=is_failed_task,
515515
issue_from=issue_from,
516516
task=task_issues_dict,

‎backend/kernelCI_app/queries/hardware.py‎

Lines changed: 150 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from typing import TypedDict
22
from datetime import datetime
33
from django.db import connection
4+
from itertools import repeat
45

56
from kernelCI_app.helpers.database import dict_fetchall
67
from kernelCI_app.cache import get_query_cache, set_query_cache
@@ -339,15 +340,145 @@ def get_hardware_details_data(
339340
return records
340341

341342

343+
def get_hardware_details_summary(
344+
*,
345+
hardware_id: str,
346+
origin: str,
347+
trees_with_selected_commits: list[Tree],
348+
start_datetime: datetime,
349+
end_datetime: datetime,
350+
):
351+
352+
cache_key = "hardwareDetailsSummary"
353+
354+
tests_cache_params = {
355+
"hardware_id": hardware_id,
356+
"origin": origin,
357+
"trees": trees_with_selected_commits,
358+
"start_date": start_datetime,
359+
"end_date": end_datetime,
360+
}
361+
362+
query_rows = get_query_cache(cache_key, tests_cache_params)
363+
364+
if query_rows:
365+
return query_rows
366+
367+
commit_hashes = [tree.head_git_commit_hash for tree in trees_with_selected_commits]
368+
369+
query = """
370+
(SELECT
371+
COUNT(distinct builds.id) AS count,
372+
checkouts.origin,
373+
builds.status AS status,
374+
count(incidents.id) AS known_issues,
375+
array[builds.compiler, builds.architecture] AS compiler_arch,
376+
builds.config_name,
377+
builds.misc->>'runtime' AS lab,
378+
tests.environment_misc->>'platform' AS platform,
379+
tests.environment_compatible,
380+
checkouts.origin,
381+
checkouts.tree_name,
382+
checkouts.git_repository_url,
383+
checkouts.git_commit_tags,
384+
checkouts.git_commit_name,
385+
checkouts.git_repository_branch,
386+
checkouts.git_commit_hash,
387+
true AS is_build,
388+
false AS is_test,
389+
false AS is_boot
390+
FROM
391+
builds
392+
INNER JOIN tests ON
393+
tests.build_id = builds.id
394+
INNER JOIN checkouts ON
395+
builds.checkout_id = checkouts.id
396+
LEFT OUTER JOIN incidents ON
397+
builds.id = incidents.build_id
398+
WHERE
399+
(
400+
builds.config_name IS NOT NULL
401+
AND builds.id not like 'maestro:dummy_%%'
402+
AND (tests.environment_compatible @> ARRAY[%s]::TEXT[]
403+
OR tests.environment_misc ->> 'platform' = %s)
404+
)
405+
AND builds.origin = %s
406+
AND builds.start_time >= %s
407+
AND builds.start_time <= %s
408+
AND checkouts.git_commit_hash IN ({0})
409+
GROUP BY checkouts.id, builds.status, tests.environment_compatible, compiler_arch,
410+
builds.config_name, lab, platform, is_boot)
411+
UNION ALL
412+
(SELECT
413+
COUNT(*) AS tests_count,
414+
checkouts.origin,
415+
tests.status AS status,
416+
count(incidents.id) AS known_issues,
417+
array[builds.compiler, builds.architecture] AS compiler_arch,
418+
builds.config_name,
419+
tests.misc->>'runtime' AS lab,
420+
tests.environment_misc->>'platform' AS platform,
421+
tests.environment_compatible,
422+
checkouts.origin,
423+
checkouts.tree_name,
424+
checkouts.git_repository_url,
425+
checkouts.git_commit_tags,
426+
checkouts.git_commit_name,
427+
checkouts.git_repository_branch,
428+
checkouts.git_commit_hash,
429+
false AS is_build,
430+
true AS is_test,
431+
(tests.path like 'boot.%%' or tests.path = 'boot') AS is_boot
432+
FROM
433+
builds
434+
inner JOIN tests ON
435+
tests.build_id = builds.id
436+
INNER JOIN checkouts ON
437+
builds.checkout_id = checkouts.id
438+
LEFT OUTER JOIN incidents ON
439+
tests.id = incidents.test_id
440+
WHERE
441+
(
442+
builds.config_name IS NOT NULL
443+
AND builds.id not like 'maestro:dummy_%%'
444+
AND (tests.environment_compatible @> ARRAY[%s]::TEXT[]
445+
OR tests.environment_misc ->> 'platform' = %s)
446+
)
447+
AND tests.origin = %s
448+
AND tests.start_time >= %s
449+
AND tests.start_time <= %s
450+
AND checkouts.git_commit_hash IN ({0})
451+
GROUP BY checkouts.id, tests.status, tests.environment_compatible, compiler_arch,
452+
builds.config_name, lab, platform, is_boot);
453+
""".format(
454+
",".join(repeat("%s", len(commit_hashes)))
455+
)
456+
457+
params = [
458+
hardware_id,
459+
hardware_id,
460+
origin,
461+
start_datetime,
462+
end_datetime,
463+
*commit_hashes,
464+
]
465+
466+
# TODO: check if we can reuse parameters to avoid double passing
467+
params = [*params, *params]
468+
469+
with connection.cursor() as cursor:
470+
cursor.execute(query, params)
471+
query_rows = dict_fetchall(cursor)
472+
set_query_cache(key=cache_key, params=tests_cache_params, rows=query_rows)
473+
return query_rows
474+
475+
342476
def query_records(
343477
*, hardware_id: str, origin: str, trees: list[Tree], start_date: int, end_date: int
344478
) -> list[dict] | None:
345479
commit_hashes = [tree.head_git_commit_hash for tree in trees]
346480

347-
# TODO Treat commit_hash collision (it can happen between repos)
348-
with connection.cursor() as cursor:
349-
cursor.execute(
350-
"""
481+
query = """
351482
SELECT
352483
tests.id,
353484
tests.origin AS test_origin,
@@ -411,19 +542,22 @@ def query_records(
411542
ORDER BY
412543
issues."_timestamp" DESC
413544
""".format(
414-
",".join(["%s"] * len(commit_hashes))
415-
),
416-
[
417-
hardware_id,
418-
hardware_id,
419-
origin,
420-
start_date,
421-
end_date,
422-
]
423-
+ commit_hashes,
424-
)
545+
",".join(["%s"] * len(commit_hashes))
546+
)
425547

426-
return dict_fetchall(cursor)
548+
params = [
549+
hardware_id,
550+
hardware_id,
551+
origin,
552+
start_date,
553+
end_date,
554+
] + commit_hashes
555+
556+
# TODO Treat commit_hash collision (it can happen between repos)
557+
with connection.cursor() as cursor:
558+
cursor.execute(query, params)
559+
query_rows = dict_fetchall(cursor)
560+
return query_rows
427561

428562

429563
def get_hardware_summary_data(

‎backend/kernelCI_app/typeModels/common.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@ class StatusCount(BaseModel):
1414
DONE: Optional[int] = 0
1515
NULL: Optional[int] = 0
1616

17-
def increment(self, status: Optional[str]) -> None:
17+
def increment(self, status: Optional[str], value=1) -> None:
1818
if status is None:
1919
status = "NULL"
2020

2121
try:
22-
setattr(self, status.upper(), getattr(self, status.upper()) + 1)
22+
setattr(self, status.upper(), getattr(self, status.upper()) + value)
2323
except AttributeError:
2424
log_message(f"Unknown status: {status}")
2525

‎backend/kernelCI_app/typeModels/commonDetails.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,28 @@ class TestArchSummaryItem(BaseModel):
3939
class BuildArchitectures(StatusCount):
4040
compilers: Optional[list[str]] = []
4141

42+
def __iadd__(self, other: StatusCount) -> "BuildArchitectures":
43+
self.PASS += other.PASS
44+
self.ERROR += other.ERROR
45+
self.FAIL += other.FAIL
46+
self.SKIP += other.SKIP
47+
self.MISS += other.MISS
48+
self.DONE += other.DONE
49+
self.NULL += other.NULL
50+
return self
51+
52+
def __add__(self, other: StatusCount) -> "BuildArchitectures":
53+
return BuildArchitectures(
54+
PASS=self.PASS + other.PASS,
55+
ERROR=self.ERROR + other.ERROR,
56+
FAIL=self.FAIL + other.FAIL,
57+
SKIP=self.SKIP + other.SKIP,
58+
MISS=self.MISS + other.MISS,
59+
DONE=self.DONE + other.DONE,
60+
NULL=self.NULL + other.NULL,
61+
compilers=self.compilers,
62+
)
63+
4264

4365
class TestHistoryItem(BaseModel):
4466
id: str

0 commit comments

Comments
 (0)