Skip to content

Commit 9c304aa

Browse files
alanpeixinhoalanpeixinho
committed
Semi aggregated query for hardware details summary endpoint
1 parent 1a4fbe5 commit 9c304aa

6 files changed

Lines changed: 486 additions & 284 deletions

File tree

‎backend/kernelCI_app/helpers/hardwareDetails.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -496,8 +496,8 @@ def process_issue(
496496
incident_test_id=record["incidents__test_id"],
497497
build_status=record["build__status"],
498498
test_status=record["status"],
499-
issue_comment=record["incidents__issue__comment"],
500-
issue_report_url=record["incidents__issue__report_url"],
499+
issue_comment=record.get("incidents__issue__comment"),
500+
issue_report_url=record.get("incidents__issue__report_url"),
501501
is_failed_task=is_failed_task,
502502
issue_from=issue_from,
503503
task=task_issues_dict,

‎backend/kernelCI_app/queries/hardware.py‎

Lines changed: 134 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from typing import TypedDict
22
from datetime import datetime
33
from django.db import connection
4+
from itertools import repeat
45

56
from kernelCI_app.helpers.database import dict_fetchall
67
from kernelCI_app.cache import get_query_cache, set_query_cache
@@ -339,15 +340,129 @@ def get_hardware_details_data(
339340
return records
340341

341342

343+
def get_hardware_details_summary(
344+
*,
345+
hardware_id: str,
346+
origin: str,
347+
trees_with_selected_commits: list[Tree],
348+
start_datetime: datetime,
349+
end_datetime: datetime,
350+
):
351+
352+
commit_hashes = [tree.head_git_commit_hash for tree in trees_with_selected_commits]
353+
354+
query = """
355+
(SELECT
356+
COUNT(distinct builds.id) AS count,
357+
checkouts.origin,
358+
builds.status AS status,
359+
count(incidents.id) AS known_issues,
360+
array[builds.compiler, builds.architecture] AS compiler_arch,
361+
builds.config_name,
362+
builds.misc->>'runtime' AS lab,
363+
tests.environment_misc->>'platform' AS platform,
364+
tests.environment_compatible,
365+
checkouts.origin,
366+
checkouts.tree_name,
367+
checkouts.git_repository_url,
368+
checkouts.git_commit_tags,
369+
checkouts.git_commit_name,
370+
checkouts.git_repository_branch,
371+
checkouts.git_commit_hash,
372+
true AS is_build,
373+
false AS is_test,
374+
false AS is_boot
375+
FROM
376+
builds
377+
INNER JOIN tests ON
378+
tests.build_id = builds.id
379+
INNER JOIN checkouts ON
380+
builds.checkout_id = checkouts.id
381+
LEFT OUTER JOIN incidents ON
382+
builds.id = incidents.build_id
383+
WHERE
384+
(
385+
builds.config_name IS NOT NULL
386+
AND builds.id not like 'maestro:dummy_%%'
387+
AND (tests.environment_compatible @> ARRAY[%s]::TEXT[]
388+
OR tests.environment_misc ->> 'platform' = %s)
389+
)
390+
AND builds.origin = %s
391+
AND builds.start_time >= %s
392+
AND builds.start_time <= %s
393+
AND checkouts.git_commit_hash IN ({0})
394+
GROUP BY checkouts.id, builds.status, tests.environment_compatible, compiler_arch,
395+
builds.config_name, lab, platform, is_boot)
396+
UNION ALL
397+
(SELECT
398+
COUNT(*) AS tests_count,
399+
checkouts.origin,
400+
tests.status AS status,
401+
count(incidents.id) AS known_issues,
402+
array[builds.compiler, builds.architecture] AS compiler_arch,
403+
builds.config_name,
404+
tests.misc->>'runtime' AS lab,
405+
tests.environment_misc->>'platform' AS platform,
406+
tests.environment_compatible,
407+
checkouts.origin,
408+
checkouts.tree_name,
409+
checkouts.git_repository_url,
410+
checkouts.git_commit_tags,
411+
checkouts.git_commit_name,
412+
checkouts.git_repository_branch,
413+
checkouts.git_commit_hash,
414+
false AS is_build,
415+
true AS is_test,
416+
(tests.path like 'boot.%%' or tests.path = 'boot') AS is_boot
417+
FROM
418+
builds
419+
inner JOIN tests ON
420+
tests.build_id = builds.id
421+
INNER JOIN checkouts ON
422+
builds.checkout_id = checkouts.id
423+
LEFT OUTER JOIN incidents ON
424+
tests.id = incidents.test_id
425+
WHERE
426+
(
427+
builds.config_name IS NOT NULL
428+
AND builds.id not like 'maestro:dummy_%%'
429+
AND (tests.environment_compatible @> ARRAY[%s]::TEXT[]
430+
OR tests.environment_misc ->> 'platform' = %s)
431+
)
432+
AND tests.origin = %s
433+
AND tests.start_time >= %s
434+
AND tests.start_time <= %s
435+
AND checkouts.git_commit_hash IN ({0})
436+
GROUP BY checkouts.id, tests.status, tests.environment_compatible, compiler_arch,
437+
builds.config_name, lab, platform, is_boot);
438+
""".format(
439+
",".join(repeat("%s", len(commit_hashes)))
440+
)
441+
442+
params = [
443+
hardware_id,
444+
hardware_id,
445+
origin,
446+
start_datetime,
447+
end_datetime,
448+
*commit_hashes,
449+
]
450+
451+
# TODO: check if we can reuse parameters to avoid double passing
452+
params = [*params, *params]
453+
454+
with connection.cursor() as cursor:
455+
cursor.execute(query, params)
456+
query_rows = dict_fetchall(cursor)
457+
return query_rows
458+
459+
342460
def query_records(
343461
*, hardware_id: str, origin: str, trees: list[Tree], start_date: int, end_date: int
344462
) -> list[dict] | None:
345463
commit_hashes = [tree.head_git_commit_hash for tree in trees]
346464

347-
# TODO Treat commit_hash collision (it can happen between repos)
348-
with connection.cursor() as cursor:
349-
cursor.execute(
350-
"""
465+
query = """
351466
SELECT
352467
tests.id,
353468
tests.origin AS test_origin,
@@ -411,19 +526,22 @@ def query_records(
411526
ORDER BY
412527
issues."_timestamp" DESC
413528
""".format(
414-
",".join(["%s"] * len(commit_hashes))
415-
),
416-
[
417-
hardware_id,
418-
hardware_id,
419-
origin,
420-
start_date,
421-
end_date,
422-
]
423-
+ commit_hashes,
424-
)
529+
",".join(["%s"] * len(commit_hashes))
530+
)
425531

426-
return dict_fetchall(cursor)
532+
params = [
533+
hardware_id,
534+
hardware_id,
535+
origin,
536+
start_date,
537+
end_date,
538+
] + commit_hashes
539+
540+
# TODO Treat commit_hash collision (it can happen between repos)
541+
with connection.cursor() as cursor:
542+
cursor.execute(query, params)
543+
query_rows = dict_fetchall(cursor)
544+
return query_rows
427545

428546

429547
def get_hardware_summary_data(

‎backend/kernelCI_app/typeModels/common.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@ class StatusCount(BaseModel):
1414
DONE: Optional[int] = 0
1515
NULL: Optional[int] = 0
1616

17-
def increment(self, status: Optional[str]) -> None:
17+
def increment(self, status: Optional[str], value=1) -> None:
1818
if status is None:
1919
status = "NULL"
2020

2121
try:
22-
setattr(self, status.upper(), getattr(self, status.upper()) + 1)
22+
setattr(self, status.upper(), getattr(self, status.upper()) + value)
2323
except AttributeError:
2424
log_message(f"Unknown status: {status}")
2525

‎backend/kernelCI_app/typeModels/commonDetails.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,28 @@ class TestArchSummaryItem(BaseModel):
3939
class BuildArchitectures(StatusCount):
4040
compilers: Optional[list[str]] = []
4141

42+
def __iadd__(self, other: StatusCount) -> "BuildArchitectures":
43+
self.PASS += other.PASS
44+
self.ERROR += other.ERROR
45+
self.FAIL += other.FAIL
46+
self.SKIP += other.SKIP
47+
self.MISS += other.MISS
48+
self.DONE += other.DONE
49+
self.NULL += other.NULL
50+
return self
51+
52+
def __add__(self, other: StatusCount) -> "BuildArchitectures":
53+
return BuildArchitectures(
54+
PASS=self.PASS + other.PASS,
55+
ERROR=self.ERROR + other.ERROR,
56+
FAIL=self.FAIL + other.FAIL,
57+
SKIP=self.SKIP + other.SKIP,
58+
MISS=self.MISS + other.MISS,
59+
DONE=self.DONE + other.DONE,
60+
NULL=self.NULL + other.NULL,
61+
compilers=self.compilers,
62+
)
63+
4264

4365
class TestHistoryItem(BaseModel):
4466
id: str

0 commit comments

Comments
 (0)