Skip to content

Commit db4e585

Browse files
author
ci bot
committed
Merge branch 'aarthy/qa' into 'enterprise'
fix(test-results): error on refresh score See merge request dkinternal/testgen/dataops-testgen!349
2 parents 0840123 + 9b54583 commit db4e585

14 files changed

Lines changed: 52 additions & 41 deletions

File tree

‎testgen/commands/queries/rollup_scores_query.py‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,16 @@ def rollup_profiling_scores(self) -> list[tuple[str, dict]]:
3535
queries.append(self._get_query("rollup_scores_profile_table_group.sql"))
3636
return queries
3737

38-
def rollup_test_scores(self) -> list[tuple[str, dict]]:
38+
def rollup_test_scores(self, update_prevalence: bool = False, update_table_group: bool = False) -> list[tuple[str, dict]]:
3939
# Runs on App database
40-
return [
41-
self._get_query("calc_prevalence_test_results.sql", no_bind=True),
42-
self._get_query("rollup_scores_test_run.sql"),
43-
self._get_query("rollup_scores_test_table_group.sql"),
44-
]
40+
queries = []
41+
42+
if update_prevalence:
43+
queries.append(self._get_query("calc_prevalence_test_results.sql", no_bind=True))
44+
45+
queries.append(self._get_query("rollup_scores_test_run.sql"))
46+
47+
if update_table_group:
48+
queries.append(self._get_query("rollup_scores_test_table_group.sql"))
49+
50+
return queries

‎testgen/commands/run_rollup_scores.py‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,14 @@
88

99

1010
def run_profile_rollup_scoring_queries(project_code: str, run_id: str, table_group_id: str | None = None):
11-
LOG.info("CurrentStep: Initializing Profiling Scores Rollup")
1211
sql_generator = RollupScoresSQL(run_id, table_group_id)
13-
14-
LOG.info("CurrentStep: Rolling up profiling scores")
1512
execute_db_queries(sql_generator.rollup_profiling_scores())
1613
run_refresh_score_cards_results(project_code=project_code)
1714

1815

1916
def run_test_rollup_scoring_queries(project_code: str, run_id: str, table_group_id: str | None = None):
20-
LOG.info("CurrentStep: Initializing Testing Scores Rollup")
2117
sql_generator = RollupScoresSQL(run_id, table_group_id)
22-
23-
queries = [sql_generator.GetRollupScoresTestRunQuery()]
24-
if table_group_id:
25-
queries.append(sql_generator.GetRollupScoresTestTableGroupQuery())
26-
27-
LOG.info("CurrentStep: Rolling up testing scores")
28-
execute_db_queries(queries)
18+
execute_db_queries(
19+
sql_generator.rollup_test_scores(update_table_group=table_group_id is not None)
20+
)
2921
run_refresh_score_cards_results(project_code=project_code)

‎testgen/commands/run_test_execution.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ def run_test_execution(test_suite_id: str | UUID, username: str | None = None, r
9595
test_run.set_progress(
9696
"validation",
9797
"Warning" if invalid_count else "Completed",
98-
error=f"{invalid_count} test{'s' if invalid_count > 1 else ''} had errors" if invalid_count else None,
98+
error=f"{invalid_count} test{'s' if invalid_count > 1 else ''} had errors. See details in results." if invalid_count else None,
9999
)
100100

101101
if valid_test_defs:
@@ -118,6 +118,7 @@ def run_test_execution(test_suite_id: str | UUID, username: str | None = None, r
118118
if (run_test_defs := [td for td in valid_test_defs if td.run_type == run_type]):
119119
run_functions[run_type](run_test_defs)
120120
else:
121+
test_run.set_progress(run_type, "Completed")
121122
LOG.info(f"No {run_type} tests to run")
122123
else:
123124
LOG.info("No valid tests to run")
@@ -174,7 +175,7 @@ def update_test_progress(progress: ThreadedProgress) -> None:
174175
run_type,
175176
"Running",
176177
detail=f"{progress['processed']} of {progress['total']}",
177-
error=f"{progress['errors']} test{'s' if progress['errors'] > 1 else ''} had errors"
178+
error=f"{progress['errors']} test{'s' if progress['errors'] > 1 else ''} had errors. See details in results."
178179
if progress["errors"]
179180
else None,
180181
)
@@ -299,7 +300,7 @@ def update_single_progress(progress: ThreadedProgress) -> None:
299300
test_run.set_progress(
300301
"CAT",
301302
"Warning" if error_count else "Completed",
302-
error=f"{error_count} test{'s' if error_count > 1 else ''} had errors"
303+
error=f"{error_count} test{'s' if error_count > 1 else ''} had errors. See details in results."
303304
if error_count
304305
else None,
305306
)
@@ -308,8 +309,9 @@ def update_single_progress(progress: ThreadedProgress) -> None:
308309
def _rollup_test_scores(test_run: TestRun, table_group: TableGroup) -> None:
309310
try:
310311
LOG.info("Rolling up test scores")
312+
sql_generator = RollupScoresSQL(test_run.id, table_group.id)
311313
execute_db_queries(
312-
RollupScoresSQL(test_run.id, table_group.id).rollup_test_scores(),
314+
sql_generator.rollup_test_scores(update_prevalence=True, update_table_group=True),
313315
)
314316
run_refresh_score_cards_results(
315317
project_code=table_group.project_code,

‎testgen/common/date_service.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,3 @@ def get_timezoned_timestamp(streamlit_session, value, dateformat="%b %-d, %-I:%M
3838
df["value"] = df["value"].dt.tz_localize("UTC").dt.tz_convert(timezone).dt.strftime(dateformat)
3939
ret = df.iloc[0, 0]
4040
return ret
41-
42-
43-
def get_timezoned_now(streamlit_session):
44-
value = datetime.now(UTC)
45-
return get_timezoned_timestamp(streamlit_session, value)

‎testgen/common/models/profiling_run.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,8 +244,8 @@ def cancel_all_running(cls) -> None:
244244
cls.clear_cache()
245245

246246
@classmethod
247-
def update_status(cls, run_id: str | UUID, status: ProfilingRunStatus) -> None:
248-
query = update(cls).where(cls.id == run_id).values(status=status)
247+
def cancel_run(cls, run_id: str | UUID) -> None:
248+
query = update(cls).where(cls.id == run_id).values(status="Cancelled", profiling_endtime=datetime.now(UTC))
249249
db_session = get_current_session()
250250
db_session.execute(query)
251251
db_session.commit()

‎testgen/common/models/test_run.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -245,8 +245,8 @@ def cancel_all_running(cls) -> None:
245245
cls.clear_cache()
246246

247247
@classmethod
248-
def update_status(cls, run_id: str | UUID, status: TestRunStatus) -> None:
249-
query = update(cls).where(cls.id == run_id).values(status=status)
248+
def cancel_run(cls, run_id: str | UUID) -> None:
249+
query = update(cls).where(cls.id == run_id).values(status="Cancelled", test_endtime=datetime.now(UTC))
250250
db_session = get_current_session()
251251
db_session.execute(query)
252252
db_session.commit()

‎testgen/ui/components/frontend/js/pages/profiling_runs.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,7 @@ const ProfilingRunItem = (
314314
)
315315
: div(
316316
{ class: 'text-caption mt-1' },
317-
runningStep
317+
item.status === 'Running' && runningStep
318318
? [
319319
div(
320320
runningStep.label,
@@ -352,7 +352,7 @@ const ProfilingRunItem = (
352352
)
353353
: null,
354354
),
355-
item.status !== 'Running' && item.column_ct ? Link({
355+
item.status === 'Complete' && item.column_ct ? Link({
356356
label: 'View results',
357357
href: 'profiling-runs:results',
358358
params: { 'run_id': item.id },

‎testgen/ui/components/frontend/js/pages/quality_dashboard.js‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,23 @@ const QualityDashboard = (/** @type {Properties} */ props) => {
3838

3939
const sortedBy = van.state('name');
4040
const filterTerm = van.state('');
41+
42+
const scoreToNumber = (score) => score ? (score.startsWith('>') ? 99.99 : Number(score)) : 101;
43+
const sortFunctions = {
44+
name: (a, b) => caseInsensitiveSort(a.name, b.name),
45+
score: (a, b) => {
46+
const scoreA = Math.min(scoreToNumber(a.score), scoreToNumber(a.cde_score));
47+
const scoreB = Math.min(scoreToNumber(b.score), scoreToNumber(b.cde_score));
48+
return scoreA - scoreB;
49+
},
50+
};
51+
4152
const scores = van.derive(() => {
4253
const sort = getValue(sortedBy) ?? 'name';
4354
const filter = getValue(filterTerm) ?? '';
4455
return getValue(props.scores)
4556
.filter(score => caseInsensitiveIncludes(score.name, filter))
46-
.sort((a, b) => caseInsensitiveSort(a[sort], b[sort]));
57+
.sort(sortFunctions[sort]);
4758
});
4859

4960
return div(

‎testgen/ui/components/frontend/js/pages/test_runs.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@ const TestRunItem = (
325325
)
326326
: div(
327327
{ class: 'text-caption mt-1' },
328-
runningStep
328+
item.status === 'Running' && runningStep
329329
? [
330330
div(
331331
runningStep.label,

‎testgen/ui/components/widgets/download_dialog.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import tempfile
22
from collections.abc import Callable, Iterable
3+
from datetime import datetime
34
from io import BytesIO
45
from typing import TypedDict
56
from zipfile import ZipFile
@@ -54,7 +55,11 @@ def get_excel_file_data(
5455

5556
# Timestamp
5657
worksheet.write("A3", "Exported on", details_key_format)
57-
worksheet.write("B3", date_service.get_timezoned_now(st.session_state), details_value_format)
58+
worksheet.write(
59+
"B3",
60+
date_service.get_timezoned_timestamp(st.session_state, datetime.utcnow()),
61+
details_value_format,
62+
)
5863

5964
# Details
6065
if details:

0 commit comments

Comments
 (0)