From 45cc9c892dc8d9ed0d48db10ca59dcca04ab12d2 Mon Sep 17 00:00:00 2001 From: saagpatel Date: Sat, 11 Jul 2026 03:05:47 -0700 Subject: [PATCH 1/2] feat(scoring): disclose partial-run basis --- src/excel_all_repos_helpers.py | 6 +++- src/excel_dashboard_helpers.py | 2 +- src/excel_styles.py | 5 +-- src/models.py | 4 +++ src/report_state.py | 2 ++ src/scorer.py | 12 ++++++- src/web_export.py | 9 +++-- tests/test_cli_hardening.py | 61 +++++++++++++++++++++++++++++++++- tests/test_excel_enhanced.py | 32 ++++++++++++++++++ tests/test_scoring_profiles.py | 21 ++++++++++++ tests/test_web_export.py | 16 +++++++++ 11 files changed, 161 insertions(+), 9 deletions(-) diff --git a/src/excel_all_repos_helpers.py b/src/excel_all_repos_helpers.py index 2e57917..1455fae 100644 --- a/src/excel_all_repos_helpers.py +++ b/src/excel_all_repos_helpers.py @@ -52,6 +52,8 @@ "Size (KB)", "Trend", "Risk Tier", + "Scored Dimensions", + "Scored Weight Sum", ] ALL_REPOS_LONG_TEXT_HEADERS = ("Description", "Topics", "Badges") @@ -60,7 +62,7 @@ ALL_REPOS_INTEREST_COLUMN = 4 ALL_REPOS_TIER_COLUMN = 7 ALL_REPOS_PATTERN_COLUMN = 12 -ALL_REPOS_TREND_COLUMN_OFFSET = 1 +ALL_REPOS_TREND_COLUMN_OFFSET = 3 ALL_REPOS_LONG_TEXT_WIDTH = 60 @@ -144,6 +146,8 @@ def build_all_repo_rows( metadata.get("size_kb", 0), render_sparkline(trend_scores), (risk_lookup or {}).get(str(repo_name), ""), + ", ".join(audit.get("scored_dimensions", [])), + round(audit.get("scored_weight_sum", 0), 3), ], } ) diff --git a/src/excel_dashboard_helpers.py b/src/excel_dashboard_helpers.py index 53cb3c7..b5b58da 100644 --- a/src/excel_dashboard_helpers.py +++ b/src/excel_dashboard_helpers.py @@ -774,7 +774,7 @@ def _write_dashboard_distribution_charts( grade_dist = {} for audit in audits: - grade = audit.get("grade", "F") + grade = audit.get("grade", "F").split(" ", 1)[0] grade_dist[grade] = grade_dist.get(grade, 0) + 1 grade_label_col = 27 grade_value_col = 28 diff --git a/src/excel_styles.py b/src/excel_styles.py index ea4ca43..0384f06 100644 --- a/src/excel_styles.py +++ b/src/excel_styles.py @@ -270,8 +270,9 @@ def write_kpi_card( def color_grade_cell(cell, grade: str) -> None: """Apply grade-specific coloring to a cell.""" - if grade in GRADE_FILLS: - cell.fill = GRADE_FILLS[grade] + grade_letter = grade.split(" ", 1)[0] + if grade_letter in GRADE_FILLS: + cell.fill = GRADE_FILLS[grade_letter] cell.font = Font("Calibri", 10, bold=True, color=WHITE) diff --git a/src/models.py b/src/models.py index 762f499..46b81f2 100644 --- a/src/models.py +++ b/src/models.py @@ -97,6 +97,8 @@ class RepoAudit: interest_score: float = 0.0 interest_tier: str = "mundane" grade: str = "F" + scored_dimensions: list[str] = field(default_factory=list) + scored_weight_sum: float = 0.0 interest_grade: str = "F" badges: list[str] = field(default_factory=list) next_badges: list[dict] = field(default_factory=list) @@ -122,6 +124,8 @@ def to_dict(self) -> dict: "completeness_tier": self.completeness_tier, "interest_tier": self.interest_tier, "grade": self.grade, + "scored_dimensions": self.scored_dimensions, + "scored_weight_sum": round(self.scored_weight_sum, 3), "interest_grade": self.interest_grade, "badges": self.badges, "next_badges": self.next_badges, diff --git a/src/report_state.py b/src/report_state.py index 9757c80..fbb43e9 100644 --- a/src/report_state.py +++ b/src/report_state.py @@ -57,6 +57,8 @@ def audit_from_dict(data: dict) -> RepoAudit: interest_score=data.get("interest_score", 0), interest_tier=data.get("interest_tier", "mundane"), grade=data.get("grade", "F"), interest_grade=data.get("interest_grade", "F"), + scored_dimensions=data.get("scored_dimensions", []), + scored_weight_sum=data.get("scored_weight_sum", 0), badges=data.get("badges", []), next_badges=data.get("next_badges", []), flags=data.get("flags", []), lenses=data.get("lenses", {}), hotspots=data.get("hotspots", []), action_candidates=data.get("action_candidates", []), security_posture=data.get("security_posture", {}), diff --git a/src/scorer.py b/src/scorer.py index 78c5a64..c93f25c 100644 --- a/src/scorer.py +++ b/src/scorer.py @@ -49,6 +49,7 @@ STALE_THRESHOLD_DAYS = 730 # 2 years GRADE_THRESHOLDS = [(0.80, "A"), (0.70, "B"), (0.55, "C"), (0.35, "D"), (0.0, "F")] +PARTIAL_RUN_BASIS_THRESHOLD = 0.85 def letter_grade( score: float, grade_thresholds: list[tuple[float, str]] | None = None @@ -105,6 +106,8 @@ def score_repo( weight_sum += weight overall_score = weighted_sum / weight_sum if weight_sum > 0 else 0.0 + scored_dimensions = [dimension for dimension in weights if dimension in score_map] + full_weight_sum = sum(weights.values()) # Compute interest score (separate axis, from interest analyzer) interest_score = score_map.get("interest", 0.0) @@ -166,6 +169,11 @@ def score_repo( tier = "skeleton" flags.append("readme-only") + base_grade = letter_grade(overall_score, grade_thresholds) + grade = base_grade + if weight_sum < full_weight_sum * PARTIAL_RUN_BASIS_THRESHOLD: + grade = f"{base_grade} on {len(scored_dimensions)}/{len(weights)} dimensions" + audit = RepoAudit( metadata=metadata, analyzer_results=results, @@ -173,7 +181,9 @@ def score_repo( interest_score=interest_score, completeness_tier=tier, interest_tier=interest_tier, - grade=letter_grade(overall_score, grade_thresholds), + grade=grade, + scored_dimensions=scored_dimensions, + scored_weight_sum=weight_sum, interest_grade=letter_grade(interest_score, grade_thresholds), flags=flags, ) diff --git a/src/web_export.py b/src/web_export.py index e00c562..83bcfc9 100644 --- a/src/web_export.py +++ b/src/web_export.py @@ -180,6 +180,8 @@ def _render_html( { "name": entry["name"], "grade": a.get("grade", "F"), + "scored_dimensions": a.get("scored_dimensions", []), + "scored_weight_sum": a.get("scored_weight_sum", 0), "score": round(a.get("overall_score", 0), 3), "interest": round(a.get("interest_score", 0), 3), "tier": a.get("completeness_tier", ""), @@ -243,7 +245,7 @@ def _render_html( # ── KPI and header sections ─────────────────────────────────────────── def _header_section(username: str, date: str, repos: int, grade: str) -> str: - color = GRADE_COLORS_CSS.get(grade, "#6B7280") + color = GRADE_COLORS_CSS.get(grade.split(" ", 1)[0], "#6B7280") return f"""

Portfolio Dashboard: {escape(username)}

@@ -923,6 +925,7 @@ def _repo_table( explanation = a.get("score_explanation") or build_score_explanation(a) or {} name = m.get("name", "") grade = a.get("grade", "F") + grade_letter = grade.split(" ", 1)[0] score = a.get("overall_score", 0) interest = a.get("interest_score", 0) profile_score = entry["profile_score"] @@ -930,7 +933,7 @@ def _repo_table( lang = m.get("language") or "" desc = (m.get("description") or "")[:60] collections = ", ".join(entry["collections"]) - gc = GRADE_COLORS_CSS.get(grade, "#6B7280") + gc = GRADE_COLORS_CSS.get(grade_letter, "#6B7280") tc = TIER_COLORS_CSS.get(tier, "#6B7280") spark = "" @@ -974,7 +977,7 @@ def _repo_table( rows.append( f' Namespace: @@ -1738,3 +1739,61 @@ def _record_score_repo(*a, **kwargs): assert captured[0]["scorecard_enabled"] is True assert captured[0]["security_offline"] is True + + +def test_analyze_repos_forwards_real_scoring_profile_overrides(monkeypatch, sample_metadata): + args = _make_args() + captured: list[dict[str, object]] = [] + + class _CloneContext: + def __enter__(self): + return {sample_metadata.name: Path("/tmp/repo")} + + def __exit__(self, exc_type, exc, tb): + return False + + monkeypatch.setattr(run_audit, "clone_workspace", lambda *a, **k: _CloneContext()) + monkeypatch.setattr(run_audit, "create_progress", lambda: None) + monkeypatch.setattr(run_audit, "run_all_analyzers", lambda *a, **k: []) + + def _record_score_repo(*a, **kwargs): + captured.append(kwargs) + return RepoAudit( + metadata=sample_metadata, + analyzer_results=[], + overall_score=0.5, + completeness_tier="functional", + ) + + monkeypatch.setattr(run_audit, "score_repo", _record_score_repo) + profile = run_audit.ScoringProfile( + dict(WEIGHTS), overrides={"stale_threshold_days": 1} + ) + + run_audit._analyze_repos( + [sample_metadata], + args=args, + client=run_audit.GitHubClient(token=None, cache=None), + portfolio_lang_freq={}, + custom_weights=profile, + ) + + assert captured[0]["scoring_profile"] == profile.overrides + + +def test_partial_run_basis_survives_report_readback(sample_metadata): + audit = RepoAudit( + metadata=sample_metadata, + analyzer_results=[], + overall_score=0.72, + completeness_tier="functional", + grade="B on 7/10 dimensions", + scored_dimensions=["readme", "testing"], + scored_weight_sum=0.30, + ) + + restored = audit_from_dict(audit.to_dict()) + + assert restored.grade == "B on 7/10 dimensions" + assert restored.scored_dimensions == ["readme", "testing"] + assert restored.scored_weight_sum == 0.30 diff --git a/tests/test_excel_enhanced.py b/tests/test_excel_enhanced.py index 54738f4..3f8b0d5 100644 --- a/tests/test_excel_enhanced.py +++ b/tests/test_excel_enhanced.py @@ -891,6 +891,38 @@ def test_headers_include_interest_breakdown(self): for label in ("Tech Novelty", "Burst", "Ambition", "Storytelling"): assert label in header_values, f"Missing header: {label}" + def test_partial_run_basis_is_exported(self): + report = _make_report() + report["audits"][0].update( + { + "grade": "B on 7/10 dimensions", + "scored_dimensions": ["readme", "testing"], + "scored_weight_sum": 0.3, + } + ) + wb = Workbook() + _build_all_repos(wb, report) + ws = wb["All Repos"] + headers = {ws.cell(row=1, column=column).value: column for column in range(1, 50)} + repo_row = next(row for row in range(2, 10) if ws.cell(row=row, column=1).value == "RepoA") + + assert "Scored Dimensions" in headers + assert "Scored Weight Sum" in headers + assert ws.cell(row=repo_row, column=headers["Grade"]).value == "B on 7/10 dimensions" + assert ws.cell(row=repo_row, column=headers["Scored Dimensions"]).value == "readme, testing" + assert ws.cell(row=repo_row, column=headers["Scored Weight Sum"]).value == 0.3 + + def test_dashboard_grade_distribution_uses_qualified_grade_letter(self): + report = _make_report() + report["audits"][0]["grade"] = "B on 7/10 dimensions" + + wb = Workbook() + _build_dashboard(wb, report) + ws = wb["Dashboard"] + + assert ws.cell(row=11, column=27).value == "B" + assert ws.cell(row=11, column=28).value == 1 + def test_trend_is_last_column(self): ws = self._build() # Trend is dynamically the last header column (currently 36) diff --git a/tests/test_scoring_profiles.py b/tests/test_scoring_profiles.py index 4d7b4a8..1648440 100644 --- a/tests/test_scoring_profiles.py +++ b/tests/test_scoring_profiles.py @@ -69,6 +69,27 @@ def test_custom_weights_none_uses_default(self): audit_default = score_repo(_make_metadata(), results) assert audit_none.overall_score == audit_default.overall_score + def test_partial_run_grade_discloses_scored_basis(self): + scored_dimensions = list(WEIGHTS)[:6] + ["documentation"] + results = _make_results({dim: 0.72 for dim in scored_dimensions}) + + audit = score_repo(_make_metadata(), results) + + assert audit.grade == "B on 7/10 dimensions" + assert audit.scored_dimensions == scored_dimensions + assert audit.scored_weight_sum == 0.75 + assert audit.to_dict()["scored_dimensions"] == scored_dimensions + assert audit.to_dict()["scored_weight_sum"] == 0.75 + + def test_full_run_grade_remains_unqualified(self): + results = _make_results({dim: 0.72 for dim in WEIGHTS}) + + audit = score_repo(_make_metadata(), results) + + assert audit.grade == "B" + assert audit.scored_dimensions == list(WEIGHTS) + assert audit.scored_weight_sum == 1.0 + def test_profile_json_valid(self, tmp_path): """Verify profile JSONs sum to ~1.0.""" from pathlib import Path diff --git a/tests/test_web_export.py b/tests/test_web_export.py index 8a73ef6..51cc668 100644 --- a/tests/test_web_export.py +++ b/tests/test_web_export.py @@ -656,6 +656,22 @@ def test_repo_table_includes_next_action_explainability(self): html = _render_html(_make_report()) assert "Next:" in html or "Gap:" in html + def test_partial_run_grade_is_displayed_but_filterable_by_letter(self): + report = _make_report() + report["audits"][0].update( + { + "grade": "B on 7/10 dimensions", + "scored_dimensions": ["readme", "testing"], + "scored_weight_sum": 0.3, + } + ) + + html = _render_html(report) + + assert "B on 7/10 dimensions" in html + assert 'data-grade="B"' in html + assert 'data-grade="B on 7/10 dimensions"' not in html + def test_run_changes_section_is_rendered(self): report = _make_report(run_change_summary="One repo improved and one regressed.") html = _render_html( From de393c0dfe6022925b1f24d304b7cf90da21ee00 Mon Sep 17 00:00:00 2001 From: saagpatel Date: Sun, 12 Jul 2026 01:05:41 -0700 Subject: [PATCH 2/2] fix(scoring): keep partial-run grades canonical --- src/excel_all_repos_helpers.py | 22 ++++++++++++++++++++-- src/scorer.py | 6 +----- src/web_export.py | 21 ++++++++++++++++++++- tests/test_cli_hardening.py | 5 +++-- tests/test_excel_enhanced.py | 12 +++++++++--- tests/test_scoring_profiles.py | 3 ++- tests/test_web_export.py | 6 +++--- 7 files changed, 58 insertions(+), 17 deletions(-) diff --git a/src/excel_all_repos_helpers.py b/src/excel_all_repos_helpers.py index 1455fae..d9bd09f 100644 --- a/src/excel_all_repos_helpers.py +++ b/src/excel_all_repos_helpers.py @@ -10,6 +10,7 @@ from openpyxl.utils import get_column_letter from openpyxl.worksheet.datavalidation import DataValidation from src.scoring_dimensions import display_dimension +from src.scorer import PARTIAL_RUN_BASIS_THRESHOLD, WEIGHTS ALL_REPOS_HEADERS = [ "Repo", @@ -103,7 +104,7 @@ def build_all_repo_rows( "commit_pattern": activity.get("commit_pattern", ""), "values": [ repo_name, - audit.get("grade", "F"), + _build_display_grade(audit), round(audit.get("overall_score", 0), 3), round(audit.get("interest_score", 0), 3), audit.get("interest_grade", "—"), @@ -166,13 +167,30 @@ def _build_biggest_drag(audit: dict[str, Any]) -> str: return f"{display_dimension(worst_dimension)} ({dimension_scores[worst_dimension]:.1f})" +def _build_display_grade(audit: dict[str, Any]) -> str: + """Render partial-run basis disclosure without changing the model grade.""" + grade = str(audit.get("grade", "F")) + if " on " in grade: + return grade + + scored_dimensions = audit.get("scored_dimensions") or [] + scored_weight_sum = audit.get("scored_weight_sum") + if ( + scored_dimensions + and isinstance(scored_weight_sum, (int, float)) + and scored_weight_sum < sum(WEIGHTS.values()) * PARTIAL_RUN_BASIS_THRESHOLD + ): + return f"{grade} on {len(scored_dimensions)}/{len(WEIGHTS)} dimensions" + return grade + + def _build_grade_reason(audit: dict[str, Any]) -> str: dimension_scores = { result["dimension"]: result["score"] for result in audit.get("analyzer_results", []) if result["dimension"] != "interest" } - grade = audit.get("grade", "F") + grade = _build_display_grade(audit) if not dimension_scores: return grade weakest_dimensions = sorted(dimension_scores.items(), key=lambda item: item[1])[:2] diff --git a/src/scorer.py b/src/scorer.py index c93f25c..649f1d9 100644 --- a/src/scorer.py +++ b/src/scorer.py @@ -107,7 +107,6 @@ def score_repo( overall_score = weighted_sum / weight_sum if weight_sum > 0 else 0.0 scored_dimensions = [dimension for dimension in weights if dimension in score_map] - full_weight_sum = sum(weights.values()) # Compute interest score (separate axis, from interest analyzer) interest_score = score_map.get("interest", 0.0) @@ -169,10 +168,7 @@ def score_repo( tier = "skeleton" flags.append("readme-only") - base_grade = letter_grade(overall_score, grade_thresholds) - grade = base_grade - if weight_sum < full_weight_sum * PARTIAL_RUN_BASIS_THRESHOLD: - grade = f"{base_grade} on {len(scored_dimensions)}/{len(weights)} dimensions" + grade = letter_grade(overall_score, grade_thresholds) audit = RepoAudit( metadata=metadata, diff --git a/src/web_export.py b/src/web_export.py index 83bcfc9..3cacce8 100644 --- a/src/web_export.py +++ b/src/web_export.py @@ -58,6 +58,7 @@ no_baseline_summary, no_linked_artifact_summary, ) +from src.scorer import PARTIAL_RUN_BASIS_THRESHOLD, WEIGHTS from src.sparkline import sparkline as render_sparkline from src.terminology import ACTION_SYNC_CANONICAL_LABELS from src.weekly_scheduling_overlay import resolve_weekly_story_value @@ -253,6 +254,23 @@ def _header_section(username: str, date: str, repos: int, grade: str) -> str:
""" +def _build_display_grade(audit: dict) -> str: + """Render partial-run basis disclosure without changing the model grade.""" + grade = str(audit.get("grade", "F")) + if " on " in grade: + return grade + + scored_dimensions = audit.get("scored_dimensions") or [] + scored_weight_sum = audit.get("scored_weight_sum") + if ( + scored_dimensions + and isinstance(scored_weight_sum, (int, float)) + and scored_weight_sum < sum(WEIGHTS.values()) * PARTIAL_RUN_BASIS_THRESHOLD + ): + return f"{grade} on {len(scored_dimensions)}/{len(WEIGHTS)} dimensions" + return grade + + def _kpi_section(data: dict) -> str: tiers = data.get("tier_distribution", {}) avg = data.get("average_score", 0) @@ -926,6 +944,7 @@ def _repo_table( name = m.get("name", "") grade = a.get("grade", "F") grade_letter = grade.split(" ", 1)[0] + display_grade = _build_display_grade(a) score = a.get("overall_score", 0) interest = a.get("interest_score", 0) profile_score = entry["profile_score"] @@ -984,7 +1003,7 @@ def _repo_table( f'data-profile="{profile_score:.3f}">' f"{link}" f'{profile_score:.3f}' - f'{escape(grade)}' + f'{escape(display_grade)}' f'{score:.3f}' f'{interest:.3f}' f'{safe_tier}' diff --git a/tests/test_cli_hardening.py b/tests/test_cli_hardening.py index 8f491c5..3ba60b9 100644 --- a/tests/test_cli_hardening.py +++ b/tests/test_cli_hardening.py @@ -1787,13 +1787,14 @@ def test_partial_run_basis_survives_report_readback(sample_metadata): analyzer_results=[], overall_score=0.72, completeness_tier="functional", - grade="B on 7/10 dimensions", + grade="B", scored_dimensions=["readme", "testing"], scored_weight_sum=0.30, ) restored = audit_from_dict(audit.to_dict()) - assert restored.grade == "B on 7/10 dimensions" + assert restored.grade == "B" + assert restored.to_dict()["grade"] == "B" assert restored.scored_dimensions == ["readme", "testing"] assert restored.scored_weight_sum == 0.30 diff --git a/tests/test_excel_enhanced.py b/tests/test_excel_enhanced.py index 3f8b0d5..96bcaae 100644 --- a/tests/test_excel_enhanced.py +++ b/tests/test_excel_enhanced.py @@ -895,7 +895,7 @@ def test_partial_run_basis_is_exported(self): report = _make_report() report["audits"][0].update( { - "grade": "B on 7/10 dimensions", + "grade": "B", "scored_dimensions": ["readme", "testing"], "scored_weight_sum": 0.3, } @@ -908,13 +908,19 @@ def test_partial_run_basis_is_exported(self): assert "Scored Dimensions" in headers assert "Scored Weight Sum" in headers - assert ws.cell(row=repo_row, column=headers["Grade"]).value == "B on 7/10 dimensions" + assert ws.cell(row=repo_row, column=headers["Grade"]).value == "B on 2/10 dimensions" assert ws.cell(row=repo_row, column=headers["Scored Dimensions"]).value == "readme, testing" assert ws.cell(row=repo_row, column=headers["Scored Weight Sum"]).value == 0.3 def test_dashboard_grade_distribution_uses_qualified_grade_letter(self): report = _make_report() - report["audits"][0]["grade"] = "B on 7/10 dimensions" + report["audits"][0].update( + { + "grade": "B", + "scored_dimensions": ["readme", "testing"], + "scored_weight_sum": 0.3, + } + ) wb = Workbook() _build_dashboard(wb, report) diff --git a/tests/test_scoring_profiles.py b/tests/test_scoring_profiles.py index 1648440..a1df357 100644 --- a/tests/test_scoring_profiles.py +++ b/tests/test_scoring_profiles.py @@ -75,11 +75,12 @@ def test_partial_run_grade_discloses_scored_basis(self): audit = score_repo(_make_metadata(), results) - assert audit.grade == "B on 7/10 dimensions" + assert audit.grade == "B" assert audit.scored_dimensions == scored_dimensions assert audit.scored_weight_sum == 0.75 assert audit.to_dict()["scored_dimensions"] == scored_dimensions assert audit.to_dict()["scored_weight_sum"] == 0.75 + assert audit.to_dict()["grade"] == "B" def test_full_run_grade_remains_unqualified(self): results = _make_results({dim: 0.72 for dim in WEIGHTS}) diff --git a/tests/test_web_export.py b/tests/test_web_export.py index 51cc668..aa9b855 100644 --- a/tests/test_web_export.py +++ b/tests/test_web_export.py @@ -660,7 +660,7 @@ def test_partial_run_grade_is_displayed_but_filterable_by_letter(self): report = _make_report() report["audits"][0].update( { - "grade": "B on 7/10 dimensions", + "grade": "B", "scored_dimensions": ["readme", "testing"], "scored_weight_sum": 0.3, } @@ -668,9 +668,9 @@ def test_partial_run_grade_is_displayed_but_filterable_by_letter(self): html = _render_html(report) - assert "B on 7/10 dimensions" in html + assert "B on 2/10 dimensions" in html assert 'data-grade="B"' in html - assert 'data-grade="B on 7/10 dimensions"' not in html + assert 'data-grade="B on 2/10 dimensions"' not in html def test_run_changes_section_is_rendered(self): report = _make_report(run_change_summary="One repo improved and one regressed.")