diff --git a/.github/workflows/sync-cloud-run-env.yml b/.github/workflows/sync-cloud-run-env.yml index 98d20cc..fa9c6d6 100644 --- a/.github/workflows/sync-cloud-run-env.yml +++ b/.github/workflows/sync-cloud-run-env.yml @@ -73,6 +73,7 @@ jobs: CLOUD_RUN_SERVICES: ${{ vars.CLOUD_RUN_SERVICES }} CLOUD_RUN_SERVICE_TARGETS_JSON: ${{ vars.CLOUD_RUN_SERVICE_TARGETS_JSON }} CLOUD_RUN_ENV_SYNC_WAIT_FOR_COMMIT: ${{ vars.CLOUD_RUN_ENV_SYNC_WAIT_FOR_COMMIT }} + CLOUD_SCHEDULER_LOCATION: ${{ vars.CLOUD_SCHEDULER_LOCATION }} TELEGRAM_TOKEN_SECRET_NAME: ${{ vars.TELEGRAM_TOKEN_SECRET_NAME }} RUNTIME_TARGET_JSON: ${{ vars.RUNTIME_TARGET_JSON }} ACCOUNT_GROUP: ${{ vars.ACCOUNT_GROUP }} @@ -852,6 +853,59 @@ jobs: gcloud "${gcloud_args[@]}" done + - name: Sync Cloud Scheduler timezone + if: steps.config.outputs.env_sync_enabled == 'true' + env: + SYNC_PLAN_JSON: ${{ steps.strategy_requirements.outputs.sync_plan_json }} + run: | + set -euo pipefail + + scheduler_location="${CLOUD_SCHEDULER_LOCATION:-${CLOUD_RUN_REGION}}" + if [ -z "${scheduler_location}" ]; then + echo "Cloud Scheduler timezone sync requires CLOUD_RUN_REGION or CLOUD_SCHEDULER_LOCATION." >&2 + exit 1 + fi + + mapfile -t scheduler_updates < <(python - <<'PY' + import json + import os + + plan = json.loads(os.environ["SYNC_PLAN_JSON"]) + for target in plan["targets"]: + service_name = str(target["service_name"]).strip() + env = target.get("env") or {} + timezone = str(env.get("IBKR_MARKET_TIMEZONE") or "").strip() + market = str(env.get("IBKR_MARKET") or "").strip().upper() + if not timezone: + timezone = "Asia/Hong_Kong" if market == "HK" else "America/New_York" + print(f"{service_name}\t{timezone}") + PY + ) + + for update in "${scheduler_updates[@]}"; do + IFS=$'\t' read -r cloud_run_service market_timezone <<< "${update}" + if [ -z "${cloud_run_service}" ] || [ -z "${market_timezone}" ]; then + continue + fi + + for suffix in scheduler probe-scheduler precheck-scheduler; do + job_name="${cloud_run_service}-${suffix}" + if ! gcloud scheduler jobs describe "${job_name}" \ + --project="${GCP_PROJECT_ID}" \ + --location="${scheduler_location}" >/dev/null 2>&1; then + echo "Cloud Scheduler job ${job_name} was not found in ${scheduler_location}; skipping timezone sync." + continue + fi + + echo "Updating Cloud Scheduler job ${job_name} timezone to ${market_timezone}." + gcloud scheduler jobs update http "${job_name}" \ + --project="${GCP_PROJECT_ID}" \ + --location="${scheduler_location}" \ + --time-zone="${market_timezone}" \ + --quiet + done + done + - name: Prune old Cloud Run revisions if: steps.config.outputs.enabled == 'true' env: diff --git a/scripts/execution_report_heartbeat.py b/scripts/execution_report_heartbeat.py index 066166c..f01c8b5 100644 --- a/scripts/execution_report_heartbeat.py +++ b/scripts/execution_report_heartbeat.py @@ -137,15 +137,29 @@ def _load_required_services( since: dt.datetime | None = None, now: dt.datetime | None = None, ) -> list[str]: + services, _skip_reason, _scheduler_checked = _resolve_required_services( + project=project, + since=since, + now=now, + ) + return services + + +def _resolve_required_services( + *, + project: str | None = None, + since: dt.datetime | None = None, + now: dt.datetime | None = None, +) -> tuple[list[str], str | None, bool]: services, explicit = _load_required_service_candidates() if explicit or not services: - return services + return services, None, False if not _env_bool("RUNTIME_HEARTBEAT_SCHEDULER_AWARE", True): - return services + return services, None, False if since is None or now is None: - return services + return services, None, False try: - return _filter_scheduler_due_services( + due_services = _filter_scheduler_due_services( services, project=project, since=since, @@ -157,7 +171,14 @@ def _load_required_services( "falling back to all configured services.", file=sys.stderr, ) - return services + return services, None, False + if not due_services: + return ( + [], + "no configured Cloud Scheduler main job was due in the heartbeat lookback window", + True, + ) + return due_services, None, True def _unique_values(values: list[str]) -> list[str]: @@ -522,7 +543,7 @@ def _send_telegram(message: str) -> bool: return ok -def main() -> int: +def main(now: dt.datetime | None = None) -> int: project = ( os.environ.get("RUNTIME_HEARTBEAT_GCP_PROJECT_ID") or os.environ.get("GCP_PROJECT_ID") @@ -533,9 +554,19 @@ def main() -> int: max_reports = int(os.environ.get("RUNTIME_HEARTBEAT_MAX_REPORTS_TO_READ") or "20") fail_workflow = _env_bool("RUNTIME_HEARTBEAT_FAIL_WORKFLOW_ON_ALERT", True) - now = dt.datetime.now(dt.timezone.utc) + now = now or dt.datetime.now(dt.timezone.utc) + if now.tzinfo is None: + now = now.replace(tzinfo=dt.timezone.utc) + now = now.astimezone(dt.timezone.utc) since = now - dt.timedelta(hours=lookback_hours) - required_services = _load_required_services(project=project, since=since, now=now) + required_services, scheduler_skip_reason, _scheduler_checked = _resolve_required_services( + project=project, + since=since, + now=now, + ) + if scheduler_skip_reason: + print(f"Execution report heartbeat skipped for {name}: {scheduler_skip_reason}") + return 0 globs = _report_globs(since, now) if not globs: raise SystemExit("No heartbeat GCS report URI configured") diff --git a/tests/test_execution_report_heartbeat.py b/tests/test_execution_report_heartbeat.py index d2ed260..fb4ebe9 100644 --- a/tests/test_execution_report_heartbeat.py +++ b/tests/test_execution_report_heartbeat.py @@ -3,6 +3,8 @@ import datetime as dt import json +import pytest + from scripts import execution_report_heartbeat as heartbeat @@ -114,3 +116,36 @@ def test_scheduler_aware_required_services_include_monthly_service_when_due(monk ) assert required == ["svc-monthly"] + + +def test_main_skips_when_no_scheduler_main_job_is_due(monkeypatch, capsys): + monkeypatch.delenv("RUNTIME_HEARTBEAT_REQUIRED_SERVICES", raising=False) + monkeypatch.setenv("GCP_PROJECT_ID", "interactivebrokersquant") + monkeypatch.setenv("RUNTIME_HEARTBEAT_NAME", "Monthly runtime") + monkeypatch.setenv("RUNTIME_HEARTBEAT_REPORT_PLATFORM", "interactive_brokers") + monkeypatch.setenv("CLOUD_RUN_SERVICE", "ibkr-monthly-service") + monkeypatch.setenv("RUNTIME_HEARTBEAT_GCS_URIS", "gs://bucket/execution-reports") + monkeypatch.setattr( + heartbeat, + "_list_scheduler_jobs", + lambda **_kwargs: [ + { + "state": "ENABLED", + "schedule": "45 15 26 * *", + "timeZone": "America/New_York", + "httpTarget": {"uri": "https://ibkr-monthly-service.example.run.app/"}, + }, + ], + ) + monkeypatch.setattr( + heartbeat, + "_list_gcs_objects", + lambda *_args, **_kwargs: pytest.fail("GCS should not be queried when no scheduler job is due"), + ) + + result = heartbeat.main(now=dt.datetime(2026, 6, 10, 1, 35, tzinfo=dt.timezone.utc)) + + assert result == 0 + output = capsys.readouterr().out + assert "Execution report heartbeat skipped for Monthly runtime" in output + assert "no configured Cloud Scheduler main job was due" in output diff --git a/tests/test_sync_cloud_run_env_workflow.sh b/tests/test_sync_cloud_run_env_workflow.sh index 07bfdfa..4b04aa4 100644 --- a/tests/test_sync_cloud_run_env_workflow.sh +++ b/tests/test_sync_cloud_run_env_workflow.sh @@ -28,6 +28,7 @@ grep -Fq 'CLOUD_RUN_SERVICE: ${{ vars.CLOUD_RUN_SERVICE }}' "$workflow_file" grep -Fq 'CLOUD_RUN_SERVICES: ${{ vars.CLOUD_RUN_SERVICES }}' "$workflow_file" grep -Fq 'CLOUD_RUN_SERVICE_TARGETS_JSON: ${{ vars.CLOUD_RUN_SERVICE_TARGETS_JSON }}' "$workflow_file" grep -Fq 'CLOUD_RUN_ENV_SYNC_WAIT_FOR_COMMIT: ${{ vars.CLOUD_RUN_ENV_SYNC_WAIT_FOR_COMMIT }}' "$workflow_file" +grep -Fq 'CLOUD_SCHEDULER_LOCATION: ${{ vars.CLOUD_SCHEDULER_LOCATION }}' "$workflow_file" grep -Fq 'RUNTIME_TARGET_JSON: ${{ vars.RUNTIME_TARGET_JSON }}' "$workflow_file" grep -Fq 'ACCOUNT_GROUP: ${{ vars.ACCOUNT_GROUP }}' "$workflow_file" grep -Fq 'IB_ACCOUNT_GROUP_CONFIG_SECRET_NAME: ${{ vars.IB_ACCOUNT_GROUP_CONFIG_SECRET_NAME }}' "$workflow_file" @@ -82,6 +83,15 @@ grep -Fq -- '--concurrency 1' "$workflow_file" grep -Fq -- '--max-instances 1' "$workflow_file" grep -Fq -- '--remove-env-vars "$(IFS=,; echo "${remove_env_vars[*]}")' "$workflow_file" grep -Fq -- '--update-env-vars "^|^$(join_by_delimiter "|" "${env_pairs[@]}")' "$workflow_file" +grep -Fq 'Sync Cloud Scheduler timezone' "$workflow_file" +grep -Fq 'scheduler_location="${CLOUD_SCHEDULER_LOCATION:-${CLOUD_RUN_REGION}}"' "$workflow_file" +grep -Fq 'timezone = str(env.get("IBKR_MARKET_TIMEZONE") or "").strip()' "$workflow_file" +grep -Fq 'timezone = "Asia/Hong_Kong" if market == "HK" else "America/New_York"' "$workflow_file" +grep -Fq 'print(f"{service_name}\t{timezone}")' "$workflow_file" +grep -Fq 'for suffix in scheduler probe-scheduler precheck-scheduler; do' "$workflow_file" +grep -Fq 'gcloud scheduler jobs describe "${job_name}"' "$workflow_file" +grep -Fq 'gcloud scheduler jobs update http "${job_name}"' "$workflow_file" +grep -Fq -- '--time-zone="${market_timezone}"' "$workflow_file" grep -Fq '"CRISIS_ALERT_GOOGLE_VOICE_TO"' "$workflow_file" grep -Fq '"CRISIS_ALERT_GOOGLE_VOICE_SENDER_PASSWORD"' "$workflow_file"