Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions .github/workflows/sync-cloud-run-env.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ jobs:
CLOUD_RUN_SERVICES: ${{ vars.CLOUD_RUN_SERVICES }}
CLOUD_RUN_SERVICE_TARGETS_JSON: ${{ vars.CLOUD_RUN_SERVICE_TARGETS_JSON }}
CLOUD_RUN_ENV_SYNC_WAIT_FOR_COMMIT: ${{ vars.CLOUD_RUN_ENV_SYNC_WAIT_FOR_COMMIT }}
CLOUD_SCHEDULER_LOCATION: ${{ vars.CLOUD_SCHEDULER_LOCATION }}
TELEGRAM_TOKEN_SECRET_NAME: ${{ vars.TELEGRAM_TOKEN_SECRET_NAME }}
RUNTIME_TARGET_JSON: ${{ vars.RUNTIME_TARGET_JSON }}
ACCOUNT_GROUP: ${{ vars.ACCOUNT_GROUP }}
Expand Down Expand Up @@ -852,6 +853,59 @@ jobs:
gcloud "${gcloud_args[@]}"
done

- name: Sync Cloud Scheduler timezone
if: steps.config.outputs.env_sync_enabled == 'true'
env:
SYNC_PLAN_JSON: ${{ steps.strategy_requirements.outputs.sync_plan_json }}
run: |
set -euo pipefail

scheduler_location="${CLOUD_SCHEDULER_LOCATION:-${CLOUD_RUN_REGION}}"
if [ -z "${scheduler_location}" ]; then
echo "Cloud Scheduler timezone sync requires CLOUD_RUN_REGION or CLOUD_SCHEDULER_LOCATION." >&2
exit 1
fi

mapfile -t scheduler_updates < <(python - <<'PY'
import json
import os

plan = json.loads(os.environ["SYNC_PLAN_JSON"])
for target in plan["targets"]:
service_name = str(target["service_name"]).strip()
env = target.get("env") or {}
timezone = str(env.get("IBKR_MARKET_TIMEZONE") or "").strip()
market = str(env.get("IBKR_MARKET") or "").strip().upper()
if not timezone:
timezone = "Asia/Hong_Kong" if market == "HK" else "America/New_York"
print(f"{service_name}\t{timezone}")
PY
)

for update in "${scheduler_updates[@]}"; do
IFS=$'\t' read -r cloud_run_service market_timezone <<< "${update}"
if [ -z "${cloud_run_service}" ] || [ -z "${market_timezone}" ]; then
continue
fi

for suffix in scheduler probe-scheduler precheck-scheduler; do
job_name="${cloud_run_service}-${suffix}"
if ! gcloud scheduler jobs describe "${job_name}" \
--project="${GCP_PROJECT_ID}" \
--location="${scheduler_location}" >/dev/null 2>&1; then
echo "Cloud Scheduler job ${job_name} was not found in ${scheduler_location}; skipping timezone sync."
continue
fi

echo "Updating Cloud Scheduler job ${job_name} timezone to ${market_timezone}."
gcloud scheduler jobs update http "${job_name}" \
--project="${GCP_PROJECT_ID}" \
--location="${scheduler_location}" \
--time-zone="${market_timezone}" \
--quiet
done
done

- name: Prune old Cloud Run revisions
if: steps.config.outputs.enabled == 'true'
env:
Expand Down
47 changes: 39 additions & 8 deletions scripts/execution_report_heartbeat.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,15 +137,29 @@ def _load_required_services(
since: dt.datetime | None = None,
now: dt.datetime | None = None,
) -> list[str]:
services, _skip_reason, _scheduler_checked = _resolve_required_services(
project=project,
since=since,
now=now,
)
return services


def _resolve_required_services(
*,
project: str | None = None,
since: dt.datetime | None = None,
now: dt.datetime | None = None,
) -> tuple[list[str], str | None, bool]:
services, explicit = _load_required_service_candidates()
if explicit or not services:
return services
return services, None, False
if not _env_bool("RUNTIME_HEARTBEAT_SCHEDULER_AWARE", True):
return services
return services, None, False
if since is None or now is None:
return services
return services, None, False
try:
return _filter_scheduler_due_services(
due_services = _filter_scheduler_due_services(
services,
project=project,
since=since,
Expand All @@ -157,7 +171,14 @@ def _load_required_services(
"falling back to all configured services.",
file=sys.stderr,
)
return services
return services, None, False
if not due_services:
return (
[],
"no configured Cloud Scheduler main job was due in the heartbeat lookback window",
True,
)
return due_services, None, True


def _unique_values(values: list[str]) -> list[str]:
Expand Down Expand Up @@ -522,7 +543,7 @@ def _send_telegram(message: str) -> bool:
return ok


def main() -> int:
def main(now: dt.datetime | None = None) -> int:
project = (
os.environ.get("RUNTIME_HEARTBEAT_GCP_PROJECT_ID")
or os.environ.get("GCP_PROJECT_ID")
Expand All @@ -533,9 +554,19 @@ def main() -> int:
max_reports = int(os.environ.get("RUNTIME_HEARTBEAT_MAX_REPORTS_TO_READ") or "20")
fail_workflow = _env_bool("RUNTIME_HEARTBEAT_FAIL_WORKFLOW_ON_ALERT", True)

now = dt.datetime.now(dt.timezone.utc)
now = now or dt.datetime.now(dt.timezone.utc)
if now.tzinfo is None:
now = now.replace(tzinfo=dt.timezone.utc)
now = now.astimezone(dt.timezone.utc)
since = now - dt.timedelta(hours=lookback_hours)
required_services = _load_required_services(project=project, since=since, now=now)
required_services, scheduler_skip_reason, _scheduler_checked = _resolve_required_services(
project=project,
since=since,
now=now,
)
if scheduler_skip_reason:
print(f"Execution report heartbeat skipped for {name}: {scheduler_skip_reason}")
return 0
globs = _report_globs(since, now)
if not globs:
raise SystemExit("No heartbeat GCS report URI configured")
Expand Down
35 changes: 35 additions & 0 deletions tests/test_execution_report_heartbeat.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import datetime as dt
import json

import pytest

from scripts import execution_report_heartbeat as heartbeat


Expand Down Expand Up @@ -114,3 +116,36 @@ def test_scheduler_aware_required_services_include_monthly_service_when_due(monk
)

assert required == ["svc-monthly"]


def test_main_skips_when_no_scheduler_main_job_is_due(monkeypatch, capsys):
monkeypatch.delenv("RUNTIME_HEARTBEAT_REQUIRED_SERVICES", raising=False)
monkeypatch.setenv("GCP_PROJECT_ID", "interactivebrokersquant")
monkeypatch.setenv("RUNTIME_HEARTBEAT_NAME", "Monthly runtime")
monkeypatch.setenv("RUNTIME_HEARTBEAT_REPORT_PLATFORM", "interactive_brokers")
monkeypatch.setenv("CLOUD_RUN_SERVICE", "ibkr-monthly-service")
monkeypatch.setenv("RUNTIME_HEARTBEAT_GCS_URIS", "gs://bucket/execution-reports")
monkeypatch.setattr(
heartbeat,
"_list_scheduler_jobs",
lambda **_kwargs: [
{
"state": "ENABLED",
"schedule": "45 15 26 * *",
"timeZone": "America/New_York",
"httpTarget": {"uri": "https://ibkr-monthly-service.example.run.app/"},
},
],
)
monkeypatch.setattr(
heartbeat,
"_list_gcs_objects",
lambda *_args, **_kwargs: pytest.fail("GCS should not be queried when no scheduler job is due"),
)

result = heartbeat.main(now=dt.datetime(2026, 6, 10, 1, 35, tzinfo=dt.timezone.utc))

assert result == 0
output = capsys.readouterr().out
assert "Execution report heartbeat skipped for Monthly runtime" in output
assert "no configured Cloud Scheduler main job was due" in output
10 changes: 10 additions & 0 deletions tests/test_sync_cloud_run_env_workflow.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ grep -Fq 'CLOUD_RUN_SERVICE: ${{ vars.CLOUD_RUN_SERVICE }}' "$workflow_file"
grep -Fq 'CLOUD_RUN_SERVICES: ${{ vars.CLOUD_RUN_SERVICES }}' "$workflow_file"
grep -Fq 'CLOUD_RUN_SERVICE_TARGETS_JSON: ${{ vars.CLOUD_RUN_SERVICE_TARGETS_JSON }}' "$workflow_file"
grep -Fq 'CLOUD_RUN_ENV_SYNC_WAIT_FOR_COMMIT: ${{ vars.CLOUD_RUN_ENV_SYNC_WAIT_FOR_COMMIT }}' "$workflow_file"
grep -Fq 'CLOUD_SCHEDULER_LOCATION: ${{ vars.CLOUD_SCHEDULER_LOCATION }}' "$workflow_file"
grep -Fq 'RUNTIME_TARGET_JSON: ${{ vars.RUNTIME_TARGET_JSON }}' "$workflow_file"
grep -Fq 'ACCOUNT_GROUP: ${{ vars.ACCOUNT_GROUP }}' "$workflow_file"
grep -Fq 'IB_ACCOUNT_GROUP_CONFIG_SECRET_NAME: ${{ vars.IB_ACCOUNT_GROUP_CONFIG_SECRET_NAME }}' "$workflow_file"
Expand Down Expand Up @@ -82,6 +83,15 @@ grep -Fq -- '--concurrency 1' "$workflow_file"
grep -Fq -- '--max-instances 1' "$workflow_file"
grep -Fq -- '--remove-env-vars "$(IFS=,; echo "${remove_env_vars[*]}")' "$workflow_file"
grep -Fq -- '--update-env-vars "^|^$(join_by_delimiter "|" "${env_pairs[@]}")' "$workflow_file"
grep -Fq 'Sync Cloud Scheduler timezone' "$workflow_file"
grep -Fq 'scheduler_location="${CLOUD_SCHEDULER_LOCATION:-${CLOUD_RUN_REGION}}"' "$workflow_file"
grep -Fq 'timezone = str(env.get("IBKR_MARKET_TIMEZONE") or "").strip()' "$workflow_file"
grep -Fq 'timezone = "Asia/Hong_Kong" if market == "HK" else "America/New_York"' "$workflow_file"
grep -Fq 'print(f"{service_name}\t{timezone}")' "$workflow_file"
grep -Fq 'for suffix in scheduler probe-scheduler precheck-scheduler; do' "$workflow_file"
grep -Fq 'gcloud scheduler jobs describe "${job_name}"' "$workflow_file"
grep -Fq 'gcloud scheduler jobs update http "${job_name}"' "$workflow_file"
grep -Fq -- '--time-zone="${market_timezone}"' "$workflow_file"

grep -Fq '"CRISIS_ALERT_GOOGLE_VOICE_TO"' "$workflow_file"
grep -Fq '"CRISIS_ALERT_GOOGLE_VOICE_SENDER_PASSWORD"' "$workflow_file"
Expand Down