From 81de0d4726f5454391e502a665323600368f7a7d Mon Sep 17 00:00:00 2001 From: Ealynn Hsu Date: Mon, 20 Jul 2026 19:45:05 +0000 Subject: [PATCH 1/3] docs: Add show_metrics, stream_logs, and job notifications documentation --- .../finetuning_hyperpod.rst | 74 +++++++++++++++++++ .../finetuning_serverful.rst | 45 +++++++++++ .../model_customization.rst | 39 ++++++++++ 3 files changed, 158 insertions(+) diff --git a/docs/model_customization/finetuning_hyperpod.rst b/docs/model_customization/finetuning_hyperpod.rst index 5d35674a88..94ef2d1b5d 100644 --- a/docs/model_customization/finetuning_hyperpod.rst +++ b/docs/model_customization/finetuning_hyperpod.rst @@ -171,6 +171,80 @@ Set Hyperparameters and Submit print(f"HyperPod job submitted: {sft_job}") +Monitor the Job +---------------- +show_metrics() +~~~~~~~~~~~~~~ + +Plot training metrics parsed from CloudWatch logs for your HyperPod cluster: + +.. code-block:: python + + # After training completes + df = trainer.show_metrics() + + # Plot specific metrics + df = trainer.show_metrics(metrics=["training_loss", "reward_score"]) + + # Filter by step range + df = trainer.show_metrics(starting_step=50, ending_step=200) + + # Filter by time window, this can help speed up completion + from datetime import datetime + df = trainer.show_metrics( + start_time=datetime(2026, 7, 1, 10, 0, 0), + end_time=datetime(2026, 7, 2, 12, 0, 0), + ) + +.. code-block:: python + + # After a kernel restart, set up a trainer with the compute config to retrieve metrics. + from sagemaker.train import SFTTrainer + from sagemaker.core.training.configs import HyperPodCompute + + # Create trainer with the same compute config (not used for training) + trainer = SFTTrainer( + model="nova-textgeneration-micro", + training_dataset="s3://dataset-unused-for-metrics", # not used for metrics + compute=HyperPodCompute( + cluster_name="my-cluster", + instance_type="ml.p5.48xlarge", + ) + ) + + # Set the job name manually + trainer._latest_training_job = "my-hp-job-20260716153000" + + # Now show_metrics works — it uses compute.cluster_name to find logs + df = trainer.show_metrics() + + +stream_logs() +~~~~~~~~~~~~~ + +Stream CloudWatch logs from the HyperPod cluster in real-time: + +.. code-block:: python + + # Start training + job = trainer.train(wait=False) + + # Stream logs (blocks until user manually runs Ctrl+C) + trainer.stream_logs() + + # Custom polling interval in seconds + trainer.stream_logs(poll=10) + + # Stream from a specific start time + trainer.stream_logs(start_time=datetime(2026, 1, 1, 15, 0, 0)) + +.. note:: + + HyperPod log streaming runs until you press Ctrl+C (unlike SMTJ which + auto-stops when the job reaches a terminal state). Logs may take a few + minutes to propagate to CloudWatch. + + Interactive Notebook --------------------- diff --git a/docs/model_customization/finetuning_serverful.rst b/docs/model_customization/finetuning_serverful.rst index 49e9555947..8b41667bba 100644 --- a/docs/model_customization/finetuning_serverful.rst +++ b/docs/model_customization/finetuning_serverful.rst @@ -131,6 +131,51 @@ Monitor the Job print(f"Status: {job.training_job_status}") print(f"Secondary Status: {job.secondary_status}") +show_metrics() +~~~~~~~~~~~~~~ + +Plot training metrics after a job completes. For Nova models, metrics are parsed from +CloudWatch logs. For open-weight models, metrics are pulled from MLflow if set up. + +.. code-block:: python + + # Plot all available metrics + df = trainer.show_metrics() + + # Plot specific metrics + df = trainer.show_metrics(metrics=["training_loss", "lr"]) + + # Filter by step range + df = trainer.show_metrics(starting_step=20, ending_step=100) + + # Filter by time window + from datetime import datetime + df = trainer.show_metrics( + start_time=datetime(2026, 1, 1, 1, 0, 0), + end_time=datetime(2026, 1, 1, 2, 0, 0), + ) + + # After a kernel restart, use the standalone function with job name + from sagemaker.train import plot_training_metrics + plot_training_metrics("my-sft-job-20260101195119") + + +stream_logs() +~~~~~~~~~~~~~ + +Stream CloudWatch logs in real-time while a job is running: + +.. code-block:: python + + # Start training non-blocking + job = trainer.train(wait=False) + + # Stream logs directly to the terminal (blocking) + trainer.stream_logs() + + # Custom polling interval (seconds) + trainer.stream_logs(poll=10) + Interactive Notebook --------------------- diff --git a/docs/model_customization/model_customization.rst b/docs/model_customization/model_customization.rst index 3a36a3ff28..bd58d5659e 100644 --- a/docs/model_customization/model_customization.rst +++ b/docs/model_customization/model_customization.rst @@ -53,6 +53,45 @@ Key Features with clear precedence. Use ``get_resolved_recipe()`` to inspect the merged configuration before job submission. See :doc:`finetuning_serverful` and :doc:`finetuning_hyperpod` for examples. +**Job Notifications** + Receive SNS notifications when training jobs complete, fail, or stop. Uses EventBridge + rules to route SageMaker Training Job status changes to your SNS topic. + + The SDK creates one rule per unique config (topic + events + prefix). Re-running with + the same config reuses the existing rule. Different configs create separate rules. + + .. note:: + Supported for SMTJ (serverful and serverless) compute only. HyperPod is not currently supported. + + .. code-block:: python + + trainer = SFTTrainer( + model="nova-textgeneration-micro", + training_dataset="s3://my-bucket/train.jsonl", + accept_eula=True, + notifications={ + "sns_topic_arn": "arn:aws:sns:us-east-1:123456789012:my-topic", # Required + "events": ["Completed", "Failed"], # Optional (default: Completed, Failed, Stopped) + "job_name_prefix": "my-team-sft-", # Optional: filter by job name + "event_bus_arn": "arn:aws:events:us-east-1:123456789012:event-bus/custom-bus" # Optional + }, + ) + job = trainer.train(wait=False) # Notification sent on completion + + # Access the rule ARN + print(trainer.notification_rule_arn) + + # List and manage rules + rules = trainer.list_notification_rules() + trainer.delete_notification_rule(rule_arn=trainer.notification_rule_arn) + + **Prerequisites:** + + - An SNS topic (and subscription) with a resource policy allowing ``events.amazonaws.com``. + To set up a topic and subscription, see `Creating an SNS topic and subscription `_. + - IAM permissions: ``events:PutRule``, ``events:PutTargets``, ``events:ListRules``, + ``events:RemoveTargets``, ``events:DeleteRule`` + .. toctree:: :maxdepth: 1 From 1128464965297cc1684c75c0a7b308d9feab15b1 Mon Sep 17 00:00:00 2001 From: Ealynn Hsu Date: Mon, 20 Jul 2026 20:01:30 +0000 Subject: [PATCH 2/3] Update example job names --- docs/model_customization/finetuning_hyperpod.rst | 2 +- docs/model_customization/finetuning_serverful.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/model_customization/finetuning_hyperpod.rst b/docs/model_customization/finetuning_hyperpod.rst index 94ef2d1b5d..d1d0ec185e 100644 --- a/docs/model_customization/finetuning_hyperpod.rst +++ b/docs/model_customization/finetuning_hyperpod.rst @@ -213,7 +213,7 @@ Plot training metrics parsed from CloudWatch logs for your HyperPod cluster: ) # Set the job name manually - trainer._latest_training_job = "my-hp-job-20260716153000" + trainer._latest_training_job = "my-hp-job" # Now show_metrics works — it uses compute.cluster_name to find logs df = trainer.show_metrics() diff --git a/docs/model_customization/finetuning_serverful.rst b/docs/model_customization/finetuning_serverful.rst index 8b41667bba..634b4f895b 100644 --- a/docs/model_customization/finetuning_serverful.rst +++ b/docs/model_customization/finetuning_serverful.rst @@ -157,7 +157,7 @@ CloudWatch logs. For open-weight models, metrics are pulled from MLflow if set u # After a kernel restart, use the standalone function with job name from sagemaker.train import plot_training_metrics - plot_training_metrics("my-sft-job-20260101195119") + plot_training_metrics("my-sft-job") stream_logs() From 0ca6035cdae43a15b697656006bea8ece2a6ddad Mon Sep 17 00:00:00 2001 From: Ealynn Hsu Date: Mon, 20 Jul 2026 20:44:13 +0000 Subject: [PATCH 3/3] Move monitoring capabilities to model_customization --- .../finetuning_hyperpod.rst | 74 ------------------- .../finetuning_serverful.rst | 45 ----------- .../model_customization.rst | 71 ++++++++++++++++++ 3 files changed, 71 insertions(+), 119 deletions(-) diff --git a/docs/model_customization/finetuning_hyperpod.rst b/docs/model_customization/finetuning_hyperpod.rst index d1d0ec185e..5d35674a88 100644 --- a/docs/model_customization/finetuning_hyperpod.rst +++ b/docs/model_customization/finetuning_hyperpod.rst @@ -171,80 +171,6 @@ Set Hyperparameters and Submit print(f"HyperPod job submitted: {sft_job}") -Monitor the Job ----------------- -show_metrics() -~~~~~~~~~~~~~~ - -Plot training metrics parsed from CloudWatch logs for your HyperPod cluster: - -.. code-block:: python - - # After training completes - df = trainer.show_metrics() - - # Plot specific metrics - df = trainer.show_metrics(metrics=["training_loss", "reward_score"]) - - # Filter by step range - df = trainer.show_metrics(starting_step=50, ending_step=200) - - # Filter by time window, this can help speed up completion - from datetime import datetime - df = trainer.show_metrics( - start_time=datetime(2026, 7, 1, 10, 0, 0), - end_time=datetime(2026, 7, 2, 12, 0, 0), - ) - -.. code-block:: python - - # After a kernel restart, set up a trainer with the compute config to retrieve metrics. - from sagemaker.train import SFTTrainer - from sagemaker.core.training.configs import HyperPodCompute - - # Create trainer with the same compute config (not used for training) - trainer = SFTTrainer( - model="nova-textgeneration-micro", - training_dataset="s3://dataset-unused-for-metrics", # not used for metrics - compute=HyperPodCompute( - cluster_name="my-cluster", - instance_type="ml.p5.48xlarge", - ) - ) - - # Set the job name manually - trainer._latest_training_job = "my-hp-job" - - # Now show_metrics works — it uses compute.cluster_name to find logs - df = trainer.show_metrics() - - -stream_logs() -~~~~~~~~~~~~~ - -Stream CloudWatch logs from the HyperPod cluster in real-time: - -.. code-block:: python - - # Start training - job = trainer.train(wait=False) - - # Stream logs (blocks until user manually runs Ctrl+C) - trainer.stream_logs() - - # Custom polling interval in seconds - trainer.stream_logs(poll=10) - - # Stream from a specific start time - trainer.stream_logs(start_time=datetime(2026, 1, 1, 15, 0, 0)) - -.. note:: - - HyperPod log streaming runs until you press Ctrl+C (unlike SMTJ which - auto-stops when the job reaches a terminal state). Logs may take a few - minutes to propagate to CloudWatch. - - Interactive Notebook --------------------- diff --git a/docs/model_customization/finetuning_serverful.rst b/docs/model_customization/finetuning_serverful.rst index 634b4f895b..49e9555947 100644 --- a/docs/model_customization/finetuning_serverful.rst +++ b/docs/model_customization/finetuning_serverful.rst @@ -131,51 +131,6 @@ Monitor the Job print(f"Status: {job.training_job_status}") print(f"Secondary Status: {job.secondary_status}") -show_metrics() -~~~~~~~~~~~~~~ - -Plot training metrics after a job completes. For Nova models, metrics are parsed from -CloudWatch logs. For open-weight models, metrics are pulled from MLflow if set up. - -.. code-block:: python - - # Plot all available metrics - df = trainer.show_metrics() - - # Plot specific metrics - df = trainer.show_metrics(metrics=["training_loss", "lr"]) - - # Filter by step range - df = trainer.show_metrics(starting_step=20, ending_step=100) - - # Filter by time window - from datetime import datetime - df = trainer.show_metrics( - start_time=datetime(2026, 1, 1, 1, 0, 0), - end_time=datetime(2026, 1, 1, 2, 0, 0), - ) - - # After a kernel restart, use the standalone function with job name - from sagemaker.train import plot_training_metrics - plot_training_metrics("my-sft-job") - - -stream_logs() -~~~~~~~~~~~~~ - -Stream CloudWatch logs in real-time while a job is running: - -.. code-block:: python - - # Start training non-blocking - job = trainer.train(wait=False) - - # Stream logs directly to the terminal (blocking) - trainer.stream_logs() - - # Custom polling interval (seconds) - trainer.stream_logs(poll=10) - Interactive Notebook --------------------- diff --git a/docs/model_customization/model_customization.rst b/docs/model_customization/model_customization.rst index bd58d5659e..938349bdd7 100644 --- a/docs/model_customization/model_customization.rst +++ b/docs/model_customization/model_customization.rst @@ -92,6 +92,77 @@ Key Features - IAM permissions: ``events:PutRule``, ``events:PutTargets``, ``events:ListRules``, ``events:RemoveTargets``, ``events:DeleteRule`` +**Monitoring: show_metrics()** + Plot training metrics after a job completes. Works across all compute types. + + - **Nova models**: Metrics parsed from CloudWatch logs. + - **OSS models**: Metrics pulled from MLflow. + + .. code-block:: python + + # Plot all available metrics + df = trainer.show_metrics() + + # Plot specific metrics + df = trainer.show_metrics(metrics=["training_loss", "lr"]) + + # Filter by step range + df = trainer.show_metrics(starting_step=10, ending_step=100) + + # Filter by time window + from datetime import datetime + df = trainer.show_metrics( + start_time=datetime(2026, 1, 1, 10, 0, 0), + end_time=datetime(2026, 1, 1, 12, 0, 0), + ) + + **After a kernel restart:** + + .. code-block:: python + + # Standalone (SMTJ) + from sagemaker.train import plot_training_metrics + plot_training_metrics("my-sft-job") + + # Re-attach (HyperPod — needs cluster name for log group resolution) + from sagemaker.train import SFTTrainer + from sagemaker.core.training.configs import HyperPodCompute + + trainer = SFTTrainer( + model="nova-textgeneration-micro", + training_dataset="s3://unused", + compute=HyperPodCompute(cluster_name="my-cluster", instance_type="ml.p5.48xlarge") + ) + trainer._latest_training_job = "my-hp-job" + df = trainer.show_metrics() + +**Monitoring: stream_logs()** + Stream CloudWatch logs in real-time while a job is running. + + .. code-block:: python + + # Start training non-blocking + job = trainer.train(wait=False) + + # Stream logs (blocks until job completes or Ctrl+C) + trainer.stream_logs() + + # Custom polling interval (seconds) + trainer.stream_logs(poll=10) + + # Stream from a specific start time - providing this will speed up execution. + from datetime import datetime + trainer.stream_logs(start_time=datetime(2026, 1, 1, 15, 0, 0)) + + .. note:: + + - **SMTJ**: Streaming auto-stops when the job reaches a terminal state. + - **HyperPod**: Streaming runs until you press Ctrl+C. Logs may take a few minutes + to propagate to CloudWatch on first run. + + +---- + .. toctree:: :maxdepth: 1