From facdcb7fb9b6537cc88201d2db2ac987a326daac Mon Sep 17 00:00:00 2001 From: Lisa Ni Date: Mon, 20 Jul 2026 16:07:10 +0000 Subject: [PATCH] documentation: add dry-run and resource reuse docs to existing RST pages --- .../deploy_sagemaker_endpoint.rst | 62 +++++++++++++++++++ docs/model_customization/evaluation.rst | 41 ++++++++++++ .../model_customization.rst | 27 ++++++++ .../sagemaker/serve/bedrock_model_builder.py | 5 +- .../bedrock-modelbuilder-deployment.ipynb | 2 +- 5 files changed, 133 insertions(+), 4 deletions(-) diff --git a/docs/model_customization/deploy_sagemaker_endpoint.rst b/docs/model_customization/deploy_sagemaker_endpoint.rst index 5b2108a6bf..b420830797 100644 --- a/docs/model_customization/deploy_sagemaker_endpoint.rst +++ b/docs/model_customization/deploy_sagemaker_endpoint.rst @@ -75,3 +75,65 @@ production deployments. model_builder = ModelBuilder(model=model_package) model = model_builder.build(model_name="my-registered-model") endpoint = model_builder.deploy(endpoint_name="my-endpoint") + + +Reuse Deployed Resources +-------------------------- + +Pass ``reuse_resources=True`` to ``build()`` and ``deploy()`` to avoid creating duplicate +endpoints when deploying the same model source multiple times. + +On first deploy, the SDK tags the endpoint with ``sagemaker.amazonaws.com/model-source`` +derived from the model's stable source identifier (model package ARN, escrow URI, S3 path, +or JumpStart model ID). On subsequent deploys with ``reuse_resources=True``, the SDK discovers +the existing endpoint by that tag and returns it instead of creating a new one. + +.. list-table:: + :header-rows: 1 + + * - Model input style + - Source ID used for tag + * - ``ModelPackage`` + - Model package ARN + * - ``BaseTrainer`` (with completed training job) + - Model package ARN if available, otherwise escrow resolution + * - ``TrainingJob`` + - Via model package ARN or escrow resolution + * - Raw S3 URI string + - The S3 path itself + * - JumpStart model ID string + - The model ID string + +.. code-block:: python + + from sagemaker.serve import ModelBuilder + + builder = ModelBuilder( + model=my_trainer, # or ModelPackage, TrainingJob, S3 URI, etc. + role_arn="arn:aws:iam::123456789012:role/MySageMakerRole", + instance_type="ml.p4d.24xlarge", + image_uri="my-inference-image:latest", + ) + + # build() checks for an existing Model with matching source tag + builder.build(region="us-east-1", reuse_resources=True) + + # deploy() checks for an existing Endpoint with matching source tag + endpoint = builder.deploy( + endpoint_name="my-endpoint", + instance_type="ml.p4d.24xlarge", + reuse_resources=True, + ) + # If a match is found: returns the existing endpoint + # If no match: creates a new endpoint as normal + +Without ``reuse_resources=True`` (the default), every deploy creates a new endpoint. The +model-source tag is still applied so that future deploys with reuse enabled can discover it. + +.. note:: + + The ``reuse_resources`` flag must be passed to each call independently — it is not + inherited between ``build()`` and ``deploy()``. + + Inference component builds (``modelbuilder_list``) manage their own reuse by component + name and bypass the endpoint-return reuse gate. diff --git a/docs/model_customization/evaluation.rst b/docs/model_customization/evaluation.rst index 81bb83741e..f5e3f6a2c8 100644 --- a/docs/model_customization/evaluation.rst +++ b/docs/model_customization/evaluation.rst @@ -17,3 +17,44 @@ Launch evaluation jobs with the following options: ../../v3-examples/model-customization-examples/custom_scorer_demo ../../v3-examples/model-customization-examples/benchmark_demo ../../v3-examples/nova-examples/evaluation-benchmark-and-custom-scorer + + +Dry-Run Validation +------------------- + +Pass ``dry_run=True`` to ``evaluate()`` to validate your evaluation configuration without +submitting a job or consuming compute. The SDK runs all validation (IAM role resolution, +model resolution, dataset path existence) and then stops before launching the evaluation +pipeline. Returns ``None`` on success, raises ``ValueError`` on validation failure. + +Supported on all evaluators: ``BenchMarkEvaluator``, ``CustomScorerEvaluator``, and +``LLMAsJudgeEvaluator``. + +.. code-block:: python + + from sagemaker.train.evaluate import BenchMarkEvaluator, get_benchmarks + + Benchmark = get_benchmarks() + + evaluator = BenchMarkEvaluator( + benchmark=Benchmark.MMLU, + model="arn:aws:sagemaker:us-east-1:123456789012:model-package/my-models/3", + s3_output_path="s3://my-bucket/eval-output/", + ) + + # Validate without launching — returns None on success + evaluator.evaluate(dry_run=True) + +.. code-block:: python + + from sagemaker.train.evaluate import CustomScorerEvaluator + + evaluator = CustomScorerEvaluator( + model="my-model-package-arn", + evaluation_dataset="s3://my-bucket/eval-data.jsonl", + s3_output_path="s3://my-bucket/custom-eval-output/", + scorer_function=my_scorer, + ) + + # Raises ValueError if dataset path does not exist + evaluator.evaluate(dry_run=True) diff --git a/docs/model_customization/model_customization.rst b/docs/model_customization/model_customization.rst index 3a36a3ff28..c943d343c6 100644 --- a/docs/model_customization/model_customization.rst +++ b/docs/model_customization/model_customization.rst @@ -53,6 +53,33 @@ Key Features with clear precedence. Use ``get_resolved_recipe()`` to inspect the merged configuration before job submission. See :doc:`finetuning_serverful` and :doc:`finetuning_hyperpod` for examples. +**Dry-Run Validation** + Pass ``dry_run=True`` to ``train()`` to run the validation steps without submitting a job + or consuming compute. Returns ``None`` on success, raises ``ValueError`` on validation failure. + + Supported on all trainers (SFT, DPO, RLVR, RLAIF, CPT) and ``ModelTrainer.train()``. + Works across serverless, serverful (``TrainingJobCompute``), and HyperPod + (``HyperPodCompute``) compute modes. Validates S3 URIs, DataSet ARNs, and ``DataSet`` + objects. + + Also available on evaluators — see :doc:`evaluation` for details. + + .. code-block:: python + + from sagemaker.train import SFTTrainer + from sagemaker.train.common import TrainingType + + trainer = SFTTrainer( + model="meta-textgeneration-llama-3-2-1b-instruct", + training_type=TrainingType.LORA, + model_package_group="my-finetuned-models", + training_dataset="s3://my-bucket/train.jsonl", + accept_eula=True, + ) + + # Validate without submitting — returns None on success + trainer.train(dry_run=True) + .. toctree:: :maxdepth: 1 diff --git a/sagemaker-serve/src/sagemaker/serve/bedrock_model_builder.py b/sagemaker-serve/src/sagemaker/serve/bedrock_model_builder.py index aedd622269..6b989c9290 100644 --- a/sagemaker-serve/src/sagemaker/serve/bedrock_model_builder.py +++ b/sagemaker-serve/src/sagemaker/serve/bedrock_model_builder.py @@ -103,9 +103,8 @@ class BedrockModelBuilder: tags each custom model it creates with the model source and, on a subsequent deploy of the same source, reuses the existing custom model (and its active deployment if present) instead of creating duplicates, - logging a warning rather than raising. This matters because Bedrock - custom-model import is slow and consumes the limited imported-model - quota per account/region. + logging a warning rather than raising. To avoid redeploying the same + model artifacts, opt into resource reuse with ``reuse_resources=True``. Args: model: The model to deploy. Can be a ModelTrainer, MultiTurnRLTrainer, diff --git a/v3-examples/model-customization-examples/bedrock-modelbuilder-deployment.ipynb b/v3-examples/model-customization-examples/bedrock-modelbuilder-deployment.ipynb index 9598dd53e0..62b5581e52 100644 --- a/v3-examples/model-customization-examples/bedrock-modelbuilder-deployment.ipynb +++ b/v3-examples/model-customization-examples/bedrock-modelbuilder-deployment.ipynb @@ -275,7 +275,7 @@ "source": [ "## Reusing an Existing Custom Model\n", "\n", - "Importing a custom model into Bedrock is slow and consumes the limited imported-model quota per account/region. If you redeploy the *same* model artifacts, you can avoid a duplicate import by opting into resource reuse with `reuse_resources=True`.\n", + "To avoid redeploying the *same* model artifacts, you can opt into resource reuse with `reuse_resources=True`.\n", "\n", "When enabled, `BedrockModelBuilder` tags each custom model it creates with the model source and, on a later deploy of the same source, reuses the existing custom model (and its active deployment if one exists) instead of creating duplicates. It logs a warning rather than raising, and the returned response includes the `modelArn` that was reused.\n", "\n",