diff --git a/README.md b/README.md index 6622a9a..77b722c 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,11 @@ vsi_user_data | string | Optional | User data to be made available when setting vsi_boot_vol_capacity | string | Optional | The capacity to use for the volume (in gigabytes). Must be at least the image's minimum_provisioned_size. The maximum value may increase in the future. vsi_boot_vol_profile | string | Optional | User can provide the available profile for the boot volume. Supported profiles: `general-purpose`, `5iops-tier`, `10iops-tier`, `sdp`, `custom`. Refer https://cloud.ibm.com/docs/vpc?topic=vpc-block-storage-profiles&interface=ui for profile info. Requires `vsi_boot_vol_capacity` to be set, except when creating from a snapshot (`vsi_boot_snapshot_id`), where the restored volume inherits the snapshot's size if no capacity is given. Cannot be combined with `vsi_boot_volume_id` (an existing volume keeps its own profile). vsi_boot_vol_iops | number | Optional | The maximum I/O operations per second (IOPS) for the boot volume. Only honored when `vsi_boot_vol_profile` is `custom` or `sdp`; the tiered profiles derive IOPS from capacity. Must be within the chosen profile's range for the volume size (enforced by IBM Cloud). Cannot be combined with `vsi_boot_volume_id`. -vsi_boot_vol_bandwidth | number | Optional | The maximum bandwidth (in megabits per second) for the boot volume. Only honored when `vsi_boot_vol_profile` is `custom` or `sdp`. If unset, IBM Cloud assigns a default for the profile. Cannot be combined with `vsi_boot_volume_id`. +vsi_boot_vol_bandwidth | number | Optional | The maximum bandwidth (in megabits per second) for the boot volume. Only honored when `vsi_boot_vol_profile` is `sdp`. If unset, IBM Cloud assigns a default for the profile. Cannot be combined with `vsi_boot_volume_id`. +vsi_data_vol_capacity | number | Optional | Capacity (in gigabytes, 10–32000) of an ephemeral scratch data volume attached to the builder instance. The volume is created with the instance and deleted with it, and is **never part of the captured image** (the image is captured from the boot volume only). Use it to keep large transient build artifacts — package/module caches, downloads, build trees — off the boot volume so they are not exported at image-capture time. Mount it in a provisioner (the disk appears as an unformatted block device) and point your cache/build directories at it; if you do not mount it and redirect writes to it, it has no effect on the captured image. If unset, no data volume is attached. +vsi_data_vol_profile | string | Optional | Profile for the data volume. Supported profiles: `general-purpose`, `5iops-tier`, `10iops-tier`, `sdp`, `custom`. Requires `vsi_data_vol_capacity` to be set. Defaults to `general-purpose`. +vsi_data_vol_iops | number | Optional | The maximum I/O operations per second (IOPS) for the data volume. Only honored when `vsi_data_vol_profile` is `custom` or `sdp`; the tiered profiles derive IOPS from capacity. Must be within the chosen profile's range for the volume size (enforced by IBM Cloud). +vsi_data_vol_bandwidth | number | Optional | The maximum bandwidth (in megabits per second) for the data volume. Only honored when `vsi_data_vol_profile` is `sdp`. If unset, IBM Cloud assigns a default for the profile. image_name | string | Optional | The name of the resulting custom image that will appear in your account. Required. encryption_key_crn | string | Optional | The CRN of the [Key Protect Root Key](https://cloud.ibm.com/docs/key-protect?topic=key-protect-getting-started-tutorial) or [Hyper Protect Crypto Services Root Key](https://cloud.ibm.com/docs/hs-crypto?topic=hs-crypto-get-started) for this resource. communicator | string | Required | Communicators are the mechanism Packer uses to upload files, execute scripts, etc. with the machine being created. Choose between "ssh" (for Linux) and "winrm" (for Windows). Required. diff --git a/builder/ibmcloud/vpc/config.go b/builder/ibmcloud/vpc/config.go index 64ed8da..d4d2702 100644 --- a/builder/ibmcloud/vpc/config.go +++ b/builder/ibmcloud/vpc/config.go @@ -43,6 +43,10 @@ type Config struct { VSIBootBandwidth int `mapstructure:"vsi_boot_vol_bandwidth"` VSIBootVolumeID string `mapstructure:"vsi_boot_volume_id"` VSIBootSnapshotID string `mapstructure:"vsi_boot_snapshot_id"` + VSIDataCapacity int `mapstructure:"vsi_data_vol_capacity"` + VSIDataProfile string `mapstructure:"vsi_data_vol_profile"` + VSIDataIops int `mapstructure:"vsi_data_vol_iops"` + VSIDataBandwidth int `mapstructure:"vsi_data_vol_bandwidth"` VSIProfile string `mapstructure:"vsi_profile"` VSIInterface string `mapstructure:"vsi_interface"` VSIUserDataFile string `mapstructure:"vsi_user_data_file"` @@ -123,12 +127,14 @@ func (c *Config) Prepare(raws ...interface{}) ([]string, error) { if c.VSIBootProfile != "" && !slices.Contains(allowedBootProfiles, c.VSIBootProfile) { errs = packer.MultiErrorAppend(errs, errors.New("vsi_boot_vol_profile must be one of: general-purpose, 5iops-tier, 10iops-tier, sdp, custom")) } - // iops/bandwidth are only honored by the custom and sdp profiles; the tiered - // profiles derive them from capacity. This validation is the single source of - // truth for that rule (the bootVolumePrototype helpers do not re-enforce it). - customOrSdp := c.VSIBootProfile == "custom" || c.VSIBootProfile == "sdp" - if (c.VSIBootIops != 0 || c.VSIBootBandwidth != 0) && !customOrSdp { - errs = packer.MultiErrorAppend(errs, errors.New("vsi_boot_vol_iops/vsi_boot_vol_bandwidth require vsi_boot_vol_profile to be 'custom' or 'sdp'")) + // iops is honored by the custom and sdp profiles; bandwidth only by sdp; the + // tiered profiles derive both from capacity. This validation is the single + // source of truth for that rule (the bootVolumePrototype helpers do not re-enforce it). + if c.VSIBootIops != 0 && c.VSIBootProfile != "custom" && c.VSIBootProfile != "sdp" { + errs = packer.MultiErrorAppend(errs, errors.New("vsi_boot_vol_iops requires vsi_boot_vol_profile to be 'custom' or 'sdp'")) + } + if c.VSIBootBandwidth != 0 && c.VSIBootProfile != "sdp" { + errs = packer.MultiErrorAppend(errs, errors.New("vsi_boot_vol_bandwidth requires vsi_boot_vol_profile to be 'sdp'")) } bootVolumeTuned := c.VSIBootProfile != "" || c.VSIBootIops != 0 || c.VSIBootBandwidth != 0 // The by-image and catalog-offering paths only attach a boot volume (and thus @@ -149,6 +155,37 @@ func (c *Config) Prepare(raws ...interface{}) ([]string, error) { errs = packer.MultiErrorAppend(errs, errors.New("vsi_boot_vol_iops and vsi_boot_vol_bandwidth must not be negative")) } + // Data-volume validation mirrors the boot-volume rules above. The data volume + // is an ephemeral scratch disk attached to the builder VSI and deleted with it + // (DeleteVolumeOnInstanceDelete); it never enters the captured image, which is + // taken from the boot volume only. It lets a build keep large transient writes + // (build caches, downloads) off the boot volume so they are not exported at + // capture time. + if c.VSIDataCapacity != 0 && (c.VSIDataCapacity < 10 || c.VSIDataCapacity > 32000) { + errs = packer.MultiErrorAppend(errs, errors.New("data capacity out of bound: provide a valid capacity between 10 and 32000")) + } + allowedDataProfiles := []string{"general-purpose", "5iops-tier", "10iops-tier", "sdp", "custom"} + if c.VSIDataProfile != "" && !slices.Contains(allowedDataProfiles, c.VSIDataProfile) { + errs = packer.MultiErrorAppend(errs, errors.New("vsi_data_vol_profile must be one of: general-purpose, 5iops-tier, 10iops-tier, sdp, custom")) + } + // iops is honored by the custom and sdp profiles; bandwidth only by sdp. The + // tiered profiles derive both from capacity, same rule as the boot volume above. + if c.VSIDataIops != 0 && c.VSIDataProfile != "custom" && c.VSIDataProfile != "sdp" { + errs = packer.MultiErrorAppend(errs, errors.New("vsi_data_vol_iops requires vsi_data_vol_profile to be 'custom' or 'sdp'")) + } + if c.VSIDataBandwidth != 0 && c.VSIDataProfile != "sdp" { + errs = packer.MultiErrorAppend(errs, errors.New("vsi_data_vol_bandwidth requires vsi_data_vol_profile to be 'sdp'")) + } + // The data volume is attached only when a capacity is set; without it the + // profile/iops/bandwidth would be silently dropped, so require capacity. + dataVolumeTuned := c.VSIDataProfile != "" || c.VSIDataIops != 0 || c.VSIDataBandwidth != 0 + if dataVolumeTuned && c.VSIDataCapacity == 0 { + errs = packer.MultiErrorAppend(errs, errors.New("vsi_data_vol_profile/vsi_data_vol_iops/vsi_data_vol_bandwidth require vsi_data_vol_capacity to be set")) + } + if c.VSIDataIops < 0 || c.VSIDataBandwidth < 0 { + errs = packer.MultiErrorAppend(errs, errors.New("vsi_data_vol_iops and vsi_data_vol_bandwidth must not be negative")) + } + var oneOfInput int // validation for mutually exclusive fields. if c.VSIBaseImageID != "" { diff --git a/builder/ibmcloud/vpc/config.hcl2spec.go b/builder/ibmcloud/vpc/config.hcl2spec.go index 0ef76de..d5a2c8e 100644 --- a/builder/ibmcloud/vpc/config.hcl2spec.go +++ b/builder/ibmcloud/vpc/config.hcl2spec.go @@ -89,6 +89,10 @@ type FlatConfig struct { VSIBootBandwidth *int `mapstructure:"vsi_boot_vol_bandwidth" cty:"vsi_boot_vol_bandwidth" hcl:"vsi_boot_vol_bandwidth"` VSIBootVolumeID *string `mapstructure:"vsi_boot_volume_id" cty:"vsi_boot_volume_id" hcl:"vsi_boot_volume_id"` VSIBootSnapshotID *string `mapstructure:"vsi_boot_snapshot_id" cty:"vsi_boot_snapshot_id" hcl:"vsi_boot_snapshot_id"` + VSIDataCapacity *int `mapstructure:"vsi_data_vol_capacity" cty:"vsi_data_vol_capacity" hcl:"vsi_data_vol_capacity"` + VSIDataProfile *string `mapstructure:"vsi_data_vol_profile" cty:"vsi_data_vol_profile" hcl:"vsi_data_vol_profile"` + VSIDataIops *int `mapstructure:"vsi_data_vol_iops" cty:"vsi_data_vol_iops" hcl:"vsi_data_vol_iops"` + VSIDataBandwidth *int `mapstructure:"vsi_data_vol_bandwidth" cty:"vsi_data_vol_bandwidth" hcl:"vsi_data_vol_bandwidth"` VSIProfile *string `mapstructure:"vsi_profile" cty:"vsi_profile" hcl:"vsi_profile"` VSIInterface *string `mapstructure:"vsi_interface" cty:"vsi_interface" hcl:"vsi_interface"` VSIUserDataFile *string `mapstructure:"vsi_user_data_file" cty:"vsi_user_data_file" hcl:"vsi_user_data_file"` @@ -201,6 +205,10 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { "vsi_boot_vol_bandwidth": &hcldec.AttrSpec{Name: "vsi_boot_vol_bandwidth", Type: cty.Number, Required: false}, "vsi_boot_volume_id": &hcldec.AttrSpec{Name: "vsi_boot_volume_id", Type: cty.String, Required: false}, "vsi_boot_snapshot_id": &hcldec.AttrSpec{Name: "vsi_boot_snapshot_id", Type: cty.String, Required: false}, + "vsi_data_vol_capacity": &hcldec.AttrSpec{Name: "vsi_data_vol_capacity", Type: cty.Number, Required: false}, + "vsi_data_vol_profile": &hcldec.AttrSpec{Name: "vsi_data_vol_profile", Type: cty.String, Required: false}, + "vsi_data_vol_iops": &hcldec.AttrSpec{Name: "vsi_data_vol_iops", Type: cty.Number, Required: false}, + "vsi_data_vol_bandwidth": &hcldec.AttrSpec{Name: "vsi_data_vol_bandwidth", Type: cty.Number, Required: false}, "vsi_profile": &hcldec.AttrSpec{Name: "vsi_profile", Type: cty.String, Required: false}, "vsi_interface": &hcldec.AttrSpec{Name: "vsi_interface", Type: cty.String, Required: false}, "vsi_user_data_file": &hcldec.AttrSpec{Name: "vsi_user_data_file", Type: cty.String, Required: false}, diff --git a/builder/ibmcloud/vpc/config_test.go b/builder/ibmcloud/vpc/config_test.go index 9f181ee..af06bc6 100644 --- a/builder/ibmcloud/vpc/config_test.go +++ b/builder/ibmcloud/vpc/config_test.go @@ -84,22 +84,25 @@ func TestPrepareBootVolumeProfile(t *testing.T) { } func TestPrepareBootVolumeIopsBandwidth(t *testing.T) { - const wantMsg = "require vsi_boot_vol_profile to be 'custom' or 'sdp'" + // iops is honored by custom and sdp; bandwidth only by sdp. + const iopsMsg = "vsi_boot_vol_iops requires vsi_boot_vol_profile to be 'custom' or 'sdp'" + const bandwidthMsg = "vsi_boot_vol_bandwidth requires vsi_boot_vol_profile to be 'sdp'" cases := []struct { - name string - profile string - iops int - bandwidth int - wantReject bool + name string + profile string + iops int + bandwidth int + wantMsg string // empty => must be accepted }{ - {"iops with sdp", "sdp", 10000, 0, false}, - {"bandwidth with sdp", "sdp", 0, 4000, false}, - {"iops with custom", "custom", 5000, 0, false}, - {"none set", "general-purpose", 0, 0, false}, - {"iops without a profile", "", 5000, 0, true}, - {"iops with a tiered profile", "general-purpose", 5000, 0, true}, - {"bandwidth with a tiered profile", "10iops-tier", 0, 2000, true}, + {"iops with sdp", "sdp", 10000, 0, ""}, + {"bandwidth with sdp", "sdp", 0, 4000, ""}, + {"iops with custom", "custom", 5000, 0, ""}, + {"bandwidth with custom", "custom", 0, 4000, bandwidthMsg}, + {"none set", "general-purpose", 0, 0, ""}, + {"iops without a profile", "", 5000, 0, iopsMsg}, + {"iops with a tiered profile", "general-purpose", 5000, 0, iopsMsg}, + {"bandwidth with a tiered profile", "10iops-tier", 0, 2000, bandwidthMsg}, } for _, tc := range cases { @@ -109,10 +112,10 @@ func TestPrepareBootVolumeIopsBandwidth(t *testing.T) { c.VSIBootIops = tc.iops c.VSIBootBandwidth = tc.bandwidth _, err := c.Prepare() - rejected := err != nil && strings.Contains(err.Error(), wantMsg) - if rejected != tc.wantReject { - t.Errorf("profile=%q iops=%d bandwidth=%d rejected=%v, want %v (err=%v)", - tc.profile, tc.iops, tc.bandwidth, rejected, tc.wantReject, err) + rejected := err != nil && tc.wantMsg != "" && strings.Contains(err.Error(), tc.wantMsg) + if rejected != (tc.wantMsg != "") { + t.Errorf("profile=%q iops=%d bandwidth=%d rejected=%v, want %q (err=%v)", + tc.profile, tc.iops, tc.bandwidth, rejected, tc.wantMsg, err) } }) } @@ -346,3 +349,254 @@ func TestSnapshotBootVolumePrototype(t *testing.T) { } }) } + +func TestPrepareDataVolumeCapacity(t *testing.T) { + const wantMsg = "data capacity out of bound" + + cases := []struct { + name string + capacity int + wantReject bool + }{ + {"zero means no data volume", 0, false}, + {"minimum 10", 10, false}, + {"just below minimum", 9, true}, + {"mid range", 60, false}, + {"maximum 32000", 32000, false}, + {"just above maximum", 32001, true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := validVPCConfig() + c.VSIDataCapacity = tc.capacity + _, err := c.Prepare() + rejected := err != nil && strings.Contains(err.Error(), wantMsg) + if rejected != tc.wantReject { + t.Errorf("VSIDataCapacity=%d rejected=%v, want %v (err=%v)", + tc.capacity, rejected, tc.wantReject, err) + } + }) + } +} + +func TestPrepareDataVolumeProfile(t *testing.T) { + const wantMsg = "vsi_data_vol_profile must be one of" + + cases := []struct { + name string + profile string + wantReject bool + }{ + {"empty uses default", "", false}, + {"general-purpose", "general-purpose", false}, + {"5iops-tier", "5iops-tier", false}, + {"10iops-tier", "10iops-tier", false}, + {"sdp", "sdp", false}, + {"custom", "custom", false}, + {"unknown profile", "platinum-tier", true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := validVPCConfig() + c.VSIDataCapacity = 60 + c.VSIDataProfile = tc.profile + _, err := c.Prepare() + rejected := err != nil && strings.Contains(err.Error(), wantMsg) + if rejected != tc.wantReject { + t.Errorf("profile=%q rejected=%v, want %v (err=%v)", tc.profile, rejected, tc.wantReject, err) + } + }) + } +} + +func TestPrepareDataVolumeIopsBandwidth(t *testing.T) { + // iops is honored by custom and sdp; bandwidth only by sdp. + const iopsMsg = "vsi_data_vol_iops requires vsi_data_vol_profile to be 'custom' or 'sdp'" + const bandwidthMsg = "vsi_data_vol_bandwidth requires vsi_data_vol_profile to be 'sdp'" + + cases := []struct { + name string + profile string + iops int + bandwidth int + wantMsg string // empty => must be accepted + }{ + {"iops with sdp", "sdp", 10000, 0, ""}, + {"bandwidth with sdp", "sdp", 0, 2000, ""}, + {"iops with custom", "custom", 5000, 0, ""}, + {"bandwidth with custom", "custom", 0, 2000, bandwidthMsg}, + {"none set", "general-purpose", 0, 0, ""}, + {"iops without a profile", "", 5000, 0, iopsMsg}, + {"iops with a tiered profile", "general-purpose", 5000, 0, iopsMsg}, + {"bandwidth with a tiered profile", "10iops-tier", 0, 2000, bandwidthMsg}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := validVPCConfig() + c.VSIDataCapacity = 60 + c.VSIDataProfile = tc.profile + c.VSIDataIops = tc.iops + c.VSIDataBandwidth = tc.bandwidth + _, err := c.Prepare() + rejected := err != nil && tc.wantMsg != "" && strings.Contains(err.Error(), tc.wantMsg) + if rejected != (tc.wantMsg != "") { + t.Errorf("profile=%q iops=%d bandwidth=%d rejected=%v, want %q (err=%v)", + tc.profile, tc.iops, tc.bandwidth, rejected, tc.wantMsg, err) + } + }) + } +} + +func TestPrepareDataVolumeRequiresCapacity(t *testing.T) { + const wantMsg = "require vsi_data_vol_capacity to be set" + + cases := []struct { + name string + profile string + iops int + bandwidth int + capacity int + wantReject bool + }{ + {"profile without capacity", "sdp", 0, 0, 0, true}, + {"iops without capacity", "sdp", 10000, 0, 0, true}, + {"bandwidth without capacity", "sdp", 0, 2000, 0, true}, + {"profile with capacity", "sdp", 0, 0, 60, false}, + {"nothing set, no capacity", "", 0, 0, 0, false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := validVPCConfig() + c.VSIDataCapacity = tc.capacity + c.VSIDataProfile = tc.profile + c.VSIDataIops = tc.iops + c.VSIDataBandwidth = tc.bandwidth + _, err := c.Prepare() + rejected := err != nil && strings.Contains(err.Error(), wantMsg) + if rejected != tc.wantReject { + t.Errorf("profile=%q iops=%d bandwidth=%d capacity=%d rejected=%v, want %v (err=%v)", + tc.profile, tc.iops, tc.bandwidth, tc.capacity, rejected, tc.wantReject, err) + } + }) + } +} + +func TestPrepareDataVolumeNegative(t *testing.T) { + const wantMsg = "vsi_data_vol_iops and vsi_data_vol_bandwidth must not be negative" + + cases := []struct { + name string + iops int + bandwidth int + wantReject bool + }{ + {"negative iops", -1, 0, true}, + {"negative bandwidth", 0, -1, true}, + {"positive values", 10000, 2000, false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := validVPCConfig() + c.VSIDataCapacity = 60 + c.VSIDataProfile = "sdp" + c.VSIDataIops = tc.iops + c.VSIDataBandwidth = tc.bandwidth + _, err := c.Prepare() + rejected := err != nil && strings.Contains(err.Error(), wantMsg) + if rejected != tc.wantReject { + t.Errorf("iops=%d bandwidth=%d rejected=%v, want %v (err=%v)", + tc.iops, tc.bandwidth, rejected, tc.wantReject, err) + } + }) + } +} + +func TestDataVolumeAttachments(t *testing.T) { + t.Run("no capacity attaches nothing", func(t *testing.T) { + if att := dataVolumeAttachments(&Config{}); att != nil { + t.Errorf("attachments = %v, want nil when no data volume is configured", att) + } + }) + + t.Run("default profile, deleted with instance, no iops or bandwidth", func(t *testing.T) { + att := dataVolumeAttachments(&Config{VSIDataCapacity: 60}) + if len(att) != 1 { + t.Fatalf("len(attachments) = %d, want 1", len(att)) + } + if att[0].DeleteVolumeOnInstanceDelete == nil || !*att[0].DeleteVolumeOnInstanceDelete { + t.Error("DeleteVolumeOnInstanceDelete should be true so the scratch volume is torn down with the VSI") + } + vol := att[0].Volume.(*vpcv1.VolumeAttachmentPrototypeVolumeVolumePrototypeInstanceContext) + if got := *vol.Profile.(*vpcv1.VolumeProfileIdentity).Name; got != "general-purpose" { + t.Errorf("profile = %q, want general-purpose", got) + } + if got := *vol.Capacity; got != 60 { + t.Errorf("capacity = %d, want 60", got) + } + if vol.Iops != nil { + t.Errorf("Iops = %d, want nil (unset)", *vol.Iops) + } + if vol.Bandwidth != nil { + t.Errorf("Bandwidth = %d, want nil (unset)", *vol.Bandwidth) + } + }) + + // iops and bandwidth are set from two independent blocks, so verify each is + // honored on its own (guards against a copy-paste bug coupling the two). + t.Run("custom profile with iops only", func(t *testing.T) { + att := dataVolumeAttachments(&Config{ + VSIDataCapacity: 60, + VSIDataProfile: "custom", + VSIDataIops: 5000, + }) + vol := att[0].Volume.(*vpcv1.VolumeAttachmentPrototypeVolumeVolumePrototypeInstanceContext) + if got := *vol.Profile.(*vpcv1.VolumeProfileIdentity).Name; got != "custom" { + t.Errorf("profile = %q, want custom", got) + } + if vol.Iops == nil || *vol.Iops != 5000 { + t.Errorf("Iops = %v, want 5000", vol.Iops) + } + if vol.Bandwidth != nil { + t.Errorf("Bandwidth = %d, want nil (unset)", *vol.Bandwidth) + } + }) + + t.Run("sdp profile with bandwidth only", func(t *testing.T) { + att := dataVolumeAttachments(&Config{ + VSIDataCapacity: 60, + VSIDataProfile: "sdp", + VSIDataBandwidth: 2000, + }) + vol := att[0].Volume.(*vpcv1.VolumeAttachmentPrototypeVolumeVolumePrototypeInstanceContext) + if vol.Bandwidth == nil || *vol.Bandwidth != 2000 { + t.Errorf("Bandwidth = %v, want 2000", vol.Bandwidth) + } + if vol.Iops != nil { + t.Errorf("Iops = %d, want nil (unset)", *vol.Iops) + } + }) + + t.Run("sdp profile with iops and bandwidth", func(t *testing.T) { + att := dataVolumeAttachments(&Config{ + VSIDataCapacity: 60, + VSIDataProfile: "sdp", + VSIDataIops: 10000, + VSIDataBandwidth: 2000, + }) + vol := att[0].Volume.(*vpcv1.VolumeAttachmentPrototypeVolumeVolumePrototypeInstanceContext) + if got := *vol.Profile.(*vpcv1.VolumeProfileIdentity).Name; got != "sdp" { + t.Errorf("profile = %q, want sdp", got) + } + if vol.Iops == nil || *vol.Iops != 10000 { + t.Errorf("Iops = %v, want 10000", vol.Iops) + } + if vol.Bandwidth == nil || *vol.Bandwidth != 2000 { + t.Errorf("Bandwidth = %v, want 2000", vol.Bandwidth) + } + }) +} diff --git a/builder/ibmcloud/vpc/step_create_instance.go b/builder/ibmcloud/vpc/step_create_instance.go index 1077009..20485ea 100644 --- a/builder/ibmcloud/vpc/step_create_instance.go +++ b/builder/ibmcloud/vpc/step_create_instance.go @@ -83,6 +83,7 @@ func (step *stepCreateInstance) Run(_ context.Context, state multistep.StateBag) Volume: bootVolumePrototype(&config), } } + instancePrototypeModel.VolumeAttachments = dataVolumeAttachments(&config) instancePrototypeModel.CatalogOffering = catalogOfferingPrototype userDataFilePath := config.VSIUserDataFile @@ -152,6 +153,7 @@ func (step *stepCreateInstance) Run(_ context.Context, state multistep.StateBag) Volume: bootVolumePrototype(&config), } } + instancePrototypeModel.VolumeAttachments = dataVolumeAttachments(&config) userDataFilePath := config.VSIUserDataFile userDataString := config.VSIUserDataString @@ -219,6 +221,7 @@ func (step *stepCreateInstance) Run(_ context.Context, state multistep.StateBag) PrimaryNetworkInterface: networkInterfacePrototypeModel, Zone: zoneIdentityModel, } + instancePrototypeModel.VolumeAttachments = dataVolumeAttachments(&config) userDataFilePath := config.VSIUserDataFile userDataString := config.VSIUserDataString @@ -286,6 +289,7 @@ func (step *stepCreateInstance) Run(_ context.Context, state multistep.StateBag) PrimaryNetworkInterface: networkInterfacePrototypeModel, Zone: zoneIdentityModel, } + instancePrototypeModel.VolumeAttachments = dataVolumeAttachments(&config) userDataFilePath := config.VSIUserDataFile userDataString := config.VSIUserDataString @@ -490,7 +494,8 @@ func bootVolumePrototype(config *Config) *vpcv1.VolumePrototypeInstanceByImageCo Profile: &vpcv1.VolumeProfileIdentity{Name: &profile}, } // iops/bandwidth are passed through whenever set; Config.Prepare is the gate - // that restricts them to the custom/sdp profiles IBM honors them on. + // that restricts iops to the custom/sdp profiles and bandwidth to sdp, the + // profiles IBM honors them on. if config.VSIBootIops != 0 { iops := int64(config.VSIBootIops) vol.Iops = &iops @@ -502,6 +507,46 @@ func bootVolumePrototype(config *Config) *vpcv1.VolumePrototypeInstanceByImageCo return vol } +// dataVolumeAttachments builds the data-volume attachments for the builder VSI, +// or nil when no data volume is configured. The volume is created with the +// instance and DeleteVolumeOnInstanceDelete=true, so it is deleted together with +// the builder VSI when the instance is torn down. It is never part of the +// captured image (capture is taken from the boot volume only), so a build can +// keep large transient writes — build caches, downloads, from-source build +// trees — off the boot volume and out of the exported image. Call this from +// every create path so the data volume is attached regardless of how the builder +// VSI is sourced. +func dataVolumeAttachments(config *Config) []vpcv1.VolumeAttachmentPrototype { + if config.VSIDataCapacity == 0 { + return nil + } + capacity := int64(config.VSIDataCapacity) + profile := "general-purpose" + if config.VSIDataProfile != "" { + profile = config.VSIDataProfile + } + vol := &vpcv1.VolumeAttachmentPrototypeVolumeVolumePrototypeInstanceContext{ + Capacity: &capacity, + Profile: &vpcv1.VolumeProfileIdentity{Name: &profile}, + } + // iops/bandwidth are passed through whenever set; Config.Prepare is the gate + // that restricts iops to the custom/sdp profiles and bandwidth to sdp, the + // profiles IBM honors them on. + if config.VSIDataIops != 0 { + iops := int64(config.VSIDataIops) + vol.Iops = &iops + } + if config.VSIDataBandwidth != 0 { + bandwidth := int64(config.VSIDataBandwidth) + vol.Bandwidth = &bandwidth + } + deleteWithInstance := true + return []vpcv1.VolumeAttachmentPrototype{{ + DeleteVolumeOnInstanceDelete: &deleteWithInstance, + Volume: vol, + }} +} + // snapshotBootVolumePrototype builds the boot volume for the // create-from-snapshot path. It mirrors bootVolumePrototype but for the // snapshot SDK type, which is why the helper cannot be shared. Unlike the @@ -522,7 +567,8 @@ func snapshotBootVolumePrototype(config *Config, sourceSnapshot vpcv1.SnapshotId vol.Capacity = &capacity } // iops/bandwidth are passed through whenever set; Config.Prepare is the gate - // that restricts them to the custom/sdp profiles IBM honors them on. + // that restricts iops to the custom/sdp profiles and bandwidth to sdp, the + // profiles IBM honors them on. if config.VSIBootIops != 0 { iops := int64(config.VSIBootIops) vol.Iops = &iops diff --git a/examples/build.vpc.data.volume.pkr.hcl b/examples/build.vpc.data.volume.pkr.hcl new file mode 100644 index 0000000..f61cca6 --- /dev/null +++ b/examples/build.vpc.data.volume.pkr.hcl @@ -0,0 +1,82 @@ +packer { + required_plugins { + ibmcloud = { + version = ">=v3.0.0" + source = "github.com/IBM/ibmcloud" + } + } +} + +variable "ibm_api_key" { + type = string + default = "" +} + +locals { + timestamp = regex_replace(timestamp(), "[- TZ:]", "") +} + +# Attaches an ephemeral scratch data volume to the builder instance. The data +# volume is deleted with the instance and is NOT part of the captured image +# (the image is captured from the boot volume only). Writing large transient +# build artifacts there — package/module caches, downloads, build trees — +# keeps them off the boot volume so they are not exported at image-capture time, +# which on VPC scales with how much of the boot volume has been written. +source "ibmcloud-vpc" "data-volume" { + api_key = "${var.ibm_api_key}" + region = "us-south" + + subnet_id = "0717-4ad0af5f-8084-469d-a10e-49c444caa312" + resource_group_id = "1984ce401571473492918ea987dd1e6f" + security_group_id = "" + + vsi_base_image_name = "ibm-ubuntu-24-04-amd64" + vsi_profile = "bx2-2x8" + vsi_boot_vol_capacity = 30 + vsi_interface = "public" + image_name = "packer-${local.timestamp}" + + # Scratch data volume for build caches. Deleted with the builder instance; + # never captured into the image. sdp lets you pin IOPS independently of size, + # and is the only profile that also honors a set bandwidth (left default here). + vsi_data_vol_capacity = 60 + vsi_data_vol_profile = "sdp" + vsi_data_vol_iops = 10000 + + communicator = "ssh" + ssh_username = "root" + ssh_port = 22 + ssh_timeout = "15m" + + timeout = "30m" +} + +build { + sources = [ + "source.ibmcloud-vpc.data-volume" + ] + + # Mount the scratch volume and point cache/build directories at it BEFORE the + # heavy provisioners run, so their writes never touch the boot volume. The data + # volume is the disk that does not hold the root filesystem; format and mount + # it, then symlink the directories that accumulate large transient data. This + # assumes exactly one data volume is attached. + provisioner "shell" { + execute_command = "{{.Vars}} bash '{{.Path}}'" + inline = [ + "set -euo pipefail", + "root_disk=/dev/$(lsblk -no PKNAME \"$(findmnt -no SOURCE /)\")", + "data_disk=$(lsblk -dpno NAME | grep -vx \"$root_disk\" | head -1)", + "mkfs.ext4 -q \"$data_disk\"", + "mkdir -p /scratch && mount \"$data_disk\" /scratch", + "for d in /root/.cache /var/cache/apt/archives /tmp/build; do mkdir -p \"/scratch$d\" \"$(dirname \"$d\")\"; rm -rf \"$d\"; ln -s \"/scratch$d\" \"$d\"; done", + ] + } + + provisioner "shell" { + execute_command = "{{.Vars}} bash '{{.Path}}'" + inline = [ + "echo 'Build artifacts written under the relocated cache dirs stay on /scratch and are not captured.'", + ] + } +}