Updates garbage collection for orphaned ReleasePayloads to use coordinated deletion via remove__ prefixed tags in quay.io

Author: deepsm007

cmd/release-controller/sync.go

@@ -490,8 +490,14 @@ func (c *Controller) syncReady(release *releasecontroller.Release) error {
 		mirror, err := releasecontroller.GetMirror(release, releaseTag.Name, c.releaseLister)
 		if err != nil {
 			klog.Errorf("Failed to identify `from` mirror for creation of release mirror job: %v", err)
-		} else if _, err := c.ensureReleaseMirrorJob(release, releaseTag.Name, mirror); err != nil {
-			klog.Errorf("Failed to create release mirror job: %v", err)
+		} else {
+			if _, err := c.ensureReleaseMirrorJob(release, releaseTag.Name, mirror); err != nil {
+				klog.Errorf("Failed to create release mirror job: %v", err)
+			}
+			// For CI payloads, also create rc_payload__ tag for pruner to preserve component images
+			if _, err := c.ensureRCPayloadTagJob(release, releaseTag.Name, mirror); err != nil {
+				klog.Errorf("Failed to create rc_payload__ tag job: %v", err)
+			}
 		}
 
 		if err := c.ensureReleaseUpgradeJobs(release, releaseTag); err != nil {

cmd/release-controller/sync_gc.go

@@ -122,11 +122,27 @@ func (c *Controller) garbageCollectSync() error {
 		}
 	}
 
-	// all releasepayloads created for releases that no longer exist should be deleted
+	// all releasepayloads created for releases that no longer exist should be garbage collected
 	for _, payload := range payloads {
 		if active.Has(payload.Name) {
 			continue
 		}
+
+		// Get the release config from the ImageStream to check if alternate repository is configured
+		imageStream, err := c.releaseLister.ImageStreams(payload.Spec.PayloadCoordinates.Namespace).Get(payload.Spec.PayloadCoordinates.ImagestreamName)
+		if err == nil {
+			release, ok, err := releasecontroller.ReleaseDefinition(imageStream, c.parsedReleaseConfigCache, c.eventRecorder, *c.releaseLister)
+			if err == nil && ok && len(release.Config.AlternateImageRepository) > 0 && len(release.Config.AlternateImageRepositorySecretName) > 0 {
+				_, err := c.ensureRemoveTagJob(payload, release)
+				if err != nil {
+					klog.V(2).Infof("Failed to create remove tag job for releasepayload %s/%s: %v, proceeding with direct deletion", payload.Namespace, payload.Name, err)
+				} else {
+					klog.V(2).Infof("Created remove tag job for orphaned releasepayload %s/%s, pruner will handle quay.io tag deletion", payload.Namespace, payload.Name)
+				}
+			}
+		}
+
+		// Delete the ReleasePayload
 		klog.V(2).Infof("Removing orphaned releasepayload %s/%s", payload.Namespace, payload.Name)
 		if err := c.releasePayloadClient.ReleasePayloads(payload.Namespace).Delete(context.TODO(), payload.Name, metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
 			utilruntime.HandleError(fmt.Errorf("can't delete orphaned releasepayload %s/%s: %v", payload.Namespace, payload.Name, err))

cmd/release-controller/sync_release.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"strings"
 	"time"
 
+	"github.com/openshift/release-controller/pkg/apis/release/v1alpha1"
 	releasecontroller "github.com/openshift/release-controller/pkg/release-controller"
 
 	batchv1 "k8s.io/api/batch/v1"
@@ -417,3 +419,95 @@ func (c *Controller) ensureReleaseMirrorJob(release *releasecontroller.Release,
 func releaseMirrorJobName(tagName string) string {
 	return fmt.Sprintf("%s-alternate-mirror", tagName)
 }
+
+// ensureRCPayloadTagJob creates a job to mirror the release to rc_payload__{version} in quay.io for CI payloads
+// This is required for the pruner to preserve component images referenced by CI payloads
+func (c *Controller) ensureRCPayloadTagJob(release *releasecontroller.Release, name string, mirror *imagev1.ImageStream) (*batchv1.Job, error) {
+	if len(release.Config.AlternateImageRepository) == 0 || len(release.Config.AlternateImageRepositorySecretName) == 0 {
+		return nil, fmt.Errorf("alternate repository or secret not configured")
+	}
+
+	// Only create rc_payload__ tags for CI payloads (tags containing .ci-)
+	// Nightly payloads reference ART images that are permanently stored
+	if !strings.Contains(name, ".ci-") {
+		return nil, nil
+	}
+
+	jobName := fmt.Sprintf("%s-rc-payload-tag", name)
+	return c.ensureJob(jobName, nil, func() (*batchv1.Job, error) {
+		fromImage := fmt.Sprintf("%s:%s", release.Target.Status.PublicDockerImageRepository, name)
+		rcPayloadTag := fmt.Sprintf("rc_payload__%s", name)
+		toImage := fmt.Sprintf("%s:%s", release.Config.AlternateImageRepository, rcPayloadTag)
+
+		cliImage := fmt.Sprintf("%s:cli", mirror.Status.DockerImageRepository)
+		if len(release.Config.OverrideCLIImage) > 0 {
+			cliImage = release.Config.OverrideCLIImage
+		}
+
+		job, prefix := newReleaseJobBase(jobName, cliImage, release.Config.AlternateImageRepositorySecretName)
+
+		manifestListMode := "false"
+		if c.manifestListMode && !release.Config.DisableManifestListMode {
+			manifestListMode = "true"
+		}
+
+		job.Spec.Template.Spec.Containers[0].Command = []string{
+			"/bin/bash", "-c",
+			prefix + `
+			oc image mirror --keep-manifest-list=$1 $2 $3
+			`,
+			"",
+			manifestListMode, fromImage, toImage,
+		}
+
+		job.Annotations[releasecontroller.ReleaseAnnotationSource] = mirror.Annotations[releasecontroller.ReleaseAnnotationSource]
+		job.Annotations[releasecontroller.ReleaseAnnotationTarget] = mirror.Annotations[releasecontroller.ReleaseAnnotationTarget]
+		job.Annotations[releasecontroller.ReleaseAnnotationGeneration] = strconv.FormatInt(release.Target.Generation, 10)
+		job.Annotations[releasecontroller.ReleaseAnnotationReleaseTag] = mirror.Annotations[releasecontroller.ReleaseAnnotationReleaseTag]
+
+		klog.V(2).Infof("Creating rc_payload__ tag job %s/%s to mirror %s to %s", c.jobNamespace, job.Name, fromImage, toImage)
+		return job, nil
+	})
+}
+
+// ensureRemoveTagJob creates a job to copy the release tag to remove__rc_payload__{version} in quay.io
+func (c *Controller) ensureRemoveTagJob(payload *v1alpha1.ReleasePayload, release *releasecontroller.Release) (*batchv1.Job, error) {
+	if len(release.Config.AlternateImageRepository) == 0 || len(release.Config.AlternateImageRepositorySecretName) == 0 {
+		return nil, fmt.Errorf("alternate repository or secret not configured")
+	}
+
+	jobName := fmt.Sprintf("%s-remove-tag", payload.Name)
+	return c.ensureJob(jobName, nil, func() (*batchv1.Job, error) {
+		// Get cli image from mirror or config
+		cliImage := "registry.ci.openshift.org/ocp/4.21:cli"
+		if mirror, err := c.releaseLister.ImageStreams(release.Target.Namespace).Get(release.Target.Name); err == nil {
+			cliImage = fmt.Sprintf("%s:cli", mirror.Status.DockerImageRepository)
+		}
+		if len(release.Config.OverrideCLIImage) > 0 {
+			cliImage = release.Config.OverrideCLIImage
+		}
+
+		job, prefix := newReleaseJobBase(jobName, cliImage, release.Config.AlternateImageRepositorySecretName)
+
+		// Mirror from the actual release tag (which exists in quay.io) to the removal request tag
+		// The pruner only cares about the tag name, not the content
+		removeTag := fmt.Sprintf("remove__rc_payload__%s", payload.Name)
+		fromImage := fmt.Sprintf("%s:%s", release.Config.AlternateImageRepository, payload.Name)
+		toImage := fmt.Sprintf("%s:%s", release.Config.AlternateImageRepository, removeTag)
+
+		job.Spec.Template.Spec.Containers[0].Command = []string{
+			"/bin/bash", "-c",
+			prefix + `
+			oc image mirror --keep-manifest-list=true $1 $2
+			`,
+			"",
+			fromImage, toImage,
+		}
+
+		job.Annotations[releasecontroller.ReleaseAnnotationReleaseTag] = payload.Name
+		job.Annotations[releasecontroller.ReleaseAnnotationTarget] = fmt.Sprintf("%s/%s", payload.Spec.PayloadCoordinates.Namespace, payload.Spec.PayloadCoordinates.ImagestreamName)
+
+		klog.V(2).Infof("Creating remove tag job %s/%s to copy %s to %s", c.jobNamespace, job.Name, fromImage, toImage)
+		return job, nil
+	})
+}

pkg/release-controller/types.go

@@ -152,7 +152,7 @@ type ReleaseConfig struct {
 	// AlternateImageRepository is the full path to an external Image Repository where we
 	// will mirror Accepted releases to.
 	// For example:
-	//   "alternateImageRepository": "quay.io/openshift-release-dev/dev-release"
+	//   "alternateImageRepository": "quay.io/openshift/ci"
 	AlternateImageRepository string `json:"alternateImageRepository"`
 
 	// AlternateImageRepositorySecret is the name of the secret containing credentials to the