From 6c9972ce9bddeae551a1f0b41344fbf42346ef74 Mon Sep 17 00:00:00 2001 From: Dalia Khater Date: Tue, 19 May 2026 13:12:19 -0500 Subject: [PATCH] rhcos: Add osImageStream support for stream-specific file paths This change enables support for RHEL 10 (coreos10-) and RHEL 9 (coreos-) file path differentiation in machine-os-images container and throughout the agent installer workflow. Changes: - Add OSImageStream field to installConfigOverrides in AgentClusterInstall - Add stream-aware helper functions for file paths in releaseextract.go - Update ReleasePayload interface and BaseIso to accept osImageStream parameter - Update GetMetalArtifact to use osImageStream for FetchCoreOSBuild calls - Add GetOSImageStream() method to AgentManifests following existing patterns - Update getHashFromInstaller() to accept and use osImageStream parameter - Fix test to use explicit OSImageStream value instead of empty string - Rely on SetInstallConfigDefaults() to provide default osImageStream value This implements the foundation for RHEL 10 support in the agent installer by allowing the installer to distinguish between RHEL 9 and RHEL 10 boot artifacts based on the osImageStream annotation. Depends on #10570 which sets the default OSImageStream centrally. --- pkg/asset/agent/image/agentimage.go | 3 +- pkg/asset/agent/image/baseiso.go | 9 ++- pkg/asset/agent/image/ignition.go | 7 ++- .../agent/image/unconfigured_ignition.go | 4 +- pkg/asset/agent/manifests/agent.go | 33 +++++++++++ .../agent/manifests/agentclusterinstall.go | 8 +++ pkg/asset/imagebased/image/baseiso.go | 4 +- pkg/asset/rhcos/iso.go | 24 ++++---- pkg/asset/rhcos/iso_test.go | 8 ++- pkg/asset/rhcos/releaseextract.go | 57 +++++++++++++------ pkg/infrastructure/baremetal/bootstrap.go | 12 ++-- 11 files changed, 124 insertions(+), 45 deletions(-) diff --git a/pkg/asset/agent/image/agentimage.go b/pkg/asset/agent/image/agentimage.go index e481492a474..c0255e93422 100644 --- a/pkg/asset/agent/image/agentimage.go +++ b/pkg/asset/agent/image/agentimage.go @@ -106,7 +106,8 @@ func (a *AgentImage) Generate(ctx context.Context, dependencies asset.Parents) e logrus.Debugf("Using custom rootfs URL: %s", a.rootFSURL) } else { // Default to the URL from the RHCOS streams file - defaultRootFSURL, err := baseIso.getRootFSURL(ctx, a.cpuArch) + osImageStream := agentManifests.GetOSImageStream() + defaultRootFSURL, err := baseIso.getRootFSURL(ctx, a.cpuArch, osImageStream) if err != nil { return err } diff --git a/pkg/asset/agent/image/baseiso.go b/pkg/asset/agent/image/baseiso.go index e6581145357..78d3145f290 100644 --- a/pkg/asset/agent/image/baseiso.go +++ b/pkg/asset/agent/image/baseiso.go @@ -36,8 +36,8 @@ func (i *BaseIso) Name() string { } // Fetch RootFS URL using the rhcos.json. -func (i *BaseIso) getRootFSURL(ctx context.Context, archName string) (string, error) { - metal, err := rhcos.GetMetalArtifact(ctx, archName) +func (i *BaseIso) getRootFSURL(ctx context.Context, archName string, osImageStream types.OSImageStream) (string, error) { + metal, err := rhcos.GetMetalArtifact(ctx, archName, osImageStream) if err != nil { return "", err } @@ -69,8 +69,11 @@ func (i *BaseIso) Generate(ctx context.Context, dependencies asset.Parents) erro clusterInfo := &joiner.ClusterInfo{} dependencies.Get(agentManifests, registriesConf, agentWorkflow, clusterInfo) + // Extract osImageStream from AgentClusterInstall annotation + osImageStream := agentManifests.GetOSImageStream() + baseIsoFileName, err := rhcos.NewBaseISOFetcher( - i.getRelease(agentManifests, registriesConf.MirrorConfig)).GetBaseISOFilename(ctx, agentManifests.InfraEnv.Spec.CpuArchitecture) + i.getRelease(agentManifests, registriesConf.MirrorConfig), osImageStream).GetBaseISOFilename(ctx, agentManifests.InfraEnv.Spec.CpuArchitecture) if err == nil { logrus.Debugf("Using base ISO image %s", baseIsoFileName) diff --git a/pkg/asset/agent/image/ignition.go b/pkg/asset/agent/image/ignition.go index 5095b1ed75b..1727b672636 100644 --- a/pkg/asset/agent/image/ignition.go +++ b/pkg/asset/agent/image/ignition.go @@ -269,7 +269,8 @@ func (a *Ignition) Generate(ctx context.Context, dependencies asset.Parents) err infraEnvID := infraEnvAsset.ID logrus.Debug("Generated random infra-env id ", infraEnvID) - osImage, err := getOSImagesInfo(ctx, archName, openshiftVersion) + osImageStream := agentManifests.GetOSImageStream() + osImage, err := getOSImagesInfo(ctx, archName, openshiftVersion, osImageStream) if err != nil { return err } @@ -745,13 +746,13 @@ func addExtraManifests(config *igntypes.Config, extraManifests *manifests.ExtraM return nil } -func getOSImagesInfo(ctx context.Context, cpuArch string, openshiftVersion string) (*models.OsImage, error) { +func getOSImagesInfo(ctx context.Context, cpuArch string, openshiftVersion string, osImageStream types.OSImageStream) (*models.OsImage, error) { osImage := &models.OsImage{ CPUArchitecture: &cpuArch, } osImage.OpenshiftVersion = &openshiftVersion - artifacts, err := rhcos.GetMetalArtifact(ctx, cpuArch) + artifacts, err := rhcos.GetMetalArtifact(ctx, cpuArch, osImageStream) if err != nil { return nil, err } diff --git a/pkg/asset/agent/image/unconfigured_ignition.go b/pkg/asset/agent/image/unconfigured_ignition.go index 5f8698a3c2c..0655dbce54e 100644 --- a/pkg/asset/agent/image/unconfigured_ignition.go +++ b/pkg/asset/agent/image/unconfigured_ignition.go @@ -22,6 +22,7 @@ import ( "github.com/openshift/installer/pkg/asset/agent/workflow" "github.com/openshift/installer/pkg/asset/ignition" "github.com/openshift/installer/pkg/asset/ignition/bootstrap" + "github.com/openshift/installer/pkg/rhcos" "github.com/openshift/installer/pkg/types" agenttypes "github.com/openshift/installer/pkg/types/agent" "github.com/openshift/installer/pkg/version" @@ -155,7 +156,8 @@ func (a *UnconfiguredIgnition) Generate(ctx context.Context, dependencies asset. if err != nil { return err } - osImage, err := getOSImagesInfo(ctx, archName, openshiftVersion) + // Use default OS image stream for unconfigured ignition workflow + osImage, err := getOSImagesInfo(ctx, archName, openshiftVersion, rhcos.DefaultOSImageStream) if err != nil { return err } diff --git a/pkg/asset/agent/manifests/agent.go b/pkg/asset/agent/manifests/agent.go index 55566272112..bf94d72a8ac 100644 --- a/pkg/asset/agent/manifests/agent.go +++ b/pkg/asset/agent/manifests/agent.go @@ -2,10 +2,12 @@ package manifests import ( "context" + "encoding/json" "fmt" "reflect" "github.com/pkg/errors" + "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/validation/field" @@ -16,6 +18,8 @@ import ( "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/agent/workflow" workflowreport "github.com/openshift/installer/pkg/asset/agent/workflow/report" + "github.com/openshift/installer/pkg/rhcos" + "github.com/openshift/installer/pkg/types" ) const ( @@ -118,6 +122,35 @@ func (m *AgentManifests) GetPullSecretData() string { return m.PullSecret.StringData[".dockerconfigjson"] } +// GetOSImageStream extracts the osImageStream from the AgentClusterInstall +// installConfigOverrides annotation, or returns the default if not present. +func (m *AgentManifests) GetOSImageStream() types.OSImageStream { + if m.AgentClusterInstall == nil { + return rhcos.DefaultOSImageStream + } + + if m.AgentClusterInstall.Annotations == nil { + return rhcos.DefaultOSImageStream + } + + overridesJSON, ok := m.AgentClusterInstall.Annotations[installConfigOverrides] + if !ok { + return rhcos.DefaultOSImageStream + } + + var overrides agentClusterInstallInstallConfigOverrides + if err := json.Unmarshal([]byte(overridesJSON), &overrides); err != nil { + logrus.Debugf("Failed to parse installConfigOverrides: %v", err) + return rhcos.DefaultOSImageStream + } + + if overrides.OSImageStream == nil || *overrides.OSImageStream == "" { + return rhcos.DefaultOSImageStream + } + + return *overrides.OSImageStream +} + func (m *AgentManifests) finish() error { if err := m.validateAgentManifests().ToAggregate(); err != nil { return errors.Wrapf(err, "invalid agent configuration") diff --git a/pkg/asset/agent/manifests/agentclusterinstall.go b/pkg/asset/agent/manifests/agentclusterinstall.go index eef1c71b10c..f35e8cd119a 100644 --- a/pkg/asset/agent/manifests/agentclusterinstall.go +++ b/pkg/asset/agent/manifests/agentclusterinstall.go @@ -29,6 +29,7 @@ import ( "github.com/openshift/installer/pkg/asset/agent/agentconfig" "github.com/openshift/installer/pkg/asset/agent/workflow" "github.com/openshift/installer/pkg/ipnet" + "github.com/openshift/installer/pkg/rhcos" "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/types/baremetal" "github.com/openshift/installer/pkg/types/defaults" @@ -131,6 +132,8 @@ type agentClusterInstallInstallConfigOverrides struct { FeatureSet configv1.FeatureSet `json:"featureSet,omitempty"` // Allow override of FeatureGates FeatureGates []string `json:"featureGates,omitempty"` + // OSImageStream is the OS Image Stream to be used for all machines in the cluster + OSImageStream *types.OSImageStream `json:"osImageStream,omitempty"` } var _ asset.WritableAsset = (*AgentClusterInstall)(nil) @@ -397,6 +400,11 @@ func (a *AgentClusterInstall) Generate(_ context.Context, dependencies asset.Par icOverrides.AdditionalTrustBundlePolicy = installConfig.Config.AdditionalTrustBundlePolicy } + if installConfig.Config.OSImageStream != rhcos.DefaultOSImageStream && installConfig.Config.OSImageStream != "" { + icOverridden = true + icOverrides.OSImageStream = &installConfig.Config.OSImageStream + } + if icOverridden { overrides, err := json.Marshal(icOverrides) if err != nil { diff --git a/pkg/asset/imagebased/image/baseiso.go b/pkg/asset/imagebased/image/baseiso.go index 09314b23407..af2ae9bc48f 100644 --- a/pkg/asset/imagebased/image/baseiso.go +++ b/pkg/asset/imagebased/image/baseiso.go @@ -10,6 +10,7 @@ import ( "github.com/openshift/installer/pkg/asset" assetrhcos "github.com/openshift/installer/pkg/asset/rhcos" + "github.com/openshift/installer/pkg/rhcos" "github.com/openshift/installer/pkg/rhcos/cache" "github.com/openshift/installer/pkg/types" ) @@ -80,7 +81,8 @@ func (i *BaseIso) Load(f asset.FileFetcher) (bool, error) { // Download the RHCOS base ISO via rhcos.json. func (i *BaseIso) downloadBaseIso(ctx context.Context, archName string) (string, error) { - metal, err := assetrhcos.GetMetalArtifact(ctx, archName) + // Use default OS image stream for image-based installer + metal, err := assetrhcos.GetMetalArtifact(ctx, archName, rhcos.DefaultOSImageStream) if err != nil { return "", err } diff --git a/pkg/asset/rhcos/iso.go b/pkg/asset/rhcos/iso.go index a38f86f38a9..72d854b0100 100644 --- a/pkg/asset/rhcos/iso.go +++ b/pkg/asset/rhcos/iso.go @@ -22,14 +22,16 @@ import ( // BaseIso generates the base ISO file for the image. type BaseIso struct { - ocRelease ReleasePayload + ocRelease ReleasePayload + osImageStream types.OSImageStream } // NewBaseISOFetcher returns a struct that can be used to fetch a base ISO using -// the default method. -func NewBaseISOFetcher(ocRelease ReleasePayload) *BaseIso { +// the default method with the specified OS image stream. +func NewBaseISOFetcher(ocRelease ReleasePayload, osImageStream types.OSImageStream) *BaseIso { return &BaseIso{ - ocRelease: ocRelease, + ocRelease: ocRelease, + osImageStream: osImageStream, } } @@ -52,12 +54,12 @@ func (i *BaseIso) GetBaseISOFilename(ctx context.Context, arch string) (baseIsoF } // GetMetalArtifact returns the CoreOS metal artifacts for a given arch -// using the embedded stream metadata. -func GetMetalArtifact(ctx context.Context, archName string) (stream.PlatformArtifacts, error) { +// using the embedded stream metadata with the specified stream. +func GetMetalArtifact(ctx context.Context, archName string, osImageStream types.OSImageStream) (stream.PlatformArtifacts, error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - st, err := rhcos.FetchCoreOSBuild(ctx, rhcos.DefaultOSImageStream) + st, err := rhcos.FetchCoreOSBuild(ctx, osImageStream) if err != nil { return stream.PlatformArtifacts{}, err } @@ -77,7 +79,7 @@ func GetMetalArtifact(ctx context.Context, archName string) (stream.PlatformArti // Download the ISO using the URL in rhcos.json. func (i *BaseIso) downloadIso(ctx context.Context, archName string) (string, error) { - metal, err := GetMetalArtifact(ctx, archName) + metal, err := GetMetalArtifact(ctx, archName, i.osImageStream) if err != nil { return "", err } @@ -101,14 +103,14 @@ func (i *BaseIso) checkReleasePayloadBaseISOVersion(ctx context.Context, r Relea logrus.Debugf("Checking release payload base ISO version") // Get current release payload CoreOS version - payloadRelease, err := r.GetBaseIsoVersion(archName) + payloadRelease, err := r.GetBaseIsoVersion(archName, i.osImageStream) if err != nil { logrus.Warnf("unable to determine base ISO version: %s", err.Error()) return } // Get pinned version from installer - metal, err := GetMetalArtifact(ctx, archName) + metal, err := GetMetalArtifact(ctx, archName, i.osImageStream) if err != nil { logrus.Warnf("unable to determine base ISO version: %s", err.Error()) return @@ -133,7 +135,7 @@ func (i *BaseIso) retrieveBaseIso(ctx context.Context, archName string) (string, if err := workflowreport.GetReport(ctx).SubStage(workflow.StageFetchBaseISOExtract); err != nil { return "", err } - baseIsoFileName, err := i.ocRelease.GetBaseIso(archName) + baseIsoFileName, err := i.ocRelease.GetBaseIso(archName, i.osImageStream) if err == nil { if err := workflowreport.GetReport(ctx).SubStage(workflow.StageFetchBaseISOVerify); err != nil { return "", err diff --git a/pkg/asset/rhcos/iso_test.go b/pkg/asset/rhcos/iso_test.go index 5375024d0d7..34fc5e4a6b2 100644 --- a/pkg/asset/rhcos/iso_test.go +++ b/pkg/asset/rhcos/iso_test.go @@ -10,6 +10,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/openshift/installer/pkg/types" ) func TestBaseIso(t *testing.T) { @@ -69,7 +71,7 @@ func TestBaseIso(t *testing.T) { isoBaseVersion: ocReleaseImage, baseIsoFileName: ocBaseIsoFilename, baseIsoError: tc.getIsoError, - }) + }, types.OSImageStreamRHCOS9) filename, err := fetcher.GetBaseISOFilename(context.Background(), "") if tc.expectedError == "" { @@ -88,14 +90,14 @@ type mockRelease struct { baseIsoError error } -func (m *mockRelease) GetBaseIso(architecture string) (string, error) { +func (m *mockRelease) GetBaseIso(architecture string, osImageStream types.OSImageStream) (string, error) { if m.baseIsoError != nil { return "", m.baseIsoError } return m.baseIsoFileName, nil } -func (m *mockRelease) GetBaseIsoVersion(architecture string) (string, error) { +func (m *mockRelease) GetBaseIsoVersion(architecture string, osImageStream types.OSImageStream) (string, error) { return m.isoBaseVersion, nil } diff --git a/pkg/asset/rhcos/releaseextract.go b/pkg/asset/rhcos/releaseextract.go index 4f8d0f6a640..3b0b58df2de 100644 --- a/pkg/asset/rhcos/releaseextract.go +++ b/pkg/asset/rhcos/releaseextract.go @@ -28,16 +28,38 @@ import ( ) const ( - machineOsImageName = "machine-os-images" - coreOsFileName = "/coreos/coreos-%s.iso" - coreOsSha256FileName = "/coreos/coreos-%s.iso.sha256" - coreOsStreamFileName = "/coreos/coreos-stream.json" + machineOsImageName = "machine-os-images" // ocDefaultTries is the number of times to execute the oc command on failures. ocDefaultTries = 5 // ocDefaultRetryDelay is the time between retries. ocDefaultRetryDelay = time.Second * 5 ) +// getCoreOsFileName returns the ISO file path in machine-os-images based on stream. +func getCoreOsFileName(stream types.OSImageStream, architecture string) string { + // rhel-10 uses coreos10- prefix, rhel-9 uses coreos- prefix + if stream == types.OSImageStreamRHCOS10 { + return fmt.Sprintf("/coreos/coreos10-%s.iso", architecture) + } + return fmt.Sprintf("/coreos/coreos-%s.iso", architecture) +} + +// getCoreOsSha256FileName returns the ISO checksum file path based on stream. +func getCoreOsSha256FileName(stream types.OSImageStream, architecture string) string { + if stream == types.OSImageStreamRHCOS10 { + return fmt.Sprintf("/coreos/coreos10-%s.iso.sha256", architecture) + } + return fmt.Sprintf("/coreos/coreos-%s.iso.sha256", architecture) +} + +// getCoreOsStreamFileName returns the stream metadata file path based on stream. +func getCoreOsStreamFileName(stream types.OSImageStream) string { + if stream == types.OSImageStreamRHCOS10 { + return "/coreos/coreos10-stream.json" + } + return "/coreos/coreos-stream.json" +} + // ExtractConfig is used to set up the retries for extracting the base ISO. type ExtractConfig struct { MaxTries uint @@ -46,8 +68,8 @@ type ExtractConfig struct { // ReleasePayload is the interface to use the oc command to the get image info. type ReleasePayload interface { - GetBaseIso(architecture string) (string, error) - GetBaseIsoVersion(architecture string) (string, error) + GetBaseIso(architecture string, osImageStream types.OSImageStream) (string, error) + GetBaseIsoVersion(architecture string, osImageStream types.OSImageStream) (string, error) ExtractFile(image string, filename string, architecture string) ([]string, error) } @@ -94,7 +116,7 @@ func (r *releasePayload) ExtractFile(image string, filename string, architecture } // Get the CoreOS ISO from the releaseImage. -func (r *releasePayload) GetBaseIso(architecture string) (string, error) { +func (r *releasePayload) GetBaseIso(architecture string, osImageStream types.OSImageStream) (string, error) { // Get the machine-os-images pullspec from the release and use that to get the CoreOS ISO image, err := r.getImageFromRelease(machineOsImageName, architecture) if err != nil { @@ -106,7 +128,7 @@ func (r *releasePayload) GetBaseIso(architecture string) (string, error) { return "", err } - filename := fmt.Sprintf(coreOsFileName, architecture) + filename := getCoreOsFileName(osImageStream, architecture) // Check if file is already cached cachedFile, err := cache.GetFileFromCache(path.Base(filename), cacheDir) if err != nil { @@ -114,7 +136,7 @@ func (r *releasePayload) GetBaseIso(architecture string) (string, error) { } if cachedFile != "" { logrus.Info("Verifying cached file") - valid, err := r.verifyCacheFile(image, cachedFile, architecture) + valid, err := r.verifyCacheFile(image, cachedFile, architecture, osImageStream) if err != nil { return "", err } @@ -133,14 +155,15 @@ func (r *releasePayload) GetBaseIso(architecture string) (string, error) { return path[0], err } -func (r *releasePayload) GetBaseIsoVersion(architecture string) (string, error) { - files, err := r.ExtractFile(machineOsImageName, coreOsStreamFileName, architecture) +func (r *releasePayload) GetBaseIsoVersion(architecture string, osImageStream types.OSImageStream) (string, error) { + streamFileName := getCoreOsStreamFileName(osImageStream) + files, err := r.ExtractFile(machineOsImageName, streamFileName, architecture) if err != nil { return "", err } if len(files) > 1 { - return "", fmt.Errorf("too many files found for %s", coreOsStreamFileName) + return "", fmt.Errorf("too many files found for %s", streamFileName) } rawData, err := os.ReadFile(files[0]) @@ -266,11 +289,11 @@ func (r *releasePayload) extractFileFromImage(image, file, cacheDir string, arch } // Get hash from rhcos.json. -func (r *releasePayload) getHashFromInstaller(architecture string) (bool, string) { +func (r *releasePayload) getHashFromInstaller(architecture string, osImageStream types.OSImageStream) (bool, string) { ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) defer cancel() - st, err := rhcos.FetchCoreOSBuild(ctx, rhcos.DefaultOSImageStream) + st, err := rhcos.FetchCoreOSBuild(ctx, osImageStream) if err != nil { return false, "" } @@ -298,7 +321,7 @@ func matchingHash(imageSha []byte, sha string) bool { } // Check if there is a different base ISO in the release payload. -func (r *releasePayload) verifyCacheFile(image, file, architecture string) (bool, error) { +func (r *releasePayload) verifyCacheFile(image, file, architecture string, osImageStream types.OSImageStream) (bool, error) { // Get hash of cached file f, err := os.Open(file) if err != nil { @@ -313,7 +336,7 @@ func (r *releasePayload) verifyCacheFile(image, file, architecture string) (bool fileSha := h.Sum(nil) // Check if the hash of cached file matches hash in rhcos.json - found, rhcosSha := r.getHashFromInstaller(architecture) + found, rhcosSha := r.getHashFromInstaller(architecture, osImageStream) if found && matchingHash(fileSha, rhcosSha) { logrus.Debug("Found matching hash in installer metadata") return true, nil @@ -327,7 +350,7 @@ func (r *releasePayload) verifyCacheFile(image, file, architecture string) (bool defer os.RemoveAll(tempDir) - shaFilename := fmt.Sprintf(coreOsSha256FileName, architecture) + shaFilename := getCoreOsSha256FileName(osImageStream, architecture) shaFile, err := r.extractFileFromImage(image, shaFilename, tempDir, architecture) if err != nil { logrus.Debug("Could not get SHA from payload for cache comparison") diff --git a/pkg/infrastructure/baremetal/bootstrap.go b/pkg/infrastructure/baremetal/bootstrap.go index ce2e6e0f79e..70a5f9ac561 100644 --- a/pkg/infrastructure/baremetal/bootstrap.go +++ b/pkg/infrastructure/baremetal/bootstrap.go @@ -17,7 +17,8 @@ import ( "github.com/openshift/assisted-image-service/pkg/isoeditor" "github.com/openshift/installer/pkg/asset/ignition" - "github.com/openshift/installer/pkg/asset/rhcos" + assetrhcos "github.com/openshift/installer/pkg/asset/rhcos" + "github.com/openshift/installer/pkg/rhcos" ) func newDomain(name string) libvirtxml.Domain { @@ -171,13 +172,14 @@ func createStoragePool(virConn *libvirt.Libvirt, config baremetalConfig) (libvir } func getLiveISO(config baremetalConfig, arch string) (string, error) { - fetcher := rhcos.NewBaseISOFetcher( - rhcos.NewReleasePayload( - rhcos.ExtractConfig{}, + // Use default OS image stream for IPI bare metal workflow + fetcher := assetrhcos.NewBaseISOFetcher( + assetrhcos.NewReleasePayload( + assetrhcos.ExtractConfig{}, config.ReleaseImagePullSpec, config.PullSecret, config.MirrorConfig, - )) + ), rhcos.DefaultOSImageStream) return fetcher.GetBaseISOFilename(context.Background(), arch) }