diff --git a/api/appbean/AppDetail.go b/api/appbean/AppDetail.go index a7ebb1dbe7..3c2913ce6a 100644 --- a/api/appbean/AppDetail.go +++ b/api/appbean/AppDetail.go @@ -60,6 +60,7 @@ type GitMaterial struct { GitRepoUrl string `json:"gitRepoUrl,notnull" validate:"required"` CheckoutPath string `json:"checkoutPath,notnull" validate:"required"` FetchSubmodules bool `json:"fetchSubmodules"` + CloningMode string `json:"cloningMode" validate:"omitempty,oneof=FULL SHALLOW"` } type DockerConfig struct { diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 2f0e6286c5..00bd2656b4 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -496,6 +496,7 @@ func (handler CoreAppRestHandlerImpl) buildAppGitMaterials(appId int) ([]*appBea GitRepoUrl: gitMaterial.Url, CheckoutPath: gitMaterial.CheckoutPath, FetchSubmodules: gitMaterial.FetchSubmodules, + CloningMode: gitMaterial.CloningMode, GitProviderUrl: gitRegistry.Url, }) } @@ -1312,6 +1313,7 @@ func (handler CoreAppRestHandlerImpl) createGitMaterials(appId int, gitMaterials GitProviderId: gitProvider.Id, CheckoutPath: material.CheckoutPath, FetchSubmodules: material.FetchSubmodules, + CloningMode: material.CloningMode, } createMaterialRequest.Material = append(createMaterialRequest.Material, gitMaterialRequest) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index c0907bc8c2..4405e8d23e 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -273,12 +273,14 @@ func (impl *AppCloneServiceImpl) CloneGitRepo(oldAppId, newAppId int, userId int gitMaterialsMap := make(map[int]int) for _, material := range originalApp.Material { gitMaterial := &bean.GitMaterial{ - Name: material.Name, - Url: material.Url, - Id: 0, - GitProviderId: material.GitProviderId, - CheckoutPath: material.CheckoutPath, - FilterPattern: material.FilterPattern, + Name: material.Name, + Url: material.Url, + Id: 0, + GitProviderId: material.GitProviderId, + CheckoutPath: material.CheckoutPath, + FetchSubmodules: material.FetchSubmodules, + CloningMode: material.CloningMode, + FilterPattern: material.FilterPattern, } createMaterial.Material = []*bean.GitMaterial{gitMaterial} // append(createMaterial.Material, gitMaterial) createMaterialres, err := impl.pipelineBuilder.CreateMaterialsForApp(createMaterial) diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 82b6c1e0fc..81a6df99e5 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -91,11 +91,23 @@ type GitMaterial struct { GitProviderId int `json:"gitProviderId,omitempty" validate:"gt=0"` CheckoutPath string `json:"checkoutPath" validate:"checkout-path-component"` FetchSubmodules bool `json:"fetchSubmodules"` + CloningMode string `json:"cloningMode" validate:"omitempty,oneof=FULL SHALLOW"` IsUsedInCiConfig bool `json:"isUsedInCiConfig"` FilterPattern []string `json:"filterPattern"` CreateBackup bool `json:"createBackup"` } +const ( + GitMaterialCloningModeFull = "FULL" + GitMaterialCloningModeShallow = "SHALLOW" +) + +func (m *GitMaterial) SetDefaultCloningMode() { + if m.CloningMode == "" { + m.CloningMode = GitMaterialCloningModeFull + } +} + // UpdateSanitisedGitRepoUrl will remove all trailing slashes , leading and trailing spaces from git repository url func (m *GitMaterial) UpdateSanitisedGitRepoUrl() { for strings.HasSuffix(m.Url, "/") { diff --git a/pkg/bean/git_material_test.go b/pkg/bean/git_material_test.go new file mode 100644 index 0000000000..6dbc240b63 --- /dev/null +++ b/pkg/bean/git_material_test.go @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2026. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bean + +import "testing" + +func TestGitMaterialSetDefaultCloningMode(t *testing.T) { + t.Run("defaults an omitted mode to full", func(t *testing.T) { + material := &GitMaterial{} + + material.SetDefaultCloningMode() + + if material.CloningMode != GitMaterialCloningModeFull { + t.Fatalf("expected %q, got %q", GitMaterialCloningModeFull, material.CloningMode) + } + }) + + t.Run("preserves an explicitly selected mode", func(t *testing.T) { + material := &GitMaterial{CloningMode: GitMaterialCloningModeShallow} + + material.SetDefaultCloningMode() + + if material.CloningMode != GitMaterialCloningModeShallow { + t.Fatalf("expected %q, got %q", GitMaterialCloningModeShallow, material.CloningMode) + } + }) +} diff --git a/pkg/build/git/gitMaterial/repository/MaterialRepository.go b/pkg/build/git/gitMaterial/repository/MaterialRepository.go index 21cfa4bb87..5e9ca5b7d0 100644 --- a/pkg/build/git/gitMaterial/repository/MaterialRepository.go +++ b/pkg/build/git/gitMaterial/repository/MaterialRepository.go @@ -34,6 +34,7 @@ type GitMaterial struct { Name string `sql:"name, omitempty"` CheckoutPath string `sql:"checkout_path, omitempty"` FetchSubmodules bool `sql:"fetch_submodules,notnull"` + CloningMode string `sql:"cloning_mode,notnull"` FilterPattern []string `sql:"filter_pattern"` sql.AuditLog App *app.App diff --git a/pkg/build/trigger/HandlerService.go b/pkg/build/trigger/HandlerService.go index a4b97ce611..fa40cde60b 100644 --- a/pkg/build/trigger/HandlerService.go +++ b/pkg/build/trigger/HandlerService.go @@ -1020,6 +1020,7 @@ func (impl *HandlerServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineCo MaterialName: ciMaterial.GitMaterial.Name, CheckoutPath: ciMaterial.GitMaterial.CheckoutPath, FetchSubmodules: ciMaterial.GitMaterial.FetchSubmodules, + CloningMode: ciMaterial.GitMaterial.CloningMode, CommitHash: commitHashForPipelineId.Commit, Author: commitHashForPipelineId.Author, SourceType: ciMaterial.Type, diff --git a/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go index 8ca77e9e66..10a3fd7128 100644 --- a/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go +++ b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go @@ -556,6 +556,7 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow MaterialName: gitMaterial.Name, CheckoutPath: gitMaterial.CheckoutPath, FetchSubmodules: gitMaterial.FetchSubmodules, + CloningMode: gitMaterial.CloningMode, SourceType: m.Type, SourceValue: m.Value, Type: string(m.Type), diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 82222aaffd..11a1b7b51b 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -121,6 +121,8 @@ type CiCdPipelineOrchestrator interface { GetWorkflowCacheConfig(appType helper.AppType, pipelineType string, pipelineWorkflowCacheConfig common2.WorkflowCacheConfigType) bean.WorkflowCacheConfig } +const gitMaterialCreationAdvisoryLockNamespace = 6344 + type CiCdPipelineOrchestratorImpl struct { appRepository app2.AppRepository logger *zap.SugaredLogger @@ -1365,7 +1367,7 @@ func (impl CiCdPipelineOrchestratorImpl) DeleteApp(appId int, userId int32) erro impl.logger.Debug("deleting materials in git_sensor") for _, m := range materials { - err = impl.updateRepositoryToGitSensor(m, "", false) + err = impl.updateRepositoryToGitSensor(m, false) if err != nil { impl.logger.Errorw("error in updating to git-sensor", "err", err) return err @@ -1426,35 +1428,60 @@ func (impl CiCdPipelineOrchestratorImpl) CreateMaterials(createMaterialRequest * return nil, err } defer tx.Rollback() + // Material creation commits before git-sensor imports the repository. Serializing requests per app makes + // retries safe while a large repository is still being imported by the original request. + _, err = tx.Exec("SELECT pg_advisory_xact_lock(?, ?)", gitMaterialCreationAdvisoryLockNamespace, createMaterialRequest.AppId) + if err != nil { + impl.logger.Errorw("error acquiring git material creation lock", "appId", createMaterialRequest.AppId, "err", err) + return nil, err + } existingMaterials, err := impl.materialRepository.FindByAppId(createMaterialRequest.AppId) if err != nil { impl.logger.Errorw("err", "err", err) return nil, err } checkoutPaths := make(map[int]string) + existingMaterialByIdentity := make(map[string]*repository6.GitMaterial, len(existingMaterials)) impl.logger.Debugw("existing materials", "material", existingMaterials) for _, material := range existingMaterials { checkoutPaths[material.Id] = material.CheckoutPath + existingMaterialByIdentity[gitMaterialIdentity(material.Url, material.GitProviderId, material.CheckoutPath)] = material } + var materialsToCreate []*bean.GitMaterial for i, material := range createMaterialRequest.Material { if material.CheckoutPath == "" { material.CheckoutPath = "./" } - checkoutPaths[i*-1] = material.CheckoutPath + material.UpdateSanitisedGitRepoUrl() + material.SetDefaultCloningMode() + if err = validateGitMaterialCloningMode(material.CloningMode); err != nil { + return nil, err + } + existingMaterial := existingMaterialByIdentity[gitMaterialIdentity(material.Url, material.GitProviderId, material.CheckoutPath)] + if existingMaterial != nil { + if !gitMaterialOptionsMatch(existingMaterial, material) { + return nil, fmt.Errorf("git material for checkout path %q already exists; use update material to change its options", material.CheckoutPath) + } + material.Id = existingMaterial.Id + material.Name = existingMaterial.Name + continue + } + checkoutPaths[(i+1)*-1] = material.CheckoutPath + materialsToCreate = append(materialsToCreate, material) } duplicatePathErr := impl.validateCheckoutPathsForMultiGit(checkoutPaths) if duplicatePathErr != nil { - impl.logger.Errorw("duplicate checkout paths", "err", err) + impl.logger.Errorw("duplicate checkout paths", "err", duplicatePathErr) return nil, duplicatePathErr } var materials []*bean.GitMaterial - for _, inputMaterial := range createMaterialRequest.Material { - inputMaterial.UpdateSanitisedGitRepoUrl() + for _, inputMaterial := range materialsToCreate { m, err := impl.createMaterial(tx, inputMaterial, createMaterialRequest.AppId, createMaterialRequest.UserId) - inputMaterial.Id = m.Id if err != nil { return nil, err } + inputMaterial.Id = m.Id + inputMaterial.Name = m.Name materials = append(materials, inputMaterial) } // moving transaction before addRepositoryToGitSensor as commiting transaction after addRepositoryToGitSensor was causing problems @@ -1464,7 +1491,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateMaterials(createMaterialRequest * impl.logger.Errorw("error in committing tx Create material", "err", err, "materials", materials) return nil, err } - err = impl.addRepositoryToGitSensor(materials, "") + err = impl.addRepositoryToGitSensor(materials) if err != nil { impl.logger.Errorw("error in updating to sensor", "err", err) return nil, err @@ -1490,8 +1517,7 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateMaterial(updateMaterialDTO *bean. return nil, err } - err = impl.updateRepositoryToGitSensor(updatedMaterial, "", - updateMaterialDTO.Material.CreateBackup) + err = impl.updateRepositoryToGitSensor(updatedMaterial, updateMaterialDTO.Material.CreateBackup) if err != nil { impl.logger.Errorw("error in updating to git-sensor", "err", err) return nil, err @@ -1499,8 +1525,7 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateMaterial(updateMaterialDTO *bean. return updateMaterialDTO, nil } -func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *repository6.GitMaterial, - cloningMode string, createBackup bool) error { +func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *repository6.GitMaterial, createBackup bool) error { sensorMaterial := &gitSensor.GitMaterial{ Name: material.Name, Url: material.Url, @@ -1510,7 +1535,7 @@ func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *r Deleted: !material.Active, FetchSubmodules: material.FetchSubmodules, FilterPattern: material.FilterPattern, - CloningMode: cloningMode, + CloningMode: material.CloningMode, CreateBackup: createBackup, } timeout := 10 * time.Minute @@ -1523,8 +1548,8 @@ func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *r return impl.GitSensorClient.UpdateRepo(ctx, sensorMaterial) } -func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*bean.GitMaterial, cloningMode string) error { - var sensorMaterials []*gitSensor.GitMaterial +func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*bean.GitMaterial) error { + sensorMaterialsByCloningMode := make(map[string][]*gitSensor.GitMaterial) for _, material := range materials { sensorMaterial := &gitSensor.GitMaterial{ Name: material.Name, @@ -1534,13 +1559,25 @@ func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*b Deleted: false, FetchSubmodules: material.FetchSubmodules, FilterPattern: material.FilterPattern, - CloningMode: cloningMode, + CloningMode: material.CloningMode, } - sensorMaterials = append(sensorMaterials, sensorMaterial) + sensorMaterialsByCloningMode[material.CloningMode] = append(sensorMaterialsByCloningMode[material.CloningMode], sensorMaterial) } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - return impl.GitSensorClient.AddRepo(ctx, sensorMaterials) + // git-sensor currently applies the first material's cloning mode to an AddRepo batch. Keep batches homogeneous + // so applications with multiple sources can configure FULL and SHALLOW independently. + for _, cloningMode := range []string{bean.GitMaterialCloningModeFull, bean.GitMaterialCloningModeShallow} { + sensorMaterials := sensorMaterialsByCloningMode[cloningMode] + if len(sensorMaterials) == 0 { + continue + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + err := impl.GitSensorClient.AddRepo(ctx, sensorMaterials) + cancel() + if err != nil { + return err + } + } + return nil } // FIXME: not thread safe @@ -1642,6 +1679,32 @@ func (impl CiCdPipelineOrchestratorImpl) validateCheckoutPathsForMultiGit(allPat return nil } +func gitMaterialIdentity(url string, gitProviderId int, checkoutPath string) string { + url = strings.TrimRight(strings.TrimSpace(url), "/") + if checkoutPath == "" { + checkoutPath = "./" + } + return fmt.Sprintf("%d\x00%s\x00%s", gitProviderId, url, checkoutPath) +} + +func gitMaterialOptionsMatch(existingMaterial *repository6.GitMaterial, requestedMaterial *bean.GitMaterial) bool { + existingCloningMode := existingMaterial.CloningMode + if existingCloningMode == "" { + existingCloningMode = bean.GitMaterialCloningModeFull + } + return existingMaterial.FetchSubmodules == requestedMaterial.FetchSubmodules && + existingCloningMode == requestedMaterial.CloningMode && + slices.Equal(existingMaterial.FilterPattern, requestedMaterial.FilterPattern) +} + +func validateGitMaterialCloningMode(cloningMode string) error { + if cloningMode != bean.GitMaterialCloningModeFull && cloningMode != bean.GitMaterialCloningModeShallow { + return fmt.Errorf("unsupported git material cloning mode %q; supported values are %s and %s", + cloningMode, bean.GitMaterialCloningModeFull, bean.GitMaterialCloningModeShallow) + } + return nil +} + func (impl CiCdPipelineOrchestratorImpl) updateMaterial(tx *pg.Tx, updateMaterialDTO *bean.UpdateMaterialDTO) (*repository6.GitMaterial, error) { existingMaterials, err := impl.materialRepository.FindByAppId(updateMaterialDTO.AppId) if err != nil { @@ -1662,6 +1725,13 @@ func (impl CiCdPipelineOrchestratorImpl) updateMaterial(tx *pg.Tx, updateMateria if currentMaterial == nil { return nil, errors.New("material to be updated does not exist") } + if updateMaterialDTO.Material.CloningMode == "" { + updateMaterialDTO.Material.CloningMode = currentMaterial.CloningMode + updateMaterialDTO.Material.SetDefaultCloningMode() + } + if err = validateGitMaterialCloningMode(updateMaterialDTO.Material.CloningMode); err != nil { + return nil, err + } if updateMaterialDTO.Material.CheckoutPath == "" { updateMaterialDTO.Material.CheckoutPath = "./" } @@ -1680,6 +1750,7 @@ func (impl CiCdPipelineOrchestratorImpl) updateMaterial(tx *pg.Tx, updateMateria currentMaterial.GitProviderId = updateMaterialDTO.Material.GitProviderId currentMaterial.CheckoutPath = updateMaterialDTO.Material.CheckoutPath currentMaterial.FetchSubmodules = updateMaterialDTO.Material.FetchSubmodules + currentMaterial.CloningMode = updateMaterialDTO.Material.CloningMode currentMaterial.FilterPattern = updateMaterialDTO.Material.FilterPattern currentMaterial.AuditLog = sql.AuditLog{UpdatedBy: updateMaterialDTO.UserId, CreatedBy: currentMaterial.CreatedBy, UpdatedOn: time.Now(), CreatedOn: currentMaterial.CreatedOn} @@ -1706,6 +1777,7 @@ func (impl CiCdPipelineOrchestratorImpl) createMaterial(tx *pg.Tx, inputMaterial Active: true, CheckoutPath: inputMaterial.CheckoutPath, FetchSubmodules: inputMaterial.FetchSubmodules, + CloningMode: inputMaterial.CloningMode, FilterPattern: inputMaterial.FilterPattern, AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, } diff --git a/pkg/pipeline/CiMaterialConfigService.go b/pkg/pipeline/CiMaterialConfigService.go index 8f38cee15b..2896440601 100644 --- a/pkg/pipeline/CiMaterialConfigService.go +++ b/pkg/pipeline/CiMaterialConfigService.go @@ -247,6 +247,7 @@ func (impl *CiMaterialConfigServiceImpl) GetMaterialsForAppId(appId int) []*bean GitProviderId: material.GitProviderId, CheckoutPath: material.CheckoutPath, FetchSubmodules: material.FetchSubmodules, + CloningMode: material.CloningMode, FilterPattern: material.FilterPattern, } //check if git material is deletable or not diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 231d3e40e1..c93c0341af 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -172,11 +172,14 @@ func (impl *PipelineBuilderImpl) getGitMaterialsForApp(appId int) ([]*bean.GitMa gitUrl = u.String() } gitMaterial := &bean.GitMaterial{ - Id: material.Id, - Url: gitUrl, - GitProviderId: material.GitProviderId, - Name: material.Name[strings.Index(material.Name, "-")+1:], - CheckoutPath: material.CheckoutPath, + Id: material.Id, + Url: gitUrl, + GitProviderId: material.GitProviderId, + Name: material.Name[strings.Index(material.Name, "-")+1:], + CheckoutPath: material.CheckoutPath, + FetchSubmodules: material.FetchSubmodules, + CloningMode: material.CloningMode, + FilterPattern: material.FilterPattern, } gitMaterials = append(gitMaterials, gitMaterial) } diff --git a/pkg/pipeline/bean/workFlowRequestBean.go b/pkg/pipeline/bean/workFlowRequestBean.go index 16e42db83a..154b015191 100644 --- a/pkg/pipeline/bean/workFlowRequestBean.go +++ b/pkg/pipeline/bean/workFlowRequestBean.go @@ -94,6 +94,7 @@ type CiProjectDetails struct { MaterialName string `json:"materialName"` CheckoutPath string `json:"checkoutPath"` FetchSubmodules bool `json:"fetchSubmodules"` + CloningMode string `json:"cloningMode"` CommitHash string `json:"commitHash"` GitTag string `json:"gitTag"` CommitTime string `json:"commitTime"` diff --git a/pkg/pipeline/git_material_cloning_test.go b/pkg/pipeline/git_material_cloning_test.go new file mode 100644 index 0000000000..62637d9bd5 --- /dev/null +++ b/pkg/pipeline/git_material_cloning_test.go @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2026. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pipeline + +import ( + "context" + "testing" + + "github.com/devtron-labs/devtron/client/gitSensor" + mock_gitSensor "github.com/devtron-labs/devtron/client/gitSensor/mocks" + "github.com/devtron-labs/devtron/pkg/bean" + gitMaterialRepository "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" + "github.com/golang/mock/gomock" +) + +func TestValidateGitMaterialCloningMode(t *testing.T) { + for _, cloningMode := range []string{bean.GitMaterialCloningModeFull, bean.GitMaterialCloningModeShallow} { + if err := validateGitMaterialCloningMode(cloningMode); err != nil { + t.Fatalf("expected cloning mode %q to be valid: %v", cloningMode, err) + } + } + if err := validateGitMaterialCloningMode("DEPTH_10"); err == nil { + t.Fatal("expected an unsupported cloning mode to fail validation") + } +} + +func TestGitMaterialIdentityNormalizesRetryValues(t *testing.T) { + first := gitMaterialIdentity(" https://github.com/devtron-labs/devtron.git/ ", 1, "") + second := gitMaterialIdentity("https://github.com/devtron-labs/devtron.git", 1, "./") + if first != second { + t.Fatalf("expected equivalent material identities, got %q and %q", first, second) + } +} + +func TestGitMaterialOptionsMatch(t *testing.T) { + existingMaterial := &gitMaterialRepository.GitMaterial{ + FetchSubmodules: true, + CloningMode: bean.GitMaterialCloningModeShallow, + FilterPattern: []string{"services/api/**"}, + } + requestedMaterial := &bean.GitMaterial{ + FetchSubmodules: true, + CloningMode: bean.GitMaterialCloningModeShallow, + FilterPattern: []string{"services/api/**"}, + } + if !gitMaterialOptionsMatch(existingMaterial, requestedMaterial) { + t.Fatal("expected identical retry options to match") + } + + requestedMaterial.CloningMode = bean.GitMaterialCloningModeFull + if gitMaterialOptionsMatch(existingMaterial, requestedMaterial) { + t.Fatal("expected a changed cloning mode not to match") + } +} + +func TestAddRepositoryToGitSensorGroupsMaterialsByCloningMode(t *testing.T) { + controller := gomock.NewController(t) + gitSensorClient := mock_gitSensor.NewMockClient(controller) + orchestrator := &CiCdPipelineOrchestratorImpl{GitSensorClient: gitSensorClient} + + gomock.InOrder( + gitSensorClient.EXPECT().AddRepo(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, materials []*gitSensor.GitMaterial) error { + assertMaterialsUseCloningMode(t, materials, bean.GitMaterialCloningModeFull) + return nil + }, + ), + gitSensorClient.EXPECT().AddRepo(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, materials []*gitSensor.GitMaterial) error { + assertMaterialsUseCloningMode(t, materials, bean.GitMaterialCloningModeShallow) + return nil + }, + ), + ) + + err := orchestrator.addRepositoryToGitSensor([]*bean.GitMaterial{ + {Id: 1, CloningMode: bean.GitMaterialCloningModeShallow}, + {Id: 2, CloningMode: bean.GitMaterialCloningModeFull}, + {Id: 3, CloningMode: bean.GitMaterialCloningModeShallow}, + }) + if err != nil { + t.Fatalf("unexpected add repository error: %v", err) + } +} + +func assertMaterialsUseCloningMode(t *testing.T, materials []*gitSensor.GitMaterial, cloningMode string) { + t.Helper() + if len(materials) == 0 { + t.Fatal("expected a non-empty git material batch") + } + for _, material := range materials { + if material.CloningMode != cloningMode { + t.Fatalf("expected a homogeneous %q batch, got material %d with %q", cloningMode, material.Id, material.CloningMode) + } + } +} diff --git a/pkg/pipeline/history/GitMaterialHistoryService.go b/pkg/pipeline/history/GitMaterialHistoryService.go index 3972d14cd2..36f330c85f 100644 --- a/pkg/pipeline/history/GitMaterialHistoryService.go +++ b/pkg/pipeline/history/GitMaterialHistoryService.go @@ -55,6 +55,7 @@ func (impl GitMaterialHistoryServiceImpl) CreateMaterialHistory(tx *pg.Tx, input Active: inputMaterial.Active, CheckoutPath: inputMaterial.CheckoutPath, FetchSubmodules: inputMaterial.FetchSubmodules, + CloningMode: inputMaterial.CloningMode, FilterPattern: inputMaterial.FilterPattern, AuditLog: sql.AuditLog{UpdatedBy: inputMaterial.UpdatedBy, CreatedBy: inputMaterial.CreatedBy, UpdatedOn: inputMaterial.UpdatedOn, CreatedOn: inputMaterial.CreatedOn}, } @@ -82,6 +83,7 @@ func (impl GitMaterialHistoryServiceImpl) CreateDeleteMaterialHistory(materials Name: material.Name, CheckoutPath: material.CheckoutPath, FetchSubmodules: material.FetchSubmodules, + CloningMode: material.CloningMode, FilterPattern: material.FilterPattern, AuditLog: sql.AuditLog{ CreatedOn: material.CreatedOn, diff --git a/pkg/pipeline/history/repository/GitMaterialHistoryRepository.go b/pkg/pipeline/history/repository/GitMaterialHistoryRepository.go index 6673788679..06860b113e 100644 --- a/pkg/pipeline/history/repository/GitMaterialHistoryRepository.go +++ b/pkg/pipeline/history/repository/GitMaterialHistoryRepository.go @@ -32,6 +32,7 @@ type GitMaterialHistory struct { Name string `sql:"name, omitempty"` CheckoutPath string `sql:"checkout_path, omitempty"` FetchSubmodules bool `sql:"fetch_submodules,notnull"` + CloningMode string `sql:"cloning_mode,notnull"` FilterPattern []string `sql:"filter_pattern"` sql.AuditLog } diff --git a/scripts/sql/36104700_git_material_cloning_mode.down.sql b/scripts/sql/36104700_git_material_cloning_mode.down.sql new file mode 100644 index 0000000000..39fcff2c5e --- /dev/null +++ b/scripts/sql/36104700_git_material_cloning_mode.down.sql @@ -0,0 +1,7 @@ +ALTER TABLE public.git_material_history + DROP CONSTRAINT IF EXISTS git_material_history_cloning_mode_check, + DROP COLUMN IF EXISTS cloning_mode; + +ALTER TABLE public.git_material + DROP CONSTRAINT IF EXISTS git_material_cloning_mode_check, + DROP COLUMN IF EXISTS cloning_mode; diff --git a/scripts/sql/36104700_git_material_cloning_mode.up.sql b/scripts/sql/36104700_git_material_cloning_mode.up.sql new file mode 100644 index 0000000000..9ef7fb2d77 --- /dev/null +++ b/scripts/sql/36104700_git_material_cloning_mode.up.sql @@ -0,0 +1,7 @@ +ALTER TABLE public.git_material + ADD COLUMN cloning_mode VARCHAR(16) NOT NULL DEFAULT 'FULL', + ADD CONSTRAINT git_material_cloning_mode_check CHECK (cloning_mode IN ('FULL', 'SHALLOW')); + +ALTER TABLE public.git_material_history + ADD COLUMN cloning_mode VARCHAR(16) NOT NULL DEFAULT 'FULL', + ADD CONSTRAINT git_material_history_cloning_mode_check CHECK (cloning_mode IN ('FULL', 'SHALLOW')); diff --git a/specs/application/get-app.yaml b/specs/application/get-app.yaml index 6b05f0311a..3bb8ab428f 100644 --- a/specs/application/get-app.yaml +++ b/specs/application/get-app.yaml @@ -339,6 +339,10 @@ components: fetchSubmodules: type: boolean description: whether to fetch submodules + cloningMode: + type: string + enum: [FULL, SHALLOW] + description: Git history cloning strategy ErrorResponse: required: - code @@ -401,10 +405,12 @@ components: gitRepoUrl: "https://github.com/user1/example-repo" checkoutPath: "./" fetchSubmodules: true + cloningMode: SHALLOW - gitProviderUrl: "https://gitlab.com" gitRepoUrl: "https://gitlab.com/user2/new-repo" checkoutPath: "./a" fetchSubmodules: false + cloningMode: FULL dockerConfig: dockerRegistry: "my-dockerhub" dockerRepository: "user/test" @@ -923,4 +929,4 @@ components: type: boolean nullable: true description: Whether infrastructure metrics are enabled - example: true \ No newline at end of file + example: true diff --git a/specs/application/labels.yaml b/specs/application/labels.yaml index ab96a62941..8c150778d4 100644 --- a/specs/application/labels.yaml +++ b/specs/application/labels.yaml @@ -327,6 +327,9 @@ components: type: string fetchSubmodules: type: boolean + cloningMode: + type: string + enum: [FULL, SHALLOW] isUsedInCiConfig: type: boolean @@ -1124,4 +1127,4 @@ tags: - name: Application Metadata description: Operations related to application metadata - name: Helm Application Metadata - description: Operations related to Helm application metadata \ No newline at end of file + description: Operations related to Helm application metadata diff --git a/specs/application/material-management.yaml b/specs/application/material-management.yaml index bb19243f81..0c37aa34e1 100644 --- a/specs/application/material-management.yaml +++ b/specs/application/material-management.yaml @@ -90,6 +90,12 @@ components: description: Whether to fetch git submodules example: false default: false + cloningMode: + type: string + enum: [FULL, SHALLOW] + description: Git history cloning strategy. SHALLOW minimizes data transferred for large repositories. + example: SHALLOW + default: FULL filterPattern: type: array items: @@ -151,6 +157,10 @@ components: fetchSubmodules: type: boolean description: Whether submodules are fetched + cloningMode: + type: string + enum: [FULL, SHALLOW] + description: Git history cloning strategy filterPattern: type: array items: @@ -198,6 +208,7 @@ paths: checkoutPath: "./" gitProviderId: 1 fetchSubmodules: false + cloningMode: SHALLOW filterPattern: ["*.yaml", "!test/*"] multiple_materials: summary: Multiple materials example @@ -208,11 +219,13 @@ paths: checkoutPath: "./frontend" gitProviderId: 1 fetchSubmodules: false + cloningMode: FULL filterPattern: ["src/**", "!src/test/**"] - url: "https://github.com/user/backend.git" checkoutPath: "./backend" gitProviderId: 1 fetchSubmodules: true + cloningMode: SHALLOW filterPattern: ["*.go", "!*_test.go"] responses: '200': @@ -279,6 +292,7 @@ paths: checkoutPath: "./src" gitProviderId: 1 fetchSubmodules: true + cloningMode: SHALLOW filterPattern: ["src/**", "!src/test/**"] responses: '200': diff --git a/specs/notifications/webhooks.yaml b/specs/notifications/webhooks.yaml index 07b94695e9..c24f00308a 100644 --- a/specs/notifications/webhooks.yaml +++ b/specs/notifications/webhooks.yaml @@ -738,6 +738,12 @@ components: description: Whether to fetch git submodules example: false default: false + cloningMode: + type: string + enum: [FULL, SHALLOW] + description: Git history cloning strategy used for the material + example: SHALLOW + default: FULL commitHash: type: string description: Git commit hash diff --git a/specs/pipeline/docker-build.yaml b/specs/pipeline/docker-build.yaml index 9fa7edee3e..40a9b989a0 100644 --- a/specs/pipeline/docker-build.yaml +++ b/specs/pipeline/docker-build.yaml @@ -369,6 +369,9 @@ components: type: string fetchSubmodules: type: boolean + cloningMode: + type: string + enum: [FULL, SHALLOW] DockerConfig: type: object properties: @@ -442,4 +445,4 @@ components: description: Error message userDetailedMessage: type: string - description: Detailed error message for user \ No newline at end of file + description: Detailed error message for user